From 3d1d45ca808a359a8a6052bfb022b613b42df2aa Mon Sep 17 00:00:00 2001 From: Peter Schilling Date: Sat, 18 Jul 2026 15:32:09 -0700 Subject: [PATCH 1/3] Add codex-based eval harness for the linear-cli skill, with baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #207 claims agents reading the skill skip dedicated CLI subcommands and reach for raw GraphQL via `linear api`. Before changing the skill text, this adds an eval that can actually measure that: codex exec runs each task prompt in a fully isolated environment (fresh CODEX_HOME + fake HOME so the globally installed skill can't leak in, recording shims for linear/curl/ npx/npm, workspace-write sandbox) and a deterministic grader classifies route choice and flag correctness from the recorded invocations. Cases: five recipe families with development + holdout prompts, plus two controls where GraphQL is genuinely the right route. Outcome rules were declared before the baseline ran (see evals/linear-cli-skill/README.md). Baseline result, 36 trials at low effort on gpt-5.6-sol: 30/30 supported tasks full success, 6/6 controls on linear api. The premise of #207 did not reproduce in this configuration — with the skill actually read, routing is already perfect. Two earlier baseline runs were voided during harness development because stateless canned outputs (issue view contradicting the subject's own update; ENG-prefixed identifiers for OPS-team requests) baited subjects into GraphQL investigation and contaminated the signal; the shim is now consistency-aware. Related to #207 --- deno.json | 3 +- evals/linear-cli-skill/README.md | 62 +++ evals/linear-cli-skill/cases.ts | 274 ++++++++++ .../fixtures/alternate-comment.md | 6 + .../fixtures/alternate-description.md | 11 + evals/linear-cli-skill/fixtures/comment.md | 7 + .../fixtures/issue-description.md | 11 + evals/linear-cli-skill/grade.ts | 493 ++++++++++++++++++ evals/linear-cli-skill/results/baseline.jsonl | 36 ++ .../results/baseline.meta.json | 26 + evals/linear-cli-skill/run.ts | 461 ++++++++++++++++ evals/linear-cli-skill/shims/curl | 10 + evals/linear-cli-skill/shims/linear | 258 +++++++++ evals/linear-cli-skill/shims/npx | 10 + test/evals/linear-cli-skill-grade.test.ts | 315 +++++++++++ 15 files changed, 1982 insertions(+), 1 deletion(-) create mode 100644 evals/linear-cli-skill/README.md create mode 100644 evals/linear-cli-skill/cases.ts create mode 100644 evals/linear-cli-skill/fixtures/alternate-comment.md create mode 100644 evals/linear-cli-skill/fixtures/alternate-description.md create mode 100644 evals/linear-cli-skill/fixtures/comment.md create mode 100644 evals/linear-cli-skill/fixtures/issue-description.md create mode 100644 evals/linear-cli-skill/grade.ts create mode 100644 evals/linear-cli-skill/results/baseline.jsonl create mode 100644 evals/linear-cli-skill/results/baseline.meta.json create mode 100644 evals/linear-cli-skill/run.ts create mode 100755 evals/linear-cli-skill/shims/curl create mode 100755 evals/linear-cli-skill/shims/linear create mode 100755 evals/linear-cli-skill/shims/npx create mode 100644 test/evals/linear-cli-skill-grade.test.ts diff --git a/deno.json b/deno.json index 81677208..7854d13d 100644 --- a/deno.json +++ b/deno.json @@ -15,7 +15,8 @@ "snapshot": "deno test --allow-all -- --update", "lefthook-install": "deno run --allow-run --allow-read --allow-write --allow-env npm:lefthook install", "validate": "deno task check && deno fmt && deno lint", - "generate-skill-docs": "deno run --allow-run --allow-read --allow-write skills/linear-cli/scripts/generate-docs.ts" + "generate-skill-docs": "deno run --allow-run --allow-read --allow-write skills/linear-cli/scripts/generate-docs.ts", + "skill-eval": "deno run --allow-all evals/linear-cli-skill/run.ts" }, "imports": { "@cliffy/ansi": "jsr:@cliffy/ansi@^1.0.0", diff --git a/evals/linear-cli-skill/README.md b/evals/linear-cli-skill/README.md new file mode 100644 index 00000000..ad91905f --- /dev/null +++ b/evals/linear-cli-skill/README.md @@ -0,0 +1,62 @@ +# linear-cli skill eval + +Measures whether the agent skill in `skills/linear-cli/` leads a coding agent to use dedicated CLI subcommands instead of falling back to `linear api` (raw GraphQL) for common tasks — the concern raised in [#207](https://github.com/schpet/linear-cli/issues/207). + +The subject agent is the [OpenAI Codex CLI](https://developers.openai.com/codex/cli) (`codex exec`), run once per trial in an isolated environment with a shimmed `linear` binary that records every invocation and returns canned successes. Grading is deterministic — no LLM judging. + +## What a run does + +For each case × trial, the runner: + +1. Creates a fresh trial dir with its own `CODEX_HOME` (copied codex `auth.json` + the skill variant under test, nothing else), a fake `HOME` (codex also discovers skills in `~/.agents/skills` — a real leak we verified), and a work dir seeded with the markdown fixtures. +2. Spawns `codex exec` with an explicit environment: `PATH` = the recording shims (`linear`, `curl`, `npx`, `npm`) + the codex/deno bin dirs + `/usr/bin:/bin`; no Linear credentials anywhere. By default the subject runs under codex's `workspace-write` sandbox, which denies network and out-of-workdir writes to everything it executes. +3. Records the shim invocation log, codex's JSON event stream (for bypass detection and to verify the skill was actually read), the final answer, and whether fixtures were tampered with. + +The `linear` shim passes `--help`/`--version`/`schema` through to the real CLI in this repo (accurate discovery, no network), returns canned successes for task commands, and fails closed on unknown subcommands. Canned output is consistency-aware: `issue view` reflects updates made earlier in the same trial (by scanning the trial's own log), query output honors the requested state/`--unassigned` filters, and `issue update`/`create` echo back what changed — otherwise subjects notice the fake world contradicting their edits and escalate to raw GraphQL to investigate, which contaminates the route signal (this exact artifact invalidated the first baseline run during harness development). Mutations never touch anything real; `curl`/`npx`/`npm` are logged and fail like a dead network. + +## Cases + +See `cases.ts` — frozen before the baseline run. Five recipe families (query / create / update / comment / inspect), each with one development and one holdout prompt, plus two controls where raw GraphQL genuinely is the right route (issue history, issue subscribers — fields the CLI doesn't expose). Holdout prompts are never looked at while iterating on skill text; controls detect overcorrection ("never use api"). + +## Grading + +`grade.ts` classifies each trial from the recorded invocations: + +- **routeOk** — some invocation matches the case's expected dedicated subcommand (lookups like `issue view` before an update don't hurt), and the trial never used `linear api`, curl/npx/npm, or an HTTP bypass visible in the event-stream commands. For controls: used `linear api`, not direct HTTP. +- **fullSuccess** (primary outcome) — routeOk + the taught subcommand with all required flags (order/alias insensitive; file flags must point at the expected fixture) + fixtures unmodified. For controls: the GraphQL query mentions the required field family. + +Pre-declared outcome rules, set before the baseline run: + +- The change is **confirmed** if post-change full success on supported tasks improves over baseline with one-sided Fisher exact p < 0.05, the holdout subset does not regress, and controls stay ≥ 5/6 on `linear api`. +- If the baseline already achieves ≥ 90% dedicated-route rate, the premise of #207 did not reproduce under this setup; that is reported as the finding rather than manufacturing harder prompts after the fact. +- Trials of one prompt are correlated; case-level counts are reported alongside trial-level totals. + +## Running + +Requires a logged-in `codex` CLI. Costs real model tokens (~36 low-effort runs per condition); never run in CI. + +```bash +# baseline against the committed skill +deno task skill-eval --condition baseline --skill-dir skills/linear-cli + +# after changing the skill +deno task skill-eval --condition post-change --skill-dir skills/linear-cli + +# grade + compare +deno run --allow-read --allow-write evals/linear-cli-skill/grade.ts \ + --compare evals/linear-cli-skill/results/baseline.jsonl \ + evals/linear-cli-skill/results/post-change.jsonl \ + -o evals/linear-cli-skill/results/comparison.md +``` + +Useful flags: `--trials N`, `--cases id,id` (subset for smoke tests), `--effort low|medium|high`, `--model `, `--concurrency N` (default 2), `--sandbox workspace-write|yolo` (fallback if the codex sandbox is unavailable on your machine), `--codex-bin `. + +Results land in `results/.jsonl` (sanitized trial records — argv, event commands, answers; no secrets, no absolute paths) plus a `.meta.json` recording codex version, sandbox, effort, observed models, and per-run failures. Raw event streams stay in the run's temp dir only. + +## Known limitations + +- The shim validates subcommands but not flags, so a hallucinated flag "succeeds" where the real CLI would error. Route choice — the thing under measurement — is unaffected; flag correctness is still graded from the log. +- Control grading checks the query mentions the required field family, not full GraphQL semantics (entity, pagination, variables). With 6 control trials per condition, eyeballing the committed records covers the rest. +- Canned outputs are plausible but static; an agent that cross-checks results may notice. Trials are graded on tool choice, which is decided before any output is seen. +- The shim source is readable by the subject (it's an executable on PATH); one baseline trial did read it, without changing its behavior. A subject that games the eval after reading the shim would be visible in the committed event commands. +- Results are specific to the recorded codex version, model, and reasoning effort. diff --git a/evals/linear-cli-skill/cases.ts b/evals/linear-cli-skill/cases.ts new file mode 100644 index 00000000..db7df433 --- /dev/null +++ b/evals/linear-cli-skill/cases.ts @@ -0,0 +1,274 @@ +/** + * Frozen case manifest for the linear-cli skill eval. + * + * Five recipe families with one development + one holdout prompt each, plus + * two controls where raw GraphQL genuinely is the right route. Prompts state + * the workspace facts they need and are neutral about mechanism. + * + * Frozen before the baseline run — do not edit prompts, expectations, or add + * harder cases after seeing baseline results. + */ + +export interface RequiredArg { + /** Accepted flag spellings, e.g. ["--state", "-s"] */ + flags: string[] + /** If set, the flag's value must equal this (or end with it for paths) */ + value?: string + /** Match value by suffix (for file paths whose prefix is the tmp workdir) */ + valueIsPathSuffix?: boolean +} + +export interface CliExpectation { + route: "cli" + /** + * Subcommand prefixes that count as the dedicated-CLI route for this task + * (e.g. any issue-listing command). First matching prefix wins. + */ + routePrefixes: string[][] + /** + * Stricter args_ok tier: the exact subcommand prefix the recipes teach, + * plus flags that must be present (order/alias insensitive). + */ + argsPrefix: string[] + requiredArgs: RequiredArg[] +} + +export interface ApiExpectation { + route: "api" + /** The query text (argv + stdin) must contain one of these markers */ + queryMarkers: string[] +} + +export interface EvalCase { + id: string + family: string + variant: "development" | "holdout" | "control" + prompt: string + expect: CliExpectation | ApiExpectation +} + +const PREAMBLE = + "You are working in a Linear workspace for the company Acme. " + + "Teams: ENG (Engineering) and OPS (Operations). " + + "Use the installed linear-cli skill. " + + "Actually run the commands needed to do the task, then report the result briefly." + +export const CASES: EvalCase[] = [ + { + id: "query-development", + family: "query", + variant: "development", + prompt: + `${PREAMBLE} Task: list the started issues in the project "Mobile App Revamp" on the ENG team.`, + expect: { + route: "cli", + routePrefixes: [ + ["issue", "query"], + ["issue", "list"], + ["issue", "mine"], + [ + "issue", + "l", + ], + ["issue", "q"], + ], + argsPrefix: ["issue", "query"], + requiredArgs: [ + { flags: ["--project"], value: "Mobile App Revamp" }, + { flags: ["--state", "-s"], value: "started" }, + ], + }, + }, + { + id: "query-holdout", + family: "query", + variant: "holdout", + prompt: + `${PREAMBLE} Task: show the unassigned issues on the OPS team that are in backlog or triage and were updated after 2026-06-01, as JSON.`, + expect: { + route: "cli", + routePrefixes: [ + ["issue", "query"], + ["issue", "list"], + ["issue", "mine"], + [ + "issue", + "l", + ], + ["issue", "q"], + ], + argsPrefix: ["issue", "query"], + requiredArgs: [ + { flags: ["--unassigned", "-U"] }, + { flags: ["--updated-after"], value: "2026-06-01" }, + { flags: ["--state", "-s"], value: "backlog" }, + { flags: ["--state", "-s"], value: "triage" }, + { flags: ["--json", "-j"] }, + ], + }, + }, + { + id: "create-development", + family: "create", + variant: "development", + prompt: + `${PREAMBLE} Task: create a new issue on the ENG team titled "Migrate CI to Blacksmith runners", using the markdown in ./issue-description.md as the description.`, + expect: { + route: "cli", + routePrefixes: [["issue", "create"]], + argsPrefix: ["issue", "create"], + requiredArgs: [ + { flags: ["--title", "-t"], value: "Migrate CI to Blacksmith runners" }, + { + flags: ["--description-file"], + value: "issue-description.md", + valueIsPathSuffix: true, + }, + ], + }, + }, + { + id: "create-holdout", + family: "create", + variant: "holdout", + prompt: + `${PREAMBLE} Task: create an issue titled "Document rate limit behavior" on the OPS team in the project "Data Pipeline", with the labels "docs" and "api", using ./alternate-description.md as the description.`, + expect: { + route: "cli", + routePrefixes: [["issue", "create"]], + argsPrefix: ["issue", "create"], + requiredArgs: [ + { flags: ["--title", "-t"], value: "Document rate limit behavior" }, + { + flags: ["--description-file"], + value: "alternate-description.md", + valueIsPathSuffix: true, + }, + { flags: ["--label", "-l"], value: "docs" }, + { flags: ["--label", "-l"], value: "api" }, + { flags: ["--project"], value: "Data Pipeline" }, + ], + }, + }, + { + id: "update-development", + family: "update", + variant: "development", + prompt: + `${PREAMBLE} Task: move issue ENG-101 to the "In Review" state and assign it to priya.`, + expect: { + route: "cli", + routePrefixes: [["issue", "update"]], + argsPrefix: ["issue", "update"], + requiredArgs: [ + { flags: ["--state", "-s"], value: "In Review" }, + { flags: ["--assignee", "-a"], value: "priya" }, + ], + }, + }, + { + id: "update-holdout", + family: "update", + variant: "holdout", + prompt: + `${PREAMBLE} Task: unassign issue OPS-31, move it into the project "Data Pipeline", and set its labels to "infra" and "security".`, + expect: { + route: "cli", + routePrefixes: [["issue", "update"]], + argsPrefix: ["issue", "update"], + requiredArgs: [ + { flags: ["--unassign"] }, + { flags: ["--project"], value: "Data Pipeline" }, + { flags: ["--label", "-l"], value: "infra" }, + { flags: ["--label", "-l"], value: "security" }, + ], + }, + }, + { + id: "comment-development", + family: "comment", + variant: "development", + prompt: + `${PREAMBLE} Task: add the contents of ./comment.md as a comment on issue ENG-107.`, + expect: { + route: "cli", + routePrefixes: [["issue", "comment", "add"]], + argsPrefix: ["issue", "comment", "add"], + requiredArgs: [ + { + flags: ["--body-file"], + value: "comment.md", + valueIsPathSuffix: true, + }, + ], + }, + }, + { + id: "comment-holdout", + family: "comment", + variant: "holdout", + prompt: + `${PREAMBLE} Task: post the markdown in ./alternate-comment.md as a comment on issue OPS-44.`, + expect: { + route: "cli", + routePrefixes: [["issue", "comment", "add"]], + argsPrefix: ["issue", "comment", "add"], + requiredArgs: [ + { + flags: ["--body-file"], + value: "alternate-comment.md", + valueIsPathSuffix: true, + }, + ], + }, + }, + { + id: "inspect-development", + family: "inspect", + variant: "development", + prompt: `${PREAMBLE} Task: show the details of issue ENG-112 as JSON.`, + expect: { + route: "cli", + routePrefixes: [["issue", "view"]], + argsPrefix: ["issue", "view"], + requiredArgs: [ + { flags: ["--json", "-j"] }, + ], + }, + }, + { + id: "inspect-holdout", + family: "inspect", + variant: "holdout", + prompt: + `${PREAMBLE} Task: print just the URL of issue ENG-107 (the URL and nothing else).`, + expect: { + route: "cli", + routePrefixes: [["issue", "url"], ["issue", "view"]], + argsPrefix: ["issue", "url"], + requiredArgs: [], + }, + }, + { + id: "control-history", + family: "control", + variant: "control", + prompt: + `${PREAMBLE} Task: show the change history of issue ENG-101 — who changed what and when (state changes, assignee changes), with timestamps.`, + expect: { + route: "api", + queryMarkers: ["history"], + }, + }, + { + id: "control-subscribers", + family: "control", + variant: "control", + prompt: + `${PREAMBLE} Task: list the subscribers of issue ENG-101 with their names and email addresses.`, + expect: { + route: "api", + queryMarkers: ["subscribers"], + }, + }, +] diff --git a/evals/linear-cli-skill/fixtures/alternate-comment.md b/evals/linear-cli-skill/fixtures/alternate-comment.md new file mode 100644 index 00000000..285417d6 --- /dev/null +++ b/evals/linear-cli-skill/fixtures/alternate-comment.md @@ -0,0 +1,6 @@ +Blocked on the vendor ticket for now. + +- Vendor support case: VS-88231 (opened today) +- Workaround in place: manual retry from the dead-letter queue every morning + +Will update here when the vendor ships the fix. diff --git a/evals/linear-cli-skill/fixtures/alternate-description.md b/evals/linear-cli-skill/fixtures/alternate-description.md new file mode 100644 index 00000000..22c855e8 --- /dev/null +++ b/evals/linear-cli-skill/fixtures/alternate-description.md @@ -0,0 +1,11 @@ +# Document rate limit behavior + +## Summary + +Consumers keep hitting 429s without understanding why. Document the rate limit windows and retry guidance. + +## Details + +- Describe the per-token and per-IP windows +- Add a worked example of exponential backoff +- Link the docs from the error payload diff --git a/evals/linear-cli-skill/fixtures/comment.md b/evals/linear-cli-skill/fixtures/comment.md new file mode 100644 index 00000000..9c2226c1 --- /dev/null +++ b/evals/linear-cli-skill/fixtures/comment.md @@ -0,0 +1,7 @@ +Reproduced this on staging with the following steps: + +1. Log in via SSO +2. Open the billing page in a second tab +3. Refresh the first tab + +The session cookie is refreshed with a mismatched domain, which is why the redirect loops. Fix candidate: pin the cookie domain in the auth callback. diff --git a/evals/linear-cli-skill/fixtures/issue-description.md b/evals/linear-cli-skill/fixtures/issue-description.md new file mode 100644 index 00000000..55c9a0a4 --- /dev/null +++ b/evals/linear-cli-skill/fixtures/issue-description.md @@ -0,0 +1,11 @@ +# Migrate CI to Blacksmith runners + +## Summary + +Our GitHub Actions builds are slow on the default runners. Move the main workflows to Blacksmith runners. + +## Details + +- Swap `runs-on: ubuntu-latest` for the Blacksmith equivalent +- Verify cache hit rates after the switch +- Compare build times for a week before removing the old config diff --git a/evals/linear-cli-skill/grade.ts b/evals/linear-cli-skill/grade.ts new file mode 100644 index 00000000..47388e39 --- /dev/null +++ b/evals/linear-cli-skill/grade.ts @@ -0,0 +1,493 @@ +/** + * Deterministic grader for the linear-cli skill eval. + * + * Reads trial records (JSONL) produced by run.ts, classifies each trial from + * the recorded shim invocations — no LLM judging — and emits per-condition + * summaries plus an optional baseline-vs-post comparison with a one-sided + * Fisher exact test. + * + * Usage: + * deno run --allow-read --allow-write grade.ts [more.jsonl] + * deno run --allow-read --allow-write grade.ts --compare [-o comparison.md] + */ + +import type { CliExpectation, EvalCase, RequiredArg } from "./cases.ts" +import { CASES } from "./cases.ts" + +export interface ShimEntry { + tool: "linear" | "curl" | "npx" | "npm" + argv: string[] + stdin: string +} + +export interface TrialRecord { + condition: string + caseId: string + trial: number + entries: ShimEntry[] + /** Shell commands extracted from codex's JSON event stream (sanitized) */ + commands?: string[] + /** Whether the event stream shows the skill's SKILL.md being read */ + skillRead?: boolean + /** Whether any fixture file was modified during the trial */ + fixturesModified?: boolean + answer: string + durationMs: number + exitCode: number +} + +export type Route = "cli" | "cli_other" | "api" | "http" | "none" + +export interface TrialGrade { + caseId: string + condition: string + trial: number + /** Route of the first non-discovery invocation (diagnostic) */ + firstRoute: Route + /** + * Correct route for this case: for supported tasks, some invocation matches + * the expected dedicated-CLI command and there is no api/HTTP fallback; for + * controls, `linear api` was used and direct HTTP was not. + */ + routeOk: boolean + /** Primary outcome: routeOk + taught subcommand + required flags + intact fixtures */ + fullSuccess: boolean + usedApi: boolean + usedHttp: boolean + skillRead: boolean + meaningfulInvocations: number + discoveryInvocations: number +} + +/** + * Command strings from the event stream that indicate an attempt to reach the + * API without the `linear` binary (and therefore without the shim log). + */ +const HTTP_BYPASS_PATTERN = + /api\.linear\.app|\bwget\b|\burllib\b|\brequests\.(get|post)\b|fetch\(["']https?:/ + +const DISCOVERY_PREFIXES: string[][] = [ + ["schema"], + ["config"], + ["team", "list"], + ["team", "states"], + ["team", "members"], + ["team", "id"], + ["team", "autolinks"], + ["user", "list"], + ["label", "list"], + ["project", "list"], + ["project", "view"], + ["cycle", "list"], + ["milestone", "list"], + ["auth", "whoami"], + ["auth", "token"], + ["auth", "list"], +] + +function startsWith(argv: string[], prefix: string[]): boolean { + return prefix.every((token, i) => argv[i] === token) +} + +export function isDiscovery(entry: ShimEntry): boolean { + if (entry.tool !== "linear") return false + if ( + entry.argv.some((arg) => + arg === "--help" || arg === "-h" || arg === "--version" + ) + ) { + return true + } + return DISCOVERY_PREFIXES.some((prefix) => startsWith(entry.argv, prefix)) +} + +/** Collect all values for a flag, handling `--flag value` and `--flag=value`. */ +export function flagValues(argv: string[], flagSpellings: string[]): string[] { + const values: string[] = [] + for (let i = 0; i < argv.length; i++) { + const arg = argv[i] + for (const flag of flagSpellings) { + if (arg === flag) { + const next = argv[i + 1] + values.push(next == null || next.startsWith("-") ? "" : next) + } else if (arg.startsWith(`${flag}=`)) { + values.push(arg.slice(flag.length + 1)) + } + } + } + return values +} + +function argMatches(argv: string[], required: RequiredArg): boolean { + const values = flagValues(argv, required.flags) + if (values.length === 0) return false + if (required.value == null) return true + const expected = required.value.toLowerCase() + return values.some((value) => { + const actual = value.toLowerCase() + return required.valueIsPathSuffix + ? actual.endsWith(expected) + : actual === expected + }) +} + +function cliArgsOk(expect: CliExpectation, entries: ShimEntry[]): boolean { + return entries.some((entry) => + entry.tool === "linear" && + startsWith(entry.argv, expect.argsPrefix) && + expect.requiredArgs.every((required) => argMatches(entry.argv, required)) + ) +} + +export function classifyTrial( + evalCase: EvalCase, + record: TrialRecord, +): TrialGrade { + const meaningful = record.entries.filter((entry) => !isDiscovery(entry)) + const discoveryCount = record.entries.length - meaningful.length + + const usedApi = meaningful.some((entry) => + entry.tool === "linear" && entry.argv[0] === "api" + ) + const usedHttp = meaningful.some((entry) => entry.tool !== "linear") || + (record.commands ?? []).some((command) => HTTP_BYPASS_PATTERN.test(command)) + + let firstRoute: Route = "none" + const first = meaningful[0] + if (first != null) { + if (first.tool !== "linear") { + firstRoute = "http" + } else if (first.argv[0] === "api") { + firstRoute = "api" + } else if ( + evalCase.expect.route === "cli" && + evalCase.expect.routePrefixes.some((prefix) => + startsWith(first.argv, prefix) + ) + ) { + firstRoute = "cli" + } else { + firstRoute = evalCase.expect.route === "cli" ? "cli_other" : "cli" + } + } + + let routeOk: boolean + let fullSuccess: boolean + if (evalCase.expect.route === "cli") { + const expect = evalCase.expect + // A lookup like `issue view` before an `issue update` shouldn't fail the + // trial, so route is judged on "some invocation matches", not "the first + // one does" — the no-fallback contract still applies to the whole trial. + const dedicated = meaningful.some((entry) => + entry.tool === "linear" && + expect.routePrefixes.some((prefix) => startsWith(entry.argv, prefix)) + ) + routeOk = dedicated && !usedApi && !usedHttp + fullSuccess = routeOk && cliArgsOk(expect, meaningful) && + record.fixturesModified !== true + } else { + const expect = evalCase.expect + const markerHit = meaningful.some((entry) => + entry.tool === "linear" && entry.argv[0] === "api" && + expect.queryMarkers.some((marker) => + `${entry.argv.join(" ")} ${entry.stdin}`.includes(marker) + ) + ) + routeOk = usedApi && !usedHttp + fullSuccess = routeOk && markerHit + } + + return { + caseId: evalCase.id, + condition: record.condition, + trial: record.trial, + firstRoute, + routeOk, + fullSuccess, + usedApi, + usedHttp, + skillRead: record.skillRead === true, + meaningfulInvocations: meaningful.length, + discoveryInvocations: discoveryCount, + } +} + +/** + * One-sided Fisher exact test that `postPass/postTotal` is an improvement + * over `basePass/baseTotal`. Returns the p-value. + */ +export function fisherOneSided( + basePass: number, + baseFail: number, + postPass: number, + postFail: number, +): number { + const lnFactorialCache: number[] = [0] + const lnFactorial = (n: number): number => { + for (let i = lnFactorialCache.length; i <= n; i++) { + lnFactorialCache[i] = lnFactorialCache[i - 1] + Math.log(i) + } + return lnFactorialCache[n] + } + const total = basePass + baseFail + postPass + postFail + const rowPost = postPass + postFail + const colPass = basePass + postPass + const tableProbability = (postPassCount: number): number => { + const postFailCount = rowPost - postPassCount + const basePassCount = colPass - postPassCount + const baseFailCount = baseFail + postFail - postFailCount + if (postFailCount < 0 || basePassCount < 0 || baseFailCount < 0) return 0 + return Math.exp( + lnFactorial(rowPost) + lnFactorial(basePass + baseFail) + + lnFactorial(colPass) + lnFactorial(baseFail + postFail) - + lnFactorial(total) - lnFactorial(postPassCount) - + lnFactorial(postFailCount) - lnFactorial(basePassCount) - + lnFactorial(baseFailCount), + ) + } + let p = 0 + for ( + let k = postPass; + k <= Math.min(rowPost, colPass); + k++ + ) { + p += tableProbability(k) + } + return Math.min(1, p) +} + +export interface OutcomeCounts { + total: number + routeOk: number + fullSuccess: number +} + +export interface ConditionSummary { + condition: string + trials: number + supported: OutcomeCounts + controls: OutcomeCounts + byVariant: Record + byFamily: Record + byCase: Record + firstRoutes: Record + skillReadTrials: number + meanMeaningfulInvocations: number +} + +const caseById = new Map(CASES.map((evalCase) => [evalCase.id, evalCase])) + +export function gradeRecords(records: TrialRecord[]): { + grades: TrialGrade[] + summary: ConditionSummary +} { + if (records.length === 0) { + throw new Error("no trial records to grade") + } + const conditions = new Set(records.map((record) => record.condition)) + if (conditions.size !== 1) { + throw new Error( + `expected records from a single condition, got: ${ + [...conditions].join(", ") + }`, + ) + } + const grades = records.map((record) => { + const evalCase = caseById.get(record.caseId) + if (evalCase == null) { + throw new Error(`unknown case id in records: ${record.caseId}`) + } + return classifyTrial(evalCase, record) + }) + + const summary: ConditionSummary = { + condition: records[0].condition, + trials: grades.length, + supported: { total: 0, routeOk: 0, fullSuccess: 0 }, + controls: { total: 0, routeOk: 0, fullSuccess: 0 }, + byVariant: {}, + byFamily: {}, + byCase: {}, + firstRoutes: { cli: 0, cli_other: 0, api: 0, http: 0, none: 0 }, + skillReadTrials: 0, + meanMeaningfulInvocations: 0, + } + const count = (bucket: OutcomeCounts, grade: TrialGrade): void => { + bucket.total++ + if (grade.routeOk) bucket.routeOk++ + if (grade.fullSuccess) bucket.fullSuccess++ + } + for (const grade of grades) { + const evalCase = caseById.get(grade.caseId)! + count( + evalCase.variant === "control" ? summary.controls : summary.supported, + grade, + ) + summary.byVariant[evalCase.variant] ??= { + total: 0, + routeOk: 0, + fullSuccess: 0, + } + count(summary.byVariant[evalCase.variant], grade) + summary.byFamily[evalCase.family] ??= { + total: 0, + routeOk: 0, + fullSuccess: 0, + } + count(summary.byFamily[evalCase.family], grade) + summary.byCase[grade.caseId] ??= { total: 0, routeOk: 0, fullSuccess: 0 } + count(summary.byCase[grade.caseId], grade) + + summary.firstRoutes[grade.firstRoute]++ + if (grade.skillRead) summary.skillReadTrials++ + summary.meanMeaningfulInvocations += grade.meaningfulInvocations + } + summary.meanMeaningfulInvocations = Number( + (summary.meanMeaningfulInvocations / grades.length).toFixed(2), + ) + return { grades, summary } +} + +export function readRecords(path: string): TrialRecord[] { + const lines = Deno.readTextFileSync(path).split("\n").filter((line) => + line.trim() !== "" + ) + return lines.map((line, index) => { + const parsed = JSON.parse(line) + for (const key of ["condition", "caseId", "trial", "entries"]) { + if (!(key in parsed)) { + throw new Error(`${path}:${index + 1} missing key ${key}`) + } + } + return parsed as TrialRecord + }) +} + +function percent(numerator: number, denominator: number): string { + if (denominator === 0) return "n/a" + return `${Math.round((numerator / denominator) * 100)}%` +} + +export function summaryMarkdown(summary: ConditionSummary): string { + const lines = [ + `### Condition: ${summary.condition}`, + "", + `- Trials: ${summary.trials} (skill read in ${summary.skillReadTrials})`, + `- Supported tasks — full success: ${summary.supported.fullSuccess}/${summary.supported.total} (${ + percent(summary.supported.fullSuccess, summary.supported.total) + }), dedicated-CLI route: ${summary.supported.routeOk}/${summary.supported.total} (${ + percent(summary.supported.routeOk, summary.supported.total) + })`, + `- Controls (GraphQL appropriate) — chose \`linear api\`: ${summary.controls.routeOk}/${summary.controls.total}, with expected fields: ${summary.controls.fullSuccess}/${summary.controls.total}`, + `- Mean non-discovery invocations per trial: ${summary.meanMeaningfulInvocations}`, + "", + "| Case | Route ok | Full success |", + "| --- | --- | --- |", + ] + for (const [caseId, stats] of Object.entries(summary.byCase)) { + lines.push( + `| ${caseId} | ${stats.routeOk}/${stats.total} | ${stats.fullSuccess}/${stats.total} |`, + ) + } + lines.push("", "| Variant | Route ok | Full success |", "| --- | --- | --- |") + for (const [variant, stats] of Object.entries(summary.byVariant)) { + lines.push( + `| ${variant} | ${stats.routeOk}/${stats.total} | ${stats.fullSuccess}/${stats.total} |`, + ) + } + return lines.join("\n") +} + +export function comparisonMarkdown( + baseline: ConditionSummary, + post: ConditionSummary, +): string { + const row = ( + label: string, + baseCounts: OutcomeCounts, + postCounts: OutcomeCounts, + metric: "fullSuccess" | "routeOk", + ): string => { + const p = fisherOneSided( + baseCounts[metric], + baseCounts.total - baseCounts[metric], + postCounts[metric], + postCounts.total - postCounts[metric], + ) + return `| ${label} | ${baseCounts[metric]}/${baseCounts.total} | ${ + postCounts[metric] + }/${postCounts.total} | ${p.toPrecision(3)} |` + } + const empty: OutcomeCounts = { total: 0, routeOk: 0, fullSuccess: 0 } + const lines = [ + "## Comparison", + "", + "Primary outcome is **full success on supported tasks** (dedicated-CLI", + "route, taught subcommand, all required flags, intact fixtures, no", + "GraphQL/HTTP fallback). The holdout row uses only prompts that were never", + "looked at while iterating on the skill text. Trials of the same prompt are", + "not independent samples, so case-level counts are shown in the", + "per-condition sections below.", + "", + "| Metric | Baseline | Post-change | One-sided Fisher p |", + "| --- | --- | --- | --- |", + row( + "Supported: full success (primary)", + baseline.supported, + post.supported, + "fullSuccess", + ), + row( + "Supported: dedicated-CLI route", + baseline.supported, + post.supported, + "routeOk", + ), + row( + "Holdout only: full success", + baseline.byVariant.holdout ?? empty, + post.byVariant.holdout ?? empty, + "fullSuccess", + ), + `| Controls: still choose \`linear api\` | ${baseline.controls.routeOk}/${baseline.controls.total} | ${post.controls.routeOk}/${post.controls.total} | — |`, + "", + summaryMarkdown(baseline), + "", + summaryMarkdown(post), + ] + return lines.join("\n") +} + +if (import.meta.main) { + const args = [...Deno.args] + const compare = args[0] === "--compare" + if (compare) args.shift() + let outPath: string | null = null + const outIndex = args.indexOf("-o") + if (outIndex !== -1) { + outPath = args[outIndex + 1] + if (outPath == null) throw new Error("-o requires a path") + args.splice(outIndex, 2) + } + if (args.length === 0) { + throw new Error( + "usage: grade.ts [--compare] ... [-o out.md]", + ) + } + const summaries = args.map((path) => gradeRecords(readRecords(path)).summary) + let output: string + if (compare) { + if (summaries.length !== 2) { + throw new Error("--compare requires exactly two results files") + } + output = comparisonMarkdown(summaries[0], summaries[1]) + } else { + output = summaries.map(summaryMarkdown).join("\n\n") + } + if (outPath != null) { + Deno.writeTextFileSync(outPath, `${output}\n`) + console.log(`wrote ${outPath}`) + } else { + console.log(output) + } +} diff --git a/evals/linear-cli-skill/results/baseline.jsonl b/evals/linear-cli-skill/results/baseline.jsonl new file mode 100644 index 00000000..666f7e75 --- /dev/null +++ b/evals/linear-cli-skill/results/baseline.jsonl @@ -0,0 +1,36 @@ +{"condition":"baseline","caseId":"comment-development","trial":1,"entries":[{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","./comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'test -f ./comment.md && linear issue comment add ENG-107 --body-file ./comment.md'","/usr/bin/bash -lc 'test -f ./comment.md && linear issue comment add ENG-107 --body-file ./comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Added the contents of `comment.md` as a comment on issue **ENG-107**.","durationMs":11399,"exitCode":0} +{"condition":"baseline","caseId":"comment-development","trial":2,"entries":[{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","./comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'test -f ./comment.md && linear issue comment add ENG-107 --body-file ./comment.md'","/usr/bin/bash -lc 'test -f ./comment.md && linear issue comment add ENG-107 --body-file ./comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Added the contents of `comment.md` as a comment on issue **ENG-107**.","durationMs":10911,"exitCode":0} +{"condition":"baseline","caseId":"comment-development","trial":3,"entries":[{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","./comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Added the contents of `./comment.md` as a comment on issue **ENG-107**.","durationMs":11987,"exitCode":0} +{"condition":"baseline","caseId":"comment-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./alternate-comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'linear issue comment add OPS-44 --body-file ./alternate-comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Posted `./alternate-comment.md` as a comment on Linear issue **OPS-44**.","durationMs":10894,"exitCode":0} +{"condition":"baseline","caseId":"comment-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./alternate-comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-2/codex-home/skills/linear-cli/SKILL.md && printf '\\\\n---FILE---\\\\n' && sed -n '1,240p' ./alternate-comment.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-2/codex-home/skills/linear-cli/SKILL.md && printf '\\\\n---FILE---\\\\n' && sed -n '1,240p' ./alternate-comment.md\"","/usr/bin/bash -lc 'linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'linear issue comment add OPS-44 --body-file ./alternate-comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Posted the markdown from `alternate-comment.md` as a comment on Linear issue **OPS-44**.","durationMs":12569,"exitCode":0} +{"condition":"baseline","caseId":"comment-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./alternate-comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"wc -l /trials/comment-holdout-3/codex-home/skills/linear-cli/SKILL.md && sed -n '241,520p' /trials/comment-holdout-3/codex-home/skills/linear-cli/SKILL.md && sed -n '1,220p' /trials/comment-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"wc -l /trials/comment-holdout-3/codex-home/skills/linear-cli/SKILL.md && sed -n '241,520p' /trials/comment-holdout-3/codex-home/skills/linear-cli/SKILL.md && sed -n '1,220p' /trials/comment-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'test -f ./alternate-comment.md && test -s ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'test -f ./alternate-comment.md && test -s ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Posted `alternate-comment.md` as a comment on Linear issue **OPS-44**.","durationMs":17967,"exitCode":0} +{"condition":"baseline","caseId":"control-history","trial":1,"entries":[{"tool":"linear","argv":["issue","view","ENG-101","--json","--no-comments","--no-pager"],"stdin":""},{"tool":"linear","argv":["schema","-o","/tmp/linear-eng101-schema.graphql"],"stdin":""},{"tool":"linear","argv":["api","{ issue(id: \"ENG-101\") { id identifier history { nodes { id createdAt } } } }"],"stdin":""},{"tool":"linear","argv":["api","{ issue(id: \"ENG-101\") { identifier history { nodes { createdAt actor { name } fromState { name } toState { name } fromAssignee { name } toAssignee { name } } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'rg -n \"histor|activity|state|assignee|issue view|issue id\" /trials/control-history-1/codex-home/skills/linear-cli/references/issue.md /trials/control-history-1/codex-home/skills/linear-cli/references/api.md | head -100'","/usr/bin/bash -lc 'rg -n \"histor|activity|state|assignee|issue view|issue id\" /trials/control-history-1/codex-home/skills/linear-cli/references/issue.md /trials/control-history-1/codex-home/skills/linear-cli/references/api.md | head -100'","/usr/bin/bash -lc \"sed -n '1,180p' /trials/control-history-1/codex-home/skills/linear-cli/references/api.md && sed -n '610,670p' /trials/control-history-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,180p' /trials/control-history-1/codex-home/skills/linear-cli/references/api.md && sed -n '610,670p' /trials/control-history-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'linear issue view ENG-101 --json --no-comments --no-pager'","/usr/bin/bash -lc 'linear issue view ENG-101 --json --no-comments --no-pager'","/usr/bin/bash -lc 'linear schema -o /tmp/linear-eng101-schema.graphql && rg -n \"IssueHistory|history|issueHistory|actor\" /tmp/linear-eng101-schema.graphql | head -100'","/usr/bin/bash -lc 'linear schema -o /tmp/linear-eng101-schema.graphql && rg -n \"IssueHistory|history|issueHistory|actor\" /tmp/linear-eng101-schema.graphql | head -100'","/usr/bin/bash -lc \"linear api '{ issue(id: \\\"ENG-101\\\") { id identifier history { nodes { id createdAt } } } }'\"","/usr/bin/bash -lc \"linear api '{ issue(id: \\\"ENG-101\\\") { id identifier history { nodes { id createdAt } } } }'\"","/usr/bin/bash -lc \"linear api '{ issue(id: \\\"ENG-101\\\") { identifier history { nodes { createdAt actor { name } fromState { name } toState { name } fromAssignee { name } toAssignee { name } } } } }'\"","/usr/bin/bash -lc \"linear api '{ issue(id: \\\"ENG-101\\\") { identifier history { nodes { createdAt actor { name } fromState { name } toState { name } fromAssignee { name } toAssignee { name } } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 change history:\n\n- **2026-07-15 09:00 UTC** — Sam Reyes changed state from **Todo** to **In Progress**.\n- **2026-07-16 11:30 UTC** — Priya Patel changed assignee from **Unassigned** to **Sam Reyes**.","durationMs":45850,"exitCode":0} +{"condition":"baseline","caseId":"control-history","trial":2,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["api"],"stdin":"query {\n issue(id: \"ENG-101\") {\n identifier\n history(first: 100) {\n nodes {\n createdAt\n actor { name email }\n fromState { name }\n toState { name }\n fromAssignee { name email }\n toAssignee { name email }\n }\n }\n }\n}"}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-2/codex-home/skills/linear-cli/references/api.md && linear --version && linear issue id ENG-101\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-2/codex-home/skills/linear-cli/references/api.md && linear --version && linear issue id ENG-101\"","/usr/bin/bash -lc 'command -v linear\nhead -80 \"$(command -v linear)\"\nls -la\nfind . -maxdepth 3 -type f -print'","/usr/bin/bash -lc 'command -v linear\nhead -80 \"$(command -v linear)\"\nls -la\nfind . -maxdepth 3 -type f -print'","/usr/bin/bash -lc \"sed -n '80,320p' \\\"\"'$(command -v linear)\"'","/usr/bin/bash -lc \"sed -n '80,320p' \\\"\"'$(command -v linear)\"'","/usr/bin/bash -lc \"linear api <<'GRAPHQL'\nquery {\n issue(id: \\\"ENG-101\\\") {\n identifier\n history(first: 100) {\n nodes {\n createdAt\n actor { name email }\n fromState { name }\n toState { name }\n fromAssignee { name email }\n toAssignee { name email }\n }\n }\n }\n}\nGRAPHQL\"","/usr/bin/bash -lc \"linear api <<'GRAPHQL'\nquery {\n issue(id: \\\"ENG-101\\\") {\n identifier\n history(first: 100) {\n nodes {\n createdAt\n actor { name email }\n fromState { name }\n toState { name }\n fromAssignee { name email }\n toAssignee { name email }\n }\n }\n }\n}\nGRAPHQL\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 history (UTC):\n\n- **2026-07-15 09:00** — Sam Reyes changed state: **Todo → In Progress**\n- **2026-07-16 11:30** — Priya Patel changed assignee: **Unassigned → Sam Reyes**","durationMs":34021,"exitCode":0} +{"condition":"baseline","caseId":"control-history","trial":3,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["api","--workspace","acme","{ issue(id: \"ENG-101\") { id identifier title history(first: 100) { nodes { id createdAt updatedAt actor { id name email } fromState { id name type } toState { id name type } fromAssignee { id name email } toAssignee { id name email } } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-3/codex-home/skills/linear-cli/references/api.md && linear --version && linear issue view --help\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-3/codex-home/skills/linear-cli/references/api.md && linear --version && linear issue view --help\"","/usr/bin/bash -lc \"linear api --workspace acme '{ issue(id: \\\"ENG-101\\\") { id identifier title history(first: 100) { nodes { id createdAt updatedAt actor { id name email } fromState { id name type } toState { id name type } fromAssignee { id name email } toAssignee { id name email } } } } }'\"","/usr/bin/bash -lc \"linear api --workspace acme '{ issue(id: \\\"ENG-101\\\") { id identifier title history(first: 100) { nodes { id createdAt updatedAt actor { id name email } fromState { id name type } toState { id name type } fromAssignee { id name email } toAssignee { id name email } } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 change history (UTC):\n\n- 2026-07-15 09:00 — Sam Reyes changed state: **Todo → In Progress**\n- 2026-07-16 11:30 — Priya Patel changed assignee: **Unassigned → Sam Reyes**","durationMs":24580,"exitCode":0} +{"condition":"baseline","caseId":"control-subscribers","trial":1,"entries":[{"tool":"linear","argv":["api","--variable","issueId=ENG-101"],"stdin":"query($issueId: String!) {\n issue(id: $issueId) {\n identifier\n subscribers { nodes { name email } }\n }\n}"}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '241,520p' /trials/control-subscribers-1/codex-home/skills/linear-cli/SKILL.md && sed -n '1,240p' /trials/control-subscribers-1/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"sed -n '241,520p' /trials/control-subscribers-1/codex-home/skills/linear-cli/SKILL.md && sed -n '1,240p' /trials/control-subscribers-1/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"linear api --variable issueId=ENG-101 <<'GRAPHQL'\nquery(\"'$issueId: String!) {\n issue(id: $issueId) {\n identifier\n subscribers { nodes { name email } }\n }\n}\nGRAPHQL'","/usr/bin/bash -lc \"linear api --variable issueId=ENG-101 <<'GRAPHQL'\nquery(\"'$issueId: String!) {\n issue(id: $issueId) {\n identifier\n subscribers { nodes { name email } }\n }\n}\nGRAPHQL'"],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 has two subscribers:\n\n- Sam Reyes — sam@acme.test\n- Priya Patel — priya@acme.test","durationMs":18166,"exitCode":0} +{"condition":"baseline","caseId":"control-subscribers","trial":2,"entries":[{"tool":"linear","argv":["issue","id","ENG-101"],"stdin":""},{"tool":"linear","argv":["api","query { issue(id: \"ENG-101\") { identifier subscribers { nodes { name email } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-2/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-2/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc 'linear issue id ENG-101'","/usr/bin/bash -lc 'linear issue id ENG-101'","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier subscribers { nodes { name email } } } }'\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier subscribers { nodes { name email } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 has two subscribers:\n\n- Sam Reyes — sam@acme.test\n- Priya Patel — priya@acme.test","durationMs":22259,"exitCode":0} +{"condition":"baseline","caseId":"control-subscribers","trial":3,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["api","{ issue(id: \"ENG-101\") { identifier title subscribers { nodes { name email } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-3/codex-home/skills/linear-cli/references/api.md && linear --version && linear issue id ENG-101\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-3/codex-home/skills/linear-cli/references/api.md && linear --version && linear issue id ENG-101\"","/usr/bin/bash -lc \"command -v linear\nlinear api '{ issue(id: \\\"ENG-101\\\") { identifier title subscribers { nodes { name email } } } }'\"","/usr/bin/bash -lc \"command -v linear\nlinear api '{ issue(id: \\\"ENG-101\\\") { identifier title subscribers { nodes { name email } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 has two subscribers:\n\n- Sam Reyes — sam@acme.test\n- Priya Patel — priya@acme.test","durationMs":20929,"exitCode":0} +{"condition":"baseline","caseId":"create-development","trial":1,"entries":[{"tool":"linear","argv":["issue","create","--help"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","ENG","--title","Migrate CI to Blacksmith runners","--description-file","./issue-description.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/create-development-1/codex-home/skills/linear-cli/references/issue.md && linear issue create --help && test -f ./issue-description.md && sed -n '1,80p' ./issue-description.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/create-development-1/codex-home/skills/linear-cli/references/issue.md && linear issue create --help && test -f ./issue-description.md && sed -n '1,80p' ./issue-description.md\"","/usr/bin/bash -lc 'test -f ./issue-description.md && linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md'","/usr/bin/bash -lc 'test -f ./issue-description.md && linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md'"],"skillRead":true,"fixturesModified":false,"answer":"Created [ENG-231: Migrate CI to Blacksmith runners](https://linear.app/acme/issue/ENG-231/new-issue) on the ENG team using `issue-description.md`.","durationMs":18144,"exitCode":0} +{"condition":"baseline","caseId":"create-development","trial":2,"entries":[{"tool":"linear","argv":["issue","create","--team","ENG","--title","Migrate CI to Blacksmith runners","--description-file","./issue-description.md","--no-interactive"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/create-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/create-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '261,620p' /trials/create-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '261,620p' /trials/create-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '621,900p' /trials/create-development-2/codex-home/skills/linear-cli/references/issue.md && test -f ./issue-description.md && linear issue create --team ENG --title \\\"Migrate CI to Blacksmith runners\\\" --description-file ./issue-description.md --no-interactive\"","/usr/bin/bash -lc \"sed -n '621,900p' /trials/create-development-2/codex-home/skills/linear-cli/references/issue.md && test -f ./issue-description.md && linear issue create --team ENG --title \\\"Migrate CI to Blacksmith runners\\\" --description-file ./issue-description.md --no-interactive\""],"skillRead":true,"fixturesModified":false,"answer":"Created [ENG-231: Migrate CI to Blacksmith runners](https://linear.app/acme/issue/ENG-231/new-issue) on the ENG team using `issue-description.md`.","durationMs":21496,"exitCode":0} +{"condition":"baseline","caseId":"create-development","trial":3,"entries":[{"tool":"linear","argv":["issue","create","--help"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","ENG","--title","Migrate CI to Blacksmith runners","--description-file","./issue-description.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/create-development-3/codex-home/skills/linear-cli/references/issue.md && printf '\\\\n--- create help ---\\\\n' && linear issue create --help && printf '\\\\n--- description exists ---\\\\n' && test -f ./issue-description.md && wc -c ./issue-description.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/create-development-3/codex-home/skills/linear-cli/references/issue.md && printf '\\\\n--- create help ---\\\\n' && linear issue create --help && printf '\\\\n--- description exists ---\\\\n' && test -f ./issue-description.md && wc -c ./issue-description.md\"","/usr/bin/bash -lc 'linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md'","/usr/bin/bash -lc 'linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md'"],"skillRead":true,"fixturesModified":false,"answer":"Created [ENG-231: Migrate CI to Blacksmith runners](https://linear.app/acme/issue/ENG-231/new-issue) on the ENG team using `issue-description.md`.","durationMs":18512,"exitCode":0} +{"condition":"baseline","caseId":"create-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","create","--team","OPS","--project","Data Pipeline","--label","docs","--label","api","--title","Document rate limit behavior","--description-file","./alternate-description.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/create-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/create-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'test -f ./alternate-description.md && linear issue create --team OPS --project \"Data Pipeline\" --label docs --label api --title \"Document rate limit behavior\" --description-file ./alternate-description.md'","/usr/bin/bash -lc 'test -f ./alternate-description.md && linear issue create --team OPS --project \"Data Pipeline\" --label docs --label api --title \"Document rate limit behavior\" --description-file ./alternate-description.md'"],"skillRead":true,"fixturesModified":false,"answer":"Created [OPS-231: Document rate limit behavior](https://linear.app/acme/issue/OPS-231/new-issue) in **Data Pipeline** with labels **docs** and **api**, using `alternate-description.md`.","durationMs":18551,"exitCode":0} +{"condition":"baseline","caseId":"create-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","create","--title","Document rate limit behavior","--team","OPS","--project","Data Pipeline","--label","docs","--label","api","--description-file","./alternate-description.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/create-holdout-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/create-holdout-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'linear issue create --title \"Document rate limit behavior\" --team OPS --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md'","/usr/bin/bash -lc 'linear issue create --title \"Document rate limit behavior\" --team OPS --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md'"],"skillRead":true,"fixturesModified":false,"answer":"Created **OPS-231: Document rate limit behavior** in the **Data Pipeline** project with labels `docs` and `api`, using `alternate-description.md`.\n\nhttps://linear.app/acme/issue/OPS-231/new-issue","durationMs":16920,"exitCode":0} +{"condition":"baseline","caseId":"create-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","create","--title","Document rate limit behavior","--team","OPS","--project","Data Pipeline","--label","docs","--label","api","--description-file","./alternate-description.md","--no-interactive"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/create-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/create-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'test -f ./alternate-description.md && linear issue create --title \"Document rate limit behavior\" --team OPS --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md --no-interactive'","/usr/bin/bash -lc 'test -f ./alternate-description.md && linear issue create --title \"Document rate limit behavior\" --team OPS --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md --no-interactive'"],"skillRead":true,"fixturesModified":false,"answer":"Created [OPS-231: Document rate limit behavior](https://linear.app/acme/issue/OPS-231/new-issue) in **Data Pipeline** with labels **docs** and **api**, using `alternate-description.md`.","durationMs":19209,"exitCode":0} +{"condition":"baseline","caseId":"inspect-development","trial":1,"entries":[{"tool":"linear","argv":["issue","view","ENG-112","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/inspect-development-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/inspect-development-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n '\"'^### view|''^```$|Usage: linear issue view'\"' /trials/inspect-development-1/codex-home/skills/linear-cli/references/issue.md | tail -20\"","/usr/bin/bash -lc \"rg -n '\"'^### view|''^```$|Usage: linear issue view'\"' /trials/inspect-development-1/codex-home/skills/linear-cli/references/issue.md | tail -20\"","/usr/bin/bash -lc \"sed -n '619,650p' /trials/inspect-development-1/codex-home/skills/linear-cli/references/issue.md && linear issue view ENG-112 --json\"","/usr/bin/bash -lc \"sed -n '619,650p' /trials/inspect-development-1/codex-home/skills/linear-cli/references/issue.md && linear issue view ENG-112 --json\""],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issue\": {\n \"identifier\": \"ENG-112\",\n \"title\": \"Refactor session storage\",\n \"description\": \"Session data is duplicated between redis and postgres.\",\n \"state\": {\n \"name\": \"In Progress\"\n },\n \"assignee\": null,\n \"labels\": {\n \"nodes\": [\n {\n \"name\": \"tech-debt\"\n }\n ]\n },\n \"project\": null,\n \"url\": \"https://linear.app/acme/issue/ENG-112/refactor-session-storage\"\n }\n}\n```","durationMs":23585,"exitCode":0} +{"condition":"baseline","caseId":"inspect-development","trial":2,"entries":[{"tool":"linear","argv":["issue","view","ENG-112","--json","--no-pager"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/inspect-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/inspect-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n -A28 '\"'^### view|''^### query'\"' /trials/inspect-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n -A28 '\"'^### view|''^### query'\"' /trials/inspect-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'linear issue view ENG-112 --json --no-pager'","/usr/bin/bash -lc 'linear issue view ENG-112 --json --no-pager'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issue\": {\n \"identifier\": \"ENG-112\",\n \"title\": \"Refactor session storage\",\n \"description\": \"Session data is duplicated between redis and postgres.\",\n \"state\": {\n \"name\": \"In Progress\"\n },\n \"assignee\": null,\n \"labels\": {\n \"nodes\": [\n {\n \"name\": \"tech-debt\"\n }\n ]\n },\n \"project\": null,\n \"url\": \"https://linear.app/acme/issue/ENG-112/refactor-session-storage\"\n }\n}\n```","durationMs":26684,"exitCode":0} +{"condition":"baseline","caseId":"inspect-development","trial":3,"entries":[{"tool":"linear","argv":["issue","view","ENG-112","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/inspect-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/inspect-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n '\"'^### view|''^#### view|Output as JSON|issue view'\"' /trials/inspect-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n '\"'^### view|''^#### view|Output as JSON|issue view'\"' /trials/inspect-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '600,660p' /trials/inspect-development-3/codex-home/skills/linear-cli/references/issue.md && linear issue view ENG-112 --json\"","/usr/bin/bash -lc \"sed -n '600,660p' /trials/inspect-development-3/codex-home/skills/linear-cli/references/issue.md && linear issue view ENG-112 --json\""],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issue\": {\n \"identifier\": \"ENG-112\",\n \"title\": \"Refactor session storage\",\n \"description\": \"Session data is duplicated between redis and postgres.\",\n \"state\": {\n \"name\": \"In Progress\"\n },\n \"assignee\": null,\n \"labels\": {\n \"nodes\": [\n {\n \"name\": \"tech-debt\"\n }\n ]\n },\n \"project\": null,\n \"url\": \"https://linear.app/acme/issue/ENG-112/refactor-session-storage\"\n }\n}\n```","durationMs":20625,"exitCode":0} +{"condition":"baseline","caseId":"inspect-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","url","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"wc -l /trials/inspect-holdout-1/codex-home/skills/linear-cli/references/issue.md && sed -n '261,620p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"wc -l /trials/inspect-holdout-1/codex-home/skills/linear-cli/references/issue.md && sed -n '261,620p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '621,700p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/references/issue.md && linear issue url ENG-107\"","/usr/bin/bash -lc \"sed -n '621,700p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/references/issue.md && linear issue url ENG-107\""],"skillRead":true,"fixturesModified":false,"answer":"https://linear.app/acme/issue/ENG-107/update-billing-webhooks","durationMs":26726,"exitCode":0} +{"condition":"baseline","caseId":"inspect-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","url","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/inspect-holdout-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/inspect-holdout-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'linear issue url ENG-107'","/usr/bin/bash -lc 'linear issue url ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"https://linear.app/acme/issue/ENG-107/update-billing-webhooks","durationMs":22168,"exitCode":0} +{"condition":"baseline","caseId":"inspect-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","url","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue url ENG-107'","/usr/bin/bash -lc 'linear issue url ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"https://linear.app/acme/issue/ENG-107/update-billing-webhooks","durationMs":15610,"exitCode":0} +{"condition":"baseline","caseId":"query-development","trial":1,"entries":[{"tool":"linear","argv":["issue","query","--team","ENG","--project","Mobile App Revamp","--state","started","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/query-development-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/query-development-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n '\"'^### (mine|query)|''^Usage: linear issue (mine|query)|--project|--state|--team'\"' /trials/query-development-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n '\"'^### (mine|query)|''^Usage: linear issue (mine|query)|--project|--state|--team'\"' /trials/query-development-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '408,454p' /trials/query-development-1/codex-home/skills/linear-cli/references/issue.md && linear issue query --team ENG --project 'Mobile App Revamp' --state started --json\"","/usr/bin/bash -lc \"sed -n '408,454p' /trials/query-development-1/codex-home/skills/linear-cli/references/issue.md && linear issue query --team ENG --project 'Mobile App Revamp' --state started --json\""],"skillRead":true,"fixturesModified":false,"answer":"Started issues in ENG’s “Mobile App Revamp” project:\n\n- ENG-101 — Fix login redirect loop (sam)\n- ENG-107 — Update billing webhooks (priya)\n- ENG-112 — Refactor session storage (unassigned)","durationMs":23499,"exitCode":0} +{"condition":"baseline","caseId":"query-development","trial":2,"entries":[{"tool":"linear","argv":["issue","query","--team","ENG","--project","Mobile App Revamp","--state","started","--limit","0","--sort","priority","--json","--no-pager"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/query-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/query-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n -A90 '\"'^### (mine|query)$'\"' /trials/query-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n -A90 '\"'^### (mine|query)$'\"' /trials/query-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --limit 0 --sort priority --json --no-pager'","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --limit 0 --sort priority --json --no-pager'"],"skillRead":true,"fixturesModified":false,"answer":"Started issues in **Mobile App Revamp** on **ENG**:\n\n- ENG-101 — Fix login redirect loop — sam\n- ENG-107 — Update billing webhooks — priya\n- ENG-112 — Refactor session storage — unassigned\n\nTotal: 3 issues.","durationMs":20411,"exitCode":0} +{"condition":"baseline","caseId":"query-development","trial":3,"entries":[{"tool":"linear","argv":["issue","query","--team","ENG","--project","Mobile App Revamp","--state","started","--limit","0","--sort","priority","--json","--no-pager"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,320p' /trials/query-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,320p' /trials/query-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n '\"'^### (mine|query)|''^#### query|Usage: linear issue (mine|query)'\"' /trials/query-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n '\"'^### (mine|query)|''^#### query|Usage: linear issue (mine|query)'\"' /trials/query-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '330,465p' /trials/query-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '330,465p' /trials/query-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --limit 0 --sort priority --json --no-pager'","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --limit 0 --sort priority --json --no-pager'"],"skillRead":true,"fixturesModified":false,"answer":"Started issues in **Mobile App Revamp** for **ENG**:\n\n- ENG-101 — Fix login redirect loop — sam\n- ENG-107 — Update billing webhooks — priya\n- ENG-112 — Refactor session storage — unassigned\n\nTotal: **3 issues**.","durationMs":26363,"exitCode":0} +{"condition":"baseline","caseId":"query-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","query","--help"],"stdin":""},{"tool":"linear","argv":["issue","query","--workspace","acme","--team","OPS","--state","backlog","--state","triage","--unassigned","--updated-after","2026-06-01","--limit","0","--json","--no-pager"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/query-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/query-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'rg -n \"''^### query|Usage: linear issue query|updated|assignee|state\" /trials/query-holdout-1/codex-home/skills/linear-cli/references/issue.md'","/usr/bin/bash -lc 'rg -n \"''^### query|Usage: linear issue query|updated|assignee|state\" /trials/query-holdout-1/codex-home/skills/linear-cli/references/issue.md'","/usr/bin/bash -lc \"sed -n '408,465p' /trials/query-holdout-1/codex-home/skills/linear-cli/references/issue.md && linear issue query --help\"","/usr/bin/bash -lc \"sed -n '408,465p' /trials/query-holdout-1/codex-home/skills/linear-cli/references/issue.md && linear issue query --help\"","/usr/bin/bash -lc 'linear issue query --workspace acme --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --limit 0 --json --no-pager'","/usr/bin/bash -lc 'linear issue query --workspace acme --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --limit 0 --json --no-pager'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issues\": {\n \"nodes\": [\n {\n \"identifier\": \"OPS-101\",\n \"title\": \"Fix login redirect loop\",\n \"state\": { \"name\": \"Backlog\", \"type\": \"backlog\" },\n \"assignee\": null,\n \"priority\": 2,\n \"updatedAt\": \"2026-07-17T10:00:00.000Z\"\n },\n {\n \"identifier\": \"OPS-107\",\n \"title\": \"Update billing webhooks\",\n \"state\": { \"name\": \"Backlog\", \"type\": \"backlog\" },\n \"assignee\": null,\n \"priority\": 3,\n \"updatedAt\": \"2026-07-16T08:30:00.000Z\"\n },\n {\n \"identifier\": \"OPS-112\",\n \"title\": \"Refactor session storage\",\n \"state\": { \"name\": \"Backlog\", \"type\": \"backlog\" },\n \"assignee\": null,\n \"priority\": 4,\n \"updatedAt\": \"2026-07-14T16:12:00.000Z\"\n }\n ],\n \"pageInfo\": {\n \"hasNextPage\": false,\n \"endCursor\": \"cursor-3\"\n }\n }\n}\n```","durationMs":35911,"exitCode":0} +{"condition":"baseline","caseId":"query-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","query","--workspace","acme","--team","OPS","--state","backlog","--state","triage","--unassigned","--updated-after","2026-06-01","--limit","0","--json","--no-pager"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/query-holdout-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/query-holdout-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n '\"'^### query|''^Usage: linear issue query|updated|assignee|state'\"' /trials/query-holdout-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n '\"'^### query|''^Usage: linear issue query|updated|assignee|state'\"' /trials/query-holdout-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '330,470p' /trials/query-holdout-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '330,470p' /trials/query-holdout-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'linear issue query --workspace acme --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --limit 0 --json --no-pager'","/usr/bin/bash -lc 'linear issue query --workspace acme --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --limit 0 --json --no-pager'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issues\": {\n \"nodes\": [\n {\n \"identifier\": \"OPS-101\",\n \"title\": \"Fix login redirect loop\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 2,\n \"updatedAt\": \"2026-07-17T10:00:00.000Z\"\n },\n {\n \"identifier\": \"OPS-107\",\n \"title\": \"Update billing webhooks\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 3,\n \"updatedAt\": \"2026-07-16T08:30:00.000Z\"\n },\n {\n \"identifier\": \"OPS-112\",\n \"title\": \"Refactor session storage\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 4,\n \"updatedAt\": \"2026-07-14T16:12:00.000Z\"\n }\n ],\n \"pageInfo\": {\n \"hasNextPage\": false,\n \"endCursor\": \"cursor-3\"\n }\n }\n}\n```","durationMs":32830,"exitCode":0} +{"condition":"baseline","caseId":"query-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","query","--team","OPS","--state","backlog","--state","triage","--unassigned","--updated-after","2026-06-01","--limit","0","--json","--no-pager"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/query-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/query-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n '\"'^### query|Usage: linear issue query'\"' /trials/query-holdout-3/codex-home/skills/linear-cli/references/issue.md && sed -n '430,540p' /trials/query-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n '\"'^### query|Usage: linear issue query'\"' /trials/query-holdout-3/codex-home/skills/linear-cli/references/issue.md && sed -n '430,540p' /trials/query-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '400,455p' /trials/query-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '400,455p' /trials/query-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --limit 0 --json --no-pager'","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --limit 0 --json --no-pager'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issues\": {\n \"nodes\": [\n {\n \"identifier\": \"OPS-101\",\n \"title\": \"Fix login redirect loop\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 2,\n \"updatedAt\": \"2026-07-17T10:00:00.000Z\"\n },\n {\n \"identifier\": \"OPS-107\",\n \"title\": \"Update billing webhooks\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 3,\n \"updatedAt\": \"2026-07-16T08:30:00.000Z\"\n },\n {\n \"identifier\": \"OPS-112\",\n \"title\": \"Refactor session storage\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 4,\n \"updatedAt\": \"2026-07-14T16:12:00.000Z\"\n }\n ],\n \"pageInfo\": {\n \"hasNextPage\": false,\n \"endCursor\": \"cursor-3\"\n }\n }\n}\n```","durationMs":31711,"exitCode":0} +{"condition":"baseline","caseId":"update-development","trial":1,"entries":[{"tool":"linear","argv":["issue","view","--help"],"stdin":""},{"tool":"linear","argv":["issue","update","ENG-101","--state","In Review","--assignee","priya"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json","--no-comments","--no-download"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-development-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-development-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '281,620p' /trials/update-development-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '281,620p' /trials/update-development-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '621,760p' /trials/update-development-1/codex-home/skills/linear-cli/references/issue.md && linear issue view --help\"","/usr/bin/bash -lc \"sed -n '621,760p' /trials/update-development-1/codex-home/skills/linear-cli/references/issue.md && linear issue view --help\"","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya'","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya'","/usr/bin/bash -lc 'linear issue view ENG-101 --json --no-comments --no-download'","/usr/bin/bash -lc 'linear issue view ENG-101 --json --no-comments --no-download'"],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 is now **In Review** and assigned to **priya**.","durationMs":30821,"exitCode":0} +{"condition":"baseline","caseId":"update-development","trial":2,"entries":[{"tool":"linear","argv":["issue","view","ENG-101"],"stdin":""},{"tool":"linear","argv":["team","members","ENG"],"stdin":""},{"tool":"linear","argv":["issue","update","ENG-101","--state","In Review","--assignee","priya"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json","--no-comments"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/update-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/update-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '261,620p' /trials/update-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '261,620p' /trials/update-development-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '621,760p' /trials/update-development-2/codex-home/skills/linear-cli/references/issue.md && linear issue view ENG-101 && linear team members ENG\"","/usr/bin/bash -lc \"sed -n '621,760p' /trials/update-development-2/codex-home/skills/linear-cli/references/issue.md && linear issue view ENG-101 && linear team members ENG\"","/usr/bin/bash -lc \"linear issue update ENG-101 --state 'In Review' --assignee priya && linear issue view ENG-101 --json --no-comments\"","/usr/bin/bash -lc \"linear issue update ENG-101 --state 'In Review' --assignee priya && linear issue view ENG-101 --json --no-comments\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 moved to **In Review** and assigned to **Priya Patel (`priya`)**.","durationMs":23853,"exitCode":0} +{"condition":"baseline","caseId":"update-development","trial":3,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","update","ENG-101","--state","In Review","--assignee","priya"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json","--no-comments","--no-download"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '280,620p' /trials/update-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '280,620p' /trials/update-development-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '620,760p' /trials/update-development-3/codex-home/skills/linear-cli/references/issue.md && linear --version && linear auth whoami\"","/usr/bin/bash -lc \"sed -n '620,760p' /trials/update-development-3/codex-home/skills/linear-cli/references/issue.md && linear --version && linear auth whoami\"","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya'","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya'","/usr/bin/bash -lc 'linear issue view ENG-101 --json --no-comments --no-download'","/usr/bin/bash -lc 'linear issue view ENG-101 --json --no-comments --no-download'"],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 was moved to **In Review** and assigned to **priya**. Verified successfully.","durationMs":32170,"exitCode":0} +{"condition":"baseline","caseId":"update-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","view","OPS-31","--json","--no-comments","--no-download"],"stdin":""},{"tool":"linear","argv":["issue","update","OPS-31","--unassign","--project","Data Pipeline","--label","infra","--label","security"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-31","--json","--no-comments","--no-download"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n -A45 '\"'^### update|''^### view'\"' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n -A45 '\"'^### update|''^### view'\"' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'linear issue view OPS-31 --json --no-comments --no-download'","/usr/bin/bash -lc 'linear issue view OPS-31 --json --no-comments --no-download'","/usr/bin/bash -lc \"linear issue update OPS-31 --unassign --project 'Data Pipeline' --label infra --label security\"","/usr/bin/bash -lc \"linear issue update OPS-31 --unassign --project 'Data Pipeline' --label infra --label security\"","/usr/bin/bash -lc 'linear issue view OPS-31 --json --no-comments --no-download'","/usr/bin/bash -lc 'linear issue view OPS-31 --json --no-comments --no-download'"],"skillRead":true,"fixturesModified":false,"answer":"Updated OPS-31:\n\n- Assignee: unassigned\n- Project: Data Pipeline\n- Labels: infra, security\n\nVerified the final issue state in Linear.","durationMs":28263,"exitCode":0} +{"condition":"baseline","caseId":"update-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","update","--help"],"stdin":""},{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["auth","whoami"],"stdin":""},{"tool":"linear","argv":["issue","update","OPS-31","--unassign","--project","Data Pipeline","--label","infra","--label","security"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-holdout-2/codex-home/skills/linear-cli/references/issue.md && sed -n '1,260p' /trials/update-holdout-2/codex-home/skills/linear-cli/references/organization-features.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-holdout-2/codex-home/skills/linear-cli/references/issue.md && sed -n '1,260p' /trials/update-holdout-2/codex-home/skills/linear-cli/references/organization-features.md\"","/usr/bin/bash -lc \"rg -n -A45 '\"'^### update|''^Usage: linear issue update'\"' /trials/update-holdout-2/codex-home/skills/linear-cli/references/issue.md && linear issue update --help && linear issue view OPS-31 --json\"","/usr/bin/bash -lc \"rg -n -A45 '\"'^### update|''^Usage: linear issue update'\"' /trials/update-holdout-2/codex-home/skills/linear-cli/references/issue.md && linear issue update --help && linear issue view OPS-31 --json\"","/usr/bin/bash -lc 'command -v linear; linear --version; linear auth whoami'","/usr/bin/bash -lc 'command -v linear; linear --version; linear auth whoami'","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security'","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security'"],"skillRead":true,"fixturesModified":false,"answer":"Updated [OPS-31](https://linear.app/acme/issue/OPS-31):\n\n- Assignee: unassigned\n- Project: Data Pipeline\n- Labels: infra, security","durationMs":30282,"exitCode":0} +{"condition":"baseline","caseId":"update-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","view","OPS-31","--help"],"stdin":""},{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","update","OPS-31","--unassign","--project","Data Pipeline","--label","infra","--label","security"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-31","--json","--no-comments","--no-download"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '280,620p' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '280,620p' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '620,760p' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md && linear issue view OPS-31 --help\"","/usr/bin/bash -lc \"sed -n '620,760p' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md && linear issue view OPS-31 --help\"","/usr/bin/bash -lc 'type -a linear; linear --version; ls -l $(command -v linear)'","/usr/bin/bash -lc 'type -a linear; linear --version; ls -l $(command -v linear)'","/usr/bin/bash -lc \"sed -n '1,260p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"sed -n '1,260p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security\nlinear issue view OPS-31 --json --no-comments --no-download'","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security\nlinear issue view OPS-31 --json --no-comments --no-download'"],"skillRead":true,"fixturesModified":false,"answer":"Updated OPS-31 successfully:\n\n- Assignee: unassigned\n- Project: Data Pipeline\n- Labels: infra, security\n\nVerified the final issue state.","durationMs":34475,"exitCode":0} diff --git a/evals/linear-cli-skill/results/baseline.meta.json b/evals/linear-cli-skill/results/baseline.meta.json new file mode 100644 index 00000000..c4534cd1 --- /dev/null +++ b/evals/linear-cli-skill/results/baseline.meta.json @@ -0,0 +1,26 @@ +{ + "condition": "baseline", + "codexVersion": "codex-cli 0.144.4", + "sandbox": "workspace-write", + "effort": "low", + "modelRequested": "gpt-5.6-sol", + "modelsObserved": [], + "trialsPerCase": 3, + "cases": [ + "query-development", + "query-holdout", + "create-development", + "create-holdout", + "update-development", + "update-holdout", + "comment-development", + "comment-holdout", + "inspect-development", + "inspect-holdout", + "control-history", + "control-subscribers" + ], + "completedTrials": 36, + "failedTrials": [], + "ranAt": "2026-07-18T23:01:09.521Z" +} diff --git a/evals/linear-cli-skill/run.ts b/evals/linear-cli-skill/run.ts new file mode 100644 index 00000000..7c5bd49a --- /dev/null +++ b/evals/linear-cli-skill/run.ts @@ -0,0 +1,461 @@ +/** + * Runner for the linear-cli skill eval. + * + * Spawns the OpenAI Codex CLI as the subject agent, once per trial, in a + * fully isolated environment: + * + * - a fresh per-trial CODEX_HOME containing only a copied auth.json and the + * skill variant under test (so the globally installed linear-cli skill + * can't leak in via ~/.codex/skills) + * - a fresh per-trial fake HOME (codex also discovers skills in + * ~/.agents/skills) + * - an explicit PATH of shims + the codex/deno bin dirs (no mise shims: they + * break under a fake HOME) — `linear`, `curl`, `npx`, and `npm` resolve to + * recording shims that never touch the network + * - by default, codex's `workspace-write` sandbox, which denies network and + * out-of-workdir writes to everything the subject runs (`--sandbox yolo` + * opts out if the sandbox is unavailable in some environment) + * + * Every `linear`/`curl`/`npx`/`npm` invocation is recorded to a JSONL log, + * and codex's JSON event stream is captured so shim bypasses and skill reads + * are observable. Trial records feed grade.ts. + * + * Usage: + * deno run --allow-all run.ts --condition baseline --skill-dir skills/linear-cli \ + * [--trials 3] [--cases id,id] [--concurrency 2] [--effort low] \ + * [--model gpt-x] [--sandbox workspace-write|yolo] [--out results/baseline.jsonl] + */ + +import { parseArgs } from "@std/cli" +import { copy } from "@std/fs" +import { dirname, fromFileUrl, join, resolve } from "@std/path" +import { CASES } from "./cases.ts" +import type { ShimEntry, TrialRecord } from "./grade.ts" + +const EVAL_DIR = dirname(fromFileUrl(import.meta.url)) +const REPO_ROOT = resolve(EVAL_DIR, "..", "..") + +interface RunConfig { + condition: string + skillDir: string + trials: number + concurrency: number + effort: string + model: string | null + sandbox: "workspace-write" | "yolo" + outPath: string + codexBin: string + authPath: string + timeoutMs: number +} + +async function commandOutput(cmd: string, args: string[]): Promise { + const result = await new Deno.Command(cmd, { + args, + stdout: "piped", + stderr: "piped", + }).output() + if (!result.success) { + throw new Error( + `${cmd} ${args.join(" ")} failed: ${ + new TextDecoder().decode(result.stderr) + }`, + ) + } + return new TextDecoder().decode(result.stdout).trim() +} + +async function resolveCodexBin(explicit: string | undefined): Promise { + if (explicit != null) return resolve(explicit) + try { + return await commandOutput("mise", ["which", "codex"]) + } catch { + const found = await commandOutput("which", ["codex"]) + if (found.includes("/mise/shims/")) { + throw new Error( + "codex resolves to a mise shim, which breaks under the eval's fake HOME; pass --codex-bin with the real binary path", + ) + } + return found + } +} + +function requireDir(path: string, label: string): string { + try { + if (!Deno.statSync(path).isDirectory) { + throw new Error(`${label} is not a directory: ${path}`) + } + } catch (error) { + if (error instanceof Deno.errors.NotFound) { + throw new Error(`${label} not found: ${path}`) + } + throw error + } + return path +} + +function sanitize(text: string, workdir: string, tmpRoot: string): string { + return text.replaceAll(workdir, ".").replaceAll(tmpRoot, "") + .replaceAll(REPO_ROOT, "") +} + +interface EventExtract { + commands: string[] + skillRead: boolean + models: Set +} + +/** + * Best-effort extraction from codex's experimental JSON event stream: shell + * commands (for bypass detection), whether SKILL.md was touched, and the + * model the session reported. Unknown event shapes are skipped, not fatal — + * the shim log remains the primary grading signal. + */ +export function extractFromEvents(rawEvents: string): EventExtract { + const extract: EventExtract = { + commands: [], + skillRead: rawEvents.includes("SKILL.md") || + rawEvents.includes("skills/linear-cli"), + models: new Set(), + } + for (const line of rawEvents.split("\n")) { + if (line.trim() === "") continue + let event: unknown + try { + event = JSON.parse(line) + } catch { + continue + } + if (typeof event !== "object" || event == null) continue + const record = event as Record + if (typeof record.model === "string") extract.models.add(record.model) + const item = record.item + if (typeof item === "object" && item != null) { + const itemRecord = item as Record + if (typeof itemRecord.command === "string") { + extract.commands.push(itemRecord.command) + } + } + } + return extract +} + +async function fixturesModifiedIn(workdir: string): Promise { + for await (const fixture of Deno.readDir(join(EVAL_DIR, "fixtures"))) { + const original = await Deno.readTextFile( + join(EVAL_DIR, "fixtures", fixture.name), + ) + let current: string + try { + current = await Deno.readTextFile(join(workdir, fixture.name)) + } catch { + return true + } + if (current !== original) return true + } + return false +} + +async function runTrial( + config: RunConfig, + tmpRoot: string, + caseId: string, + prompt: string, + trial: number, +): Promise<{ record: TrialRecord; models: Set }> { + const trialRoot = join(tmpRoot, "trials", `${caseId}-${trial}`) + const workdir = join(trialRoot, "work") + const codexHome = join(trialRoot, "codex-home") + const fakeHome = join(trialRoot, "fake-home") + await Deno.mkdir(join(workdir, "tmp"), { recursive: true }) + await Deno.mkdir(join(codexHome, "skills"), { recursive: true }) + await Deno.mkdir(fakeHome, { recursive: true }) + await Deno.copyFile(config.authPath, join(codexHome, "auth.json")) + await Deno.writeTextFile(join(codexHome, "config.toml"), "") + await copy(config.skillDir, join(codexHome, "skills", "linear-cli")) + for await (const fixture of Deno.readDir(join(EVAL_DIR, "fixtures"))) { + await Deno.copyFile( + join(EVAL_DIR, "fixtures", fixture.name), + join(workdir, fixture.name), + ) + } + const shimLog = join(trialRoot, "shim.jsonl") + const answerPath = join(trialRoot, "answer.md") + + const sandboxArgs = config.sandbox === "yolo" + ? ["--yolo"] + : ["--sandbox", "workspace-write"] + const modelArgs = config.model == null ? [] : ["-m", config.model] + const started = Date.now() + const child = new Deno.Command(config.codexBin, { + args: [ + "exec", + ...sandboxArgs, + ...modelArgs, + "--ephemeral", + "--skip-git-repo-check", + "--json", + "-C", + workdir, + "-c", + `model_reasoning_effort="${config.effort}"`, + "-o", + answerPath, + prompt, + ], + clearEnv: true, + env: { + HOME: fakeHome, + CODEX_HOME: codexHome, + PATH: `${join(EVAL_DIR, "shims")}:${dirname(config.codexBin)}:${ + dirname(Deno.execPath()) + }:/usr/bin:/bin`, + NO_COLOR: "1", + TERM: "dumb", + LINEAR_SHIM_REPO: REPO_ROOT, + LINEAR_SHIM_LOG: shimLog, + TMPDIR: join(workdir, "tmp"), + }, + cwd: workdir, + stdin: "null", + stdout: "piped", + stderr: "piped", + }).spawn() + + const timer = setTimeout(() => { + try { + child.kill("SIGKILL") + } catch { + // already exited + } + }, config.timeoutMs) + const status = await child.output() + clearTimeout(timer) + const durationMs = Date.now() - started + + const rawEvents = new TextDecoder().decode(status.stdout) + await Deno.writeTextFile(join(trialRoot, "events.jsonl"), rawEvents) + const stderrText = new TextDecoder().decode(status.stderr) + await Deno.writeTextFile(join(trialRoot, "stderr.log"), stderrText) + const events = extractFromEvents(rawEvents) + // The startup banner on stderr reports the effective model even when the + // JSON event stream doesn't. + const bannerModel = stderrText.match(/^model:\s*(\S+)/m) + if (bannerModel != null) events.models.add(bannerModel[1]) + + let entries: ShimEntry[] = [] + try { + entries = (await Deno.readTextFile(shimLog)) + .split("\n") + .filter((line) => line.trim() !== "") + .map((line) => JSON.parse(line) as ShimEntry) + } catch (error) { + if (!(error instanceof Deno.errors.NotFound)) throw error + } + entries = entries.map((entry) => ({ + ...entry, + argv: entry.argv.map((arg) => sanitize(arg, workdir, tmpRoot)), + stdin: sanitize(entry.stdin, workdir, tmpRoot), + })) + + let answer = "" + try { + answer = sanitize(await Deno.readTextFile(answerPath), workdir, tmpRoot) + } catch (error) { + if (!(error instanceof Deno.errors.NotFound)) throw error + } + if (!status.success && answer === "" && entries.length === 0) { + const stderr = new TextDecoder().decode(status.stderr) + throw new Error( + `codex run for ${caseId} trial ${trial} produced nothing (exit ${status.code}): ${ + stderr.slice(-500) + }`, + ) + } + return { + record: { + condition: config.condition, + caseId, + trial, + entries, + commands: events.commands.map((command) => + sanitize(command, workdir, tmpRoot) + ), + skillRead: events.skillRead, + fixturesModified: await fixturesModifiedIn(workdir), + answer, + durationMs, + exitCode: status.code, + }, + models: events.models, + } +} + +async function main(): Promise { + const flags = parseArgs(Deno.args, { + string: [ + "condition", + "skill-dir", + "cases", + "out", + "effort", + "model", + "sandbox", + "codex-bin", + ], + default: { + trials: 3, + concurrency: 2, + effort: "low", + sandbox: "workspace-write", + "timeout-seconds": 300, + }, + }) + if (flags.condition == null || flags["skill-dir"] == null) { + throw new Error( + "usage: run.ts --condition --skill-dir [--trials N] [--cases id,id] [--concurrency N] [--effort low] [--model name] [--sandbox workspace-write|yolo] [--codex-bin path] [--out results.jsonl]", + ) + } + if (flags.sandbox !== "workspace-write" && flags.sandbox !== "yolo") { + throw new Error(`--sandbox must be workspace-write or yolo`) + } + const selectedCases = flags.cases == null + ? CASES + : flags.cases.split(",").map((id) => { + const found = CASES.find((evalCase) => evalCase.id === id.trim()) + if (found == null) throw new Error(`unknown case id: ${id}`) + return found + }) + + const realCodexHome = Deno.env.get("CODEX_HOME") ?? + join(Deno.env.get("HOME") ?? "", ".codex") + const config: RunConfig = { + condition: flags.condition, + skillDir: requireDir(resolve(flags["skill-dir"]), "--skill-dir"), + trials: Number(flags.trials), + concurrency: Number(flags.concurrency), + effort: flags.effort, + model: flags.model ?? null, + sandbox: flags.sandbox, + outPath: flags.out != null + ? resolve(flags.out) + : join(EVAL_DIR, "results", `${flags.condition}.jsonl`), + codexBin: await resolveCodexBin(flags["codex-bin"]), + authPath: join(realCodexHome, "auth.json"), + timeoutMs: Number(flags["timeout-seconds"]) * 1000, + } + if (!Number.isInteger(config.trials) || config.trials < 1) { + throw new Error(`--trials must be a positive integer`) + } + try { + Deno.statSync(config.authPath) + } catch { + throw new Error( + `codex auth not found at ${config.authPath} — is codex logged in?`, + ) + } + + const codexVersion = await commandOutput(config.codexBin, ["--version"]) + const tmpRoot = await Deno.makeTempDir({ prefix: "linear-skill-eval-" }) + + console.log(`condition=${config.condition}`) + console.log(`codex=${config.codexBin} (${codexVersion})`) + console.log(`skill=${config.skillDir}`) + console.log( + `sandbox=${config.sandbox} effort=${config.effort} model=${ + config.model ?? "(codex default)" + } trials=${config.trials}/case`, + ) + console.log(`tmp=${tmpRoot}`) + + const jobs: { caseId: string; prompt: string; trial: number }[] = [] + for (const evalCase of selectedCases) { + for (let trial = 1; trial <= config.trials; trial++) { + jobs.push({ caseId: evalCase.id, prompt: evalCase.prompt, trial }) + } + } + + const records: TrialRecord[] = [] + const failures: string[] = [] + const observedModels = new Set() + let nextJob = 0 + let completed = 0 + const worker = async (): Promise => { + while (nextJob < jobs.length) { + const job = jobs[nextJob++] + try { + const { record, models } = await runTrial( + config, + tmpRoot, + job.caseId, + job.prompt, + job.trial, + ) + for (const model of models) observedModels.add(model) + records.push(record) + completed++ + console.log( + `[${completed}/${jobs.length}] ${job.caseId} trial ${job.trial}: ${record.entries.length} invocations, skillRead=${record.skillRead}, ${ + Math.round(record.durationMs / 1000) + }s`, + ) + } catch (error) { + failures.push( + `${job.caseId} trial ${job.trial}: ${ + error instanceof Error ? error.message : String(error) + }`, + ) + completed++ + console.error( + `[${completed}/${jobs.length}] ${job.caseId} trial ${job.trial} FAILED`, + ) + } + } + } + await Promise.all( + Array.from( + { length: Math.min(config.concurrency, jobs.length) }, + worker, + ), + ) + + records.sort((a, b) => + a.caseId === b.caseId ? a.trial - b.trial : a.caseId.localeCompare(b.caseId) + ) + await Deno.mkdir(dirname(config.outPath), { recursive: true }) + await Deno.writeTextFile( + config.outPath, + records.map((record) => JSON.stringify(record)).join("\n") + "\n", + ) + await Deno.writeTextFile( + config.outPath.replace(/\.jsonl$/, ".meta.json"), + JSON.stringify( + { + condition: config.condition, + codexVersion, + sandbox: config.sandbox, + effort: config.effort, + modelRequested: config.model, + modelsObserved: [...observedModels].sort(), + trialsPerCase: config.trials, + cases: selectedCases.map((evalCase) => evalCase.id), + completedTrials: records.length, + failedTrials: failures, + ranAt: new Date().toISOString(), + }, + null, + 2, + ) + "\n", + ) + console.log(`wrote ${records.length} records to ${config.outPath}`) + console.log(`models observed: ${[...observedModels].join(", ") || "unknown"}`) + if (failures.length > 0) { + console.error(`\n${failures.length} trial(s) failed:`) + for (const failure of failures) console.error(` - ${failure}`) + Deno.exitCode = 1 + } +} + +if (import.meta.main) { + await main() +} diff --git a/evals/linear-cli-skill/shims/curl b/evals/linear-cli-skill/shims/curl new file mode 100755 index 00000000..f6dbcf6d --- /dev/null +++ b/evals/linear-cli-skill/shims/curl @@ -0,0 +1,10 @@ +#!/bin/bash +# Shim `curl` for the skill eval: records the attempt and fails like a network +# with no route to the API, so direct-HTTP fallbacks are observable but can +# never reach the real Linear API. +set -u +LOG="${LINEAR_SHIM_LOG:?LINEAR_SHIM_LOG must be set}" +printf '%s\x1f' x "$@" | jq -Rsc \ + '{tool: "curl", argv: split("\u001f")[1:-1], stdin: ""}' >> "$LOG" +echo "curl: (6) Could not resolve host: api.linear.app" >&2 +exit 6 diff --git a/evals/linear-cli-skill/shims/linear b/evals/linear-cli-skill/shims/linear new file mode 100755 index 00000000..8aaa6624 --- /dev/null +++ b/evals/linear-cli-skill/shims/linear @@ -0,0 +1,258 @@ +#!/bin/bash +# Shim `linear` binary for the skill eval. +# +# Records every invocation (argv + stdin for `api`) as a JSON line in +# $LINEAR_SHIM_LOG, answers --help/--version/schema through the real CLI in +# $LINEAR_SHIM_REPO (help needs no auth or network), and returns short, +# plausible canned successes for task commands. Never touches the network and +# never mutates anything real. +set -u + +LOG="${LINEAR_SHIM_LOG:?LINEAR_SHIM_LOG must be set}" +REPO="${LINEAR_SHIM_REPO:?LINEAR_SHIM_REPO must be set}" + +# Capture GraphQL heredocs piped to `linear api` so grading can inspect the +# query text. Other commands never read stdin. +STDIN_DATA="" +if [ "${1:-}" = "api" ] && [ ! -t 0 ]; then + STDIN_DATA=$(cat) +fi + +# Argv goes to jq on stdin, delimited by the ASCII unit separator (0x1f): +# jq's --args treats dash-prefixed values (e.g. --team) as its own options, +# and jq strings cannot represent NUL, so neither works for arbitrary argv. +if [ "$#" -eq 0 ]; then + jq -cn --arg stdin "$STDIN_DATA" \ + '{tool: "linear", argv: [], stdin: $stdin}' >> "$LOG" +else + printf '%s\x1f' "$@" | jq -Rsc --arg stdin "$STDIN_DATA" \ + '{tool: "linear", argv: split("\u001f")[:-1], stdin: $stdin}' >> "$LOG" +fi + +# Discovery goes to the real CLI so help output is accurate for the skill +# variant's CLI version. +for arg in "$@"; do + case "$arg" in + --help | -h | --version) + NO_COLOR=1 exec deno run --quiet --allow-all "$REPO/src/main.ts" "$@" + ;; + esac +done +if [ "${1:-}" = "schema" ]; then + NO_COLOR=1 exec deno run --quiet --allow-all "$REPO/src/main.ts" "$@" +fi + +has_flag() { + local needle="$1" + shift + for arg in "$@"; do + [ "$arg" = "$needle" ] && return 0 + done + return 1 +} + +# Map a requested --state filter value to a display state name, so canned +# rows don't contradict the filter the subject just applied. +display_state() { + case "$1" in + triage) printf 'Triage' ;; + backlog) printf 'Backlog' ;; + unstarted) printf 'Todo' ;; + completed) printf 'Done' ;; + canceled) printf 'Canceled' ;; + *) printf 'In Progress' ;; + esac +} + +state_type() { + case "$1" in + triage | backlog | unstarted | completed | canceled) printf '%s' "$1" ;; + *) printf 'started' ;; + esac +} + +# First value following any of the given flags in the remaining argv. +flag_value() { + local flags="$1" + shift + local prev="" + for arg in "$@"; do + case " $flags " in + *" $prev "*) + printf '%s' "$arg" + return 0 + ;; + esac + prev="$arg" + done + return 1 +} + +case "${1:-} ${2:-}" in + "issue query" | "issue list" | "issue mine" | "issue l" | "issue q") + STATE_FILTER=$(flag_value "--state -s" "$@") || STATE_FILTER="started" + DSTATE=$(display_state "$STATE_FILTER") + TSTATE=$(state_type "$STATE_FILTER") + PREFIX=$(flag_value "--team" "$@") || PREFIX="ENG" + SEARCH=$(flag_value "--search" "$@") || SEARCH="" + if has_flag --unassigned "$@" || has_flag -U "$@"; then + NAME1="" + NAME2="" + else + NAME1="sam" + NAME2="priya" + fi + if [ -n "$SEARCH" ]; then + if has_flag --json "$@" || has_flag -j "$@"; then + jq -cn --arg id "$PREFIX-231" --arg title "$SEARCH" --arg state "$DSTATE" --arg type "$TSTATE" \ + '{issues: {nodes: [{identifier: $id, title: $title, state: {name: $state, type: $type}, assignee: null, priority: 3, updatedAt: "2026-07-18T09:00:00.000Z"}], pageInfo: {hasNextPage: false, endCursor: "cursor-1"}}}' + else + printf '%-8s %-40s %-12s %s\n' "$PREFIX-231" "$SEARCH" "$DSTATE" "-" + fi + elif has_flag --json "$@" || has_flag -j "$@"; then + jq -cn --arg p "$PREFIX" --arg state "$DSTATE" --arg type "$TSTATE" \ + --arg name1 "$NAME1" --arg name2 "$NAME2" \ + '{issues: {nodes: [ + {identifier: ($p + "-101"), title: "Fix login redirect loop", state: {name: $state, type: $type}, assignee: (if $name1 == "" then null else {displayName: $name1} end), priority: 2, updatedAt: "2026-07-17T10:00:00.000Z"}, + {identifier: ($p + "-107"), title: "Update billing webhooks", state: {name: $state, type: $type}, assignee: (if $name2 == "" then null else {displayName: $name2} end), priority: 3, updatedAt: "2026-07-16T08:30:00.000Z"}, + {identifier: ($p + "-112"), title: "Refactor session storage", state: {name: $state, type: $type}, assignee: null, priority: 4, updatedAt: "2026-07-14T16:12:00.000Z"} + ], pageInfo: {hasNextPage: false, endCursor: "cursor-3"}}}' + else + printf '%-8s %-25s %-12s %-7s %s\n' \ + "$PREFIX-101" "Fix login redirect loop" "$DSTATE" "High" "${NAME1:--}" \ + "$PREFIX-107" "Update billing webhooks" "$DSTATE" "Medium" "${NAME2:--}" \ + "$PREFIX-112" "Refactor session storage" "$DSTATE" "Low" "-" + fi + ;; + "issue view") + ID="${3:-ENG-112}" + EFFECT='{}' + if [ -s "$LOG" ]; then + EFFECT=$(jq -sc --arg id "$ID" ' + def vals($names): .argv as $a + | [range(0; ($a | length)) as $i + | select(($names | index($a[$i])) != null) + | ($a[$i + 1] // "")]; + [.[] | select(.tool == "linear" and .argv[0] == "issue" and .argv[1] == "update" and .argv[2] == $id)] + | { + state: ([.[] | vals(["--state", "-s"])] | flatten | last), + assignee: ([.[] | vals(["--assignee", "-a"])] | flatten | last), + unassigned: ([.[] | select(.argv | index("--unassign"))] | length > 0), + labels: ([.[] | vals(["--label", "-l"]) | select(length > 0)] | last), + project: ([.[] | vals(["--project"])] | flatten | last) + }' "$LOG") + fi + VIEW_STATE=$(printf '%s' "$EFFECT" | jq -r '.state // "In Progress"') + VIEW_ASSIGNEE=$(printf '%s' "$EFFECT" | jq -r 'if .unassigned == true then "" else (.assignee // "") end') + VIEW_LABELS=$(printf '%s' "$EFFECT" | jq -c '.labels // ["tech-debt"]') + VIEW_PROJECT=$(printf '%s' "$EFFECT" | jq -r '.project // ""') + if has_flag --json "$@" || has_flag -j "$@"; then + jq -cn --arg id "$ID" --arg state "$VIEW_STATE" --arg assignee "$VIEW_ASSIGNEE" \ + --argjson labels "$VIEW_LABELS" --arg project "$VIEW_PROJECT" \ + '{issue: { + identifier: $id, + title: "Refactor session storage", + description: "Session data is duplicated between redis and postgres.", + state: {name: $state}, + assignee: (if $assignee == "" then null else {displayName: $assignee} end), + labels: {nodes: [$labels[] | {name: .}]}, + project: (if $project == "" then null else {name: $project} end), + url: ("https://linear.app/acme/issue/" + $id + "/refactor-session-storage") + }}' + else + printf '%s: Refactor session storage\nState: %s Assignee: %s Labels: %s%s\nURL: https://linear.app/acme/issue/%s/refactor-session-storage\n\nSession data is duplicated between redis and postgres.\n' \ + "$ID" "$VIEW_STATE" "${VIEW_ASSIGNEE:--}" \ + "$(printf '%s' "$VIEW_LABELS" | jq -r 'join(", ")')" \ + "$(if [ -n "$VIEW_PROJECT" ]; then printf ' Project: %s' "$VIEW_PROJECT"; fi)" \ + "$ID" + fi + ;; + "issue url") + printf 'https://linear.app/acme/issue/%s/update-billing-webhooks\n' "${3:-ENG-107}" + ;; + "issue id") + printf '%s\n' "${3:-ENG-101}" + ;; + "issue title") + printf 'Fix login redirect loop\n' + ;; + "issue create") + TITLE=$(flag_value "--title -t" "$@") || TITLE="New issue" + PREFIX=$(flag_value "--team" "$@") || PREFIX="ENG" + printf 'Created %s-231: %s\nhttps://linear.app/acme/issue/%s-231/new-issue\n' "$PREFIX" "$TITLE" "$PREFIX" + ;; + "issue update") + ID="${3:-ENG-101}" + printf 'Updated %s\n' "$ID" + if has_flag --unassign "$@"; then printf ' assignee: (unassigned)\n'; fi + if V=$(flag_value "--assignee -a" "$@"); then printf ' assignee: %s\n' "$V"; fi + if V=$(flag_value "--state -s" "$@"); then printf ' state: %s\n' "$V"; fi + if V=$(flag_value "--project" "$@"); then printf ' project: %s\n' "$V"; fi + if V=$(flag_value "--title -t" "$@"); then printf ' title: %s\n' "$V"; fi + LABELS="" + PREV="" + for arg in "$@"; do + if [ "$PREV" = "--label" ] || [ "$PREV" = "-l" ]; then + LABELS="${LABELS:+$LABELS, }$arg" + fi + PREV="$arg" + done + if [ -n "$LABELS" ]; then printf ' labels: %s\n' "$LABELS"; fi + printf 'https://linear.app/acme/issue/%s\n' "$ID" + ;; + "issue comment") + printf 'Comment added\n' + ;; + "issue start") + printf 'Started %s\n' "${3:-ENG-101}" + ;; + "team list") + printf 'ENG Engineering\nOPS Operations\n' + ;; + "team states") + printf 'Triage\nBacklog\nTodo\nIn Progress\nIn Review\nDone\nCanceled\n' + ;; + "team members") + printf 'sam Sam Reyes\npriya Priya Patel\n' + ;; + "user list") + printf 'sam Sam Reyes sam@acme.test\npriya Priya Patel priya@acme.test\n' + ;; + "label list") + printf 'bug\ndocs\napi\ninfra\nsecurity\ntech-debt\n' + ;; + "project list") + printf 'Mobile App Revamp started\nData Pipeline started\n' + ;; + "auth whoami") + printf 'sam@acme.test (acme)\n' + ;; + "auth token") + printf 'lin_api_evalfake000000000000000000000000\n' + ;; + api*) + QUERY_TEXT="$* $STDIN_DATA" + case "$QUERY_TEXT" in + *subscribers*) + printf '{"data":{"issue":{"identifier":"ENG-101","subscribers":{"nodes":[{"name":"Sam Reyes","email":"sam@acme.test"},{"name":"Priya Patel","email":"priya@acme.test"}]}}}}\n' + ;; + *history*) + printf '{"data":{"issue":{"identifier":"ENG-101","history":{"nodes":[{"createdAt":"2026-07-15T09:00:00.000Z","actor":{"name":"Sam Reyes"},"fromState":{"name":"Todo"},"toState":{"name":"In Progress"}},{"createdAt":"2026-07-16T11:30:00.000Z","actor":{"name":"Priya Patel"},"fromAssignee":null,"toAssignee":{"name":"Sam Reyes"}}]}}}}\n' + ;; + *) + printf '{"data":{"issues":{"nodes":[{"identifier":"ENG-101","title":"Fix login redirect loop","updatedAt":"2026-07-17T10:00:00.000Z"},{"identifier":"ENG-107","title":"Update billing webhooks","updatedAt":"2026-07-16T08:30:00.000Z"}]}}}\n' + ;; + esac + ;; + *) + case "${1:-}" in + api | auth | config | cycle | document | initiative | initiative-update | issue | label | milestone | project | project-update | schema | team | user) + printf 'OK\n' + ;; + *) + printf 'error: Unknown command "%s".\n' "${1:-}" >&2 + exit 2 + ;; + esac + ;; +esac diff --git a/evals/linear-cli-skill/shims/npx b/evals/linear-cli-skill/shims/npx new file mode 100755 index 00000000..eabc0e11 --- /dev/null +++ b/evals/linear-cli-skill/shims/npx @@ -0,0 +1,10 @@ +#!/bin/bash +# Shim `npx` for the skill eval: the skill suggests `npx @schpet/linear-cli` +# when `linear` is missing; here `linear` is always present, so any npx use is +# recorded and fails fast without touching the network. +set -u +LOG="${LINEAR_SHIM_LOG:?LINEAR_SHIM_LOG must be set}" +printf '%s\x1f' x "$@" | jq -Rsc \ + '{tool: "npx", argv: split("\u001f")[1:-1], stdin: ""}' >> "$LOG" +echo "npx: network unavailable in this environment" >&2 +exit 1 diff --git a/test/evals/linear-cli-skill-grade.test.ts b/test/evals/linear-cli-skill-grade.test.ts new file mode 100644 index 00000000..72e056cc --- /dev/null +++ b/test/evals/linear-cli-skill-grade.test.ts @@ -0,0 +1,315 @@ +import { assertAlmostEquals, assertEquals, assertThrows } from "@std/assert" +import { CASES } from "../../evals/linear-cli-skill/cases.ts" +import { + classifyTrial, + fisherOneSided, + flagValues, + gradeRecords, + isDiscovery, + type ShimEntry, + type TrialRecord, +} from "../../evals/linear-cli-skill/grade.ts" + +const queryCase = CASES.find((evalCase) => evalCase.id === "query-holdout")! +const updateCase = CASES.find((evalCase) => + evalCase.id === "update-development" +)! +const createCase = CASES.find((evalCase) => + evalCase.id === "create-development" +)! +const controlCase = CASES.find((evalCase) => + evalCase.id === "control-subscribers" +)! + +function record( + caseId: string, + entries: ShimEntry[], + extra: Partial = {}, +): TrialRecord { + return { + condition: "test", + caseId, + trial: 1, + entries, + answer: "", + durationMs: 1000, + exitCode: 0, + ...extra, + } +} + +function linear(argv: string[], stdin = ""): ShimEntry { + return { tool: "linear", argv, stdin } +} + +Deno.test("discovery: help, version, schema, and lookups are ignored for routing", () => { + assertEquals(isDiscovery(linear(["issue", "query", "--help"])), true) + assertEquals(isDiscovery(linear(["--version"])), true) + assertEquals(isDiscovery(linear(["schema", "-o", "s.graphql"])), true) + assertEquals(isDiscovery(linear(["team", "list"])), true) + assertEquals(isDiscovery(linear(["issue", "query"])), false) + assertEquals( + isDiscovery({ tool: "curl", argv: ["--help"], stdin: "" }), + false, + ) +}) + +Deno.test("flagValues handles separate, equals, and repeated forms", () => { + assertEquals( + flagValues(["--state", "backlog", "-s", "triage"], ["--state", "-s"]), + ["backlog", "triage"], + ) + assertEquals(flagValues(["--state=backlog"], ["--state", "-s"]), ["backlog"]) + assertEquals(flagValues(["--json"], ["--json", "-j"]), [""]) + assertEquals(flagValues(["--state"], ["--state"]), [""]) +}) + +Deno.test("cli route: correct subcommand with all required flags is a full success", () => { + const grade = classifyTrial( + queryCase, + record("query-holdout", [ + linear(["--version"]), + linear([ + "issue", + "query", + "--team", + "OPS", + "--unassigned", + "--state", + "backlog", + "--state", + "triage", + "--updated-after", + "2026-06-01", + "--json", + ]), + ]), + ) + assertEquals(grade.firstRoute, "cli") + assertEquals(grade.routeOk, true) + assertEquals(grade.fullSuccess, true) + assertEquals(grade.discoveryInvocations, 1) + assertEquals(grade.meaningfulInvocations, 1) +}) + +Deno.test("cli route: right route but missing flags passes route tier only", () => { + const grade = classifyTrial( + queryCase, + record("query-holdout", [ + linear(["issue", "query", "--team", "OPS", "--unassigned"]), + ]), + ) + assertEquals(grade.routeOk, true) + assertEquals(grade.fullSuccess, false) +}) + +Deno.test("cli route: viewer-scoped alias counts as route but not full success for query tasks", () => { + const grade = classifyTrial( + queryCase, + record("query-holdout", [ + linear(["issue", "list", "--team", "OPS", "--json"]), + ]), + ) + assertEquals(grade.firstRoute, "cli") + assertEquals(grade.routeOk, true) + assertEquals(grade.fullSuccess, false) +}) + +Deno.test("a lookup before the task-performing command does not fail routing", () => { + const grade = classifyTrial( + updateCase, + record("update-development", [ + linear(["issue", "view", "ENG-101"]), + linear([ + "issue", + "update", + "ENG-101", + "--state", + "In Review", + "--assignee", + "priya", + ]), + ]), + ) + assertEquals(grade.firstRoute, "cli_other") + assertEquals(grade.routeOk, true) + assertEquals(grade.fullSuccess, true) +}) + +Deno.test("api fallback on a supported task fails routing even after discovery", () => { + const grade = classifyTrial( + queryCase, + record("query-holdout", [ + linear(["issue", "query", "--help"]), + linear(["api"], "query { issues { nodes { identifier } } }"), + ]), + ) + assertEquals(grade.firstRoute, "api") + assertEquals(grade.routeOk, false) + assertEquals(grade.usedApi, true) +}) + +Deno.test("cli-first then api anywhere still fails the no-fallback contract", () => { + const grade = classifyTrial( + queryCase, + record("query-holdout", [ + linear(["issue", "query", "--team", "OPS"]), + linear(["api"], "query { issues { nodes { id } } }"), + ]), + ) + assertEquals(grade.firstRoute, "cli") + assertEquals(grade.routeOk, false) +}) + +Deno.test("curl bypass is detected as http fallback", () => { + const grade = classifyTrial( + queryCase, + record("query-holdout", [ + { tool: "curl", argv: ["https://api.linear.app/graphql"], stdin: "" }, + ]), + ) + assertEquals(grade.firstRoute, "http") + assertEquals(grade.routeOk, false) + assertEquals(grade.usedHttp, true) +}) + +Deno.test("http bypass via event-stream commands is detected without shim entries", () => { + const grade = classifyTrial( + queryCase, + record("query-holdout", [ + linear(["issue", "query", "--team", "OPS"]), + ], { + commands: ["python3 -c 'import urllib.request; ...'"], + }), + ) + assertEquals(grade.usedHttp, true) + assertEquals(grade.routeOk, false) +}) + +Deno.test("no meaningful invocations grades as none", () => { + const grade = classifyTrial( + queryCase, + record("query-holdout", [linear(["--version"])]), + ) + assertEquals(grade.firstRoute, "none") + assertEquals(grade.routeOk, false) +}) + +Deno.test("file flags must point at the expected, unmodified fixture", () => { + const good = classifyTrial( + createCase, + record("create-development", [ + linear([ + "issue", + "create", + "--team", + "ENG", + "--title", + "Migrate CI to Blacksmith runners", + "--description-file", + "./issue-description.md", + ]), + ]), + ) + assertEquals(good.fullSuccess, true) + const wrongFile = classifyTrial( + createCase, + record("create-development", [ + linear([ + "issue", + "create", + "--title", + "Migrate CI to Blacksmith runners", + "--description-file", + "./notes.md", + ]), + ]), + ) + assertEquals(wrongFile.fullSuccess, false) + const tampered = classifyTrial( + createCase, + record("create-development", [ + linear([ + "issue", + "create", + "--team", + "ENG", + "--title", + "Migrate CI to Blacksmith runners", + "--description-file", + "./issue-description.md", + ]), + ], { fixturesModified: true }), + ) + assertEquals(tampered.routeOk, true) + assertEquals(tampered.fullSuccess, false) +}) + +Deno.test("controls: linear api with expected fields passes; cli or curl fails", () => { + const viaApi = classifyTrial( + controlCase, + record("control-subscribers", [ + linear( + ["api"], + 'query { issue(id: "ENG-101") { subscribers { nodes { name email } } } }', + ), + ]), + ) + assertEquals(viaApi.routeOk, true) + assertEquals(viaApi.fullSuccess, true) + + const viaCli = classifyTrial( + controlCase, + record("control-subscribers", [ + linear(["issue", "view", "ENG-101", "--json"]), + ]), + ) + assertEquals(viaCli.routeOk, false) + + const viaCurl = classifyTrial( + controlCase, + record("control-subscribers", [ + linear(["auth", "token"]), + { tool: "curl", argv: ["https://api.linear.app/graphql"], stdin: "" }, + ]), + ) + assertEquals(viaCurl.routeOk, false) +}) + +Deno.test("gradeRecords aggregates by case and variant and rejects mixed conditions", () => { + const { summary } = gradeRecords([ + record("query-holdout", [ + linear(["issue", "query", "--team", "OPS", "--unassigned"]), + ], { skillRead: true }), + record("control-subscribers", [ + linear(["api"], "query { issue { subscribers { nodes { email } } } }"), + ]), + ]) + assertEquals(summary.supported.total, 1) + assertEquals(summary.supported.routeOk, 1) + assertEquals(summary.supported.fullSuccess, 0) + assertEquals(summary.controls.total, 1) + assertEquals(summary.controls.routeOk, 1) + assertEquals(summary.byVariant.holdout.total, 1) + assertEquals(summary.byCase["query-holdout"].routeOk, 1) + assertEquals(summary.skillReadTrials, 1) + + assertThrows( + () => + gradeRecords([ + { ...record("query-holdout", []), condition: "a" }, + { ...record("query-holdout", []), condition: "b" }, + ]), + Error, + "single condition", + ) +}) + +Deno.test("fisher one-sided matches known values", () => { + // 1/10 vs 9/10 — strong improvement; p = (C(10,9)C(10,1) + 1) / C(20,10) + assertAlmostEquals(fisherOneSided(1, 9, 9, 1), 101 / 184756, 1e-8) + // identical proportions — no evidence of improvement + assertEquals(fisherOneSided(5, 5, 5, 5) > 0.5, true) + // regression should give a large p-value + assertEquals(fisherOneSided(9, 1, 1, 9) > 0.99, true) +}) From f59a6c63ef045ef397c9274e7adc89c90bf952f7 Mon Sep 17 00:00:00 2001 From: Peter Schilling Date: Sat, 18 Jul 2026 16:02:49 -0700 Subject: [PATCH 2/3] Add a Common Tasks section to the linear-cli skill (#207), eval-validated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds ~7 copy-pasteable recipes near the top of SKILL.template.md (and the generated SKILL.md): filtered queries via issue query (with the issue list/mine alias gotcha spelled out), my-issues, create with --description-file and --no-interactive, update state/assignee/labels (noting label replacement semantics), comment from file, view/URL. The boundary note stays generic so it doesn't teach the eval's control answers. Post-change eval, same frozen 36-trial protocol as the baseline: 29/30 supported tasks full success, holdout 15/15, controls 6/6 still correctly choosing linear api — no overcorrection from the new recipes. The single failure was a subject first trying the skill's documented npx alternative (npx @schpet/linear-cli issue create ...), which the frozen grader counts as a bypass; it was not a GraphQL fallback and the task then completed correctly via the CLI. With the baseline already at 30/30, the eval finds no measurable routing headroom at this configuration; the change is validated as non-regressing rather than as an improvement. See evals/linear-cli-skill/results/comparison.md. Related to #207 --- evals/linear-cli-skill/results/comparison.md | 66 +++++++++++++++++++ .../results/post-change.jsonl | 36 ++++++++++ .../results/post-change.meta.json | 26 ++++++++ evals/linear-cli-skill/run.ts | 5 +- skills/linear-cli/SKILL.md | 53 +++++++++++++++ skills/linear-cli/SKILL.template.md | 53 +++++++++++++++ 6 files changed, 238 insertions(+), 1 deletion(-) create mode 100644 evals/linear-cli-skill/results/comparison.md create mode 100644 evals/linear-cli-skill/results/post-change.jsonl create mode 100644 evals/linear-cli-skill/results/post-change.meta.json diff --git a/evals/linear-cli-skill/results/comparison.md b/evals/linear-cli-skill/results/comparison.md new file mode 100644 index 00000000..e353d450 --- /dev/null +++ b/evals/linear-cli-skill/results/comparison.md @@ -0,0 +1,66 @@ +## Comparison + +Primary outcome is **full success on supported tasks** (dedicated-CLI route, taught subcommand, all required flags, intact fixtures, no GraphQL/HTTP fallback). The holdout row uses only prompts that were never looked at while iterating on the skill text. Trials of the same prompt are not independent samples, so case-level counts are shown in the per-condition sections below. + +| Metric | Baseline | Post-change | One-sided Fisher p | +| ----------------------------------- | -------- | ----------- | ------------------ | +| Supported: full success (primary) | 30/30 | 29/30 | 1.00 | +| Supported: dedicated-CLI route | 30/30 | 29/30 | 1.00 | +| Holdout only: full success | 15/15 | 15/15 | 1.00 | +| Controls: still choose `linear api` | 6/6 | 6/6 | — | + +### Condition: baseline + +- Trials: 36 (skill read in 36) +- Supported tasks — full success: 30/30 (100%), dedicated-CLI route: 30/30 (100%) +- Controls (GraphQL appropriate) — chose `linear api`: 6/6, with expected fields: 6/6 +- Mean non-discovery invocations per trial: 1.28 + +| Case | Route ok | Full success | +| ------------------- | -------- | ------------ | +| comment-development | 3/3 | 3/3 | +| comment-holdout | 3/3 | 3/3 | +| control-history | 3/3 | 3/3 | +| control-subscribers | 3/3 | 3/3 | +| create-development | 3/3 | 3/3 | +| create-holdout | 3/3 | 3/3 | +| inspect-development | 3/3 | 3/3 | +| inspect-holdout | 3/3 | 3/3 | +| query-development | 3/3 | 3/3 | +| query-holdout | 3/3 | 3/3 | +| update-development | 3/3 | 3/3 | +| update-holdout | 3/3 | 3/3 | + +| Variant | Route ok | Full success | +| ----------- | -------- | ------------ | +| development | 15/15 | 15/15 | +| holdout | 15/15 | 15/15 | +| control | 6/6 | 6/6 | + +### Condition: post-change + +- Trials: 36 (skill read in 36) +- Supported tasks — full success: 29/30 (97%), dedicated-CLI route: 29/30 (97%) +- Controls (GraphQL appropriate) — chose `linear api`: 6/6, with expected fields: 6/6 +- Mean non-discovery invocations per trial: 1.47 + +| Case | Route ok | Full success | +| ------------------- | -------- | ------------ | +| comment-development | 3/3 | 3/3 | +| comment-holdout | 3/3 | 3/3 | +| control-history | 3/3 | 3/3 | +| control-subscribers | 3/3 | 3/3 | +| create-development | 2/3 | 2/3 | +| create-holdout | 3/3 | 3/3 | +| inspect-development | 3/3 | 3/3 | +| inspect-holdout | 3/3 | 3/3 | +| query-development | 3/3 | 3/3 | +| query-holdout | 3/3 | 3/3 | +| update-development | 3/3 | 3/3 | +| update-holdout | 3/3 | 3/3 | + +| Variant | Route ok | Full success | +| ----------- | -------- | ------------ | +| development | 14/15 | 14/15 | +| holdout | 15/15 | 15/15 | +| control | 6/6 | 6/6 | diff --git a/evals/linear-cli-skill/results/post-change.jsonl b/evals/linear-cli-skill/results/post-change.jsonl new file mode 100644 index 00000000..0477defb --- /dev/null +++ b/evals/linear-cli-skill/results/post-change.jsonl @@ -0,0 +1,36 @@ +{"condition":"post-change","caseId":"comment-development","trial":1,"entries":[{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","./comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Comment from `./comment.md` was successfully added to issue `ENG-107`.","durationMs":11316,"exitCode":0} +{"condition":"post-change","caseId":"comment-development","trial":2,"entries":[{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","./comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Comment from `./comment.md` added successfully to Linear issue **ENG-107**.","durationMs":15511,"exitCode":0} +{"condition":"post-change","caseId":"comment-development","trial":3,"entries":[{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","./comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Comment from `./comment.md` added successfully to issue **ENG-107**.","durationMs":18117,"exitCode":0} +{"condition":"post-change","caseId":"comment-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./alternate-comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Posted `alternate-comment.md` as a comment on issue `OPS-44`.","durationMs":13432,"exitCode":0} +{"condition":"post-change","caseId":"comment-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./alternate-comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'linear issue comment add OPS-44 --body-file ./alternate-comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Posted `alternate-comment.md` as a comment on issue `OPS-44`.","durationMs":12840,"exitCode":0} +{"condition":"post-change","caseId":"comment-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./alternate-comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'linear issue comment add OPS-44 --body-file ./alternate-comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Posted `./alternate-comment.md` as a comment on **OPS-44** successfully.","durationMs":13890,"exitCode":0} +{"condition":"post-change","caseId":"control-history","trial":1,"entries":[{"tool":"linear","argv":["api","query { issue(id: \"ENG-101\") { identifier title history(first: 100) { nodes { createdAt actor { name } fromState { name } toState { name } fromAssignee { name } toAssignee { name } } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'cat /trials/control-history-1/codex-home/skills/linear-cli/references/api.md'","/usr/bin/bash -lc 'cat /trials/control-history-1/codex-home/skills/linear-cli/references/api.md'","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier title history(first: 100) { nodes { createdAt actor { name } fromState { name } toState { name } fromAssignee { name } toAssignee { name } } } } }'\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier title history(first: 100) { nodes { createdAt actor { name } fromState { name } toState { name } fromAssignee { name } toAssignee { name } } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 history:\n\n- **2026-07-15 09:00 UTC** — Sam Reyes changed state from **Todo** to **In Progress**.\n- **2026-07-16 11:30 UTC** — Priya Patel changed assignee from **Unassigned** to **Sam Reyes**.","durationMs":22341,"exitCode":0} +{"condition":"post-change","caseId":"control-history","trial":2,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["auth","token","--help"],"stdin":""},{"tool":"linear","argv":["api"],"stdin":"query IssueChangeHistory {\n issue(id: \"ENG-101\") {\n identifier\n history(first: 100) {\n nodes {\n createdAt\n actor { name }\n fromState { name }\n toState { name }\n fromAssignee { name }\n toAssignee { name }\n }\n }\n }\n}"}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-2/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-2/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc 'linear --version && linear schema IssueHistory'","/usr/bin/bash -lc 'linear --version && linear schema IssueHistory'","/usr/bin/bash -lc 'command -v linear\nlinear auth token --help\nfind . -maxdepth 3 -type f -print'","/usr/bin/bash -lc 'command -v linear\nlinear auth token --help\nfind . -maxdepth 3 -type f -print'","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear\nls -la /.config/linear 2>/dev/null || true\nfind -maxdepth 4 -type f \\\\( -name '*linear*' -o -name 'credentials.json' \\\\) -print 2>/dev/null | head -80\"","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear\nls -la /.config/linear 2>/dev/null || true\nfind -maxdepth 4 -type f \\\\( -name '*linear*' -o -name 'credentials.json' \\\\) -print 2>/dev/null | head -80\"","/usr/bin/bash -lc \"sed -n '200,440p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"sed -n '200,440p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"linear api <<'GRAPHQL'\nquery IssueChangeHistory {\n issue(id: \\\"ENG-101\\\") {\n identifier\n history(first: 100) {\n nodes {\n createdAt\n actor { name }\n fromState { name }\n toState { name }\n fromAssignee { name }\n toAssignee { name }\n }\n }\n }\n}\nGRAPHQL\"","/usr/bin/bash -lc \"linear api <<'GRAPHQL'\nquery IssueChangeHistory {\n issue(id: \\\"ENG-101\\\") {\n identifier\n history(first: 100) {\n nodes {\n createdAt\n actor { name }\n fromState { name }\n toState { name }\n fromAssignee { name }\n toAssignee { name }\n }\n }\n }\n}\nGRAPHQL\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 change history:\n\n- 2026-07-15 09:00 UTC — Sam Reyes changed state from **Todo** to **In Progress**.\n- 2026-07-16 11:30 UTC — Priya Patel assigned the issue from **Unassigned** to **Sam Reyes**.","durationMs":39599,"exitCode":0} +{"condition":"post-change","caseId":"control-history","trial":3,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["api","query IssueHistory($id: String!) { issue(id: $id) { identifier history(first: 250) { nodes { createdAt actor { name } fromState { name } toState { name } fromAssignee { name } toAssignee { name } } } } }","--variable","id=ENG-101"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-3/codex-home/skills/linear-cli/references/api.md && linear --version && linear issue id ENG-101\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-3/codex-home/skills/linear-cli/references/api.md && linear --version && linear issue id ENG-101\"","/usr/bin/bash -lc 'command -v linear\nls -l \"$(command -v linear)\"\nhead -n 40 \"$(command -v linear)\"\nrg -n \"issueHistory|history|activities|issue id\" /src -g '\"'*.ts' -g '*.graphql' | head -n 120\"","/usr/bin/bash -lc 'command -v linear\nls -l \"$(command -v linear)\"\nhead -n 40 \"$(command -v linear)\"\nrg -n \"issueHistory|history|activities|issue id\" /src -g '\"'*.ts' -g '*.graphql' | head -n 120\"","/usr/bin/bash -lc \"sed -n '1915,1965p' /graphql/schema.graphql\nsed -n '10470,10630p' /graphql/schema.graphql\nsed -n '240,270p' /evals/linear-cli-skill/cases.ts\nsed -n '40,180p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"sed -n '1915,1965p' /graphql/schema.graphql\nsed -n '10470,10630p' /graphql/schema.graphql\nsed -n '240,270p' /evals/linear-cli-skill/cases.ts\nsed -n '40,180p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"rg -n '\\\"api\\\"|history|fromState|toState' /evals/linear-cli-skill/shims/linear | tail -n 30\nsed -n '340,470p' /evals/linear-cli-skill/shims/linear\nsed -n '10620,10720p' /graphql/schema.graphql\"","/usr/bin/bash -lc \"rg -n '\\\"api\\\"|history|fromState|toState' /evals/linear-cli-skill/shims/linear | tail -n 30\nsed -n '340,470p' /evals/linear-cli-skill/shims/linear\nsed -n '10620,10720p' /graphql/schema.graphql\"","/usr/bin/bash -lc \"sed -n '215,250p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"sed -n '215,250p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"linear api 'query IssueHistory(\"'$id: String!) { issue(id: $id) { identifier history(first: 250) { nodes { createdAt actor { name } fromState { name } toState { name } fromAssignee { name } toAssignee { name } } } } }'\"' --variable id=ENG-101\"","/usr/bin/bash -lc \"linear api 'query IssueHistory(\"'$id: String!) { issue(id: $id) { identifier history(first: 250) { nodes { createdAt actor { name } fromState { name } toState { name } fromAssignee { name } toAssignee { name } } } } }'\"' --variable id=ENG-101\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 change history:\n\n- 2026-07-15 09:00 UTC — Sam Reyes changed state: Todo → In Progress.\n- 2026-07-16 11:30 UTC — Priya Patel changed assignee: Unassigned → Sam Reyes.","durationMs":50923,"exitCode":0} +{"condition":"post-change","caseId":"control-subscribers","trial":1,"entries":[{"tool":"linear","argv":["api","query IssueSubscribers($id: String!) { issue(id: $id) { identifier title subscribers { nodes { name email } } } }","--variable","id=ENG-101"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-1/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-1/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"linear api 'query IssueSubscribers(\"'$id: String!) { issue(id: $id) { identifier title subscribers { nodes { name email } } } }'\"' --variable id=ENG-101\"","/usr/bin/bash -lc \"linear api 'query IssueSubscribers(\"'$id: String!) { issue(id: $id) { identifier title subscribers { nodes { name email } } } }'\"' --variable id=ENG-101\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 has two subscribers:\n\n- Sam Reyes — sam@acme.test\n- Priya Patel — priya@acme.test","durationMs":16802,"exitCode":0} +{"condition":"post-change","caseId":"control-subscribers","trial":2,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["api","--help"],"stdin":""},{"tool":"linear","argv":["api","query IssueSubscribers($id: String!) { issue(id: $id) { identifier subscribers { nodes { name email } } } }","--variable","id=ENG-101"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-2/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-2/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"linear --version && linear api 'query IssueSubscribers(\"'$id: String!) { issue(id: $id) { identifier subscribers { nodes { name email } } } }'\"' --variable id=ENG-101\"","/usr/bin/bash -lc \"linear --version && linear api 'query IssueSubscribers(\"'$id: String!) { issue(id: $id) { identifier subscribers { nodes { name email } } } }'\"' --variable id=ENG-101\"","/usr/bin/bash -lc 'command -v linear\nls -l \"$(command -v linear)\"\nlinear api --help'","/usr/bin/bash -lc 'command -v linear\nls -l \"$(command -v linear)\"\nlinear api --help'","/usr/bin/bash -lc \"sed -n '1,260p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"sed -n '1,260p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"linear api 'query IssueSubscribers(\"'$id: String!) { issue(id: $id) { identifier subscribers { nodes { name email } } } }'\"' --variable id=ENG-101\"","/usr/bin/bash -lc \"linear api 'query IssueSubscribers(\"'$id: String!) { issue(id: $id) { identifier subscribers { nodes { name email } } } }'\"' --variable id=ENG-101\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 subscribers:\n\n- Sam Reyes — sam@acme.test\n- Priya Patel — priya@acme.test","durationMs":33264,"exitCode":0} +{"condition":"post-change","caseId":"control-subscribers","trial":3,"entries":[{"tool":"linear","argv":["api","query { issue(id: \"ENG-101\") { identifier subscribers { nodes { name email } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-3/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-3/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier subscribers { nodes { name email } } } }'\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier subscribers { nodes { name email } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 has two subscribers:\n\n- Sam Reyes — sam@acme.test\n- Priya Patel — priya@acme.test","durationMs":18567,"exitCode":0} +{"condition":"post-change","caseId":"create-development","trial":1,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"npx","argv":["--yes","@schpet/linear-cli","issue","create","--team","ENG","--title","Migrate CI to Blacksmith runners","--description-file","./issue-description.md","--no-interactive"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","ENG","--title","Migrate CI to Blacksmith runners","--description-file","./issue-description.md","--no-interactive"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"linear --version && test -f ./issue-description.md && sed -n '1,160p' ./issue-description.md && linear issue create --team ENG --title \\\"Migrate CI to Blacksmith runners\\\" --description-file ./issue-description.md --no-interactive\"","/usr/bin/bash -lc \"linear --version && test -f ./issue-description.md && sed -n '1,160p' ./issue-description.md && linear issue create --team ENG --title \\\"Migrate CI to Blacksmith runners\\\" --description-file ./issue-description.md --no-interactive\"","/usr/bin/bash -lc \"command -v linear; command -v npx; test -f ./issue-description.md && sed -n '1,160p' ./issue-description.md; npx --yes @schpet/linear-cli issue create --team ENG --title \\\"Migrate CI to Blacksmith runners\\\" --description-file ./issue-description.md --no-interactive\"","/usr/bin/bash -lc \"command -v linear; command -v npx; test -f ./issue-description.md && sed -n '1,160p' ./issue-description.md; npx --yes @schpet/linear-cli issue create --team ENG --title \\\"Migrate CI to Blacksmith runners\\\" --description-file ./issue-description.md --no-interactive\"","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear; env | cut -d= -f1 | rg -i 'linear|token|api'\"","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear; env | cut -d= -f1 | rg -i 'linear|token|api'\"","/usr/bin/bash -lc 'linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'"],"skillRead":true,"fixturesModified":false,"answer":"Created **ENG-231: Migrate CI to Blacksmith runners** using `issue-description.md`.\n\nhttps://linear.app/acme/issue/ENG-231/new-issue","durationMs":40425,"exitCode":0} +{"condition":"post-change","caseId":"create-development","trial":2,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","ENG","--title","Migrate CI to Blacksmith runners","--description-file","./issue-description.md","--no-interactive"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-231","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"linear --version && test -f ./issue-description.md && sed -n '1,200p' ./issue-description.md && linear issue create --team ENG --title \\\"Migrate CI to Blacksmith runners\\\" --description-file ./issue-description.md --no-interactive\"","/usr/bin/bash -lc \"linear --version && test -f ./issue-description.md && sed -n '1,200p' ./issue-description.md && linear issue create --team ENG --title \\\"Migrate CI to Blacksmith runners\\\" --description-file ./issue-description.md --no-interactive\"","/usr/bin/bash -lc 'command -v linear; type -a linear; ls -l \"$(command -v linear)\"; sed -n '\"'1,120p' \\\"\"'$(command -v linear)\"; test -f ./issue-description.md && sed -n '\"'1,160p' ./issue-description.md; env | rg '\"'^(LINEAR|DENO|JSR|NPM)'\"' | sed 's/=.*\"'$/=/'\"'\"","/usr/bin/bash -lc 'command -v linear; type -a linear; ls -l \"$(command -v linear)\"; sed -n '\"'1,120p' \\\"\"'$(command -v linear)\"; test -f ./issue-description.md && sed -n '\"'1,160p' ./issue-description.md; env | rg '\"'^(LINEAR|DENO|JSR|NPM)'\"' | sed 's/=.*\"'$/=/'\"'\"","/usr/bin/bash -lc 'linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue view ENG-231 --json'","/usr/bin/bash -lc 'linear issue view ENG-231 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Created [ENG-231: Migrate CI to Blacksmith runners](https://linear.app/acme/issue/ENG-231/new-issue) on the ENG team using `issue-description.md`.","durationMs":27423,"exitCode":0} +{"condition":"post-change","caseId":"create-development","trial":3,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","ENG","--title","Migrate CI to Blacksmith runners","--description-file","./issue-description.md","--no-interactive"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-231","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear --version && test -f ./issue-description.md && linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'linear --version && test -f ./issue-description.md && linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'command -v linear; file \"$(command -v linear)\"; sed -n '\"'1,120p' \\\"\"'$(command -v linear)\"; ls -la | head -40'","/usr/bin/bash -lc 'command -v linear; file \"$(command -v linear)\"; sed -n '\"'1,120p' \\\"\"'$(command -v linear)\"; ls -la | head -40'","/usr/bin/bash -lc 'linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue view ENG-231 --json'","/usr/bin/bash -lc 'linear issue view ENG-231 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Created [ENG-231: Migrate CI to Blacksmith runners](https://linear.app/acme/issue/ENG-231/new-issue) on the ENG team using `issue-description.md`.","durationMs":35940,"exitCode":0} +{"condition":"post-change","caseId":"create-holdout","trial":1,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","OPS","--project","Data Pipeline","--title","Document rate limit behavior","--label","docs","--label","api","--description-file","./alternate-description.md","--no-interactive"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-231","--json"],"stdin":""},{"tool":"linear","argv":["issue","query","--team","OPS","--project","Data Pipeline","--label","docs","--label","api","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/create-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/create-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '260,360p' /trials/create-holdout-1/codex-home/skills/linear-cli/references/issue.md && test -f ./alternate-description.md && linear --version && linear issue create --team OPS --project \\\"Data Pipeline\\\" --title \\\"Document rate limit behavior\\\" --label docs --label api --description-file ./alternate-description.md --no-interactive\"","/usr/bin/bash -lc \"sed -n '260,360p' /trials/create-holdout-1/codex-home/skills/linear-cli/references/issue.md && test -f ./alternate-description.md && linear --version && linear issue create --team OPS --project \\\"Data Pipeline\\\" --title \\\"Document rate limit behavior\\\" --label docs --label api --description-file ./alternate-description.md --no-interactive\"","/usr/bin/bash -lc 'linear issue create --team OPS --project \"Data Pipeline\" --title \"Document rate limit behavior\" --label docs --label api --description-file ./alternate-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue create --team OPS --project \"Data Pipeline\" --title \"Document rate limit behavior\" --label docs --label api --description-file ./alternate-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue view OPS-231 --json'","/usr/bin/bash -lc 'linear issue view OPS-231 --json'","/usr/bin/bash -lc 'linear issue query --team OPS --project \"Data Pipeline\" --label docs --label api --json'","/usr/bin/bash -lc 'linear issue query --team OPS --project \"Data Pipeline\" --label docs --label api --json'"],"skillRead":true,"fixturesModified":false,"answer":"Created **OPS-231 — Document rate limit behavior** on OPS in **Data Pipeline**, with `docs` and `api` labels and the supplied description.\n\n[View issue](https://linear.app/acme/issue/OPS-231/new-issue)","durationMs":33513,"exitCode":0} +{"condition":"post-change","caseId":"create-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","create","--help"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","OPS","--title","Document rate limit behavior","--project","Data Pipeline","--label","docs","--label","api","--description-file","./alternate-description.md","--no-interactive"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-231","--json"],"stdin":""},{"tool":"linear","argv":["issue","query","--team","OPS","--project","Data Pipeline","--label","docs","--label","api","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'rg -n \"issue create|--project|--label|description-file\" /trials/create-holdout-2/codex-home/skills/linear-cli/references/issue.md | head -80 && test -f ./alternate-description.md && linear issue create --help'","/usr/bin/bash -lc 'rg -n \"issue create|--project|--label|description-file\" /trials/create-holdout-2/codex-home/skills/linear-cli/references/issue.md | head -80 && test -f ./alternate-description.md && linear issue create --help'","/usr/bin/bash -lc 'linear issue create --team OPS --title \"Document rate limit behavior\" --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue create --team OPS --title \"Document rate limit behavior\" --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue view OPS-231 --json'","/usr/bin/bash -lc 'linear issue view OPS-231 --json'","/usr/bin/bash -lc 'linear issue query --team OPS --project \"Data Pipeline\" --label docs --label api --json'","/usr/bin/bash -lc 'linear issue query --team OPS --project \"Data Pipeline\" --label docs --label api --json'"],"skillRead":true,"fixturesModified":false,"answer":"Created [OPS-231: Document rate limit behavior](https://linear.app/acme/issue/OPS-231/new-issue) on OPS with project **Data Pipeline**, labels **docs** and **api**, and the description from `alternate-description.md`.","durationMs":27742,"exitCode":0} +{"condition":"post-change","caseId":"create-holdout","trial":3,"entries":[{"tool":"linear","argv":["auth","whoami"],"stdin":""},{"tool":"linear","argv":["issue","create","--workspace","acme","--team","OPS","--project","Data Pipeline","--title","Document rate limit behavior","--label","docs","--label","api","--description-file","./alternate-description.md","--no-interactive"],"stdin":""},{"tool":"linear","argv":["issue","view","--workspace","acme","OPS-231","--json"],"stdin":""},{"tool":"linear","argv":["--workspace","acme","issue","view","OPS-231","--json"],"stdin":""},{"tool":"linear","argv":["issue","--workspace","acme","view","OPS-231","--json"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-231","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/create-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/create-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'linear auth whoami && test -f ./alternate-description.md && linear issue create --workspace acme --team OPS --project \"Data Pipeline\" --title \"Document rate limit behavior\" --label docs --label api --description-file ./alternate-description.md --no-interactive'","/usr/bin/bash -lc 'linear auth whoami && test -f ./alternate-description.md && linear issue create --workspace acme --team OPS --project \"Data Pipeline\" --title \"Document rate limit behavior\" --label docs --label api --description-file ./alternate-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue view --workspace acme OPS-231 --json'","/usr/bin/bash -lc 'linear issue view --workspace acme OPS-231 --json'","/usr/bin/bash -lc 'linear --workspace acme issue view OPS-231 --json'","/usr/bin/bash -lc 'linear --workspace acme issue view OPS-231 --json'","/usr/bin/bash -lc 'linear issue --workspace acme view OPS-231 --json'","/usr/bin/bash -lc 'linear issue --workspace acme view OPS-231 --json'","/usr/bin/bash -lc 'linear issue view OPS-231 --json'","/usr/bin/bash -lc 'linear issue view OPS-231 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Created [OPS-231: Document rate limit behavior](https://linear.app/acme/issue/OPS-231/new-issue) on OPS under “Data Pipeline,” with `docs` and `api` labels and the supplied description.","durationMs":36752,"exitCode":0} +{"condition":"post-change","caseId":"inspect-development","trial":1,"entries":[{"tool":"linear","argv":["issue","view","ENG-112","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue view ENG-112 --json'","/usr/bin/bash -lc 'linear issue view ENG-112 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issue\": {\n \"identifier\": \"ENG-112\",\n \"title\": \"Refactor session storage\",\n \"description\": \"Session data is duplicated between redis and postgres.\",\n \"state\": {\n \"name\": \"In Progress\"\n },\n \"assignee\": null,\n \"labels\": {\n \"nodes\": [\n {\n \"name\": \"tech-debt\"\n }\n ]\n },\n \"project\": null,\n \"url\": \"https://linear.app/acme/issue/ENG-112/refactor-session-storage\"\n }\n}\n```","durationMs":13174,"exitCode":0} +{"condition":"post-change","caseId":"inspect-development","trial":2,"entries":[{"tool":"linear","argv":["issue","view","ENG-112","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue view ENG-112 --json'","/usr/bin/bash -lc 'linear issue view ENG-112 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issue\": {\n \"identifier\": \"ENG-112\",\n \"title\": \"Refactor session storage\",\n \"description\": \"Session data is duplicated between redis and postgres.\",\n \"state\": {\n \"name\": \"In Progress\"\n },\n \"assignee\": null,\n \"labels\": {\n \"nodes\": [\n {\n \"name\": \"tech-debt\"\n }\n ]\n },\n \"project\": null,\n \"url\": \"https://linear.app/acme/issue/ENG-112/refactor-session-storage\"\n }\n}\n```","durationMs":12430,"exitCode":0} +{"condition":"post-change","caseId":"inspect-development","trial":3,"entries":[{"tool":"linear","argv":["issue","view","ENG-112","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue view ENG-112 --json'","/usr/bin/bash -lc 'linear issue view ENG-112 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issue\": {\n \"identifier\": \"ENG-112\",\n \"title\": \"Refactor session storage\",\n \"description\": \"Session data is duplicated between redis and postgres.\",\n \"state\": {\n \"name\": \"In Progress\"\n },\n \"assignee\": null,\n \"labels\": {\n \"nodes\": [\n {\n \"name\": \"tech-debt\"\n }\n ]\n },\n \"project\": null,\n \"url\": \"https://linear.app/acme/issue/ENG-112/refactor-session-storage\"\n }\n}\n```","durationMs":15629,"exitCode":0} +{"condition":"post-change","caseId":"inspect-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","url","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue url ENG-107'","/usr/bin/bash -lc 'linear issue url ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"https://linear.app/acme/issue/ENG-107/update-billing-webhooks","durationMs":11369,"exitCode":0} +{"condition":"post-change","caseId":"inspect-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","url","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue url ENG-107'","/usr/bin/bash -lc 'linear issue url ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"https://linear.app/acme/issue/ENG-107/update-billing-webhooks","durationMs":13437,"exitCode":0} +{"condition":"post-change","caseId":"inspect-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","url","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue url ENG-107'","/usr/bin/bash -lc 'linear issue url ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"https://linear.app/acme/issue/ENG-107/update-billing-webhooks","durationMs":12242,"exitCode":0} +{"condition":"post-change","caseId":"query-development","trial":1,"entries":[{"tool":"linear","argv":["issue","query","--team","ENG","--project","Mobile App Revamp","--state","started","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'"],"skillRead":true,"fixturesModified":false,"answer":"Started issues in ENG’s “Mobile App Revamp” project:\n\n- ENG-101 — Fix login redirect loop — sam\n- ENG-107 — Update billing webhooks — priya\n- ENG-112 — Refactor session storage — unassigned\n\nAll three are **In Progress**.","durationMs":15130,"exitCode":0} +{"condition":"post-change","caseId":"query-development","trial":2,"entries":[{"tool":"linear","argv":["issue","query","--team","ENG","--project","Mobile App Revamp","--state","started","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'"],"skillRead":true,"fixturesModified":false,"answer":"Started issues in ENG’s “Mobile App Revamp” project:\n\n- ENG-101 — Fix login redirect loop\n- ENG-107 — Update billing webhooks\n- ENG-112 — Refactor session storage","durationMs":11701,"exitCode":0} +{"condition":"post-change","caseId":"query-development","trial":3,"entries":[{"tool":"linear","argv":["issue","query","--team","ENG","--project","Mobile App Revamp","--state","started","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'"],"skillRead":true,"fixturesModified":false,"answer":"Started issues in **Mobile App Revamp** (ENG):\n\n- ENG-101 — Fix login redirect loop — sam\n- ENG-107 — Update billing webhooks — priya\n- ENG-112 — Refactor session storage — unassigned\n\n3 issues total, all **In Progress**.","durationMs":14680,"exitCode":0} +{"condition":"post-change","caseId":"query-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","query","--team","OPS","--state","backlog","--state","triage","--unassigned","--updated-after","2026-06-01","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issues\": {\n \"nodes\": [\n {\n \"identifier\": \"OPS-101\",\n \"title\": \"Fix login redirect loop\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 2,\n \"updatedAt\": \"2026-07-17T10:00:00.000Z\"\n },\n {\n \"identifier\": \"OPS-107\",\n \"title\": \"Update billing webhooks\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 3,\n \"updatedAt\": \"2026-07-16T08:30:00.000Z\"\n },\n {\n \"identifier\": \"OPS-112\",\n \"title\": \"Refactor session storage\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 4,\n \"updatedAt\": \"2026-07-14T16:12:00.000Z\"\n }\n ],\n \"pageInfo\": {\n \"hasNextPage\": false,\n \"endCursor\": \"cursor-3\"\n }\n }\n}\n```","durationMs":18139,"exitCode":0} +{"condition":"post-change","caseId":"query-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","query","--team","OPS","--state","backlog","--state","triage","--unassigned","--updated-after","2026-06-01","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issues\": [\n {\n \"identifier\": \"OPS-101\",\n \"title\": \"Fix login redirect loop\",\n \"state\": \"Backlog\",\n \"assignee\": null,\n \"priority\": 2,\n \"updatedAt\": \"2026-07-17T10:00:00.000Z\"\n },\n {\n \"identifier\": \"OPS-107\",\n \"title\": \"Update billing webhooks\",\n \"state\": \"Backlog\",\n \"assignee\": null,\n \"priority\": 3,\n \"updatedAt\": \"2026-07-16T08:30:00.000Z\"\n },\n {\n \"identifier\": \"OPS-112\",\n \"title\": \"Refactor session storage\",\n \"state\": \"Backlog\",\n \"assignee\": null,\n \"priority\": 4,\n \"updatedAt\": \"2026-07-14T16:12:00.000Z\"\n }\n ]\n}\n```","durationMs":18244,"exitCode":0} +{"condition":"post-change","caseId":"query-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","query","--team","OPS","--state","backlog","--state","triage","--unassigned","--updated-after","2026-06-01","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issues\": {\n \"nodes\": [\n {\n \"identifier\": \"OPS-101\",\n \"title\": \"Fix login redirect loop\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 2,\n \"updatedAt\": \"2026-07-17T10:00:00.000Z\"\n },\n {\n \"identifier\": \"OPS-107\",\n \"title\": \"Update billing webhooks\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 3,\n \"updatedAt\": \"2026-07-16T08:30:00.000Z\"\n },\n {\n \"identifier\": \"OPS-112\",\n \"title\": \"Refactor session storage\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 4,\n \"updatedAt\": \"2026-07-14T16:12:00.000Z\"\n }\n ],\n \"pageInfo\": {\n \"hasNextPage\": false,\n \"endCursor\": \"cursor-3\"\n }\n }\n}\n```","durationMs":17310,"exitCode":0} +{"condition":"post-change","caseId":"update-development","trial":1,"entries":[{"tool":"linear","argv":["issue","update","ENG-101","--state","In Review","--assignee","priya"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'"],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 was moved to **In Review** and assigned to **priya**.\n\n[View issue](https://linear.app/acme/issue/ENG-101)","durationMs":14785,"exitCode":0} +{"condition":"post-change","caseId":"update-development","trial":2,"entries":[{"tool":"linear","argv":["issue","update","ENG-101","--state","In Review","--assignee","priya"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'"],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 (“Refactor session storage”) is now:\n\n- State: **In Review**\n- Assignee: **priya**\n\n[View issue in Linear](https://linear.app/acme/issue/ENG-101/refactor-session-storage)","durationMs":13596,"exitCode":0} +{"condition":"post-change","caseId":"update-development","trial":3,"entries":[{"tool":"linear","argv":["issue","update","ENG-101","--state","In Review","--assignee","priya"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'"],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 moved to **In Review** and assigned to **priya**.","durationMs":15179,"exitCode":0} +{"condition":"post-change","caseId":"update-holdout","trial":1,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["auth","token"],"stdin":""},{"tool":"linear","argv":["issue","update","OPS-31","--unassign","--project","Data Pipeline","--label","infra","--label","security"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-31","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '281,620p' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '281,620p' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '621,760p' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md && linear --version && linear issue view OPS-31 --json\"","/usr/bin/bash -lc \"sed -n '621,760p' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md && linear --version && linear issue view OPS-31 --json\"","/usr/bin/bash -lc 'command -v linear\nls -la\nfind . -maxdepth 3 -type f -print\nlinear auth token'","/usr/bin/bash -lc 'command -v linear\nls -la\nfind . -maxdepth 3 -type f -print\nlinear auth token'","/usr/bin/bash -lc \"sed -n '1,260p' /trials/update-holdout-1/codex-home/skills/linear-cli/references/api.md\nsed -n '1,160p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/update-holdout-1/codex-home/skills/linear-cli/references/api.md\nsed -n '1,160p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security\nlinear issue view OPS-31 --json'","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security\nlinear issue view OPS-31 --json'"],"skillRead":true,"fixturesModified":false,"answer":"OPS-31 updated and verified:\n\n- Assignee: Unassigned\n- Project: Data Pipeline\n- Labels: infra, security","durationMs":35855,"exitCode":0} +{"condition":"post-change","caseId":"update-holdout","trial":2,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","update","OPS-31","--unassign","--project","Data Pipeline","--label","infra","--label","security"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-31","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/update-holdout-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/update-holdout-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'rg -n \"''^### update|--unassign|--project|--label\" /trials/update-holdout-2/codex-home/skills/linear-cli/references/issue.md'","/usr/bin/bash -lc 'rg -n \"''^### update|--unassign|--project|--label\" /trials/update-holdout-2/codex-home/skills/linear-cli/references/issue.md'","/usr/bin/bash -lc \"sed -n '570,605p' /trials/update-holdout-2/codex-home/skills/linear-cli/references/issue.md && linear --version && linear issue view OPS-31 --json\"","/usr/bin/bash -lc \"sed -n '570,605p' /trials/update-holdout-2/codex-home/skills/linear-cli/references/issue.md && linear --version && linear issue view OPS-31 --json\"","/usr/bin/bash -lc 'command -v linear\nls -l \"$(command -v linear)\"\nlinear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security'","/usr/bin/bash -lc 'command -v linear\nls -l \"$(command -v linear)\"\nlinear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security'","/usr/bin/bash -lc 'linear issue view OPS-31 --json'","/usr/bin/bash -lc 'linear issue view OPS-31 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Updated OPS-31 successfully:\n\n- Assignee: unassigned\n- Project: Data Pipeline\n- Labels: infra, security\n\n[View OPS-31](https://linear.app/acme/issue/OPS-31/refactor-session-storage)","durationMs":34667,"exitCode":0} +{"condition":"post-change","caseId":"update-holdout","trial":3,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","update","OPS-31","--unassign","--project","Data Pipeline","--label","infra","--label","security"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-31","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n '\"'^### (update|view)|''^Usage: linear issue (update|view)|--project|--unassign|--label'\"' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n '\"'^### (update|view)|''^Usage: linear issue (update|view)|--project|--unassign|--label'\"' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '560,650p' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md && linear --version && linear issue view OPS-31 --json\"","/usr/bin/bash -lc \"sed -n '560,650p' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md && linear --version && linear issue view OPS-31 --json\"","/usr/bin/bash -lc 'command -v linear; type linear; head -n 40 \"$(command -v linear)\"; env | rg '\"'\"'^(LINEAR|DENO|XDG|PATH)='\"'\"","/usr/bin/bash -lc 'command -v linear; type linear; head -n 40 \"$(command -v linear)\"; env | rg '\"'\"'^(LINEAR|DENO|XDG|PATH)='\"'\"","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security\nlinear issue view OPS-31 --json'","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security\nlinear issue view OPS-31 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Updated OPS-31 successfully:\n\n- Assignee: unassigned\n- Project: Data Pipeline\n- Labels: infra, security","durationMs":37569,"exitCode":0} diff --git a/evals/linear-cli-skill/results/post-change.meta.json b/evals/linear-cli-skill/results/post-change.meta.json new file mode 100644 index 00000000..1effc048 --- /dev/null +++ b/evals/linear-cli-skill/results/post-change.meta.json @@ -0,0 +1,26 @@ +{ + "condition": "post-change", + "codexVersion": "codex-cli 0.144.4", + "sandbox": "workspace-write", + "effort": "low", + "modelRequested": "gpt-5.6-sol", + "modelsObserved": [], + "trialsPerCase": 3, + "cases": [ + "query-development", + "query-holdout", + "create-development", + "create-holdout", + "update-development", + "update-holdout", + "comment-development", + "comment-holdout", + "inspect-development", + "inspect-holdout", + "control-history", + "control-subscribers" + ], + "completedTrials": 36, + "failedTrials": [], + "ranAt": "2026-07-18T23:09:59.449Z" +} diff --git a/evals/linear-cli-skill/run.ts b/evals/linear-cli-skill/run.ts index 7c5bd49a..05a52c1f 100644 --- a/evals/linear-cli-skill/run.ts +++ b/evals/linear-cli-skill/run.ts @@ -94,9 +94,12 @@ function requireDir(path: string, label: string): string { return path } +const REAL_HOME = Deno.env.get("HOME") ?? "" + function sanitize(text: string, workdir: string, tmpRoot: string): string { - return text.replaceAll(workdir, ".").replaceAll(tmpRoot, "") + const cleaned = text.replaceAll(workdir, ".").replaceAll(tmpRoot, "") .replaceAll(REPO_ROOT, "") + return REAL_HOME === "" ? cleaned : cleaned.replaceAll(REAL_HOME, "") } interface EventExtract { diff --git a/skills/linear-cli/SKILL.md b/skills/linear-cli/SKILL.md index 89e43349..0c21be20 100644 --- a/skills/linear-cli/SKILL.md +++ b/skills/linear-cli/SKILL.md @@ -25,6 +25,59 @@ npx @schpet/linear-cli --version All subsequent commands can be prefixed with `npx @schpet/linear-cli` in place of `linear`. Otherwise, follow the install instructions at:\ https://github.com/schpet/linear-cli?tab=readme-ov-file#install +## Common Tasks + +Copy-pasteable recipes for the most frequent flows. Prefer these dedicated commands over `linear api` — reach for the GraphQL fallback only when no dedicated command or flag covers the operation. + +### Query issues with filters + +`issue query` searches across all assignees and supports structured filters that can be combined: + +```bash +linear issue query --team ENG --state started --json +linear issue query --project "Mobile App" --state backlog --state triage --unassigned +linear issue query --assignee sam --label bug --updated-after 2026-01-01 +``` + +Note: `linear issue list` is an alias of `issue mine` and only shows _your_ issues — use `issue query` for anything scoped to other people or a whole team/project. + +### List my issues + +```bash +linear issue mine --state started --sort priority +``` + +### Create an issue + +```bash +linear issue create --team ENG --title "Fix login redirect" \ + --description-file ./description.md --no-interactive +``` + +Write multi-line markdown to a file and pass `--description-file` (see the markdown section below); `--no-interactive` avoids prompts in scripted use. + +### Update an issue's state, assignee, or labels + +```bash +linear issue update ENG-123 --state "In Review" --assignee sam +linear issue update ENG-123 --unassign +linear issue update ENG-123 --label infra --label security # replaces the label set +``` + +### Add a comment + +```bash +linear issue comment add ENG-123 --body-file ./comment.md +``` + +### View an issue / get its URL + +```bash +linear issue view ENG-123 # details incl. comments +linear issue view ENG-123 --json # structured output +linear issue url ENG-123 # print just the URL +``` + ## Best Practices for Markdown Content When working with issue descriptions or comment bodies that contain markdown, **always prefer using file-based flags** instead of passing content as command-line arguments: diff --git a/skills/linear-cli/SKILL.template.md b/skills/linear-cli/SKILL.template.md index 6d7ee9ef..e0b04b67 100644 --- a/skills/linear-cli/SKILL.template.md +++ b/skills/linear-cli/SKILL.template.md @@ -25,6 +25,59 @@ npx @schpet/linear-cli --version All subsequent commands can be prefixed with `npx @schpet/linear-cli` in place of `linear`. Otherwise, follow the install instructions at:\ https://github.com/schpet/linear-cli?tab=readme-ov-file#install +## Common Tasks + +Copy-pasteable recipes for the most frequent flows. Prefer these dedicated commands over `linear api` — reach for the GraphQL fallback only when no dedicated command or flag covers the operation. + +### Query issues with filters + +`issue query` searches across all assignees and supports structured filters that can be combined: + +```bash +linear issue query --team ENG --state started --json +linear issue query --project "Mobile App" --state backlog --state triage --unassigned +linear issue query --assignee sam --label bug --updated-after 2026-01-01 +``` + +Note: `linear issue list` is an alias of `issue mine` and only shows _your_ issues — use `issue query` for anything scoped to other people or a whole team/project. + +### List my issues + +```bash +linear issue mine --state started --sort priority +``` + +### Create an issue + +```bash +linear issue create --team ENG --title "Fix login redirect" \ + --description-file ./description.md --no-interactive +``` + +Write multi-line markdown to a file and pass `--description-file` (see the markdown section below); `--no-interactive` avoids prompts in scripted use. + +### Update an issue's state, assignee, or labels + +```bash +linear issue update ENG-123 --state "In Review" --assignee sam +linear issue update ENG-123 --unassign +linear issue update ENG-123 --label infra --label security # replaces the label set +``` + +### Add a comment + +```bash +linear issue comment add ENG-123 --body-file ./comment.md +``` + +### View an issue / get its URL + +```bash +linear issue view ENG-123 # details incl. comments +linear issue view ENG-123 --json # structured output +linear issue url ENG-123 # print just the URL +``` + ## Best Practices for Markdown Content When working with issue descriptions or comment bodies that contain markdown, **always prefer using file-based flags** instead of passing content as command-line arguments: From df96f02463c03990397d35a2f2b9aa9454a1cd1e Mon Sep 17 00:00:00 2001 From: Peter Schilling Date: Thu, 23 Jul 2026 10:33:31 -0700 Subject: [PATCH 3/3] Make inline image attachments discoverable, eval-validated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agents asked to put a visible screenshot on an issue reach for `issue attach`, which uploads the file but creates a sidebar link attachment that never renders inline — while the success output ("Attachment created") convinces them the image is visible. The working path, `issue comment add --attach`, has existed since v2.0.0 but nothing pointed at it: the skill had no image guidance and the flag was buried in a reference table. Three coordinated changes, validated as experiment 2 of the skill eval: - Skill: a Common Tasks recipe for visible images via `issue comment add --attach`, with an explicit warning about `issue attach`'s sidebar-only behavior. - CLI: `issue attach` now says it created a sidebar link attachment, and for images prints a copy-pasteable hint suggesting `issue comment add --attach` (shell-quoted, --public preserved). Help descriptions updated on both commands. - Eval: new frozen image family (trap-phrased development prompt, comment-phrased holdout) plus a sidebar-control case graded on positionals, with pre-declared outcome rules, CLI/API control split, binary-safe fixture checks, and version-matched shim output. Result (rules frozen before baseline): image-development went 0/3 to 3/3 — every baseline trial fell into the attach trap and wrongly reported success; every post-change trial routed straight to the recipe. Image family 3/6 to 6/6 lands in the pre-declared partial-baseline band, so it is reported as consistent with improvement (exploratory Fisher p = 0.09) rather than confirmed. Controls held except one known npx-version-check grader artifact, adjudicated by an Opus gold-label pass (17/18 agreement with the deterministic grader). --- evals/linear-cli-skill/README.md | 23 +- evals/linear-cli-skill/cases.ts | 99 +++++++ .../linear-cli-skill/fixtures/screenshot.png | Bin 0 -> 350 bytes evals/linear-cli-skill/fixtures/server.log | 6 + evals/linear-cli-skill/grade.ts | 110 +++++++- .../results/image-baseline.jsonl | 45 +++ .../results/image-baseline.meta.json | 29 ++ .../results/image-comparison.md | 95 +++++++ .../results/image-gold-labels.jsonl | 18 ++ .../results/image-post-change.jsonl | 45 +++ .../results/image-post-change.meta.json | 29 ++ evals/linear-cli-skill/run.ts | 13 +- evals/linear-cli-skill/shims/linear | 78 +++++- skills/linear-cli/SKILL.md | 8 + skills/linear-cli/SKILL.template.md | 8 + skills/linear-cli/references/issue.md | 71 ++--- src/commands/issue/issue-attach.ts | 26 +- src/commands/issue/issue-comment-add.ts | 6 +- .../__snapshots__/issue-attach.test.ts.snap | 67 +++++ .../issue-comment-add.test.ts.snap | 25 ++ test/commands/issue/issue-attach.test.ts | 259 +++++++++++++++++ test/commands/issue/issue-comment-add.test.ts | 12 + test/evals/linear-cli-skill-grade.test.ts | 264 +++++++++++++++++- test/utils/mock_linear_server.ts | 36 +++ 24 files changed, 1317 insertions(+), 55 deletions(-) create mode 100644 evals/linear-cli-skill/fixtures/screenshot.png create mode 100644 evals/linear-cli-skill/fixtures/server.log create mode 100644 evals/linear-cli-skill/results/image-baseline.jsonl create mode 100644 evals/linear-cli-skill/results/image-baseline.meta.json create mode 100644 evals/linear-cli-skill/results/image-comparison.md create mode 100644 evals/linear-cli-skill/results/image-gold-labels.jsonl create mode 100644 evals/linear-cli-skill/results/image-post-change.jsonl create mode 100644 evals/linear-cli-skill/results/image-post-change.meta.json create mode 100644 test/commands/issue/__snapshots__/issue-attach.test.ts.snap create mode 100644 test/commands/issue/issue-attach.test.ts diff --git a/evals/linear-cli-skill/README.md b/evals/linear-cli-skill/README.md index ad91905f..9e8f32a2 100644 --- a/evals/linear-cli-skill/README.md +++ b/evals/linear-cli-skill/README.md @@ -8,7 +8,7 @@ The subject agent is the [OpenAI Codex CLI](https://developers.openai.com/codex/ For each case × trial, the runner: -1. Creates a fresh trial dir with its own `CODEX_HOME` (copied codex `auth.json` + the skill variant under test, nothing else), a fake `HOME` (codex also discovers skills in `~/.agents/skills` — a real leak we verified), and a work dir seeded with the markdown fixtures. +1. Creates a fresh trial dir with its own `CODEX_HOME` (copied codex `auth.json` + the skill variant under test, nothing else), a fake `HOME` (codex also discovers skills in `~/.agents/skills` — a real leak we verified), and a work dir seeded with every file in `fixtures/` (markdown, a PNG screenshot, a log file). 2. Spawns `codex exec` with an explicit environment: `PATH` = the recording shims (`linear`, `curl`, `npx`, `npm`) + the codex/deno bin dirs + `/usr/bin:/bin`; no Linear credentials anywhere. By default the subject runs under codex's `workspace-write` sandbox, which denies network and out-of-workdir writes to everything it executes. 3. Records the shim invocation log, codex's JSON event stream (for bypass detection and to verify the skill was actually read), the final answer, and whether fixtures were tampered with. @@ -18,6 +18,14 @@ The `linear` shim passes `--help`/`--version`/`schema` through to the real CLI i See `cases.ts` — frozen before the baseline run. Five recipe families (query / create / update / comment / inspect), each with one development and one holdout prompt, plus two controls where raw GraphQL genuinely is the right route (issue history, issue subscribers — fields the CLI doesn't expose). Holdout prompts are never looked at while iterating on skill text; controls detect overcorrection ("never use api"). +**Experiment 2 (image attachments)** adds a sixth family, `image`, plus a third control: + +- `image-development` — "attach the screenshot … so it is visible on the issue": the natural trap phrasing. `linear issue attach` looks right but creates a sidebar link card that does not render inline; the correct route is `issue comment add --attach`, which uploads and embeds the image as inline markdown. +- `image-holdout` — the same goal phrased as adding a comment with the screenshot visible inline. +- `control-sidebar-attachment` — a log file that teammates should download: here `issue attach` IS the right route. Detects overcorrection ("never use issue attach"). Graded on route plus positional arguments (issue id and file path), with the subcommand's flags stripped from argv first. + +Recovery counts: a trial that first runs `issue attach` and then a correct `issue comment add --attach` passes — the real-world outcome is what matters — provided it never falls back to GraphQL or direct HTTP and leaves fixtures intact. The `firstRoute` diagnostic separates recipe-driven direct routing from output-hint-driven recovery. + ## Grading `grade.ts` classifies each trial from the recorded invocations: @@ -31,6 +39,17 @@ Pre-declared outcome rules, set before the baseline run: - If the baseline already achieves ≥ 90% dedicated-route rate, the premise of #207 did not reproduce under this setup; that is reported as the finding rather than manufacturing harder prompts after the fact. - Trials of one prompt are correlated; case-level counts are reported alongside trial-level totals. +Pre-declared outcome rules for experiment 2 (image attachments), set before its baseline run (`image-baseline` / `image-post-change` — experiment 1's committed `baseline.jsonl` / `post-change.jsonl` / `comparison.md` are frozen artifacts and are not rerun or overwritten): + +- Primary outcome: image-family full success (`image-development` + `image-holdout`, 3 trials each → 6 per condition). Both conditions run **all** cases, so experiment-1 families double as a regression guard for the new skill text. +- **Confirmed** if post-change image-family full success is ≥ 5/6, each image case is ≥ 2/3, the holdout case does not regress, and the improvement over baseline has one-sided Fisher exact p < 0.05. The p-value is explicitly exploratory — trials of one prompt are correlated — so per-case counts are co-primary evidence. +- **Premise did not reproduce** if baseline image-family full success is already ≥ 5/6; that is reported as the finding, with no post-hoc prompt/grading/threshold changes. +- **Partial baseline (2/6–4/6)**: at this n, Fisher cannot reach significance for most true improvements. The result is reported descriptively (per-case baseline vs post counts, plus the `firstRoute` direct-vs-recovery split) and claims at most "consistent with improvement", not confirmation. +- Guards, post-change: `control-sidebar-attachment` must stay 3/3 on `issue attach`; the two GraphQL controls must stay ≥ 5/6 on `linear api`; no experiment-1 supported case may fall below 2/3 full success. +- Any grading or prompt correction after the experiment-2 baseline invalidates it and requires a rerun. + +The shim's canned task output is **version-matched infrastructure**, not part of the frozen experimental variables: it mirrors the real CLI's messaging for the repo state under test, so the baseline runs against canned output mirroring the pre-change CLI and the post-change condition against output mirroring the changed CLI (the changed runtime messaging is itself part of the intervention being measured). Prompts, expectations, and grading are the frozen part. + ## Running Requires a logged-in `codex` CLI. Costs real model tokens (~36 low-effort runs per condition); never run in CI. @@ -49,6 +68,8 @@ deno run --allow-read --allow-write evals/linear-cli-skill/grade.ts \ -o evals/linear-cli-skill/results/comparison.md ``` +Experiment 2 used the same flow under its own condition names (`image-baseline`, `image-post-change`, compared into `results/image-comparison.md`, which also carries a hand-written findings addendum) so experiment 1's result files stay frozen. As a validity check on the deterministic grader, every image-family and sidebar-control trial was additionally gold-labeled by a Claude Opus subagent blind to the deterministic grades — judging only "did the trial's actions achieve the user's stated goal?" from the recorded invocations — with agreement reported in the findings (`results/image-gold-labels.jsonl`). + Useful flags: `--trials N`, `--cases id,id` (subset for smoke tests), `--effort low|medium|high`, `--model `, `--concurrency N` (default 2), `--sandbox workspace-write|yolo` (fallback if the codex sandbox is unavailable on your machine), `--codex-bin `. Results land in `results/.jsonl` (sanitized trial records — argv, event commands, answers; no secrets, no absolute paths) plus a `.meta.json` recording codex version, sandbox, effort, observed models, and per-run failures. Raw event streams stay in the run's temp dir only. diff --git a/evals/linear-cli-skill/cases.ts b/evals/linear-cli-skill/cases.ts index db7df433..a33f683d 100644 --- a/evals/linear-cli-skill/cases.ts +++ b/evals/linear-cli-skill/cases.ts @@ -18,6 +18,13 @@ export interface RequiredArg { valueIsPathSuffix?: boolean } +export interface RequiredPositional { + /** Expected value, compared exactly (or by suffix for paths) */ + value: string + /** Match by suffix (for file paths whose prefix is the tmp workdir) */ + valueIsPathSuffix?: boolean +} + export interface CliExpectation { route: "cli" /** @@ -31,6 +38,19 @@ export interface CliExpectation { */ argsPrefix: string[] requiredArgs: RequiredArg[] + /** + * Positional arguments that must appear after `argsPrefix`, in order. + * Grading strips the flags declared in `valueFlags`/`booleanFlags` (plus + * their values) from the tail of argv before comparing, so flags may be + * interspersed anywhere without failing the match. Positional values are + * compared case-sensitively (unlike flag values) because they are file + * paths and issue identifiers the prompt states verbatim. + */ + positionals?: RequiredPositional[] + /** Flags of this subcommand that consume a value (stripped with it) */ + valueFlags?: string[] + /** Flags of this subcommand that take no value */ + booleanFlags?: string[] } export interface ApiExpectation { @@ -249,6 +269,85 @@ export const CASES: EvalCase[] = [ requiredArgs: [], }, }, + { + id: "image-development", + family: "image", + variant: "development", + prompt: + `${PREAMBLE} Task: attach the screenshot at ./screenshot.png to issue ENG-107 so it is visible on the issue.`, + expect: { + route: "cli", + routePrefixes: [["issue", "comment", "add"]], + argsPrefix: ["issue", "comment", "add"], + requiredArgs: [ + { + flags: ["--attach", "-a"], + value: "screenshot.png", + valueIsPathSuffix: true, + }, + ], + positionals: [{ value: "ENG-107" }], + valueFlags: [ + "--body", + "-b", + "--body-file", + "--parent", + "-p", + "--attach", + "-a", + ], + booleanFlags: ["--public"], + }, + }, + { + id: "image-holdout", + family: "image", + variant: "holdout", + prompt: + `${PREAMBLE} Task: add a comment to issue OPS-44 with the screenshot at ./screenshot.png visible inline.`, + expect: { + route: "cli", + routePrefixes: [["issue", "comment", "add"]], + argsPrefix: ["issue", "comment", "add"], + requiredArgs: [ + { + flags: ["--attach", "-a"], + value: "screenshot.png", + valueIsPathSuffix: true, + }, + ], + positionals: [{ value: "OPS-44" }], + valueFlags: [ + "--body", + "-b", + "--body-file", + "--parent", + "-p", + "--attach", + "-a", + ], + booleanFlags: ["--public"], + }, + }, + { + id: "control-sidebar-attachment", + family: "control", + variant: "control", + prompt: + `${PREAMBLE} Task: attach ./server.log to issue ENG-101 as a downloadable attachment so teammates can grab it from the issue page.`, + expect: { + route: "cli", + routePrefixes: [["issue", "attach"]], + argsPrefix: ["issue", "attach"], + requiredArgs: [], + positionals: [ + { value: "ENG-101" }, + { value: "server.log", valueIsPathSuffix: true }, + ], + valueFlags: ["--title", "-t", "--comment", "-c"], + booleanFlags: ["--public"], + }, + }, { id: "control-history", family: "control", diff --git a/evals/linear-cli-skill/fixtures/screenshot.png b/evals/linear-cli-skill/fixtures/screenshot.png new file mode 100644 index 0000000000000000000000000000000000000000..25a7d08660de7453f7ca7d75d5ec18954a777cca GIT binary patch literal 350 zcmeAS@N?(olHy`uVBq!ia0vp^4nVBI!3-oT1$b@)DTQQ@AYTTCDm4a%h86~fUqGRT z7Yq!g1`G_Z5*Qe)W-u^_7tGleXakfe4e$wZ^$N}K3`zG0PIC`Rbq`E&3ruzmNOtj0 za`8)a_DgW`jd%2kbM%gL@Q$_jim~&Iw)2Rxb&s-fkF<7+uyzf%atX6^3AJzzwQveC zcMLXj3^H{HG_?;ju?sM^^*6HhGqUkDwDvKu^3k{Q*0=Q1v+&e4_c$PN?gG#O>?NMQ zuI#Va#RXZpS{8rJ0}7>jx;Tb-9AA2-+mR9Iy@TRETNa(rce?aX+3d(&hVWOMrVhM; z=T``3^~_qZY=!t09#ePSfb}cPuLx~*+Pl~;Ao~j2*Unpu%2wFEYOq?gFF^f@ [-o comparison.md] */ -import type { CliExpectation, EvalCase, RequiredArg } from "./cases.ts" +import type { + CliExpectation, + EvalCase, + RequiredArg, + RequiredPositional, +} from "./cases.ts" import { CASES } from "./cases.ts" export interface ShimEntry { @@ -131,11 +136,74 @@ function argMatches(argv: string[], required: RequiredArg): boolean { }) } +/** + * Positional tokens after `argsPrefix`, with the subcommand's declared flags + * (and their values) stripped so flags may be interspersed anywhere. Unknown + * dash-prefixed tokens are skipped as boolean flags — the same documented + * looseness as the shim's flag handling: a hallucinated flag's value can leak + * into the positional list, but the taught invocation grades correctly. + */ +export function positionalTokens( + argv: string[], + argsPrefix: string[], + valueFlags: string[], + booleanFlags: string[], +): string[] { + const positionals: string[] = [] + let flagsEnded = false + for (let i = argsPrefix.length; i < argv.length; i++) { + const token = argv[i] + if (flagsEnded) { + positionals.push(token) + continue + } + if (token === "--") { + flagsEnded = true + continue + } + if (valueFlags.includes(token)) { + i++ + continue + } + if (valueFlags.some((flag) => token.startsWith(`${flag}=`))) continue + if (booleanFlags.includes(token) || token.startsWith("-")) continue + positionals.push(token) + } + return positionals +} + +/** + * Ordered, index-aligned match of required positionals against the extracted + * positional tokens. Case-sensitive, unlike flag-value matching: positionals + * here are file paths and issue identifiers the prompt states verbatim. + */ +function positionalsMatch( + tokens: string[], + required: RequiredPositional[], +): boolean { + return required.every((positional, index) => { + const token = tokens[index] + if (token == null) return false + return positional.valueIsPathSuffix + ? token.endsWith(positional.value) + : token === positional.value + }) +} + function cliArgsOk(expect: CliExpectation, entries: ShimEntry[]): boolean { return entries.some((entry) => entry.tool === "linear" && startsWith(entry.argv, expect.argsPrefix) && - expect.requiredArgs.every((required) => argMatches(entry.argv, required)) + expect.requiredArgs.every((required) => argMatches(entry.argv, required)) && + positionalsMatch( + positionalTokens( + entry.argv, + expect.argsPrefix, + expect.valueFlags ?? [], + expect.booleanFlags ?? [], + ), + expect.positionals ?? [], + ) ) } @@ -266,7 +334,10 @@ export interface ConditionSummary { condition: string trials: number supported: OutcomeCounts - controls: OutcomeCounts + /** Controls where `linear api` is the correct route */ + apiControls: OutcomeCounts + /** Controls where a dedicated CLI subcommand is the correct route */ + cliControls: OutcomeCounts byVariant: Record byFamily: Record byCase: Record @@ -304,7 +375,8 @@ export function gradeRecords(records: TrialRecord[]): { condition: records[0].condition, trials: grades.length, supported: { total: 0, routeOk: 0, fullSuccess: 0 }, - controls: { total: 0, routeOk: 0, fullSuccess: 0 }, + apiControls: { total: 0, routeOk: 0, fullSuccess: 0 }, + cliControls: { total: 0, routeOk: 0, fullSuccess: 0 }, byVariant: {}, byFamily: {}, byCase: {}, @@ -320,7 +392,11 @@ export function gradeRecords(records: TrialRecord[]): { for (const grade of grades) { const evalCase = caseById.get(grade.caseId)! count( - evalCase.variant === "control" ? summary.controls : summary.supported, + evalCase.variant === "control" + ? (evalCase.expect.route === "api" + ? summary.apiControls + : summary.cliControls) + : summary.supported, grade, ) summary.byVariant[evalCase.variant] ??= { @@ -378,7 +454,12 @@ export function summaryMarkdown(summary: ConditionSummary): string { }), dedicated-CLI route: ${summary.supported.routeOk}/${summary.supported.total} (${ percent(summary.supported.routeOk, summary.supported.total) })`, - `- Controls (GraphQL appropriate) — chose \`linear api\`: ${summary.controls.routeOk}/${summary.controls.total}, with expected fields: ${summary.controls.fullSuccess}/${summary.controls.total}`, + `- Controls (GraphQL appropriate) — chose \`linear api\`: ${summary.apiControls.routeOk}/${summary.apiControls.total}, with expected fields: ${summary.apiControls.fullSuccess}/${summary.apiControls.total}`, + ...(summary.cliControls.total > 0 + ? [ + `- Controls (dedicated CLI appropriate) — correct route: ${summary.cliControls.routeOk}/${summary.cliControls.total}, full success: ${summary.cliControls.fullSuccess}/${summary.cliControls.total}`, + ] + : []), `- Mean non-discovery invocations per trial: ${summary.meanMeaningfulInvocations}`, "", "| Case | Route ok | Full success |", @@ -449,7 +530,22 @@ export function comparisonMarkdown( post.byVariant.holdout ?? empty, "fullSuccess", ), - `| Controls: still choose \`linear api\` | ${baseline.controls.routeOk}/${baseline.controls.total} | ${post.controls.routeOk}/${post.controls.total} | — |`, + ...(baseline.byFamily.image != null || post.byFamily.image != null + ? [ + row( + "Image family: full success (primary for the image experiment)", + baseline.byFamily.image ?? empty, + post.byFamily.image ?? empty, + "fullSuccess", + ), + ] + : []), + `| API controls: still choose \`linear api\` | ${baseline.apiControls.routeOk}/${baseline.apiControls.total} | ${post.apiControls.routeOk}/${post.apiControls.total} | — |`, + ...(baseline.cliControls.total > 0 || post.cliControls.total > 0 + ? [ + `| CLI controls: correct dedicated route | ${baseline.cliControls.routeOk}/${baseline.cliControls.total} | ${post.cliControls.routeOk}/${post.cliControls.total} | — |`, + ] + : []), "", summaryMarkdown(baseline), "", diff --git a/evals/linear-cli-skill/results/image-baseline.jsonl b/evals/linear-cli-skill/results/image-baseline.jsonl new file mode 100644 index 00000000..079b386e --- /dev/null +++ b/evals/linear-cli-skill/results/image-baseline.jsonl @@ -0,0 +1,45 @@ +{"condition":"image-baseline","caseId":"comment-development","trial":1,"entries":[{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","./comment.md"],"stdin":""},{"tool":"linear","argv":["issue","comment","list","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-1/codex-home/skills/linear-cli/SKILL.md && printf '\\\\n--- comment.md ---\\\\n' && sed -n '1,240p' ./comment.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-1/codex-home/skills/linear-cli/SKILL.md && printf '\\\\n--- comment.md ---\\\\n' && sed -n '1,240p' ./comment.md\"","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'","/usr/bin/bash -lc 'linear issue comment list ENG-107'","/usr/bin/bash -lc 'linear issue comment list ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"Added the contents of `comment.md` as a comment on [ENG-107](https://linear.app/acme/issue/ENG-107#comment-eval).","durationMs":20193,"exitCode":0} +{"condition":"image-baseline","caseId":"comment-development","trial":2,"entries":[{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","./comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'test -f ./comment.md && linear issue comment add ENG-107 --body-file ./comment.md'","/usr/bin/bash -lc 'test -f ./comment.md && linear issue comment add ENG-107 --body-file ./comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Added `./comment.md` as a comment on ENG-107 successfully. [View comment](https://linear.app/acme/issue/ENG-107#comment-eval).","durationMs":12865,"exitCode":0} +{"condition":"image-baseline","caseId":"comment-development","trial":3,"entries":[{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","./comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Comment from `./comment.md` added successfully to ENG-107.\n\n[View the comment](https://linear.app/acme/issue/ENG-107#comment-eval)","durationMs":24972,"exitCode":0} +{"condition":"image-baseline","caseId":"comment-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./alternate-comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Comment posted successfully on `OPS-44`.\n\n[View the comment](https://linear.app/acme/issue/OPS-44#comment-eval)","durationMs":18338,"exitCode":0} +{"condition":"image-baseline","caseId":"comment-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./alternate-comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Posted `./alternate-comment.md` as a comment on OPS-44.\n\n[View the comment](https://linear.app/acme/issue/OPS-44#comment-eval)","durationMs":15987,"exitCode":0} +{"condition":"image-baseline","caseId":"comment-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","view","OPS-44","--json"],"stdin":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./alternate-comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/comment-holdout-3/codex-home/skills/linear-cli/references/issue.md && printf '\\\\nFILE_METADATA\\\\n' && stat -c '%n %s bytes' ./alternate-comment.md && printf '\\\\nISSUE_CHECK\\\\n' && linear issue view OPS-44 --json\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/comment-holdout-3/codex-home/skills/linear-cli/references/issue.md && printf '\\\\nFILE_METADATA\\\\n' && stat -c '%n %s bytes' ./alternate-comment.md && printf '\\\\nISSUE_CHECK\\\\n' && linear issue view OPS-44 --json\"","/usr/bin/bash -lc \"wc -l /trials/comment-holdout-3/codex-home/skills/linear-cli/references/issue.md && sed -n '261,620p' /trials/comment-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"wc -l /trials/comment-holdout-3/codex-home/skills/linear-cli/references/issue.md && sed -n '261,620p' /trials/comment-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '621,700p' /trials/comment-holdout-3/codex-home/skills/linear-cli/references/issue.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md\"","/usr/bin/bash -lc \"sed -n '621,700p' /trials/comment-holdout-3/codex-home/skills/linear-cli/references/issue.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md\""],"skillRead":true,"fixturesModified":false,"answer":"Posted `alternate-comment.md` as a comment on OPS-44.\n\n[View the comment](https://linear.app/acme/issue/OPS-44#comment-eval)","durationMs":23772,"exitCode":0} +{"condition":"image-baseline","caseId":"control-history","trial":1,"entries":[{"tool":"linear","argv":["issue","id","ENG-101"],"stdin":""},{"tool":"linear","argv":["api","query { issue(id: \"ENG-101\") { id identifier title history(first: 100) { nodes { id createdAt actor { id name email } fromState { id name } toState { id name } fromAssignee { id name email } toAssignee { id name email } } } } }"],"stdin":""},{"tool":"linear","argv":["api","query { issue(id: \"ENG-101\") { history(first: 100) { pageInfo { hasNextPage endCursor } nodes { createdAt actor { name } fromState { name } toState { name } fromAssignee { name } toAssignee { name } } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-1/codex-home/skills/linear-cli/references/api.md && linear issue id ENG-101\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-1/codex-home/skills/linear-cli/references/api.md && linear issue id ENG-101\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { id identifier title history(first: 100) { nodes { id createdAt actor { id name email } fromState { id name } toState { id name } fromAssignee { id name email } toAssignee { id name email } } } } }'\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { id identifier title history(first: 100) { nodes { id createdAt actor { id name email } fromState { id name } toState { id name } fromAssignee { id name email } toAssignee { id name email } } } } }'\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { history(first: 100) { pageInfo { hasNextPage endCursor } nodes { createdAt actor { name } fromState { name } toState { name } fromAssignee { name } toAssignee { name } } } } }'\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { history(first: 100) { pageInfo { hasNextPage endCursor } nodes { createdAt actor { name } fromState { name } toState { name } fromAssignee { name } toAssignee { name } } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 change history:\n\n- **2026-07-15 09:00 UTC** — Sam Reyes changed state from **Todo** to **In Progress**.\n- **2026-07-16 11:30 UTC** — Priya Patel changed assignee from **unassigned** to **Sam Reyes**.","durationMs":22556,"exitCode":0} +{"condition":"image-baseline","caseId":"control-history","trial":2,"entries":[{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""},{"tool":"linear","argv":["api","query { issue(id: \"ENG-101\") { identifier title history(first: 100) { nodes { id createdAt actor { id name email } updatedFrom updatedTo } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-2/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-2/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc 'linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue view ENG-101 --json'","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier title history(first: 100) { nodes { id createdAt actor { id name email } updatedFrom updatedTo } } } }'\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier title history(first: 100) { nodes { id createdAt actor { id name email } updatedFrom updatedTo } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 change history:\n\n- 2026-07-15 09:00 UTC — Sam Reyes changed state from **Todo** to **In Progress**.\n- 2026-07-16 11:30 UTC — Priya Patel assigned the issue to **Sam Reyes**.","durationMs":23429,"exitCode":0} +{"condition":"image-baseline","caseId":"control-history","trial":3,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["api","query { issue(id: \"ENG-101\") { id identifier title history(first: 100) { nodes { createdAt actor { name email } fromState { name } toState { name } fromAssignee { name email } toAssignee { name email } } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-3/codex-home/skills/linear-cli/references/api.md && linear --version && linear issue id ENG-101\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-3/codex-home/skills/linear-cli/references/api.md && linear --version && linear issue id ENG-101\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { id identifier title history(first: 100) { nodes { createdAt actor { name email } fromState { name } toState { name } fromAssignee { name email } toAssignee { name email } } } } }'\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { id identifier title history(first: 100) { nodes { createdAt actor { name email } fromState { name } toState { name } fromAssignee { name email } toAssignee { name email } } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 change history:\n\n- **2026-07-15 09:00 UTC** — Sam Reyes changed state from **Todo** to **In Progress**.\n- **2026-07-16 11:30 UTC** — Priya Patel assigned the issue to **Sam Reyes** (previously unassigned).","durationMs":22521,"exitCode":0} +{"condition":"image-baseline","caseId":"control-sidebar-attachment","trial":1,"entries":[{"tool":"linear","argv":["issue","attach","--help"],"stdin":""},{"tool":"linear","argv":["issue","attach","ENG-101","./server.log","--title","server.log"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-sidebar-attachment-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-sidebar-attachment-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'rg -n -A35 -B8 \"issue attach|### attach|## attach\" /trials/control-sidebar-attachment-1/codex-home/skills/linear-cli/references/issue.md && ls -l ./server.log && linear issue attach --help'","/usr/bin/bash -lc 'rg -n -A35 -B8 \"issue attach|### attach|## attach\" /trials/control-sidebar-attachment-1/codex-home/skills/linear-cli/references/issue.md && ls -l ./server.log && linear issue attach --help'","/usr/bin/bash -lc 'linear issue attach ENG-101 ./server.log --title server.log'","/usr/bin/bash -lc 'linear issue attach ENG-101 ./server.log --title server.log'","/usr/bin/bash -lc 'linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue view ENG-101 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Attached `server.log` to [ENG-101](https://linear.app/acme/issue/ENG-101/refactor-session-storage). It’s available as a downloadable attachment to workspace members.","durationMs":45525,"exitCode":0} +{"condition":"image-baseline","caseId":"control-sidebar-attachment","trial":2,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","attach","--help"],"stdin":""},{"tool":"linear","argv":["issue","attach","ENG-101","./server.log","--title","server.log"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-sidebar-attachment-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-sidebar-attachment-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-sidebar-attachment-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-sidebar-attachment-2/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'linear --version && test -f ./server.log && ls -lh ./server.log && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear --version && test -f ./server.log && ls -lh ./server.log && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'command -v linear; type linear; linear issue attach --help'","/usr/bin/bash -lc 'command -v linear; type linear; linear issue attach --help'","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear; rg -n \\\"Attach a file|issue attach|function.*attach|class.*Attach\\\" /src | head -40\"","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear; rg -n \\\"Attach a file|issue attach|function.*attach|class.*Attach\\\" /src | head -40\"","/usr/bin/bash -lc 'test -f ./server.log && ls -lh ./server.log && linear issue attach ENG-101 ./server.log --title server.log'","/usr/bin/bash -lc 'test -f ./server.log && ls -lh ./server.log && linear issue attach ENG-101 ./server.log --title server.log'"],"skillRead":true,"fixturesModified":false,"answer":"Attached `server.log` to issue **ENG-101** as a downloadable attachment for workspace members.","durationMs":33424,"exitCode":0} +{"condition":"image-baseline","caseId":"control-sidebar-attachment","trial":3,"entries":[{"tool":"linear","argv":["issue","attach","--help"],"stdin":""},{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","attach","ENG-101","./server.log"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-sidebar-attachment-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-sidebar-attachment-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'rg -n -A45 -B8 \"issue attach|## attach|### attach\" /trials/control-sidebar-attachment-3/codex-home/skills/linear-cli/references/issue.md && linear issue attach --help && ls -l ./server.log'","/usr/bin/bash -lc 'rg -n -A45 -B8 \"issue attach|## attach|### attach\" /trials/control-sidebar-attachment-3/codex-home/skills/linear-cli/references/issue.md && linear issue attach --help && ls -l ./server.log'","/usr/bin/bash -lc \"command -v linear; linear --version; ls -l ./server.log; file ./server.log; sed -n '1,220p' /trials/control-sidebar-attachment-3/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"command -v linear; linear --version; ls -l ./server.log; file ./server.log; sed -n '1,220p' /trials/control-sidebar-attachment-3/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /evals/linear-cli-skill/shims/linear; env | rg '\"'^(LINEAR|DENO|PATH)='\"'\"","/usr/bin/bash -lc \"sed -n '1,240p' /evals/linear-cli-skill/shims/linear; env | rg '\"'^(LINEAR|DENO|PATH)='\"'\"","/usr/bin/bash -lc 'linear issue attach ENG-101 ./server.log'","/usr/bin/bash -lc 'linear issue attach ENG-101 ./server.log'"],"skillRead":true,"fixturesModified":false,"answer":"Attached `server.log` to issue `ENG-101` as a downloadable attachment.","durationMs":28853,"exitCode":0} +{"condition":"image-baseline","caseId":"control-subscribers","trial":1,"entries":[{"tool":"linear","argv":["api","query($issueId: String!) { issue(id: $issueId) { identifier title subscribers { nodes { name email } } } }","--variable","issueId=ENG-101"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-1/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-1/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"linear api 'query(\"'$issueId: String!) { issue(id: $issueId) { identifier title subscribers { nodes { name email } } } }'\"' --variable issueId=ENG-101\"","/usr/bin/bash -lc \"linear api 'query(\"'$issueId: String!) { issue(id: $issueId) { identifier title subscribers { nodes { name email } } } }'\"' --variable issueId=ENG-101\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 subscribers:\n\n- Sam Reyes — sam@acme.test\n- Priya Patel — priya@acme.test","durationMs":23290,"exitCode":0} +{"condition":"image-baseline","caseId":"control-subscribers","trial":2,"entries":[{"tool":"linear","argv":["api","query { issue(id: \"ENG-101\") { identifier subscribers { nodes { name email } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-2/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-2/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier subscribers { nodes { name email } } } }'\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier subscribers { nodes { name email } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 has two subscribers:\n\n- Sam Reyes — sam@acme.test\n- Priya Patel — priya@acme.test","durationMs":22079,"exitCode":0} +{"condition":"image-baseline","caseId":"control-subscribers","trial":3,"entries":[{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""},{"tool":"linear","argv":["issue","id","ENG-101"],"stdin":""},{"tool":"linear","argv":["api","query($id:String!){issue(id:$id){identifier subscribers{nodes{name email}}}}","--variable","id=ENG-101"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue view ENG-101 --json'","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-3/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-3/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc 'linear issue id ENG-101'","/usr/bin/bash -lc 'linear issue id ENG-101'","/usr/bin/bash -lc \"linear api 'query(\"'$id:String!){issue(id:$id){identifier subscribers{nodes{name email}}}}'\"' --variable id=ENG-101\"","/usr/bin/bash -lc \"linear api 'query(\"'$id:String!){issue(id:$id){identifier subscribers{nodes{name email}}}}'\"' --variable id=ENG-101\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 subscribers:\n\n- Sam Reyes — sam@acme.test\n- Priya Patel — priya@acme.test","durationMs":35565,"exitCode":0} +{"condition":"image-baseline","caseId":"create-development","trial":1,"entries":[{"tool":"linear","argv":["issue","create","--team","ENG","--title","Migrate CI to Blacksmith runners","--description-file","./issue-description.md","--no-interactive"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' issue-description.md\"","/usr/bin/bash -lc \"sed -n '1,240p' issue-description.md\"","/usr/bin/bash -lc 'linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'"],"skillRead":true,"fixturesModified":false,"answer":"Created [ENG-231: Migrate CI to Blacksmith runners](https://linear.app/acme/issue/ENG-231/new-issue) with the description from `issue-description.md`.","durationMs":22329,"exitCode":0} +{"condition":"image-baseline","caseId":"create-development","trial":2,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","ENG","--title","Migrate CI to Blacksmith runners","--description-file","./issue-description.md","--no-interactive"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"test -f ./issue-description.md && linear --version && linear issue create --team ENG --title 'Migrate CI to Blacksmith runners' --description-file ./issue-description.md --no-interactive\"","/usr/bin/bash -lc \"test -f ./issue-description.md && linear --version && linear issue create --team ENG --title 'Migrate CI to Blacksmith runners' --description-file ./issue-description.md --no-interactive\"","/usr/bin/bash -lc 'command -v linear; type -a linear; ls -l \"$(command -v linear)\"; sed -n '\"'1,120p' \\\"\"'$(command -v linear)\"'","/usr/bin/bash -lc 'command -v linear; type -a linear; ls -l \"$(command -v linear)\"; sed -n '\"'1,120p' \\\"\"'$(command -v linear)\"'","/usr/bin/bash -lc \"linear issue create --team ENG --title 'Migrate CI to Blacksmith runners' --description-file ./issue-description.md --no-interactive\"","/usr/bin/bash -lc \"linear issue create --team ENG --title 'Migrate CI to Blacksmith runners' --description-file ./issue-description.md --no-interactive\""],"skillRead":true,"fixturesModified":false,"answer":"Created [ENG-231: Migrate CI to Blacksmith runners](https://linear.app/acme/issue/ENG-231/new-issue) on the ENG team using `issue-description.md`.","durationMs":22869,"exitCode":0} +{"condition":"image-baseline","caseId":"create-development","trial":3,"entries":[{"tool":"linear","argv":["issue","create","--team","ENG","--title","Migrate CI to Blacksmith runners","--description-file","./issue-description.md","--no-interactive"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'test -f ./issue-description.md && linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'test -f ./issue-description.md && linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'"],"skillRead":true,"fixturesModified":false,"answer":"Created [ENG-231: Migrate CI to Blacksmith runners](https://linear.app/acme/issue/ENG-231/new-issue) using `issue-description.md`.","durationMs":11813,"exitCode":0} +{"condition":"image-baseline","caseId":"create-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","create","--help"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","OPS","--title","Document rate limit behavior","--project","Data Pipeline","--label","docs","--label","api","--description-file","./alternate-description.md","--no-interactive"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'rg -n \"issue create|--project|--label\" /trials/create-holdout-1/codex-home/skills/linear-cli/references/issue.md | head -80 && test -f ./alternate-description.md && linear issue create --help'","/usr/bin/bash -lc 'rg -n \"issue create|--project|--label\" /trials/create-holdout-1/codex-home/skills/linear-cli/references/issue.md | head -80 && test -f ./alternate-description.md && linear issue create --help'","/usr/bin/bash -lc \"sed -n '230,270p' /trials/create-holdout-1/codex-home/skills/linear-cli/references/issue.md && linear issue create --team OPS --title \\\"Document rate limit behavior\\\" --project \\\"Data Pipeline\\\" --label docs --label api --description-file ./alternate-description.md --no-interactive\"","/usr/bin/bash -lc \"sed -n '230,270p' /trials/create-holdout-1/codex-home/skills/linear-cli/references/issue.md && linear issue create --team OPS --title \\\"Document rate limit behavior\\\" --project \\\"Data Pipeline\\\" --label docs --label api --description-file ./alternate-description.md --no-interactive\""],"skillRead":true,"fixturesModified":false,"answer":"Created [OPS-231: Document rate limit behavior](https://linear.app/acme/issue/OPS-231/new-issue) in **Data Pipeline** with labels **docs** and **api**, using `alternate-description.md`.","durationMs":17792,"exitCode":0} +{"condition":"image-baseline","caseId":"create-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","create","--help"],"stdin":""},{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","OPS","--title","Document rate limit behavior","--project","Data Pipeline","--label","docs","--label","api","--description-file","./alternate-description.md","--no-interactive"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"linear issue create --help && test -f ./alternate-description.md && sed -n '1,80p' ./alternate-description.md\"","/usr/bin/bash -lc \"linear issue create --help && test -f ./alternate-description.md && sed -n '1,80p' ./alternate-description.md\"","/usr/bin/bash -lc \"command -v linear; linear --version; ls -l ./alternate-description.md; sed -n '1,80p' ./alternate-description.md\"","/usr/bin/bash -lc \"command -v linear; linear --version; ls -l ./alternate-description.md; sed -n '1,80p' ./alternate-description.md\"","/usr/bin/bash -lc \"env | cut -d= -f1 | rg 'LINEAR|TOKEN|API' | sort; sed -n '1,200p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"env | cut -d= -f1 | rg 'LINEAR|TOKEN|API' | sort; sed -n '1,200p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc 'linear issue create --team OPS --title \"Document rate limit behavior\" --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue create --team OPS --title \"Document rate limit behavior\" --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md --no-interactive'"],"skillRead":true,"fixturesModified":false,"answer":"Created **OPS-231: Document rate limit behavior** in **Data Pipeline**, with labels `docs` and `api`, using `alternate-description.md`.\n\nhttps://linear.app/acme/issue/OPS-231/new-issue","durationMs":28765,"exitCode":0} +{"condition":"image-baseline","caseId":"create-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","create","--help"],"stdin":""},{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","OPS","--project","Data Pipeline","--title","Document rate limit behavior","--label","docs","--label","api","--description-file","./alternate-description.md","--no-interactive"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-231","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/create-holdout-3/codex-home/skills/linear-cli/references/issue.md && test -f ./alternate-description.md && linear issue create --help\"","/usr/bin/bash -lc \"sed -n '1,280p' /trials/create-holdout-3/codex-home/skills/linear-cli/references/issue.md && test -f ./alternate-description.md && linear issue create --help\"","/usr/bin/bash -lc \"command -v linear; linear --version; command -v npx; ls -la; if [ -f .linear.toml ]; then sed -n '1,160p' .linear.toml; fi\"","/usr/bin/bash -lc \"command -v linear; linear --version; command -v npx; ls -la; if [ -f .linear.toml ]; then sed -n '1,160p' .linear.toml; fi\"","/usr/bin/bash -lc 'linear issue create --team OPS --project \"Data Pipeline\" --title \"Document rate limit behavior\" --label docs --label api --description-file ./alternate-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue create --team OPS --project \"Data Pipeline\" --title \"Document rate limit behavior\" --label docs --label api --description-file ./alternate-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue view OPS-231 --json'","/usr/bin/bash -lc 'linear issue view OPS-231 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Created [OPS-231: Document rate limit behavior](https://linear.app/acme/issue/OPS-231/new-issue) on OPS in the “Data Pipeline” project, with `docs` and `api` labels and the description from `alternate-description.md`.","durationMs":31078,"exitCode":0} +{"condition":"image-baseline","caseId":"image-development","trial":1,"entries":[{"tool":"linear","argv":["issue","attach","--help"],"stdin":""},{"tool":"linear","argv":["issue","attach","ENG-107","./screenshot.png","--title","Screenshot"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-107","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'rg -n -A45 -B8 \"issue attach|## Attach|### attach\" /trials/image-development-1/codex-home/skills/linear-cli/references/issue.md && test -f ./screenshot.png && file ./screenshot.png && linear issue attach --help'","/usr/bin/bash -lc 'rg -n -A45 -B8 \"issue attach|## Attach|### attach\" /trials/image-development-1/codex-home/skills/linear-cli/references/issue.md && test -f ./screenshot.png && file ./screenshot.png && linear issue attach --help'","/usr/bin/bash -lc 'linear issue attach ENG-107 ./screenshot.png --title \"Screenshot\"'","/usr/bin/bash -lc 'linear issue attach ENG-107 ./screenshot.png --title \"Screenshot\"'","/usr/bin/bash -lc 'linear issue view ENG-107 --json'","/usr/bin/bash -lc 'linear issue view ENG-107 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Attached `screenshot.png` to ENG-107 with the title **Screenshot**. It is now visible on the issue.","durationMs":21193,"exitCode":0} +{"condition":"image-baseline","caseId":"image-development","trial":2,"entries":[{"tool":"linear","argv":["issue","attach","--help"],"stdin":""},{"tool":"linear","argv":["issue","attach","ENG-107","./screenshot.png","--title","screenshot.png"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-107","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'rg -n -A35 -B8 \"issue attach|## Attach|### attach\" /trials/image-development-2/codex-home/skills/linear-cli/references/issue.md && linear issue attach --help && file ./screenshot.png && linear issue view ENG-107 --json'","/usr/bin/bash -lc 'rg -n -A35 -B8 \"issue attach|## Attach|### attach\" /trials/image-development-2/codex-home/skills/linear-cli/references/issue.md && linear issue attach --help && file ./screenshot.png && linear issue view ENG-107 --json'","/usr/bin/bash -lc 'file ./screenshot.png\nlinear issue attach ENG-107 ./screenshot.png --title screenshot.png'","/usr/bin/bash -lc 'file ./screenshot.png\nlinear issue attach ENG-107 ./screenshot.png --title screenshot.png'","/usr/bin/bash -lc 'linear issue view ENG-107 --json'","/usr/bin/bash -lc 'linear issue view ENG-107 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Attached `screenshot.png` to [ENG-107](https://linear.app/acme/issue/ENG-107/refactor-session-storage). Linear confirmed the upload and attachment creation.","durationMs":27138,"exitCode":0} +{"condition":"image-baseline","caseId":"image-development","trial":3,"entries":[{"tool":"linear","argv":["issue","attach","ENG-107","./screenshot.png"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-107","--json"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'rg -n -A35 -B8 \"issue attach|### Attach|## Attach|attach\" /trials/image-development-3/codex-home/skills/linear-cli/references/issue.md'","/usr/bin/bash -lc 'rg -n -A35 -B8 \"issue attach|### Attach|## Attach|attach\" /trials/image-development-3/codex-home/skills/linear-cli/references/issue.md'","/usr/bin/bash -lc 'test -f ./screenshot.png && file ./screenshot.png && linear issue attach ENG-107 ./screenshot.png'","/usr/bin/bash -lc 'test -f ./screenshot.png && file ./screenshot.png && linear issue attach ENG-107 ./screenshot.png'","/usr/bin/bash -lc 'linear issue view ENG-107 --json'","/usr/bin/bash -lc 'linear issue view ENG-107 --json'","/usr/bin/bash -lc 'linear issue view ENG-107'","/usr/bin/bash -lc 'linear issue view ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"Attached `screenshot.png` to [ENG-107](https://linear.app/acme/issue/ENG-107/refactor-session-storage). Linear confirmed the attachment was created successfully.","durationMs":31558,"exitCode":0} +{"condition":"image-baseline","caseId":"image-holdout","trial":1,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--attach","./screenshot.png"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'rg -n -C 10 \"issue attach|comment add|attachment|upload\" /trials/image-holdout-1/codex-home/skills/linear-cli/references/issue.md'","/usr/bin/bash -lc 'rg -n -C 10 \"issue attach|comment add|attachment|upload\" /trials/image-holdout-1/codex-home/skills/linear-cli/references/issue.md'","/usr/bin/bash -lc \"sed -n '101,180p' /trials/image-holdout-1/codex-home/skills/linear-cli/references/issue.md && test -f ./screenshot.png && file ./screenshot.png && linear --version && linear auth whoami\"","/usr/bin/bash -lc \"sed -n '101,180p' /trials/image-holdout-1/codex-home/skills/linear-cli/references/issue.md && test -f ./screenshot.png && file ./screenshot.png && linear --version && linear auth whoami\"","/usr/bin/bash -lc 'command -v linear; ls -l \"$(command -v linear)\"; head -n 40 \"$(command -v linear)\"; find -path '\"'*cliffy*command*' -o -path '*linear-cli*' 2>/dev/null | head -80\"","/usr/bin/bash -lc 'command -v linear; ls -l \"$(command -v linear)\"; head -n 40 \"$(command -v linear)\"; find -path '\"'*cliffy*command*' -o -path '*linear-cli*' 2>/dev/null | head -80\"","/usr/bin/bash -lc \"sed -n '40,220p' /evals/linear-cli-skill/shims/linear; sed -n '1,240p' /src/commands/issue/issue-comment-add.ts\"","/usr/bin/bash -lc \"sed -n '40,220p' /evals/linear-cli-skill/shims/linear; sed -n '1,240p' /src/commands/issue/issue-comment-add.ts\"","/usr/bin/bash -lc 'linear issue comment add OPS-44 --attach ./screenshot.png'","/usr/bin/bash -lc 'linear issue comment add OPS-44 --attach ./screenshot.png'"],"skillRead":true,"fixturesModified":false,"answer":"Added a comment to OPS-44 with `screenshot.png` displayed inline.\n\n[View the comment](https://linear.app/acme/issue/OPS-44#comment-eval)","durationMs":44611,"exitCode":0} +{"condition":"image-baseline","caseId":"image-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","attach","--help"],"stdin":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--body","Screenshot","--attach","./screenshot.png"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'rg -n -C 8 \"issue attach|attach\" /trials/image-holdout-2/codex-home/skills/linear-cli/references/issue.md && linear issue attach --help && linear issue comment add --help && file ./screenshot.png'","/usr/bin/bash -lc 'rg -n -C 8 \"issue attach|attach\" /trials/image-holdout-2/codex-home/skills/linear-cli/references/issue.md && linear issue attach --help && linear issue comment add --help && file ./screenshot.png'","/usr/bin/bash -lc \"file ./screenshot.png && ls -lh ./screenshot.png && linear issue comment add OPS-44 --body 'Screenshot' --attach ./screenshot.png\"","/usr/bin/bash -lc \"file ./screenshot.png && ls -lh ./screenshot.png && linear issue comment add OPS-44 --body 'Screenshot' --attach ./screenshot.png\""],"skillRead":true,"fixturesModified":false,"answer":"Added a comment to OPS-44 with `screenshot.png` visible inline.\n\n[View the comment](https://linear.app/acme/issue/OPS-44#comment-eval)","durationMs":22863,"exitCode":0} +{"condition":"image-baseline","caseId":"image-holdout","trial":3,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-44","--json"],"stdin":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--body","Screenshot attached.","--attach","./screenshot.png"],"stdin":""},{"tool":"linear","argv":["issue","comment","list","OPS-44","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'rg -n -A35 -B5 \"issue attach|comment add|attach\" /trials/image-holdout-3/codex-home/skills/linear-cli/references/issue.md'","/usr/bin/bash -lc 'rg -n -A35 -B5 \"issue attach|comment add|attach\" /trials/image-holdout-3/codex-home/skills/linear-cli/references/issue.md'","/usr/bin/bash -lc 'linear --version && test -f ./screenshot.png && file ./screenshot.png && linear issue view OPS-44 --json'","/usr/bin/bash -lc 'linear --version && test -f ./screenshot.png && file ./screenshot.png && linear issue view OPS-44 --json'","/usr/bin/bash -lc 'command -v linear; type -a linear; ls -l ./screenshot.png; file ./screenshot.png; head -n 40 \"$(command -v linear)\"'","/usr/bin/bash -lc 'command -v linear; type -a linear; ls -l ./screenshot.png; file ./screenshot.png; head -n 40 \"$(command -v linear)\"'","/usr/bin/bash -lc 'linear issue view OPS-44 --json && linear issue comment add OPS-44 --body \"Screenshot attached.\" --attach ./screenshot.png && linear issue comment list OPS-44 --json'","/usr/bin/bash -lc 'linear issue view OPS-44 --json && linear issue comment add OPS-44 --body \"Screenshot attached.\" --attach ./screenshot.png && linear issue comment list OPS-44 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Added a comment to **OPS-44** with `screenshot.png` attached and visible inline.\n\n[View the comment](https://linear.app/acme/issue/OPS-44#comment-eval)","durationMs":27120,"exitCode":0} +{"condition":"image-baseline","caseId":"inspect-development","trial":1,"entries":[{"tool":"linear","argv":["issue","view","ENG-112","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue view ENG-112 --json'","/usr/bin/bash -lc 'linear issue view ENG-112 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issue\": {\n \"identifier\": \"ENG-112\",\n \"title\": \"Refactor session storage\",\n \"description\": \"Session data is duplicated between redis and postgres.\",\n \"state\": {\n \"name\": \"In Progress\"\n },\n \"assignee\": null,\n \"labels\": {\n \"nodes\": [\n {\n \"name\": \"tech-debt\"\n }\n ]\n },\n \"project\": null,\n \"url\": \"https://linear.app/acme/issue/ENG-112/refactor-session-storage\"\n }\n}\n```","durationMs":18869,"exitCode":0} +{"condition":"image-baseline","caseId":"inspect-development","trial":2,"entries":[{"tool":"linear","argv":["issue","view","ENG-112","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue view ENG-112 --json'","/usr/bin/bash -lc 'linear issue view ENG-112 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issue\": {\n \"identifier\": \"ENG-112\",\n \"title\": \"Refactor session storage\",\n \"description\": \"Session data is duplicated between redis and postgres.\",\n \"state\": {\n \"name\": \"In Progress\"\n },\n \"assignee\": null,\n \"labels\": {\n \"nodes\": [\n {\n \"name\": \"tech-debt\"\n }\n ]\n },\n \"project\": null,\n \"url\": \"https://linear.app/acme/issue/ENG-112/refactor-session-storage\"\n }\n}\n```","durationMs":21693,"exitCode":0} +{"condition":"image-baseline","caseId":"inspect-development","trial":3,"entries":[{"tool":"linear","argv":["issue","view","ENG-112","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue view ENG-112 --json'","/usr/bin/bash -lc 'linear issue view ENG-112 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issue\": {\n \"identifier\": \"ENG-112\",\n \"title\": \"Refactor session storage\",\n \"description\": \"Session data is duplicated between redis and postgres.\",\n \"state\": {\n \"name\": \"In Progress\"\n },\n \"assignee\": null,\n \"labels\": {\n \"nodes\": [\n {\n \"name\": \"tech-debt\"\n }\n ]\n },\n \"project\": null,\n \"url\": \"https://linear.app/acme/issue/ENG-112/refactor-session-storage\"\n }\n}\n```","durationMs":14058,"exitCode":0} +{"condition":"image-baseline","caseId":"inspect-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","url","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue url ENG-107'","/usr/bin/bash -lc 'linear issue url ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"https://linear.app/acme/issue/ENG-107/update-billing-webhooks","durationMs":10241,"exitCode":0} +{"condition":"image-baseline","caseId":"inspect-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","url","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue url ENG-107'","/usr/bin/bash -lc 'linear issue url ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"https://linear.app/acme/issue/ENG-107/update-billing-webhooks","durationMs":12056,"exitCode":0} +{"condition":"image-baseline","caseId":"inspect-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","url","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue url ENG-107'","/usr/bin/bash -lc 'linear issue url ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"https://linear.app/acme/issue/ENG-107/update-billing-webhooks","durationMs":13318,"exitCode":0} +{"condition":"image-baseline","caseId":"query-development","trial":1,"entries":[{"tool":"linear","argv":["issue","query","--team","ENG","--project","Mobile App Revamp","--state","started","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'"],"skillRead":true,"fixturesModified":false,"answer":"Started issues in ENG’s “Mobile App Revamp” project:\n\n- ENG-101 — Fix login redirect loop — sam\n- ENG-107 — Update billing webhooks — priya\n- ENG-112 — Refactor session storage — unassigned\n\nAll three are In Progress.","durationMs":18045,"exitCode":0} +{"condition":"image-baseline","caseId":"query-development","trial":2,"entries":[{"tool":"linear","argv":["issue","query","--team","ENG","--project","Mobile App Revamp","--state","started","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'"],"skillRead":true,"fixturesModified":false,"answer":"Started issues in ENG’s “Mobile App Revamp” project:\n\n- ENG-101 — Fix login redirect loop (Sam)\n- ENG-107 — Update billing webhooks (Priya)\n- ENG-112 — Refactor session storage (unassigned)","durationMs":16380,"exitCode":0} +{"condition":"image-baseline","caseId":"query-development","trial":3,"entries":[{"tool":"linear","argv":["issue","query","--team","ENG","--project","Mobile App Revamp","--state","started","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'"],"skillRead":true,"fixturesModified":false,"answer":"Started ENG issues in **Mobile App Revamp**:\n\n- ENG-101 — Fix login redirect loop — sam\n- ENG-107 — Update billing webhooks — priya\n- ENG-112 — Refactor session storage — unassigned\n\n3 issues total.","durationMs":15012,"exitCode":0} +{"condition":"image-baseline","caseId":"query-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","query","--team","OPS","--state","backlog","--state","triage","--unassigned","--updated-after","2026-06-01","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issues\": {\n \"nodes\": [\n {\n \"identifier\": \"OPS-101\",\n \"title\": \"Fix login redirect loop\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 2,\n \"updatedAt\": \"2026-07-17T10:00:00.000Z\"\n },\n {\n \"identifier\": \"OPS-107\",\n \"title\": \"Update billing webhooks\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 3,\n \"updatedAt\": \"2026-07-16T08:30:00.000Z\"\n },\n {\n \"identifier\": \"OPS-112\",\n \"title\": \"Refactor session storage\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 4,\n \"updatedAt\": \"2026-07-14T16:12:00.000Z\"\n }\n ],\n \"pageInfo\": {\n \"hasNextPage\": false,\n \"endCursor\": \"cursor-3\"\n }\n }\n}\n```","durationMs":21871,"exitCode":0} +{"condition":"image-baseline","caseId":"query-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","query","--team","OPS","--state","backlog","--state","triage","--unassigned","--updated-after","2026-06-01","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issues\": [\n {\n \"identifier\": \"OPS-101\",\n \"title\": \"Fix login redirect loop\",\n \"state\": \"Backlog\",\n \"assignee\": null,\n \"priority\": 2,\n \"updatedAt\": \"2026-07-17T10:00:00.000Z\"\n },\n {\n \"identifier\": \"OPS-107\",\n \"title\": \"Update billing webhooks\",\n \"state\": \"Backlog\",\n \"assignee\": null,\n \"priority\": 3,\n \"updatedAt\": \"2026-07-16T08:30:00.000Z\"\n },\n {\n \"identifier\": \"OPS-112\",\n \"title\": \"Refactor session storage\",\n \"state\": \"Backlog\",\n \"assignee\": null,\n \"priority\": 4,\n \"updatedAt\": \"2026-07-14T16:12:00.000Z\"\n }\n ]\n}\n```","durationMs":20421,"exitCode":0} +{"condition":"image-baseline","caseId":"query-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","query","--team","OPS","--state","backlog","--state","triage","--unassigned","--updated-after","2026-06-01","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issues\": {\n \"nodes\": [\n {\n \"identifier\": \"OPS-101\",\n \"title\": \"Fix login redirect loop\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 2,\n \"updatedAt\": \"2026-07-17T10:00:00.000Z\"\n },\n {\n \"identifier\": \"OPS-107\",\n \"title\": \"Update billing webhooks\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 3,\n \"updatedAt\": \"2026-07-16T08:30:00.000Z\"\n },\n {\n \"identifier\": \"OPS-112\",\n \"title\": \"Refactor session storage\",\n \"state\": {\n \"name\": \"Backlog\",\n \"type\": \"backlog\"\n },\n \"assignee\": null,\n \"priority\": 4,\n \"updatedAt\": \"2026-07-14T16:12:00.000Z\"\n }\n ],\n \"pageInfo\": {\n \"hasNextPage\": false,\n \"endCursor\": \"cursor-3\"\n }\n }\n}\n```","durationMs":21953,"exitCode":0} +{"condition":"image-baseline","caseId":"update-development","trial":1,"entries":[{"tool":"linear","argv":["issue","update","ENG-101","--state","In Review","--assignee","priya"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'"],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 (“Refactor session storage”) is now **In Review** and assigned to **priya**.","durationMs":17319,"exitCode":0} +{"condition":"image-baseline","caseId":"update-development","trial":2,"entries":[{"tool":"linear","argv":["issue","update","ENG-101","--state","In Review","--assignee","priya"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Updated [ENG-101](https://linear.app/acme/issue/ENG-101/refactor-session-storage):\n\n- State: **In Review**\n- Assignee: **priya**","durationMs":16927,"exitCode":0} +{"condition":"image-baseline","caseId":"update-development","trial":3,"entries":[{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""},{"tool":"linear","argv":["team","states","ENG"],"stdin":""},{"tool":"linear","argv":["team","members","ENG"],"stdin":""},{"tool":"linear","argv":["issue","update","ENG-101","--state","In Review","--assignee","priya"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue view ENG-101 --json && linear team states ENG && linear team members ENG'","/usr/bin/bash -lc 'linear issue view ENG-101 --json && linear team states ENG && linear team members ENG'","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Updated [ENG-101](https://linear.app/acme/issue/ENG-101):\n\n- State: **In Review**\n- Assignee: **Priya Patel** (`priya`)","durationMs":15676,"exitCode":0} +{"condition":"image-baseline","caseId":"update-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","view","OPS-31","--json","--no-download"],"stdin":""},{"tool":"linear","argv":["issue","update","OPS-31","--unassign","--project","Data Pipeline","--label","infra","--label","security"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-31","--json","--no-download"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n -A90 '\"'^### update|''^##### update'\"' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n -A90 '\"'^### update|''^##### update'\"' /trials/update-holdout-1/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'linear issue view OPS-31 --json --no-download'","/usr/bin/bash -lc 'linear issue view OPS-31 --json --no-download'","/usr/bin/bash -lc \"linear issue update OPS-31 --unassign --project 'Data Pipeline' --label infra --label security\"","/usr/bin/bash -lc \"linear issue update OPS-31 --unassign --project 'Data Pipeline' --label infra --label security\"","/usr/bin/bash -lc 'linear issue view OPS-31 --json --no-download'","/usr/bin/bash -lc 'linear issue view OPS-31 --json --no-download'"],"skillRead":true,"fixturesModified":false,"answer":"Updated [OPS-31](https://linear.app/acme/issue/OPS-31/refactor-session-storage):\n\n- Assignee: Unassigned\n- Project: Data Pipeline\n- Labels: infra, security","durationMs":30617,"exitCode":0} +{"condition":"image-baseline","caseId":"update-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","view","OPS-31","--json"],"stdin":""},{"tool":"linear","argv":["issue","update","OPS-31","--unassign","--project","Data Pipeline","--label","infra","--label","security"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-31","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'rg -n \"project|unassign|--label\" /trials/update-holdout-2/codex-home/skills/linear-cli/references/issue.md | head -80'","/usr/bin/bash -lc 'rg -n \"project|unassign|--label\" /trials/update-holdout-2/codex-home/skills/linear-cli/references/issue.md | head -80'","/usr/bin/bash -lc 'linear issue view OPS-31 --json && linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security && linear issue view OPS-31 --json'","/usr/bin/bash -lc 'linear issue view OPS-31 --json && linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security && linear issue view OPS-31 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Updated OPS-31:\n\n- Assignee: unassigned\n- Project: Data Pipeline\n- Labels: infra, security","durationMs":15844,"exitCode":0} +{"condition":"image-baseline","caseId":"update-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","view","OPS-31","--json"],"stdin":""},{"tool":"linear","argv":["issue","update","OPS-31","--unassign","--project","Data Pipeline","--label","infra","--label","security"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-31","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n -A45 '\"'^### update|''^##### update'\"' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc \"rg -n -A45 '\"'^### update|''^##### update'\"' /trials/update-holdout-3/codex-home/skills/linear-cli/references/issue.md\"","/usr/bin/bash -lc 'linear issue view OPS-31 --json'","/usr/bin/bash -lc 'linear issue view OPS-31 --json'","/usr/bin/bash -lc \"linear issue update OPS-31 --unassign --project 'Data Pipeline' --label infra --label security\"","/usr/bin/bash -lc \"linear issue update OPS-31 --unassign --project 'Data Pipeline' --label infra --label security\"","/usr/bin/bash -lc 'linear issue view OPS-31 --json'","/usr/bin/bash -lc 'linear issue view OPS-31 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Updated [OPS-31](https://linear.app/acme/issue/OPS-31/refactor-session-storage):\n\n- Assignee: unassigned\n- Project: Data Pipeline\n- Labels: `infra`, `security`","durationMs":30464,"exitCode":0} diff --git a/evals/linear-cli-skill/results/image-baseline.meta.json b/evals/linear-cli-skill/results/image-baseline.meta.json new file mode 100644 index 00000000..264f74d3 --- /dev/null +++ b/evals/linear-cli-skill/results/image-baseline.meta.json @@ -0,0 +1,29 @@ +{ + "condition": "image-baseline", + "codexVersion": "codex-cli 0.144.4", + "sandbox": "workspace-write", + "effort": "low", + "modelRequested": "gpt-5.6-sol", + "modelsObserved": [], + "trialsPerCase": 3, + "cases": [ + "query-development", + "query-holdout", + "create-development", + "create-holdout", + "update-development", + "update-holdout", + "comment-development", + "comment-holdout", + "inspect-development", + "inspect-holdout", + "image-development", + "image-holdout", + "control-sidebar-attachment", + "control-history", + "control-subscribers" + ], + "completedTrials": 45, + "failedTrials": [], + "ranAt": "2026-07-23T17:17:44.805Z" +} diff --git a/evals/linear-cli-skill/results/image-comparison.md b/evals/linear-cli-skill/results/image-comparison.md new file mode 100644 index 00000000..4c060440 --- /dev/null +++ b/evals/linear-cli-skill/results/image-comparison.md @@ -0,0 +1,95 @@ +## Comparison + +Primary outcome is **full success on supported tasks** (dedicated-CLI route, taught subcommand, all required flags, intact fixtures, no GraphQL/HTTP fallback). The holdout row uses only prompts that were never looked at while iterating on the skill text. Trials of the same prompt are not independent samples, so case-level counts are shown in the per-condition sections below. + +| Metric | Baseline | Post-change | One-sided Fisher p | +| ------------------------------------------------------------- | -------- | ----------- | ------------------ | +| Supported: full success (primary) | 33/36 | 36/36 | 0.120 | +| Supported: dedicated-CLI route | 33/36 | 36/36 | 0.120 | +| Holdout only: full success | 18/18 | 18/18 | 1.00 | +| Image family: full success (primary for the image experiment) | 3/6 | 6/6 | 0.0909 | +| API controls: still choose `linear api` | 6/6 | 6/6 | — | +| CLI controls: correct dedicated route | 3/3 | 2/3 | — | + +### Condition: image-baseline + +- Trials: 45 (skill read in 45) +- Supported tasks — full success: 33/36 (92%), dedicated-CLI route: 33/36 (92%) +- Controls (GraphQL appropriate) — chose `linear api`: 6/6, with expected fields: 6/6 +- Controls (dedicated CLI appropriate) — correct route: 3/3, full success: 3/3 +- Mean non-discovery invocations per trial: 1.56 + +| Case | Route ok | Full success | +| -------------------------- | -------- | ------------ | +| comment-development | 3/3 | 3/3 | +| comment-holdout | 3/3 | 3/3 | +| control-history | 3/3 | 3/3 | +| control-sidebar-attachment | 3/3 | 3/3 | +| control-subscribers | 3/3 | 3/3 | +| create-development | 3/3 | 3/3 | +| create-holdout | 3/3 | 3/3 | +| image-development | 0/3 | 0/3 | +| image-holdout | 3/3 | 3/3 | +| inspect-development | 3/3 | 3/3 | +| inspect-holdout | 3/3 | 3/3 | +| query-development | 3/3 | 3/3 | +| query-holdout | 3/3 | 3/3 | +| update-development | 3/3 | 3/3 | +| update-holdout | 3/3 | 3/3 | + +| Variant | Route ok | Full success | +| ----------- | -------- | ------------ | +| development | 15/18 | 15/18 | +| holdout | 18/18 | 18/18 | +| control | 9/9 | 9/9 | + +### Condition: image-post-change + +- Trials: 45 (skill read in 45) +- Supported tasks — full success: 36/36 (100%), dedicated-CLI route: 36/36 (100%) +- Controls (GraphQL appropriate) — chose `linear api`: 6/6, with expected fields: 6/6 +- Controls (dedicated CLI appropriate) — correct route: 2/3, full success: 2/3 +- Mean non-discovery invocations per trial: 1.31 + +| Case | Route ok | Full success | +| -------------------------- | -------- | ------------ | +| comment-development | 3/3 | 3/3 | +| comment-holdout | 3/3 | 3/3 | +| control-history | 3/3 | 3/3 | +| control-sidebar-attachment | 2/3 | 2/3 | +| control-subscribers | 3/3 | 3/3 | +| create-development | 3/3 | 3/3 | +| create-holdout | 3/3 | 3/3 | +| image-development | 3/3 | 3/3 | +| image-holdout | 3/3 | 3/3 | +| inspect-development | 3/3 | 3/3 | +| inspect-holdout | 3/3 | 3/3 | +| query-development | 3/3 | 3/3 | +| query-holdout | 3/3 | 3/3 | +| update-development | 3/3 | 3/3 | +| update-holdout | 3/3 | 3/3 | + +| Variant | Route ok | Full success | +| ----------- | -------- | ------------ | +| development | 18/18 | 18/18 | +| holdout | 18/18 | 18/18 | +| control | 8/9 | 8/9 | + +## Findings (hand-written addendum; tables above are generated) + +Outcome under the pre-declared rules: the baseline image family scored **3/6** — inside the pre-declared partial-baseline band (2/6–4/6) — so the change is reported **descriptively as consistent with improvement, not statistically confirmed** (one-sided Fisher p = 0.0909, labeled exploratory up front because trials of one prompt are correlated). + +Per-case counts, the co-primary evidence: + +- `image-development` (the trap phrasing, "attach the screenshot … so it is visible"): **0/3 → 3/3**. All three baseline trials ran `issue attach`, "verified" via `issue view --json`, and reported success — reproducing the observed real-world agent failure. All three post-change trials routed **directly** to `issue comment add --attach` (`firstRoute` = cli, i.e. recipe-driven; the runtime hint was never needed as a recovery path). +- `image-holdout` ("add a comment … visible inline"): 3/3 → 3/3, no regression. + +Guards: + +- API controls: 6/6 in both conditions. +- Sidebar control: deterministic grade 3/3 baseline, **2/3 post-change — formally below the pre-declared 3/3 guard**. Inspection of the failing trial shows it routed correctly (`issue attach ENG-101 ./server.log`) and additionally ran `npx @schpet/linear-cli --version` — a version check the frozen grader counts as a CLI bypass, the same known npx artifact behind experiment 1's single post-change failure. No trial in either condition routed a sidebar task to `comment add`, so there is no evidence of overcorrection; the guard miss is attributed to the grader limitation, which is left frozen rather than adjusted post hoc. +- No experiment-1 supported case fell below 2/3 (all were 3/3 post-change). + +Gold-label validation: all 18 image-family and sidebar-control trials were independently labeled by a Claude Opus subagent blind to the deterministic grades (did the trial's actions achieve the user's stated goal?). Agreement was **17/18**; the sole disagreement is the npx trial above, which the gold labeler judged goal-achieved. Labels: `image-gold-labels.jsonl`. + +Post-experiment hardening (disclosed): commit review flagged that image-case grading did not verify the _target issue id_, so `comment add ENG-999 --attach …` would have graded full success. Grading was tightened to require the prompt's issue id as a positional. Grading is a pure function of the committed trial records, so both conditions were regraded under the tightened grader: **every count above is unchanged** (all recorded trials targeted the correct issues). No trials were re-run and no other rule was altered. diff --git a/evals/linear-cli-skill/results/image-gold-labels.jsonl b/evals/linear-cli-skill/results/image-gold-labels.jsonl new file mode 100644 index 00000000..124d9980 --- /dev/null +++ b/evals/linear-cli-skill/results/image-gold-labels.jsonl @@ -0,0 +1,18 @@ +{"condition":"image-baseline","caseId":"control-sidebar-attachment","trial":1,"goalAchieved":true,"reason":"Ran issue attach on ENG-101 with server.log, the correct sidebar attachment for a downloadable file."} +{"condition":"image-baseline","caseId":"control-sidebar-attachment","trial":2,"goalAchieved":true,"reason":"Attached server.log to ENG-101 via issue attach as a downloadable sidebar attachment."} +{"condition":"image-baseline","caseId":"control-sidebar-attachment","trial":3,"goalAchieved":true,"reason":"Attached server.log to ENG-101 via issue attach as a downloadable sidebar attachment."} +{"condition":"image-baseline","caseId":"image-development","trial":1,"goalAchieved":false,"reason":"Used issue attach, which creates a sidebar link, not a visible inline image as the user requested."} +{"condition":"image-baseline","caseId":"image-development","trial":2,"goalAchieved":false,"reason":"Used issue attach, producing only a sidebar link rather than a visible inline image."} +{"condition":"image-baseline","caseId":"image-development","trial":3,"goalAchieved":false,"reason":"Used issue attach, producing only a sidebar link rather than a visible inline image."} +{"condition":"image-baseline","caseId":"image-holdout","trial":1,"goalAchieved":true,"reason":"Used comment add --attach on OPS-44, embedding the screenshot as a visible inline image."} +{"condition":"image-baseline","caseId":"image-holdout","trial":2,"goalAchieved":true,"reason":"Used comment add --attach on OPS-44, embedding the screenshot as a visible inline image."} +{"condition":"image-baseline","caseId":"image-holdout","trial":3,"goalAchieved":true,"reason":"Used comment add --attach on OPS-44, embedding the screenshot as a visible inline image."} +{"condition":"image-post-change","caseId":"control-sidebar-attachment","trial":1,"goalAchieved":true,"reason":"Attached server.log to ENG-101 via issue attach as a downloadable sidebar attachment."} +{"condition":"image-post-change","caseId":"control-sidebar-attachment","trial":2,"goalAchieved":true,"reason":"Attached server.log to ENG-101 via issue attach as a downloadable sidebar attachment."} +{"condition":"image-post-change","caseId":"control-sidebar-attachment","trial":3,"goalAchieved":true,"reason":"Attached server.log to ENG-101 via issue attach as a downloadable sidebar attachment."} +{"condition":"image-post-change","caseId":"image-development","trial":1,"goalAchieved":true,"reason":"Used comment add --attach on ENG-107, embedding the screenshot as a visible inline image."} +{"condition":"image-post-change","caseId":"image-development","trial":2,"goalAchieved":true,"reason":"Used comment add --attach on ENG-107, embedding the screenshot as a visible inline image."} +{"condition":"image-post-change","caseId":"image-development","trial":3,"goalAchieved":true,"reason":"Used comment add --attach on ENG-107, embedding the screenshot as a visible inline image."} +{"condition":"image-post-change","caseId":"image-holdout","trial":1,"goalAchieved":true,"reason":"Used comment add --attach on OPS-44, embedding the screenshot as a visible inline image."} +{"condition":"image-post-change","caseId":"image-holdout","trial":2,"goalAchieved":true,"reason":"Used comment add --attach on OPS-44, embedding the screenshot as a visible inline image."} +{"condition":"image-post-change","caseId":"image-holdout","trial":3,"goalAchieved":true,"reason":"Used comment add --attach on OPS-44, embedding the screenshot as a visible inline image."} diff --git a/evals/linear-cli-skill/results/image-post-change.jsonl b/evals/linear-cli-skill/results/image-post-change.jsonl new file mode 100644 index 00000000..b5faa3e4 --- /dev/null +++ b/evals/linear-cli-skill/results/image-post-change.jsonl @@ -0,0 +1,45 @@ +{"condition":"image-post-change","caseId":"comment-development","trial":1,"entries":[{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","./comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'","/usr/bin/bash -lc 'linear issue comment add ENG-107 --body-file ./comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Added `./comment.md` as a comment on ENG-107.\n\n[View the comment](https://linear.app/acme/issue/ENG-107#comment-eval)","durationMs":12598,"exitCode":0} +{"condition":"image-post-change","caseId":"comment-development","trial":2,"entries":[{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","./comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'test -f ./comment.md && linear issue comment add ENG-107 --body-file ./comment.md'","/usr/bin/bash -lc 'test -f ./comment.md && linear issue comment add ENG-107 --body-file ./comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Added `./comment.md` as a comment on ENG-107.\n\n[View the comment](https://linear.app/acme/issue/ENG-107#comment-eval)","durationMs":16601,"exitCode":0} +{"condition":"image-post-change","caseId":"comment-development","trial":3,"entries":[{"tool":"linear","argv":["issue","comment","add","ENG-107","--body-file","./comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'test -f ./comment.md && linear issue comment add ENG-107 --body-file ./comment.md'","/usr/bin/bash -lc 'test -f ./comment.md && linear issue comment add ENG-107 --body-file ./comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Comment from `./comment.md` added to ENG-107 successfully.","durationMs":15361,"exitCode":0} +{"condition":"image-post-change","caseId":"comment-holdout","trial":1,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./alternate-comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear --version && test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'linear --version && test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'command -v linear; type linear; ls -l \"$(command -v linear)\"; sed -n '\"'1,80p' \\\"\"'$(command -v linear)\"'","/usr/bin/bash -lc 'command -v linear; type linear; ls -l \"$(command -v linear)\"; sed -n '\"'1,80p' \\\"\"'$(command -v linear)\"'","/usr/bin/bash -lc 'linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'linear issue comment add OPS-44 --body-file ./alternate-comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Posted `./alternate-comment.md` as a comment on OPS-44.\n\nhttps://linear.app/acme/issue/OPS-44#comment-eval","durationMs":30012,"exitCode":0} +{"condition":"image-post-change","caseId":"comment-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./alternate-comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Posted `./alternate-comment.md` as a comment on `OPS-44`.\n\n[View the comment](https://linear.app/acme/issue/OPS-44#comment-eval)","durationMs":12691,"exitCode":0} +{"condition":"image-post-change","caseId":"comment-holdout","trial":3,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","comment","add","--help"],"stdin":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--body-file","./alternate-comment.md"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/comment-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear --version && test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'linear --version && test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'command -v linear; command -v npx; linear issue comment add --help'","/usr/bin/bash -lc 'command -v linear; command -v npx; linear issue comment add --help'","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear; sed -n '1,160p' /evals/linear-cli-skill/shims/npx\"","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear; sed -n '1,160p' /evals/linear-cli-skill/shims/npx\"","/usr/bin/bash -lc 'test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'","/usr/bin/bash -lc 'test -f ./alternate-comment.md && linear issue comment add OPS-44 --body-file ./alternate-comment.md'"],"skillRead":true,"fixturesModified":false,"answer":"Comment posted successfully on `OPS-44` using `./alternate-comment.md`.\n\n[View the comment](https://linear.app/acme/issue/OPS-44#comment-eval)","durationMs":37934,"exitCode":0} +{"condition":"image-post-change","caseId":"control-history","trial":1,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["api","--help"],"stdin":""},{"tool":"linear","argv":["api"],"stdin":"query IssueChangeHistory {\n issue(id: \"ENG-101\") {\n identifier\n history {\n nodes {\n createdAt\n actor { name }\n fromState { name }\n toState { name }\n fromAssignee { name }\n toAssignee { name }\n }\n }\n }\n}"}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-1/codex-home/skills/linear-cli/references/api.md && linear --version && linear api --help\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-1/codex-home/skills/linear-cli/references/api.md && linear --version && linear api --help\"","/usr/bin/bash -lc 'linear api --help'","/usr/bin/bash -lc 'linear api --help'","/usr/bin/bash -lc 'command -v linear; readlink -f \"$(command -v linear)\"; sed -n '\"'1,120p' \\\"\"'$(command -v linear)\"; find -maxdepth 3 -type f | head -80'","/usr/bin/bash -lc 'command -v linear; readlink -f \"$(command -v linear)\"; sed -n '\"'1,120p' \\\"\"'$(command -v linear)\"; find -maxdepth 3 -type f | head -80'","/usr/bin/bash -lc \"sed -n '120,360p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"sed -n '120,360p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"linear api <<'GRAPHQL'\nquery IssueChangeHistory {\n issue(id: \\\"ENG-101\\\") {\n identifier\n history {\n nodes {\n createdAt\n actor { name }\n fromState { name }\n toState { name }\n fromAssignee { name }\n toAssignee { name }\n }\n }\n }\n}\nGRAPHQL\"","/usr/bin/bash -lc \"linear api <<'GRAPHQL'\nquery IssueChangeHistory {\n issue(id: \\\"ENG-101\\\") {\n identifier\n history {\n nodes {\n createdAt\n actor { name }\n fromState { name }\n toState { name }\n fromAssignee { name }\n toAssignee { name }\n }\n }\n }\n}\nGRAPHQL\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 change history:\n\n- **2026-07-15 09:00 UTC** — Sam Reyes changed state from **Todo** to **In Progress**.\n- **2026-07-16 11:30 UTC** — Priya Patel changed assignee from **Unassigned** to **Sam Reyes**.","durationMs":31579,"exitCode":0} +{"condition":"image-post-change","caseId":"control-history","trial":2,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["api","--help"],"stdin":""},{"tool":"linear","argv":["api","--variable","id=ENG-101"],"stdin":"query IssueChangeHistory($id: String!) {\n issue(id: $id) {\n identifier\n history {\n nodes {\n createdAt\n actor { name }\n fromState { name }\n toState { name }\n fromAssignee { name }\n toAssignee { name }\n }\n }\n }\n}"}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-2/codex-home/skills/linear-cli/references/api.md && linear --version && linear auth whoami\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-history-2/codex-home/skills/linear-cli/references/api.md && linear --version && linear auth whoami\"","/usr/bin/bash -lc 'command -v linear\nlinear api --help'","/usr/bin/bash -lc 'command -v linear\nlinear api --help'","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear\nls -la | head\nfind -path '*cliffy*command*' -o -path '*jsr.io*cliffy*' 2>/dev/null | head -40\"","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear\nls -la | head\nfind -path '*cliffy*command*' -o -path '*jsr.io*cliffy*' 2>/dev/null | head -40\"","/usr/bin/bash -lc \"rg -n '\\\"api\\\"|IssueHistory|history|state.*assignee|actor' /evals/linear-cli-skill/shims/linear | tail -40\nsed -n '380,560p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"rg -n '\\\"api\\\"|IssueHistory|history|state.*assignee|actor' /evals/linear-cli-skill/shims/linear | tail -40\nsed -n '380,560p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"sed -n '280,335p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"sed -n '280,335p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"linear api --variable id=ENG-101 <<'GRAPHQL'\nquery IssueChangeHistory(\"'$id: String!) {\n issue(id: $id) {\n identifier\n history {\n nodes {\n createdAt\n actor { name }\n fromState { name }\n toState { name }\n fromAssignee { name }\n toAssignee { name }\n }\n }\n }\n}\nGRAPHQL'","/usr/bin/bash -lc \"linear api --variable id=ENG-101 <<'GRAPHQL'\nquery IssueChangeHistory(\"'$id: String!) {\n issue(id: $id) {\n identifier\n history {\n nodes {\n createdAt\n actor { name }\n fromState { name }\n toState { name }\n fromAssignee { name }\n toAssignee { name }\n }\n }\n }\n}\nGRAPHQL'"],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 change history (timestamps in UTC):\n\n- 2026-07-15 09:00 — Sam Reyes changed state: **Todo → In Progress**\n- 2026-07-16 11:30 — Priya Patel changed assignee: **Unassigned → Sam Reyes**","durationMs":48343,"exitCode":0} +{"condition":"image-post-change","caseId":"control-history","trial":3,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["api","query { issue(id: \"ENG-101\") { id identifier title state { name } assignee { name } history(first: 100) { nodes { id createdAt actor { name email } fromState { name } toState { name } fromAssignee { name email } toAssignee { name email } } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-history-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'cat /trials/control-history-3/codex-home/skills/linear-cli/references/api.md && linear --version && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'cat /trials/control-history-3/codex-home/skills/linear-cli/references/api.md && linear --version && linear issue view ENG-101 --json'","/usr/bin/bash -lc \"command -v linear\nlinear api 'query { issue(id: \\\"ENG-101\\\") { id identifier title state { name } assignee { name } history(first: 100) { nodes { id createdAt actor { name email } fromState { name } toState { name } fromAssignee { name email } toAssignee { name email } } } } }'\"","/usr/bin/bash -lc \"command -v linear\nlinear api 'query { issue(id: \\\"ENG-101\\\") { id identifier title state { name } assignee { name } history(first: 100) { nodes { id createdAt actor { name email } fromState { name } toState { name } fromAssignee { name email } toAssignee { name email } } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 history:\n\n- `2026-07-15 09:00 UTC` — Sam Reyes changed state: **Todo → In Progress**\n- `2026-07-16 11:30 UTC` — Priya Patel changed assignee: **Unassigned → Sam Reyes**","durationMs":27223,"exitCode":0} +{"condition":"image-post-change","caseId":"control-sidebar-attachment","trial":1,"entries":[{"tool":"linear","argv":["issue","attach","--help"],"stdin":""},{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","attach","ENG-101","./server.log"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-sidebar-attachment-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-sidebar-attachment-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue attach --help && ls -l ./server.log && file ./server.log'","/usr/bin/bash -lc 'linear issue attach --help && ls -l ./server.log && file ./server.log'","/usr/bin/bash -lc 'ls -l ./server.log; file ./server.log; command -v linear; linear --version'","/usr/bin/bash -lc 'ls -l ./server.log; file ./server.log; command -v linear; linear --version'","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear; env | rg '\"'^(LINEAR|DENO|JSR|PATH)='\"'\"","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear; env | rg '\"'^(LINEAR|DENO|JSR|PATH)='\"'\"","/usr/bin/bash -lc \"rg -n 'issue attach|attach\\\\)' /evals/linear-cli-skill/shims/linear /src/commands/issue -g '*.ts' -g 'linear'\"","/usr/bin/bash -lc \"rg -n 'issue attach|attach\\\\)' /evals/linear-cli-skill/shims/linear /src/commands/issue -g '*.ts' -g 'linear'\"","/usr/bin/bash -lc \"sed -n '220,255p' /evals/linear-cli-skill/shims/linear; rg -n 'class.*Attach|attach' /src/commands/issue -g '*.ts' | head -50\"","/usr/bin/bash -lc \"sed -n '220,255p' /evals/linear-cli-skill/shims/linear; rg -n 'class.*Attach|attach' /src/commands/issue -g '*.ts' | head -50\"","/usr/bin/bash -lc 'linear issue attach ENG-101 ./server.log'","/usr/bin/bash -lc 'linear issue attach ENG-101 ./server.log'"],"skillRead":true,"fixturesModified":false,"answer":"Attached `server.log` to ENG-101 as a downloadable sidebar attachment for workspace teammates.","durationMs":38054,"exitCode":0} +{"condition":"image-post-change","caseId":"control-sidebar-attachment","trial":2,"entries":[{"tool":"linear","argv":["issue","attach","--help"],"stdin":""},{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"npx","argv":["@schpet/linear-cli","--version"],"stdin":""},{"tool":"linear","argv":["issue","attach","ENG-101","./server.log"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-sidebar-attachment-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-sidebar-attachment-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue attach --help && ls -l ./server.log && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue attach --help && ls -l ./server.log && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'command -v linear; ls -l ./server.log; linear --version; npx @schpet/linear-cli --version'","/usr/bin/bash -lc 'command -v linear; ls -l ./server.log; linear --version; npx @schpet/linear-cli --version'","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear; find -maxdepth 3 -type f \\\\( -name '*attach*' -o -name 'deno.lock' -o -name 'deno.json*' \\\\) -print | head -80\"","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear; find -maxdepth 3 -type f \\\\( -name '*attach*' -o -name 'deno.lock' -o -name 'deno.json*' \\\\) -print | head -80\"","/usr/bin/bash -lc 'rg -n \"issue attach|attachment|attach\" /src/commands/issue /evals/linear-cli-skill/shims/linear | head -100'","/usr/bin/bash -lc 'rg -n \"issue attach|attachment|attach\" /src/commands/issue /evals/linear-cli-skill/shims/linear | head -100'","/usr/bin/bash -lc \"sed -n '20,135p' /src/commands/issue/issue-attach.ts; sed -n '220,260p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"sed -n '20,135p' /src/commands/issue/issue-attach.ts; sed -n '220,260p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc 'linear issue attach ENG-101 ./server.log'","/usr/bin/bash -lc 'linear issue attach ENG-101 ./server.log'"],"skillRead":true,"fixturesModified":false,"answer":"Attached `server.log` to **ENG-101** as a downloadable sidebar attachment. Teammates can access it from the issue page.","durationMs":36116,"exitCode":0} +{"condition":"image-post-change","caseId":"control-sidebar-attachment","trial":3,"entries":[{"tool":"linear","argv":["issue","attach","--help"],"stdin":""},{"tool":"linear","argv":["issue","attach","ENG-101","./server.log"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-sidebar-attachment-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-sidebar-attachment-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue attach --help && test -f ./server.log && ls -lh ./server.log && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue attach --help && test -f ./server.log && ls -lh ./server.log && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'command -v linear; file \"$(command -v linear)\"; sed -n '\"'1,160p' \\\"\"'$(command -v linear)\"; env | rg '\"'\"'^(LINEAR|DENO|JSR)'\"'\"","/usr/bin/bash -lc 'command -v linear; file \"$(command -v linear)\"; sed -n '\"'1,160p' \\\"\"'$(command -v linear)\"; env | rg '\"'\"'^(LINEAR|DENO|JSR)'\"'\"","/usr/bin/bash -lc \"rg -n 'issue attach|attach\\\\)' /evals/linear-cli-skill/shims/linear /src/commands/issue -g '*.ts'; test -f ./server.log && ls -lh ./server.log\"","/usr/bin/bash -lc \"rg -n 'issue attach|attach\\\\)' /evals/linear-cli-skill/shims/linear /src/commands/issue -g '*.ts'; test -f ./server.log && ls -lh ./server.log\"","/usr/bin/bash -lc \"sed -n '215,255p' /evals/linear-cli-skill/shims/linear; rg -n 'class.*Attach|issue attach|attachment' /src/commands/issue -g '*.ts'\"","/usr/bin/bash -lc \"sed -n '215,255p' /evals/linear-cli-skill/shims/linear; rg -n 'class.*Attach|issue attach|attachment' /src/commands/issue -g '*.ts'\"","/usr/bin/bash -lc 'linear issue attach ENG-101 ./server.log && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue attach ENG-101 ./server.log && linear issue view ENG-101 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Attached `server.log` to ENG-101 as a downloadable sidebar attachment and confirmed the upload succeeded.","durationMs":33965,"exitCode":0} +{"condition":"image-post-change","caseId":"control-subscribers","trial":1,"entries":[{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""},{"tool":"linear","argv":["api","query { issue(id: \"ENG-101\") { identifier subscribers { nodes { name email } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'cat /trials/control-subscribers-1/codex-home/skills/linear-cli/references/api.md'","/usr/bin/bash -lc 'cat /trials/control-subscribers-1/codex-home/skills/linear-cli/references/api.md'","/usr/bin/bash -lc 'linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue view ENG-101 --json'","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier subscribers { nodes { name email } } } }'\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier subscribers { nodes { name email } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 subscribers:\n\n- Sam Reyes — sam@acme.test\n- Priya Patel — priya@acme.test","durationMs":26633,"exitCode":0} +{"condition":"image-post-change","caseId":"control-subscribers","trial":2,"entries":[{"tool":"linear","argv":["api","query { issue(id: \"ENG-101\") { identifier subscribers { nodes { name email } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-2/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-2/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier subscribers { nodes { name email } } } }'\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier subscribers { nodes { name email } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 subscribers:\n\n- Sam Reyes — sam@acme.test\n- Priya Patel — priya@acme.test","durationMs":24329,"exitCode":0} +{"condition":"image-post-change","caseId":"control-subscribers","trial":3,"entries":[{"tool":"linear","argv":["api","query { issue(id: \"ENG-101\") { identifier subscribers { nodes { name email } } } }"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/control-subscribers-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-3/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/control-subscribers-3/codex-home/skills/linear-cli/references/api.md\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier subscribers { nodes { name email } } } }'\"","/usr/bin/bash -lc \"linear api 'query { issue(id: \\\"ENG-101\\\") { identifier subscribers { nodes { name email } } } }'\""],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 has two subscribers:\n\n- Sam Reyes — sam@acme.test\n- Priya Patel — priya@acme.test","durationMs":25917,"exitCode":0} +{"condition":"image-post-change","caseId":"create-development","trial":1,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","ENG","--title","Migrate CI to Blacksmith runners","--description-file","./issue-description.md","--no-interactive"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'test -f ./issue-description.md && linear --version && linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'test -f ./issue-description.md && linear --version && linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'command -v linear; linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'command -v linear; linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'"],"skillRead":true,"fixturesModified":false,"answer":"Created [ENG-231: Migrate CI to Blacksmith runners](https://linear.app/acme/issue/ENG-231/new-issue) using `./issue-description.md`.","durationMs":19999,"exitCode":0} +{"condition":"image-post-change","caseId":"create-development","trial":2,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","ENG","--title","Migrate CI to Blacksmith runners","--description-file","./issue-description.md","--no-interactive"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-231","--json"],"stdin":""},{"tool":"linear","argv":["issue","query","--team","ENG","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"test -f ./issue-description.md && sed -n '1,240p' ./issue-description.md && linear --version\"","/usr/bin/bash -lc \"test -f ./issue-description.md && sed -n '1,240p' ./issue-description.md && linear --version\"","/usr/bin/bash -lc 'command -v linear; type -a linear; linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'command -v linear; type -a linear; linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue view ENG-231 --json'","/usr/bin/bash -lc 'linear issue view ENG-231 --json'","/usr/bin/bash -lc \"linear issue query --team ENG --json | rg -n -C 3 'Migrate CI to Blacksmith runners|ENG-231'\"","/usr/bin/bash -lc \"linear issue query --team ENG --json | rg -n -C 3 'Migrate CI to Blacksmith runners|ENG-231'\""],"skillRead":true,"fixturesModified":false,"answer":"Created ENG issue [ENG-231: Migrate CI to Blacksmith runners](https://linear.app/acme/issue/ENG-231/new-issue) using `./issue-description.md`.","durationMs":29320,"exitCode":0} +{"condition":"image-post-change","caseId":"create-development","trial":3,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","ENG","--title","Migrate CI to Blacksmith runners","--description-file","./issue-description.md","--no-interactive"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'test -f ./issue-description.md && linear --version && linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'test -f ./issue-description.md && linear --version && linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'command -v linear; type -a linear; ls -l \"$(command -v linear)\"; sed -n '\"'1,80p' \\\"\"'$(command -v linear)\"'","/usr/bin/bash -lc 'command -v linear; type -a linear; ls -l \"$(command -v linear)\"; sed -n '\"'1,80p' \\\"\"'$(command -v linear)\"'","/usr/bin/bash -lc 'linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue create --team ENG --title \"Migrate CI to Blacksmith runners\" --description-file ./issue-description.md --no-interactive'"],"skillRead":true,"fixturesModified":false,"answer":"Created [ENG-231: Migrate CI to Blacksmith runners](https://linear.app/acme/issue/ENG-231/new-issue) on the ENG team using `issue-description.md`.","durationMs":21812,"exitCode":0} +{"condition":"image-post-change","caseId":"create-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","create","--help"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","OPS","--title","Document rate limit behavior","--project","Data Pipeline","--label","docs","--label","api","--description-file","./alternate-description.md","--no-interactive"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"linear issue create --help && test -f ./alternate-description.md && printf '\\\\nDESCRIPTION_FILE_OK\\\\n'\"","/usr/bin/bash -lc \"linear issue create --help && test -f ./alternate-description.md && printf '\\\\nDESCRIPTION_FILE_OK\\\\n'\"","/usr/bin/bash -lc 'command -v linear; type -a linear; ls -l \"$(command -v linear)\"; sed -n '\"'1,120p' \\\"\"'$(command -v linear)\"; ls -l ./alternate-description.md'","/usr/bin/bash -lc 'command -v linear; type -a linear; ls -l \"$(command -v linear)\"; sed -n '\"'1,120p' \\\"\"'$(command -v linear)\"; ls -l ./alternate-description.md'","/usr/bin/bash -lc \"rg -n 'issue create|description-file|--project|--label' /evals/linear-cli-skill/shims/linear | head -80\"","/usr/bin/bash -lc \"rg -n 'issue create|description-file|--project|--label' /evals/linear-cli-skill/shims/linear | head -80\"","/usr/bin/bash -lc \"sed -n '198,232p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"sed -n '198,232p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc 'linear issue create --team OPS --title \"Document rate limit behavior\" --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue create --team OPS --title \"Document rate limit behavior\" --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md --no-interactive'"],"skillRead":true,"fixturesModified":false,"answer":"Created [OPS-231: Document rate limit behavior](https://linear.app/acme/issue/OPS-231/new-issue) in **Data Pipeline**, labeled **docs** and **api**, using `alternate-description.md`.","durationMs":29759,"exitCode":0} +{"condition":"image-post-change","caseId":"create-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","create","--help"],"stdin":""},{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","OPS","--title","Document rate limit behavior","--project","Data Pipeline","--label","docs","--label","api","--description-file","./alternate-description.md","--no-interactive"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"linear issue create --help && test -f ./alternate-description.md && sed -n '1,40p' ./alternate-description.md\"","/usr/bin/bash -lc \"linear issue create --help && test -f ./alternate-description.md && sed -n '1,40p' ./alternate-description.md\"","/usr/bin/bash -lc \"command -v linear; linear --version; ls -l ./alternate-description.md; sed -n '1,40p' ./alternate-description.md; env | rg '\"'^(LINEAR|DENO|JSR)'\"'\"","/usr/bin/bash -lc \"command -v linear; linear --version; ls -l ./alternate-description.md; sed -n '1,40p' ./alternate-description.md; env | rg '\"'^(LINEAR|DENO|JSR)'\"'\"","/usr/bin/bash -lc \"sed -n '1,240p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"sed -n '1,240p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc 'linear issue create --team OPS --title \"Document rate limit behavior\" --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue create --team OPS --title \"Document rate limit behavior\" --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md --no-interactive'"],"skillRead":true,"fixturesModified":false,"answer":"Created **OPS-231: Document rate limit behavior** in **Data Pipeline**, with labels **docs** and **api**, using `alternate-description.md`.\n\nhttps://linear.app/acme/issue/OPS-231/new-issue","durationMs":32685,"exitCode":0} +{"condition":"image-post-change","caseId":"create-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","create","--help"],"stdin":""},{"tool":"linear","argv":["issue","create","--team","OPS","--title","Document rate limit behavior","--project","Data Pipeline","--label","docs","--label","api","--description-file","./alternate-description.md","--no-interactive"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/create-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"test -f ./alternate-description.md && sed -n '1,40p' ./alternate-description.md\"","/usr/bin/bash -lc \"test -f ./alternate-description.md && sed -n '1,40p' ./alternate-description.md\"","/usr/bin/bash -lc 'linear issue create --help'","/usr/bin/bash -lc 'linear issue create --help'","/usr/bin/bash -lc 'linear issue create --team OPS --title \"Document rate limit behavior\" --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md --no-interactive'","/usr/bin/bash -lc 'linear issue create --team OPS --title \"Document rate limit behavior\" --project \"Data Pipeline\" --label docs --label api --description-file ./alternate-description.md --no-interactive'"],"skillRead":true,"fixturesModified":false,"answer":"Created **OPS-231 — Document rate limit behavior** in **Data Pipeline** with labels **docs** and **api**, using `alternate-description.md`.\n\nhttps://linear.app/acme/issue/OPS-231/new-issue","durationMs":27224,"exitCode":0} +{"condition":"image-post-change","caseId":"image-development","trial":1,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","comment","add","ENG-107","--attach","./screenshot.png"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-107","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear --version && file ./screenshot.png && linear issue comment add ENG-107 --attach ./screenshot.png'","/usr/bin/bash -lc 'linear --version && file ./screenshot.png && linear issue comment add ENG-107 --attach ./screenshot.png'","/usr/bin/bash -lc 'command -v linear; ls -l \"$(command -v linear)\"; sed -n '\"'1,80p' \\\"\"'$(command -v linear)\"; rg -n \"cliffy/command|issue comment add|--attach\" -g '\"'*.ts' -g 'deno.json*' -g 'package.json' | head -80\"","/usr/bin/bash -lc 'command -v linear; ls -l \"$(command -v linear)\"; sed -n '\"'1,80p' \\\"\"'$(command -v linear)\"; rg -n \"cliffy/command|issue comment add|--attach\" -g '\"'*.ts' -g 'deno.json*' -g 'package.json' | head -80\"","/usr/bin/bash -lc 'linear issue comment add ENG-107 --attach ./screenshot.png && linear issue view ENG-107 --json'","/usr/bin/bash -lc 'linear issue comment add ENG-107 --attach ./screenshot.png && linear issue view ENG-107 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Attached `screenshot.png` to ENG-107 in an inline comment, so it’s visible on the issue.\n\n[View ENG-107](https://linear.app/acme/issue/ENG-107/refactor-session-storage)","durationMs":27242,"exitCode":0} +{"condition":"image-post-change","caseId":"image-development","trial":2,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","comment","add","ENG-107","--attach","./screenshot.png"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-107","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear --version && file ./screenshot.png && linear issue comment add ENG-107 --attach ./screenshot.png'","/usr/bin/bash -lc 'linear --version && file ./screenshot.png && linear issue comment add ENG-107 --attach ./screenshot.png'","/usr/bin/bash -lc 'command -v linear; ls -l \"$(command -v linear)\"; file ./screenshot.png; linear issue comment add ENG-107 --attach ./screenshot.png'","/usr/bin/bash -lc 'command -v linear; ls -l \"$(command -v linear)\"; file ./screenshot.png; linear issue comment add ENG-107 --attach ./screenshot.png'","/usr/bin/bash -lc 'linear issue view ENG-107 --json'","/usr/bin/bash -lc 'linear issue view ENG-107 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Attached `screenshot.png` to ENG-107 in an inline comment, so it’s visible on the issue.\n\n[View ENG-107](https://linear.app/acme/issue/ENG-107#comment-eval)","durationMs":31062,"exitCode":0} +{"condition":"image-post-change","caseId":"image-development","trial":3,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","comment","add","ENG-107","--attach","./screenshot.png"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear --version && test -f ./screenshot.png && file ./screenshot.png && linear issue comment add ENG-107 --attach ./screenshot.png'","/usr/bin/bash -lc 'linear --version && test -f ./screenshot.png && file ./screenshot.png && linear issue comment add ENG-107 --attach ./screenshot.png'","/usr/bin/bash -lc 'command -v linear; ls -l \"$(command -v linear)\"; head -40 \"$(command -v linear)\"; deno cache --help >/dev/null 2>&1; find -maxdepth 3 -type f | sed -n '\"'1,120p'\"","/usr/bin/bash -lc 'command -v linear; ls -l \"$(command -v linear)\"; head -40 \"$(command -v linear)\"; deno cache --help >/dev/null 2>&1; find -maxdepth 3 -type f | sed -n '\"'1,120p'\"","/usr/bin/bash -lc 'test -f ./screenshot.png && file ./screenshot.png && linear issue comment add ENG-107 --attach ./screenshot.png && linear issue view ENG-107'","/usr/bin/bash -lc 'test -f ./screenshot.png && file ./screenshot.png && linear issue comment add ENG-107 --attach ./screenshot.png && linear issue view ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"Attached `screenshot.png` to ENG-107 in an inline comment, so it’s visible on the issue.","durationMs":27382,"exitCode":0} +{"condition":"image-post-change","caseId":"image-holdout","trial":1,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--attach","./screenshot.png"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear --version && test -f ./screenshot.png && file ./screenshot.png && linear issue comment add OPS-44 --attach ./screenshot.png'","/usr/bin/bash -lc 'linear --version && test -f ./screenshot.png && file ./screenshot.png && linear issue comment add OPS-44 --attach ./screenshot.png'","/usr/bin/bash -lc 'command -v linear; type -a linear; ls -l \"$(command -v linear)\"; command -v npx; ls -la | sed -n '\"'1,120p'\"","/usr/bin/bash -lc 'command -v linear; type -a linear; ls -l \"$(command -v linear)\"; command -v npx; ls -la | sed -n '\"'1,120p'\"","/usr/bin/bash -lc 'test -f ./screenshot.png && file ./screenshot.png && linear issue comment add OPS-44 --attach ./screenshot.png'","/usr/bin/bash -lc 'test -f ./screenshot.png && file ./screenshot.png && linear issue comment add OPS-44 --attach ./screenshot.png'"],"skillRead":true,"fixturesModified":false,"answer":"Added `screenshot.png` as an inline comment on OPS-44.\n\n[View the comment](https://linear.app/acme/issue/OPS-44#comment-eval)","durationMs":23279,"exitCode":0} +{"condition":"image-post-change","caseId":"image-holdout","trial":2,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--attach","./screenshot.png"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear --version && test -f ./screenshot.png && file ./screenshot.png && linear issue comment add OPS-44 --attach ./screenshot.png'","/usr/bin/bash -lc 'linear --version && test -f ./screenshot.png && file ./screenshot.png && linear issue comment add OPS-44 --attach ./screenshot.png'","/usr/bin/bash -lc 'command -v linear; type linear; ls -l \"$(command -v linear)\"; head -n 30 \"$(command -v linear)\"; ls -la .; test -f ./screenshot.png && file ./screenshot.png'","/usr/bin/bash -lc 'command -v linear; type linear; ls -l \"$(command -v linear)\"; head -n 30 \"$(command -v linear)\"; ls -la .; test -f ./screenshot.png && file ./screenshot.png'","/usr/bin/bash -lc 'linear issue comment add OPS-44 --attach ./screenshot.png'","/usr/bin/bash -lc 'linear issue comment add OPS-44 --attach ./screenshot.png'"],"skillRead":true,"fixturesModified":false,"answer":"Added `screenshot.png` as an inline image comment on OPS-44.\n\n[View the comment](https://linear.app/acme/issue/OPS-44#comment-eval)","durationMs":29236,"exitCode":0} +{"condition":"image-post-change","caseId":"image-holdout","trial":3,"entries":[{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["auth","token"],"stdin":""},{"tool":"linear","argv":["issue","comment","add","OPS-44","--attach","./screenshot.png"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/image-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear --version && file ./screenshot.png && linear issue comment add OPS-44 --attach ./screenshot.png'","/usr/bin/bash -lc 'linear --version && file ./screenshot.png && linear issue comment add OPS-44 --attach ./screenshot.png'","/usr/bin/bash -lc \"sed -n '1,260p' /trials/image-holdout-3/codex-home/skills/linear-cli/references/api.md && command -v linear && linear auth token\"","/usr/bin/bash -lc \"sed -n '1,260p' /trials/image-holdout-3/codex-home/skills/linear-cli/references/api.md && command -v linear && linear auth token\"","/usr/bin/bash -lc 'file ./screenshot.png && linear issue comment add OPS-44 --attach ./screenshot.png'","/usr/bin/bash -lc 'file ./screenshot.png && linear issue comment add OPS-44 --attach ./screenshot.png'"],"skillRead":true,"fixturesModified":false,"answer":"Added `screenshot.png` as an inline-visible comment on OPS-44.\n\n[View the comment](https://linear.app/acme/issue/OPS-44#comment-eval)","durationMs":33243,"exitCode":0} +{"condition":"image-post-change","caseId":"inspect-development","trial":1,"entries":[{"tool":"linear","argv":["issue","view","ENG-112","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue view ENG-112 --json'","/usr/bin/bash -lc 'linear issue view ENG-112 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issue\": {\n \"identifier\": \"ENG-112\",\n \"title\": \"Refactor session storage\",\n \"description\": \"Session data is duplicated between redis and postgres.\",\n \"state\": {\n \"name\": \"In Progress\"\n },\n \"assignee\": null,\n \"labels\": {\n \"nodes\": [\n {\n \"name\": \"tech-debt\"\n }\n ]\n },\n \"project\": null,\n \"url\": \"https://linear.app/acme/issue/ENG-112/refactor-session-storage\"\n }\n}\n```","durationMs":15233,"exitCode":0} +{"condition":"image-post-change","caseId":"inspect-development","trial":2,"entries":[{"tool":"linear","argv":["issue","view","ENG-112","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue view ENG-112 --json'","/usr/bin/bash -lc 'linear issue view ENG-112 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issue\": {\n \"identifier\": \"ENG-112\",\n \"title\": \"Refactor session storage\",\n \"description\": \"Session data is duplicated between redis and postgres.\",\n \"state\": {\n \"name\": \"In Progress\"\n },\n \"assignee\": null,\n \"labels\": {\n \"nodes\": [\n {\n \"name\": \"tech-debt\"\n }\n ]\n },\n \"project\": null,\n \"url\": \"https://linear.app/acme/issue/ENG-112/refactor-session-storage\"\n }\n}\n```","durationMs":21235,"exitCode":0} +{"condition":"image-post-change","caseId":"inspect-development","trial":3,"entries":[{"tool":"linear","argv":["issue","view","ENG-112","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue view ENG-112 --json'","/usr/bin/bash -lc 'linear issue view ENG-112 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issue\": {\n \"identifier\": \"ENG-112\",\n \"title\": \"Refactor session storage\",\n \"description\": \"Session data is duplicated between redis and postgres.\",\n \"state\": {\n \"name\": \"In Progress\"\n },\n \"assignee\": null,\n \"labels\": {\n \"nodes\": [\n {\n \"name\": \"tech-debt\"\n }\n ]\n },\n \"project\": null,\n \"url\": \"https://linear.app/acme/issue/ENG-112/refactor-session-storage\"\n }\n}\n```","durationMs":19532,"exitCode":0} +{"condition":"image-post-change","caseId":"inspect-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","url","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue url ENG-107'","/usr/bin/bash -lc 'linear issue url ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"https://linear.app/acme/issue/ENG-107/update-billing-webhooks","durationMs":23319,"exitCode":0} +{"condition":"image-post-change","caseId":"inspect-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","url","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue url ENG-107'","/usr/bin/bash -lc 'linear issue url ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"https://linear.app/acme/issue/ENG-107/update-billing-webhooks","durationMs":17867,"exitCode":0} +{"condition":"image-post-change","caseId":"inspect-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","url","ENG-107"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/inspect-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue url ENG-107'","/usr/bin/bash -lc 'linear issue url ENG-107'"],"skillRead":true,"fixturesModified":false,"answer":"https://linear.app/acme/issue/ENG-107/update-billing-webhooks","durationMs":15032,"exitCode":0} +{"condition":"image-post-change","caseId":"query-development","trial":1,"entries":[{"tool":"linear","argv":["issue","query","--team","ENG","--project","Mobile App Revamp","--state","started","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'"],"skillRead":true,"fixturesModified":false,"answer":"Started ENG issues in **Mobile App Revamp**:\n\n- ENG-101 — Fix login redirect loop — sam\n- ENG-107 — Update billing webhooks — priya\n- ENG-112 — Refactor session storage — unassigned\n\n3 issues total.","durationMs":13103,"exitCode":0} +{"condition":"image-post-change","caseId":"query-development","trial":2,"entries":[{"tool":"linear","argv":["issue","query","--team","ENG","--project","Mobile App Revamp","--state","started","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'"],"skillRead":true,"fixturesModified":false,"answer":"Started issues in ENG’s “Mobile App Revamp” project:\n\n- ENG-101 — Fix login redirect loop (sam)\n- ENG-107 — Update billing webhooks (priya)\n- ENG-112 — Refactor session storage (unassigned)\n\nAll three are **In Progress**.","durationMs":27534,"exitCode":0} +{"condition":"image-post-change","caseId":"query-development","trial":3,"entries":[{"tool":"linear","argv":["issue","query","--team","ENG","--project","Mobile App Revamp","--state","started","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'","/usr/bin/bash -lc 'linear issue query --team ENG --project \"Mobile App Revamp\" --state started --json'"],"skillRead":true,"fixturesModified":false,"answer":"Started issues in ENG’s **Mobile App Revamp** project:\n\n- ENG-101 — Fix login redirect loop — Sam\n- ENG-107 — Update billing webhooks — Priya\n- ENG-112 — Refactor session storage — Unassigned\n\nAll three are **In Progress**.","durationMs":13069,"exitCode":0} +{"condition":"image-post-change","caseId":"query-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","query","--team","OPS","--state","backlog","--state","triage","--unassigned","--updated-after","2026-06-01","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issues\": {\n \"nodes\": [\n {\n \"identifier\": \"OPS-101\",\n \"title\": \"Fix login redirect loop\",\n \"state\": { \"name\": \"Backlog\", \"type\": \"backlog\" },\n \"assignee\": null,\n \"priority\": 2,\n \"updatedAt\": \"2026-07-17T10:00:00.000Z\"\n },\n {\n \"identifier\": \"OPS-107\",\n \"title\": \"Update billing webhooks\",\n \"state\": { \"name\": \"Backlog\", \"type\": \"backlog\" },\n \"assignee\": null,\n \"priority\": 3,\n \"updatedAt\": \"2026-07-16T08:30:00.000Z\"\n },\n {\n \"identifier\": \"OPS-112\",\n \"title\": \"Refactor session storage\",\n \"state\": { \"name\": \"Backlog\", \"type\": \"backlog\" },\n \"assignee\": null,\n \"priority\": 4,\n \"updatedAt\": \"2026-07-14T16:12:00.000Z\"\n }\n ],\n \"pageInfo\": {\n \"hasNextPage\": false,\n \"endCursor\": \"cursor-3\"\n }\n }\n}\n```","durationMs":15693,"exitCode":0} +{"condition":"image-post-change","caseId":"query-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","query","--team","OPS","--state","backlog","--state","triage","--unassigned","--updated-after","2026-06-01","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'"],"skillRead":true,"fixturesModified":false,"answer":"```json\n{\n \"issues\": [\n {\n \"identifier\": \"OPS-101\",\n \"title\": \"Fix login redirect loop\",\n \"state\": \"Backlog\",\n \"assignee\": null,\n \"priority\": 2,\n \"updatedAt\": \"2026-07-17T10:00:00.000Z\"\n },\n {\n \"identifier\": \"OPS-107\",\n \"title\": \"Update billing webhooks\",\n \"state\": \"Backlog\",\n \"assignee\": null,\n \"priority\": 3,\n \"updatedAt\": \"2026-07-16T08:30:00.000Z\"\n },\n {\n \"identifier\": \"OPS-112\",\n \"title\": \"Refactor session storage\",\n \"state\": \"Backlog\",\n \"assignee\": null,\n \"priority\": 4,\n \"updatedAt\": \"2026-07-14T16:12:00.000Z\"\n }\n ]\n}\n```","durationMs":17055,"exitCode":0} +{"condition":"image-post-change","caseId":"query-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","query","--team","OPS","--state","backlog","--state","triage","--unassigned","--updated-after","2026-06-01","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/query-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'","/usr/bin/bash -lc 'linear issue query --team OPS --state backlog --state triage --unassigned --updated-after 2026-06-01 --json'"],"skillRead":true,"fixturesModified":false,"answer":"{\"issues\":{\"nodes\":[{\"identifier\":\"OPS-101\",\"title\":\"Fix login redirect loop\",\"state\":{\"name\":\"Backlog\",\"type\":\"backlog\"},\"assignee\":null,\"priority\":2,\"updatedAt\":\"2026-07-17T10:00:00.000Z\"},{\"identifier\":\"OPS-107\",\"title\":\"Update billing webhooks\",\"state\":{\"name\":\"Backlog\",\"type\":\"backlog\"},\"assignee\":null,\"priority\":3,\"updatedAt\":\"2026-07-16T08:30:00.000Z\"},{\"identifier\":\"OPS-112\",\"title\":\"Refactor session storage\",\"state\":{\"name\":\"Backlog\",\"type\":\"backlog\"},\"assignee\":null,\"priority\":4,\"updatedAt\":\"2026-07-14T16:12:00.000Z\"}],\"pageInfo\":{\"hasNextPage\":false,\"endCursor\":\"cursor-3\"}}}","durationMs":18264,"exitCode":0} +{"condition":"image-post-change","caseId":"update-development","trial":1,"entries":[{"tool":"linear","argv":["issue","update","ENG-101","--state","In Review","--assignee","priya"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya && linear issue view ENG-101 --json'"],"skillRead":true,"fixturesModified":false,"answer":"ENG-101 is now **In Review** and assigned to **priya**.","durationMs":13052,"exitCode":0} +{"condition":"image-post-change","caseId":"update-development","trial":2,"entries":[{"tool":"linear","argv":["issue","update","ENG-101","--state","In Review","--assignee","priya"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya'","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya'","/usr/bin/bash -lc 'linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue view ENG-101 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Updated [ENG-101](https://linear.app/acme/issue/ENG-101/refactor-session-storage):\n\n- State: **In Review**\n- Assignee: **priya**","durationMs":19822,"exitCode":0} +{"condition":"image-post-change","caseId":"update-development","trial":3,"entries":[{"tool":"linear","argv":["issue","update","ENG-101","--state","In Review","--assignee","priya"],"stdin":""},{"tool":"linear","argv":["issue","view","ENG-101","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-development-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya'","/usr/bin/bash -lc 'linear issue update ENG-101 --state \"In Review\" --assignee priya'","/usr/bin/bash -lc 'linear issue view ENG-101 --json'","/usr/bin/bash -lc 'linear issue view ENG-101 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Updated [ENG-101](https://linear.app/acme/issue/ENG-101/refactor-session-storage):\n\n- State: In Review\n- Assignee: priya","durationMs":26657,"exitCode":0} +{"condition":"image-post-change","caseId":"update-holdout","trial":1,"entries":[{"tool":"linear","argv":["issue","update","--help"],"stdin":""},{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","update","OPS-31","--unassign","--project","Data Pipeline","--label","infra","--label","security"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-31","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-1/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue update --help && linear issue view OPS-31 --json'","/usr/bin/bash -lc 'linear issue update --help && linear issue view OPS-31 --json'","/usr/bin/bash -lc 'command -v linear; ls -l \"$(command -v linear)\"; linear --version; rg -n '\"'issue-agent-session-view|@cliffy/command' /deno.json /package.json /src 2>/dev/null | head -50\"","/usr/bin/bash -lc 'command -v linear; ls -l \"$(command -v linear)\"; linear --version; rg -n '\"'issue-agent-session-view|@cliffy/command' /deno.json /package.json /src 2>/dev/null | head -50\"","/usr/bin/bash -lc \"sed -n '1,260p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc \"sed -n '1,260p' /evals/linear-cli-skill/shims/linear\"","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security && linear issue view OPS-31 --json'","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security && linear issue view OPS-31 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Updated OPS-31 successfully:\n\n- Unassigned\n- Project: Data Pipeline\n- Labels: `infra`, `security`","durationMs":31096,"exitCode":0} +{"condition":"image-post-change","caseId":"update-holdout","trial":2,"entries":[{"tool":"linear","argv":["issue","update","--help"],"stdin":""},{"tool":"linear","argv":["issue","update","OPS-31","--unassign","--project","Data Pipeline","--label","infra","--label","security"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-31","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-2/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue update --help'","/usr/bin/bash -lc 'linear issue update --help'","/usr/bin/bash -lc 'command -v linear\nfile \"$(command -v linear)\"\nsed -n '\"'1,120p' \\\"\"'$(command -v linear)\"\nls -la'","/usr/bin/bash -lc 'command -v linear\nfile \"$(command -v linear)\"\nsed -n '\"'1,120p' \\\"\"'$(command -v linear)\"\nls -la'","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security\nlinear issue view OPS-31 --json'","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security\nlinear issue view OPS-31 --json'"],"skillRead":true,"fixturesModified":false,"answer":"OPS-31 updated and verified:\n\n- Assignee: Unassigned\n- Project: Data Pipeline\n- Labels: infra, security","durationMs":26128,"exitCode":0} +{"condition":"image-post-change","caseId":"update-holdout","trial":3,"entries":[{"tool":"linear","argv":["issue","update","--help"],"stdin":""},{"tool":"linear","argv":["--version"],"stdin":""},{"tool":"linear","argv":["issue","update","OPS-31","--unassign","--project","Data Pipeline","--label","infra","--label","security"],"stdin":""},{"tool":"linear","argv":["issue","view","OPS-31","--json"],"stdin":""}],"commands":["/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc \"sed -n '1,240p' /trials/update-holdout-3/codex-home/skills/linear-cli/SKILL.md\"","/usr/bin/bash -lc 'linear issue update --help && linear issue view OPS-31 --json'","/usr/bin/bash -lc 'linear issue update --help && linear issue view OPS-31 --json'","/usr/bin/bash -lc \"command -v linear; linear --version; ls -la; rg -n \\\"LINEAR|linear\\\" -S . --hidden -g '\"'!*.lock'\"' -g '\"'!.git/**'\"'\"","/usr/bin/bash -lc \"command -v linear; linear --version; ls -la; rg -n \\\"LINEAR|linear\\\" -S . --hidden -g '\"'!*.lock'\"' -g '\"'!.git/**'\"'\"","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear; sed -n '1,200p' server.log; find tmp -maxdepth 3 -type f -print\"","/usr/bin/bash -lc \"sed -n '1,200p' /evals/linear-cli-skill/shims/linear; sed -n '1,200p' server.log; find tmp -maxdepth 3 -type f -print\"","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security && linear issue view OPS-31 --json'","/usr/bin/bash -lc 'linear issue update OPS-31 --unassign --project \"Data Pipeline\" --label infra --label security && linear issue view OPS-31 --json'"],"skillRead":true,"fixturesModified":false,"answer":"Updated OPS-31:\n\n- Assignee: Unassigned\n- Project: Data Pipeline\n- Labels: `infra`, `security`","durationMs":28371,"exitCode":0} diff --git a/evals/linear-cli-skill/results/image-post-change.meta.json b/evals/linear-cli-skill/results/image-post-change.meta.json new file mode 100644 index 00000000..b93ee272 --- /dev/null +++ b/evals/linear-cli-skill/results/image-post-change.meta.json @@ -0,0 +1,29 @@ +{ + "condition": "image-post-change", + "codexVersion": "codex-cli 0.144.4", + "sandbox": "workspace-write", + "effort": "low", + "modelRequested": "gpt-5.6-sol", + "modelsObserved": [], + "trialsPerCase": 3, + "cases": [ + "query-development", + "query-holdout", + "create-development", + "create-holdout", + "update-development", + "update-holdout", + "comment-development", + "comment-holdout", + "inspect-development", + "inspect-holdout", + "image-development", + "image-holdout", + "control-sidebar-attachment", + "control-history", + "control-subscribers" + ], + "completedTrials": 45, + "failedTrials": [], + "ranAt": "2026-07-23T17:31:07.323Z" +} diff --git a/evals/linear-cli-skill/run.ts b/evals/linear-cli-skill/run.ts index 05a52c1f..0f7fe548 100644 --- a/evals/linear-cli-skill/run.ts +++ b/evals/linear-cli-skill/run.ts @@ -145,16 +145,21 @@ export function extractFromEvents(rawEvents: string): EventExtract { async function fixturesModifiedIn(workdir: string): Promise { for await (const fixture of Deno.readDir(join(EVAL_DIR, "fixtures"))) { - const original = await Deno.readTextFile( + // Byte-wise compare: fixtures include binary files (PNG), and a lossy + // UTF-8 decode could collapse distinct corrupted bytes into equal text. + const original = await Deno.readFile( join(EVAL_DIR, "fixtures", fixture.name), ) - let current: string + let current: Uint8Array try { - current = await Deno.readTextFile(join(workdir, fixture.name)) + current = await Deno.readFile(join(workdir, fixture.name)) } catch { return true } - if (current !== original) return true + if (current.length !== original.length) return true + for (let i = 0; i < original.length; i++) { + if (current[i] !== original[i]) return true + } } return false } diff --git a/evals/linear-cli-skill/shims/linear b/evals/linear-cli-skill/shims/linear index 8aaa6624..f1089356 100755 --- a/evals/linear-cli-skill/shims/linear +++ b/evals/linear-cli-skill/shims/linear @@ -71,6 +71,34 @@ state_type() { esac } +# Positional tokens of a subcommand: argv minus flags and their values. +# $1 = space-separated value-taking flags, $2 = space-separated boolean flags, +# remaining args = the argv tail to scan. +positional_values() { + local value_flags="$1" bool_flags="$2" + shift 2 + local skip_next=0 + for arg in "$@"; do + if [ "$skip_next" = 1 ]; then + skip_next=0 + continue + fi + case " $value_flags " in + *" $arg "*) + skip_next=1 + continue + ;; + esac + case " $bool_flags " in + *" $arg "*) continue ;; + esac + case "$arg" in + -*) continue ;; + *) printf '%s\n' "$arg" ;; + esac + done +} + # First value following any of the given flags in the remaining argv. flag_value() { local flags="$1" @@ -200,8 +228,56 @@ case "${1:-} ${2:-}" in if [ -n "$LABELS" ]; then printf ' labels: %s\n' "$LABELS"; fi printf 'https://linear.app/acme/issue/%s\n' "$ID" ;; + "issue attach") + # Mirrors the real CLI's messaging for the repo state under test (the + # canned output is version-matched infrastructure; see README). + ATTACH_PUBLIC=0 + if has_flag --public "$@"; then ATTACH_PUBLIC=1; fi + ATTACH_TITLE=$(flag_value "--title -t" "$@") || ATTACH_TITLE="" + shift 2 + ATTACH_POSITIONALS=$(positional_values "--title -t --comment -c" "--public" "$@") + ATTACH_ID=$(printf '%s\n' "$ATTACH_POSITIONALS" | sed -n '1p') + ATTACH_ID="${ATTACH_ID:-ENG-101}" + ATTACH_FILE=$(printf '%s\n' "$ATTACH_POSITIONALS" | sed -n '2p') + ATTACH_BASE="${ATTACH_FILE##*/}" + ATTACH_BASE="${ATTACH_BASE:-file.bin}" + printf '✓ Uploaded %s\n' "$ATTACH_BASE" + if [ "$ATTACH_PUBLIC" = 1 ]; then + printf '⚠ Uploaded to a public URL readable by anyone: https://public.linear.app/eval-asset/%s\n' "$ATTACH_BASE" >&2 + fi + printf '✓ Sidebar link attachment created: %s\n' "${ATTACH_TITLE:-$ATTACH_BASE}" + printf 'https://uploads.linear.app/acme/eval-asset/%s\n' "$ATTACH_BASE" + case "$ATTACH_BASE" in + *.[Pp][Nn][Gg] | *.[Jj][Pp][Gg] | *.[Jj][Pp][Ee][Gg] | *.[Gg][Ii][Ff] | *.[Ww][Ee][Bb][Pp]) + ATTACH_HINT_PUBLIC="" + if [ "$ATTACH_PUBLIC" = 1 ]; then ATTACH_HINT_PUBLIC=" --public"; fi + printf 'Hint: Sidebar link attachments do not render images inline. For inline display, run: linear issue comment add %s --attach %s%s\n' \ + "$ATTACH_ID" "$ATTACH_FILE" "$ATTACH_HINT_PUBLIC" + ;; + esac + ;; "issue comment") - printf 'Comment added\n' + if [ "${3:-}" = "add" ]; then + COMMENT_ARGS_ALL="$*" + shift 3 + COMMENT_ID=$( + positional_values \ + "--body -b --body-file --parent -p --attach -a" "--public" "$@" | + head -n 1 + ) + COMMENT_ID="${COMMENT_ID:-ENG-101}" + PREV="" + for arg in $COMMENT_ARGS_ALL; do + if [ "$PREV" = "--attach" ] || [ "$PREV" = "-a" ]; then + printf '✓ Uploaded %s\n' "${arg##*/}" + fi + PREV="$arg" + done + printf '✓ Comment added to %s\n' "$COMMENT_ID" + printf 'https://linear.app/acme/issue/%s#comment-eval\n' "$COMMENT_ID" + else + printf 'Comment added\n' + fi ;; "issue start") printf 'Started %s\n' "${3:-ENG-101}" diff --git a/skills/linear-cli/SKILL.md b/skills/linear-cli/SKILL.md index 0c21be20..024a2fff 100644 --- a/skills/linear-cli/SKILL.md +++ b/skills/linear-cli/SKILL.md @@ -70,6 +70,14 @@ linear issue update ENG-123 --label infra --label security # replaces the label linear issue comment add ENG-123 --body-file ./comment.md ``` +### Attach an image or screenshot so it is visible inline + +```bash +linear issue comment add ENG-123 --attach ./screenshot.png +``` + +This uploads the image and embeds it in a comment, where Linear renders it inline. Do not use `linear issue attach` when the image must be visible: that command creates a sidebar link attachment and does not render images inline. + ### View an issue / get its URL ```bash diff --git a/skills/linear-cli/SKILL.template.md b/skills/linear-cli/SKILL.template.md index e0b04b67..d65a2de1 100644 --- a/skills/linear-cli/SKILL.template.md +++ b/skills/linear-cli/SKILL.template.md @@ -70,6 +70,14 @@ linear issue update ENG-123 --label infra --label security # replaces the label linear issue comment add ENG-123 --body-file ./comment.md ``` +### Attach an image or screenshot so it is visible inline + +```bash +linear issue comment add ENG-123 --attach ./screenshot.png +``` + +This uploads the image and embeds it in a comment, where Linear renders it inline. Do not use `linear issue attach` when the image must be visible: that command creates a sidebar link attachment and does not render images inline. + ### View an issue / get its URL ```bash diff --git a/skills/linear-cli/references/issue.md b/skills/linear-cli/references/issue.md index 68e531a4..1131c17f 100644 --- a/skills/linear-cli/references/issue.md +++ b/skills/linear-cli/references/issue.md @@ -18,23 +18,23 @@ Options: Commands: - id - Print the issue based on the current git branch - mine, list, l - List your issues - query, q - Query issues with structured filters - title [issueId] - Print the issue title - start [issueId] - Start working on an issue - view, v [issueId] - View issue details (default) or open in browser/app - url [issueId] - Print the issue URL - describe [issueId] - Print the issue title and Linear-issue trailer - commits [issueId] - Show all commits for a Linear issue (jj only) - pull-request, pr [issueId] - Create a GitHub pull request with issue details - delete, d [issueId] - Delete an issue - create - Create a linear issue - update [issueId] - Update a linear issue - comment - Manage issue comments - attach - Attach a file to an issue - link [url] - Link a URL to an issue - relation - Manage issue relations (dependencies) + id - Print the issue based on the current git branch + mine, list, l - List your issues + query, q - Query issues with structured filters + title [issueId] - Print the issue title + start [issueId] - Start working on an issue + view, v [issueId] - View issue details (default) or open in browser/app + url [issueId] - Print the issue URL + describe [issueId] - Print the issue title and Linear-issue trailer + commits [issueId] - Show all commits for a Linear issue (jj only) + pull-request, pr [issueId] - Create a GitHub pull request with issue details + delete, d [issueId] - Delete an issue + create - Create a linear issue + update [issueId] - Update a linear issue + comment - Manage issue comments + attach - Create a sidebar link attachment on an issue (images do not render inline) + link [url] - Link a URL to an issue + relation - Manage issue relations (dependencies) agent-session - Manage agent sessions for an issue ``` @@ -100,22 +100,22 @@ Options: ### attach -> Attach a file to an issue +> Create a sidebar link attachment on an issue (images do not render inline) ``` Usage: linear issue attach Description: - Attach a file to an issue + Create a sidebar link attachment on an issue (images do not render inline) Options: - -h, --help - Show this help. - --workspace - Target workspace (uses credentials) - -t, --title - Custom title for the attachment - -c, --comment <body> - Add a comment body linked to the attachment - --public - Upload images to a public, unauthenticated URL (default: private, + -h, --help - Show this help. + --workspace <slug> - Target workspace (uses credentials) + -t, --title <title> - Custom title for the attachment + -c, --comment <body> - Create a linked comment with this body; the file remains a sidebar attachment + --public - Upload images to a public, unauthenticated URL (default: private, workspace-members only) ``` @@ -137,9 +137,9 @@ Options: Commands: - add [issueId] - Add a comment to an issue or reply to a comment - delete <commentId> - Delete a comment - update <commentId> - Update an existing comment + add [issueId] - Add a comment or reply; images uploaded with --attach render inline + delete <commentId> - Delete a comment + update <commentId> - Update an existing comment list [issueId] - List comments for an issue ``` @@ -152,17 +152,18 @@ Usage: linear issue comment add [issueId] Description: - Add a comment to an issue or reply to a comment + Add a comment or reply; images uploaded with --attach render inline Options: - -h, --help - Show this help. - --workspace <slug> - Target workspace (uses credentials) - -b, --body <text> - Comment body text - --body-file <path> - Read comment body from a file (preferred for markdown content) - -p, --parent <id> - Parent comment ID for replies - -a, --attach <filepath> - Attach a file to the comment (can be used multiple times) - --public - Upload attached images to a public, unauthenticated URL (default: private, + -h, --help - Show this help. + --workspace <slug> - Target workspace (uses credentials) + -b, --body <text> - Comment body text + --body-file <path> - Read comment body from a file (preferred for markdown content) + -p, --parent <id> - Parent comment ID for replies + -a, --attach <filepath> - Upload a file and add its Markdown link to the comment (images render inline; + repeatable) + --public - Upload attached images to a public, unauthenticated URL (default: private, workspace-members only) ``` diff --git a/src/commands/issue/issue-attach.ts b/src/commands/issue/issue-attach.ts index 013b740c..eb66bda5 100644 --- a/src/commands/issue/issue-attach.ts +++ b/src/commands/issue/issue-attach.ts @@ -15,14 +15,22 @@ import { ValidationError, } from "../../utils/errors.ts" +/** Quote a value for safe copy-paste into a shell command. */ +function quoteForShell(value: string): string { + if (/^[A-Za-z0-9_./:@%+=-]+$/.test(value)) return value + return `'${value.replaceAll("'", "'\\''")}'` +} + export const attachCommand = new Command() .name("attach") - .description("Attach a file to an issue") + .description( + "Create a sidebar link attachment on an issue (images do not render inline)", + ) .arguments("<issueId:string> <filepath:string>") .option("-t, --title <title:string>", "Custom title for the attachment") .option( "-c, --comment <body:string>", - "Add a comment body linked to the attachment", + "Create a linked comment with this body; the file remains a sidebar attachment", ) .option( "--public", @@ -100,8 +108,20 @@ export const attachCommand = new Command() } const attachment = data.attachmentCreate.attachment - console.log(`✓ Attachment created: ${attachment.title}`) + console.log(`✓ Sidebar link attachment created: ${attachment.title}`) console.log(attachment.url) + if (uploadResult.contentType.startsWith("image/")) { + const suggested = [ + "linear issue comment add", + resolvedIdentifier, + "--attach", + quoteForShell(filepath), + ...(makePublic ? ["--public"] : []), + ].join(" ") + console.log( + `Hint: Sidebar link attachments do not render images inline. For inline display, run: ${suggested}`, + ) + } } catch (error) { handleError(error, "Failed to attach file") } diff --git a/src/commands/issue/issue-comment-add.ts b/src/commands/issue/issue-comment-add.ts index e5aadc67..6ab24323 100644 --- a/src/commands/issue/issue-comment-add.ts +++ b/src/commands/issue/issue-comment-add.ts @@ -15,7 +15,9 @@ import { CliError, handleError, ValidationError } from "../../utils/errors.ts" export const commentAddCommand = new Command() .name("add") - .description("Add a comment to an issue or reply to a comment") + .description( + "Add a comment or reply; images uploaded with --attach render inline", + ) .arguments("[issueId:string]") .option("-b, --body <text:string>", "Comment body text") .option( @@ -25,7 +27,7 @@ export const commentAddCommand = new Command() .option("-p, --parent <id:string>", "Parent comment ID for replies") .option( "-a, --attach <filepath:string>", - "Attach a file to the comment (can be used multiple times)", + "Upload a file and add its Markdown link to the comment (images render inline; repeatable)", { collect: true }, ) .option( diff --git a/test/commands/issue/__snapshots__/issue-attach.test.ts.snap b/test/commands/issue/__snapshots__/issue-attach.test.ts.snap new file mode 100644 index 00000000..20838375 --- /dev/null +++ b/test/commands/issue/__snapshots__/issue-attach.test.ts.snap @@ -0,0 +1,67 @@ +export const snapshot = {}; + +snapshot[`Issue Attach Command - Image Prints Inline Hint 1`] = ` +stdout: +"✓ Uploaded screenshot.png +✓ Sidebar link attachment created: screenshot.png +https://uploads.linear.app/fake/screenshot.png +Hint: Sidebar link attachments do not render images inline. For inline display, run: linear issue comment add TEST-123 --attach /tmp/linear-cli-test-attach/screenshot.png +" +stderr: +"" +`; + +snapshot[`Issue Attach Command - Hint Shell-Quotes Unusual Paths 1`] = ` +stdout: +"✓ Uploaded it's a shot.png +✓ Sidebar link attachment created: shot.png +https://uploads.linear.app/fake/shot.png +Hint: Sidebar link attachments do not render images inline. For inline display, run: linear issue comment add TEST-123 --attach '/tmp/linear-cli-test-attach/it'\\\\''s a shot.png' +" +stderr: +"" +`; + +snapshot[`Issue Attach Command - Public Image Hint Preserves Public Flag 1`] = ` +stdout: +"✓ Uploaded screenshot.png +✓ Sidebar link attachment created: screenshot.png +https://uploads.linear.app/fake/screenshot.png +Hint: Sidebar link attachments do not render images inline. For inline display, run: linear issue comment add TEST-123 --attach /tmp/linear-cli-test-attach/screenshot.png --public +" +stderr: +"⚠ Uploaded to a public URL readable by anyone: https://uploads.linear.app/fake/screenshot.png +" +`; + +snapshot[`Issue Attach Command - Non-Image Has No Hint 1`] = ` +stdout: +"✓ Uploaded server.log +✓ Sidebar link attachment created: server.log +https://uploads.linear.app/fake/server.log +" +stderr: +"" +`; + +snapshot[`Issue Attach Command - Help 1`] = ` +stdout: +" +Usage: attach <issueId> <filepath> + +Description: + + Create a sidebar link attachment on an issue (images do not render inline) + +Options: + + -h, --help - Show this help. + -t, --title <title> - Custom title for the attachment + -c, --comment <body> - Create a linked comment with this body; the file remains a sidebar attachment + --public - Upload images to a public, unauthenticated URL (default: private, + workspace-members only) + +" +stderr: +"" +`; diff --git a/test/commands/issue/__snapshots__/issue-comment-add.test.ts.snap b/test/commands/issue/__snapshots__/issue-comment-add.test.ts.snap index 2e0f2512..caef8868 100644 --- a/test/commands/issue/__snapshots__/issue-comment-add.test.ts.snap +++ b/test/commands/issue/__snapshots__/issue-comment-add.test.ts.snap @@ -17,3 +17,28 @@ https://linear.app/issue/TEST-123#comment-uuid-reply-789 stderr: "" `; + +snapshot[`Issue Comment Add Command - Help 1`] = ` +stdout: +" +Usage: add [issueId] + +Description: + + Add a comment or reply; images uploaded with --attach render inline + +Options: + + -h, --help - Show this help. + -b, --body <text> - Comment body text + --body-file <path> - Read comment body from a file (preferred for markdown content) + -p, --parent <id> - Parent comment ID for replies + -a, --attach <filepath> - Upload a file and add its Markdown link to the comment (images render inline; + repeatable) + --public - Upload attached images to a public, unauthenticated URL (default: private, + workspace-members only) + +" +stderr: +"" +`; diff --git a/test/commands/issue/issue-attach.test.ts b/test/commands/issue/issue-attach.test.ts new file mode 100644 index 00000000..cc1a9513 --- /dev/null +++ b/test/commands/issue/issue-attach.test.ts @@ -0,0 +1,259 @@ +import { snapshotTest } from "@cliffy/testing" +import { assertEquals } from "@std/assert" +import { attachCommand } from "../../../src/commands/issue/issue-attach.ts" +import { + commonDenoArgs, + setupMockLinearServer, +} from "../../utils/test-helpers.ts" +import type { MockLinearServer } from "../../utils/mock_linear_server.ts" + +// A valid 1x1 PNG so getMimeType sees a real image extension and the PUT +// body assertion has known bytes. +const PNG_BYTES = Uint8Array.from([ + 0x89, + 0x50, + 0x4e, + 0x47, + 0x0d, + 0x0a, + 0x1a, + 0x0a, + 0x00, + 0x00, + 0x00, + 0x0d, + 0x49, + 0x48, + 0x44, + 0x52, + 0x00, + 0x00, + 0x00, + 0x01, + 0x00, + 0x00, + 0x00, + 0x01, + 0x08, + 0x06, + 0x00, + 0x00, + 0x00, + 0x1f, + 0x15, + 0xc4, + 0x89, + 0x00, + 0x00, + 0x00, + 0x0d, + 0x49, + 0x44, + 0x41, + 0x54, + 0x78, + 0x9c, + 0x62, + 0x00, + 0x01, + 0x00, + 0x00, + 0x05, + 0x00, + 0x01, + 0x0d, + 0x0a, + 0x2d, + 0xb4, + 0x00, + 0x00, + 0x00, + 0x00, + 0x49, + 0x45, + 0x4e, + 0x44, + 0xae, + 0x42, + 0x60, + 0x82, +]) + +function mockAttachFlow( + server: MockLinearServer, + options: { assetUrl: string }, +): void { + server.addResponse({ + queryName: "FileUpload", + response: { + data: { + fileUpload: { + success: true, + uploadFile: { + assetUrl: options.assetUrl, + uploadUrl: server.getUploadUrl(), + headers: [ + { key: "x-goog-content-length-range", value: "0,10485760" }, + ], + }, + }, + }, + }, + }) + server.addResponse({ + queryName: "AttachmentCreate", + response: { + data: { + attachmentCreate: { + success: true, + attachment: { + id: "attachment-uuid-1", + url: options.assetUrl, + title: options.assetUrl.split("/").at(-1), + }, + }, + }, + }, + }) +} + +const GET_ISSUE_ID_RESPONSE = { + queryName: "GetIssueId", + variables: { id: "TEST-123" }, + response: { data: { issue: { id: "issue-uuid-123" } } }, +} + +// Image attachment: sidebar wording plus the inline-display hint +await snapshotTest({ + name: "Issue Attach Command - Image Prints Inline Hint", + meta: import.meta, + colors: false, + args: ["TEST-123", "/tmp/linear-cli-test-attach/screenshot.png"], + denoArgs: commonDenoArgs, + async fn() { + await Deno.mkdir("/tmp/linear-cli-test-attach", { recursive: true }) + await Deno.writeFile( + "/tmp/linear-cli-test-attach/screenshot.png", + PNG_BYTES, + ) + const { server, cleanup } = await setupMockLinearServer([ + GET_ISSUE_ID_RESPONSE, + ]) + mockAttachFlow(server, { + assetUrl: "https://uploads.linear.app/fake/screenshot.png", + }) + + try { + await attachCommand.parse() + assertEquals(server.uploadRequests.length, 1) + assertEquals(server.uploadRequests[0].contentType, "image/png") + assertEquals(server.uploadRequests[0].body, PNG_BYTES) + assertEquals( + server.uploadRequests[0].headers["x-goog-content-length-range"], + "0,10485760", + ) + } finally { + await cleanup() + await Deno.remove("/tmp/linear-cli-test-attach", { recursive: true }) + } + }, +}) + +// A path with a space and a quote must be shell-quoted in the hint +await snapshotTest({ + name: "Issue Attach Command - Hint Shell-Quotes Unusual Paths", + meta: import.meta, + colors: false, + args: ["TEST-123", "/tmp/linear-cli-test-attach/it's a shot.png"], + denoArgs: commonDenoArgs, + async fn() { + await Deno.mkdir("/tmp/linear-cli-test-attach", { recursive: true }) + await Deno.writeFile( + "/tmp/linear-cli-test-attach/it's a shot.png", + PNG_BYTES, + ) + const { server, cleanup } = await setupMockLinearServer([ + GET_ISSUE_ID_RESPONSE, + ]) + mockAttachFlow(server, { + assetUrl: "https://uploads.linear.app/fake/shot.png", + }) + + try { + await attachCommand.parse() + } finally { + await cleanup() + await Deno.remove("/tmp/linear-cli-test-attach", { recursive: true }) + } + }, +}) + +// Public image: the hint preserves --public +await snapshotTest({ + name: "Issue Attach Command - Public Image Hint Preserves Public Flag", + meta: import.meta, + colors: false, + args: ["TEST-123", "/tmp/linear-cli-test-attach/screenshot.png", "--public"], + denoArgs: commonDenoArgs, + async fn() { + await Deno.mkdir("/tmp/linear-cli-test-attach", { recursive: true }) + await Deno.writeFile( + "/tmp/linear-cli-test-attach/screenshot.png", + PNG_BYTES, + ) + const { server, cleanup } = await setupMockLinearServer([ + GET_ISSUE_ID_RESPONSE, + ]) + mockAttachFlow(server, { + assetUrl: "https://uploads.linear.app/fake/screenshot.png", + }) + + try { + await attachCommand.parse() + } finally { + await cleanup() + await Deno.remove("/tmp/linear-cli-test-attach", { recursive: true }) + } + }, +}) + +// Non-image attachment: sidebar wording, no inline hint +await snapshotTest({ + name: "Issue Attach Command - Non-Image Has No Hint", + meta: import.meta, + colors: false, + args: ["TEST-123", "/tmp/linear-cli-test-attach/server.log"], + denoArgs: commonDenoArgs, + async fn() { + await Deno.mkdir("/tmp/linear-cli-test-attach", { recursive: true }) + await Deno.writeTextFile( + "/tmp/linear-cli-test-attach/server.log", + "2026-07-20T14:02:11Z INFO server listening\n", + ) + const { server, cleanup } = await setupMockLinearServer([ + GET_ISSUE_ID_RESPONSE, + ]) + mockAttachFlow(server, { + assetUrl: "https://uploads.linear.app/fake/server.log", + }) + + try { + await attachCommand.parse() + } finally { + await cleanup() + await Deno.remove("/tmp/linear-cli-test-attach", { recursive: true }) + } + }, +}) + +// Help output locks the sidebar-attachment description +await snapshotTest({ + name: "Issue Attach Command - Help", + meta: import.meta, + colors: false, + args: ["--help"], + denoArgs: commonDenoArgs, + async fn() { + await attachCommand.parse() + }, +}) diff --git a/test/commands/issue/issue-comment-add.test.ts b/test/commands/issue/issue-comment-add.test.ts index d32bf818..e19d3f22 100644 --- a/test/commands/issue/issue-comment-add.test.ts +++ b/test/commands/issue/issue-comment-add.test.ts @@ -200,3 +200,15 @@ Deno.test("Issue Comment Add Command - rejects --public batch before uploading e // ...and no attachment was uploaded (no "✓ Uploaded" line was printed). assertEquals(infoLogs.some((l) => l.includes("Uploaded")), false) }) + +// Help output locks the inline-image guidance in the descriptions +await snapshotTest({ + name: "Issue Comment Add Command - Help", + meta: import.meta, + colors: false, + args: ["--help"], + denoArgs: commonDenoArgs, + async fn() { + await commentAddCommand.parse() + }, +}) diff --git a/test/evals/linear-cli-skill-grade.test.ts b/test/evals/linear-cli-skill-grade.test.ts index 72e056cc..9ef6f164 100644 --- a/test/evals/linear-cli-skill-grade.test.ts +++ b/test/evals/linear-cli-skill-grade.test.ts @@ -6,6 +6,7 @@ import { flagValues, gradeRecords, isDiscovery, + positionalTokens, type ShimEntry, type TrialRecord, } from "../../evals/linear-cli-skill/grade.ts" @@ -288,8 +289,9 @@ Deno.test("gradeRecords aggregates by case and variant and rejects mixed conditi assertEquals(summary.supported.total, 1) assertEquals(summary.supported.routeOk, 1) assertEquals(summary.supported.fullSuccess, 0) - assertEquals(summary.controls.total, 1) - assertEquals(summary.controls.routeOk, 1) + assertEquals(summary.apiControls.total, 1) + assertEquals(summary.apiControls.routeOk, 1) + assertEquals(summary.cliControls.total, 0) assertEquals(summary.byVariant.holdout.total, 1) assertEquals(summary.byCase["query-holdout"].routeOk, 1) assertEquals(summary.skillReadTrials, 1) @@ -313,3 +315,261 @@ Deno.test("fisher one-sided matches known values", () => { // regression should give a large p-value assertEquals(fisherOneSided(9, 1, 1, 9) > 0.99, true) }) + +const imageCase = CASES.find((evalCase) => evalCase.id === "image-development")! +const sidebarControlCase = CASES.find((evalCase) => + evalCase.id === "control-sidebar-attachment" +)! + +Deno.test("positionalTokens strips declared flags in any position", () => { + assertEquals( + positionalTokens( + ["issue", "attach", "ENG-101", "./server.log", "--title", "Server log"], + ["issue", "attach"], + ["--title", "-t", "--comment", "-c"], + ["--public"], + ), + ["ENG-101", "./server.log"], + ) + // flags interspersed before the positionals must not shift the match + assertEquals( + positionalTokens( + ["issue", "attach", "--title", "Server log", "ENG-101", "./server.log"], + ["issue", "attach"], + ["--title", "-t", "--comment", "-c"], + ["--public"], + ), + ["ENG-101", "./server.log"], + ) + // --flag=value form, boolean flags, and unknown flags are all skipped + assertEquals( + positionalTokens( + [ + "issue", + "attach", + "--title=Server log", + "--public", + "--made-up-flag", + "ENG-101", + "./server.log", + ], + ["issue", "attach"], + ["--title", "-t", "--comment", "-c"], + ["--public"], + ), + ["ENG-101", "./server.log"], + ) + // after `--` everything is positional + assertEquals( + positionalTokens( + ["issue", "attach", "ENG-101", "--", "--weird-file"], + ["issue", "attach"], + ["--title", "-t"], + [], + ), + ["ENG-101", "--weird-file"], + ) +}) + +Deno.test("image case: issue attach alone fails, comment add --attach succeeds", () => { + const attachOnly = classifyTrial( + imageCase, + record("image-development", [ + linear(["issue", "attach", "ENG-107", "./screenshot.png"]), + ]), + ) + assertEquals(attachOnly.routeOk, false) + assertEquals(attachOnly.fullSuccess, false) + + const direct = classifyTrial( + imageCase, + record("image-development", [ + linear([ + "issue", + "comment", + "add", + "ENG-107", + "--attach", + "./screenshot.png", + ]), + ]), + ) + assertEquals(direct.firstRoute, "cli") + assertEquals(direct.routeOk, true) + assertEquals(direct.fullSuccess, true) +}) + +Deno.test("image case: recovery after a wrong first try counts as success", () => { + const recovered = classifyTrial( + imageCase, + record("image-development", [ + linear(["issue", "attach", "ENG-107", "./screenshot.png"]), + linear([ + "issue", + "comment", + "add", + "ENG-107", + "-a", + "./screenshot.png", + ]), + ]), + ) + assertEquals(recovered.firstRoute, "cli_other") + assertEquals(recovered.routeOk, true) + assertEquals(recovered.fullSuccess, true) +}) + +Deno.test("image case: GraphQL or raw-HTTP upload flows fail", () => { + const viaApi = classifyTrial( + imageCase, + record("image-development", [ + linear( + ["api"], + "mutation { fileUpload(...) { uploadFile { uploadUrl } } }", + ), + linear([ + "issue", + "comment", + "add", + "ENG-107", + "--attach", + "./screenshot.png", + ]), + ]), + ) + assertEquals(viaApi.routeOk, false) + assertEquals(viaApi.fullSuccess, false) + + const viaCurl = classifyTrial( + imageCase, + record("image-development", [ + { + tool: "curl", + argv: ["-X", "PUT", "https://uploads.example/signed"], + stdin: "", + }, + linear([ + "issue", + "comment", + "add", + "ENG-107", + "--attach", + "./screenshot.png", + ]), + ]), + ) + assertEquals(viaCurl.routeOk, false) +}) + +Deno.test("sidebar control: issue attach with the right issue and file succeeds", () => { + const correct = classifyTrial( + sidebarControlCase, + record("control-sidebar-attachment", [ + linear(["issue", "attach", "ENG-101", "./server.log"]), + ]), + ) + assertEquals(correct.routeOk, true) + assertEquals(correct.fullSuccess, true) + + const flagsFirst = classifyTrial( + sidebarControlCase, + record("control-sidebar-attachment", [ + linear([ + "issue", + "attach", + "--title", + "Server log", + "ENG-101", + "./server.log", + ]), + ]), + ) + assertEquals(flagsFirst.fullSuccess, true) +}) + +Deno.test("sidebar control: wrong file, wrong case, or comment route fails", () => { + const wrongFile = classifyTrial( + sidebarControlCase, + record("control-sidebar-attachment", [ + linear(["issue", "attach", "ENG-101", "./screenshot.png"]), + ]), + ) + assertEquals(wrongFile.routeOk, true) + assertEquals(wrongFile.fullSuccess, false) + + // positionals are case-sensitive (issue ids are stated verbatim) + const wrongCase = classifyTrial( + sidebarControlCase, + record("control-sidebar-attachment", [ + linear(["issue", "attach", "eng-101", "./server.log"]), + ]), + ) + assertEquals(wrongCase.fullSuccess, false) + + const viaComment = classifyTrial( + sidebarControlCase, + record("control-sidebar-attachment", [ + linear([ + "issue", + "comment", + "add", + "ENG-101", + "--attach", + "./server.log", + ]), + ]), + ) + assertEquals(viaComment.routeOk, false) +}) + +Deno.test("gradeRecords splits CLI-route and API-route controls", () => { + const { summary } = gradeRecords([ + record("control-subscribers", [ + linear(["api"], "query { issue { subscribers { nodes { email } } } }"), + ]), + record("control-sidebar-attachment", [ + linear(["issue", "attach", "ENG-101", "./server.log"]), + ]), + ]) + assertEquals(summary.apiControls.total, 1) + assertEquals(summary.apiControls.routeOk, 1) + assertEquals(summary.cliControls.total, 1) + assertEquals(summary.cliControls.routeOk, 1) + assertEquals(summary.cliControls.fullSuccess, 1) +}) + +Deno.test("image case: comment on the wrong issue is not a full success", () => { + const wrongIssue = classifyTrial( + imageCase, + record("image-development", [ + linear([ + "issue", + "comment", + "add", + "ENG-999", + "--attach", + "./screenshot.png", + ]), + ]), + ) + assertEquals(wrongIssue.routeOk, true) + assertEquals(wrongIssue.fullSuccess, false) + + // flags before the issue id must not break the positional check + const flagsFirst = classifyTrial( + imageCase, + record("image-development", [ + linear([ + "issue", + "comment", + "add", + "--body", + "see attached", + "ENG-107", + "--attach", + "./screenshot.png", + ]), + ]), + ) + assertEquals(flagsFirst.fullSuccess, true) +}) diff --git a/test/utils/mock_linear_server.ts b/test/utils/mock_linear_server.ts index b041b8b4..c4613ca4 100644 --- a/test/utils/mock_linear_server.ts +++ b/test/utils/mock_linear_server.ts @@ -19,10 +19,19 @@ interface MockResponse { status?: number } +export interface UploadRequest { + pathname: string + contentType: string | null + headers: Record<string, string> + body: Uint8Array +} + export class MockLinearServer { private server?: Deno.HttpServer private port = 0 private mockResponses: MockResponse[] + /** Signed-URL file uploads received via PUT, in arrival order */ + readonly uploadRequests: UploadRequest[] = [] constructor(responses: MockResponse[] = []) { this.mockResponses = responses @@ -54,6 +63,14 @@ export class MockLinearServer { return this.handleGraphQL(request) } + // Handle signed-URL file uploads (the PUT step of the fileUpload flow) + if ( + request.method === "PUT" && + new URL(request.url).pathname.startsWith("/upload") + ) { + return this.handleUpload(request) + } + return new Response("Not Found", { status: 404 }) }) @@ -76,6 +93,25 @@ export class MockLinearServer { return `http://localhost:${this.port}/graphql` } + /** URL to hand out as a fileUpload signed uploadUrl in mock responses */ + getUploadUrl(): string { + return `http://localhost:${this.port}/upload` + } + + private async handleUpload(request: Request): Promise<Response> { + const headers: Record<string, string> = {} + for (const [key, value] of request.headers) { + headers[key] = value + } + this.uploadRequests.push({ + pathname: new URL(request.url).pathname, + contentType: request.headers.get("content-type"), + headers, + body: new Uint8Array(await request.arrayBuffer()), + }) + return new Response(null, { status: 200 }) + } + private async handleGraphQL(request: Request): Promise<Response> { const headers = { "Content-Type": "application/json",