From 7d10265ecf2c01a9c34893fc05b0371f6d9aba0c Mon Sep 17 00:00:00 2001
From: KageBinary <ianjakobhock@gmail.com>
Date: Sun, 30 Aug 2026 13:42:47 -0700
Subject: [PATCH] test(tools): make test:live actually test something

`IX_LIVE_TESTS` appeared in exactly three places -- the `test:live` script, a
CI comment and a docstring -- and no test ever read it. There was no gate for
the variable to open, so `bun run test:live` ran the identical 91 offline
tests, and no tool had ever been exercised against a real `ix`.

That left every tool covered only on its fallback path: the suite drives them
at a directory where `ix` cannot succeed and asserts they return a string
rather than throwing. A regression in argument shape, output parsing or
exit-code handling would pass the whole suite.

Add a `LiveIxCli` block behind `describe.skipIf(!IX_LIVE_TESTS)` covering
ix-stats, ix-health and a deliberate ix-locate miss. The load-bearing
assertion is the absence of each tool's unavailable marker, which is what
separates a real success from the fallback the offline tests already cover.

Note ix-health uses its own marker (`Status: UNAVAILABLE`, not the
`ix unavailable` string the others use). Asserting the wrong one made that test
pass with no `ix` on PATH at all -- verified each test fails when `ix` is
absent, so none of them is vacuous.

Also corrects the CI comment and docstring, which described an @live suite that
did not exist.

Closes #18
---
 .github/workflows/ci.yml |  4 ++-
 tests/tools.test.ts      | 59 ++++++++++++++++++++++++++++++++++++++--
 2 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 59f7154..6e99c53 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -19,8 +19,10 @@ jobs:
   # This is a Bun project (package.json `test` = `bun test`), so CI installs
   # the Bun toolchain rather than Node. Bun executes the TypeScript test files
   # directly — there is no separate build/typecheck step (no build script and
-  # no tsconfig in the repo). `bun test` skips the @live suite unless
+  # no tsconfig in the repo). `bun test` skips the LiveIxCli suite unless
   # IX_LIVE_TESTS=1 is set, so CI runs the offline contract/fallback tests.
+  # The live suite needs a real `ix` on PATH and a reachable backend, neither
+  # of which this runner has, so it stays opt-in (`bun run test:live`).
   test:
     name: Test (bun)
     runs-on: ubuntu-latest
diff --git a/tests/tools.test.ts b/tests/tools.test.ts
index 3740afe..f52d738 100644
--- a/tests/tools.test.ts
+++ b/tests/tools.test.ts
@@ -8,8 +8,8 @@
  * 2. All tools return strings (never throw) when ix is unavailable
  * 3. Tool parameter schemas are valid JSON Schema objects
  *
- * Tests that require a live OpenCode session or ix CLI are marked @live
- * and skipped in CI unless IX_LIVE_TESTS=1 is set.
+ * Tests that require a live ix CLI live in the `LiveIxCli` block at the bottom
+ * and are skipped unless IX_LIVE_TESTS=1 is set (`bun run test:live`).
  */
 
 import { describe, test, expect } from "bun:test";
@@ -180,3 +180,58 @@ describe("PluginHookContract", () => {
     expect(typeof reg["tool.execute.after"]).toBe("function");
   });
 });
+
+// ─── LiveIxCli (@live) ───────────────────────────────────────────────────────
+//
+// Every test above drives the tools at a directory where `ix` cannot succeed,
+// so they only ever prove the *fallback* path: that a tool returns a string
+// rather than throwing. Nothing exercised a successful `ix` invocation, which
+// means a regression in argument shape, output parsing or exit-code handling
+// would pass the whole suite (#18).
+//
+// These are opt-in because they need a real `ix` on PATH and a reachable
+// backend. Enable with `bun run test:live` (IX_LIVE_TESTS=1).
+//
+// The load-bearing assertion is `not.toContain("ix unavailable")` — that string
+// is exactly what every tool emits when the CLI call fails, so asserting its
+// absence is what distinguishes a real success from the fallback the offline
+// tests already cover.
+
+const LIVE = Boolean(process.env.IX_LIVE_TESTS);
+const LIVE_CTX = { directory: process.cwd() };
+
+describe.skipIf(!LIVE)("LiveIxCli", () => {
+  test("ix-stats reaches the CLI and renders a real report", async () => {
+    const output = await ixStats.execute({}, LIVE_CTX);
+
+    expect(typeof output).toBe("string");
+    expect(output).toContain("## ix-stats");
+    expect(output).not.toContain("ix unavailable");
+    expect(output).not.toContain("Failed to parse output");
+  });
+
+  test("ix-health reaches the CLI", async () => {
+    const output = await ixHealth.execute({}, LIVE_CTX);
+
+    expect(typeof output).toBe("string");
+    // ix-health has its own marker -- it reports `Status: UNAVAILABLE` rather
+    // than the `ix unavailable` string the other tools use. Asserting the wrong
+    // one here made this test pass with no `ix` on PATH at all.
+    expect(output).not.toContain("Status: UNAVAILABLE");
+    expect(output).not.toContain("ix CLI not found");
+  });
+
+  test("a miss is a real answer, not the unavailable fallback", async () => {
+    // A symbol that cannot exist. `ix` answers "no match" and exits without
+    // error, so the tool must render that answer rather than reporting the CLI
+    // as unavailable -- the two are easy to conflate and only a live run
+    // separates them.
+    const output = await ixLocate.execute(
+      { symbol: "IxDefinitelyMissingSymbol_99999" },
+      LIVE_CTX,
+    );
+
+    expect(typeof output).toBe("string");
+    expect(output).not.toContain("ix unavailable");
+  });
+});