fix(catalog): surface real extraction errors and self-heal CI labels (#68)

extractModelCatalog kept the first candidate failure private and threw a generic 'Could not evaluate model catalog', so a new minified bundle shape (e.g. the $R is not defined regression) required manual reproduction to diagnose. The error now names the missing binding.

catalog-sync.yml failed to open catalog-break issues whenever the repo labels were missing (issues disabled at the time); the workflow now force-creates catalog-break and automation labels before every sync run.

Author: BrainerVirus

.github/workflows/catalog-sync.yml

@@ -31,6 +31,15 @@ jobs:
       - name: Install dependencies
         run: bun install --frozen-lockfile
 
+      - name: Ensure automation labels
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_SYNC_TOKEN || secrets.CATALOG_PUSH_TOKEN || secrets.GITHUB_TOKEN }}
+        run: |
+          gh label create catalog-break --force --color B60205 \
+            --description "Catalog model extraction failed in CI" || true
+          gh label create automation --force --color 0E8A16 \
+            --description "Automated bot activity" || true
+
       - name: Open catalog PR
         env:
           CI: "true"

src/catalog.ts

@@ -396,16 +396,20 @@ export function extractModelCatalog(
   }
   if (candidates.length === 0) throw new Error("Could not locate model catalog object");
 
+  let lastError: unknown = null;
   for (const raw of candidates) {
     try {
       const value = evaluateWithContext(normalizeForEval(raw), ctx);
       if (isModelCatalog(value)) return value;
-    } catch {
-      // try next span
+    } catch (err) {
+      // keep the first real failure (e.g. "$R is not defined") — it names the
+      // missing binding; later candidates usually fail on the same cause
+      lastError ??= err;
     }
   }
 
-  throw new Error("Could not evaluate model catalog");
+  const detail = lastError instanceof Error ? `: ${lastError.message}` : "";
+  throw new Error(`Could not evaluate model catalog${detail}`);
 }
 
 function isCostMap(value: unknown): value is Record<string, CostEntry[]> {

tests/unit/catalog.test.ts

@@ -273,6 +273,17 @@ describe("loadCatalogFromBundle", () => {
     expect(sonnet!.reasoningEfforts).toEqual(["low", "high"]);
   });
 
+  test("surfaces the real eval error when no candidate evaluates", () => {
+    // Bundle shape where the catalog references an unbindable identifier:
+    // the thrown error must name it, not the generic extraction failure.
+    const source = [
+      'var KR="chatComplete",qR="responses";',
+      'var Sn={SONNET_4_6:{id:"claude-sonnet-4-6",provider:$TOTALLY_MISSING,spec:KR,label:"Sonnet",name:"Claude Sonnet 4.6",description:"d"},GPT_X:{id:"gpt-5.5",provider:"openai",spec:KR,label:"GPT",name:"GPT-5.5",description:"d"}};',
+    ].join("");
+
+    expect(() => loadCatalogFromBundle(source)).toThrow(/\$TOTALLY_MISSING/);
+  });
+
   test("returns models when cost extraction fails", () => {
     const source = [
       '(Wt={ANTHROPIC:"anthropic",OPENAI:"openai",VERCEL_AI_GATEWAY:"vercel-ai-gateway"});',

tests/unit/release-workflow.test.ts

@@ -153,6 +153,23 @@ describe("catalog-sync.yml", () => {
     expect(blob).not.toContain("semantic-release");
     expect(blob).not.toContain("publish-if-needed");
   });
+
+  test("creates the catalog-break labels before syncing so break issues never fail", () => {
+    const wf = Bun.YAML.parse(read(".github/workflows/catalog-sync.yml")) as {
+      jobs: { sync: { steps: Array<{ name?: string; run?: string }> } };
+    };
+    const labelStep = wf.jobs.sync.steps.find((s) => s.name === "Ensure automation labels");
+    expect(labelStep).toBeDefined();
+    const run = labelStep!.run ?? "";
+    expect(run).toContain("gh label create catalog-break");
+    expect(run).toContain("gh label create automation");
+    expect(run).toContain("--force");
+    // must run before the sync step that opens break issues
+    const steps = wf.jobs.sync.steps;
+    expect(steps.findIndex((s) => s.name === "Ensure automation labels")).toBeLessThan(
+      steps.findIndex((s) => (s.run ?? "").includes("catalog-sync-ci.ts")),
+    );
+  });
 });
 
 describe("release.config.cjs", () => {