From 832d92eae55070dbc39d771d984d3f309111b1b4 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 10 Sep 2026 15:33:43 -0700 Subject: [PATCH 1/2] feat(vscode-lm): add five model families and refresh stale context budgets --- .../types/src/__tests__/vscode-llm.spec.ts | 86 +++++++++++---- packages/types/src/providers/vscode-llm.ts | 101 ++++++++++++------ src/api/providers/__tests__/vscode-lm.spec.ts | 79 ++++++++++++++ .../hooks/__tests__/useSelectedModel.spec.ts | 44 ++++++++ 4 files changed, 257 insertions(+), 53 deletions(-) diff --git a/packages/types/src/__tests__/vscode-llm.spec.ts b/packages/types/src/__tests__/vscode-llm.spec.ts index 3f79fb39272..71c0f7f5db8 100644 --- a/packages/types/src/__tests__/vscode-llm.spec.ts +++ b/packages/types/src/__tests__/vscode-llm.spec.ts @@ -2,14 +2,24 @@ import { describe, it, expect } from "vitest" import { vscodeLlmModels, vscodeLlmDefaultModelId } from "../providers/vscode-llm.js" +// The five families added and the two refreshed on 2026-09-10 (VS Code 1.137.0). +const SCOPED_ROWS = { + "gpt-6-astra": { contextWindow: 871793, maxInputTokens: 271783, supportsImages: true }, + "grok-4.5": { contextWindow: 424794, maxInputTokens: 199783, supportsImages: false }, + "grok-4.6": { contextWindow: 424794, maxInputTokens: 199784, supportsImages: false }, + "gemini-3.7-flash": { contextWindow: 935793, maxInputTokens: 935783, supportsImages: true }, + "gemini-3.8-flash": { contextWindow: 982833, maxInputTokens: 955113, supportsImages: true }, + "claude-opus-5": { contextWindow: 935793, maxInputTokens: 680456, supportsImages: true }, + "gemini-3.6-flash": { contextWindow: 935793, maxInputTokens: 935785, supportsImages: true }, +} as const + +const REFRESHED_FAMILIES = Object.keys(SCOPED_ROWS) as Array + describe("vscodeLlmModels", () => { it("exposes the opus-4.8 row with its measured maxInputTokens and contextWindow", () => { - // The VS Code LM API exposes only maxInputTokens; that is the value the UI reads from this - // table (useSelectedModel.ts). For claude-opus-4.8 the two fields intentionally DIVERGE: - // maxInputTokens (197897) is the enforced input ceiling, while contextWindow (679560) records - // the larger advertised window. The UI reads maxInputTokens, so the divergence is a deliberate - // tripwire — assert the actual on-disk literals rather than forcing equality. - // See GitHub issue simurg79/Roo-Code#10. + // claude-opus-4.8 intentionally DIVERGES: maxInputTokens (197897) is the enforced ceiling the + // UI reads, contextWindow (679560) the larger advertised window. Assert the on-disk literals + // rather than forcing equality. expect(vscodeLlmModels).toHaveProperty("claude-opus-4.8") expect(vscodeLlmModels["claude-opus-4.8"].contextWindow).toBe(679560) expect(vscodeLlmModels["claude-opus-4.8"].maxInputTokens).toBe(197897) @@ -23,10 +33,9 @@ describe("vscodeLlmModels", () => { }) it("includes the 2026-07-14 additions with their measured single-message ceilings", () => { - // Measured via single-message binary search on VS Code 1.126.0 (largest input the backend - // accepts). claude-sonnet-5 accepts nearly its full advertised window (925449), unlike the - // older claude rows that cap at ~197.9K — this divergence is exactly why the values are - // measured rather than inferred from a sibling row. + // Measured by single-message binary search on VS Code 1.126.0. claude-sonnet-5 accepts nearly + // its full window (925449) unlike older claude rows capped at ~197.9K — which is why values + // are measured, not inferred from a sibling row. expect(vscodeLlmModels["claude-sonnet-5"].maxInputTokens).toBe(925449) expect(vscodeLlmModels["claude-sonnet-5"].contextWindow).toBe(925449) expect(vscodeLlmModels["gpt-5.6-luna"].maxInputTokens).toBe(199753) @@ -34,22 +43,53 @@ describe("vscodeLlmModels", () => { expect(vscodeLlmModels["gpt-5.6-terra"].maxInputTokens).toBe(271785) }) - it("caps the 2026-07-26 model-picker-cache additions at the conservative measured ceiling", () => { - // Sourced from `chat.cachedLanguageModels` in User/globalStorage/state.vscdb; not yet - // binary-searched, so contextWindow records the advertised window while maxInputTokens holds - // the lowest measured enforced ceiling to avoid overflowing on an unverified number. - expect(vscodeLlmModels["claude-opus-5"].maxInputTokens).toBe(197897) + it("replaces the conservative 197897 placeholders with the 2026-09-10 measured ceilings", () => { + // These rows previously carried the placeholder 197897 (a borrowed sibling ceiling, not a + // measurement); the 2026-09-10 bracket-derived values supersede it. contextWindow is + // unchanged, so a failure here means the ceiling reverted to the placeholder. + expect(vscodeLlmModels["claude-opus-5"].maxInputTokens).toBe(680456) expect(vscodeLlmModels["claude-opus-5"].contextWindow).toBe(935793) - expect(vscodeLlmModels["gemini-3.6-flash"].maxInputTokens).toBe(197897) + expect(vscodeLlmModels["gemini-3.6-flash"].maxInputTokens).toBe(935785) expect(vscodeLlmModels["gemini-3.6-flash"].contextWindow).toBe(935793) }) + it("leaves no 2026-09-10 scoped family pinned to the 197897 placeholder", () => { + // Untouched rows (e.g. claude-opus-4.7) legitimately still measure 197897, so this guard is + // limited to the families this refresh touched. + for (const family of REFRESHED_FAMILIES) { + expect(vscodeLlmModels[family].maxInputTokens, `${family} must not be the 197897 placeholder`).not.toBe( + 197897, + ) + } + }) + + it("pins every 2026-09-10 row's windows and capability flags", () => { + for (const [family, expected] of Object.entries(SCOPED_ROWS)) { + const model = vscodeLlmModels[family as keyof typeof vscodeLlmModels] + + expect(model.maxInputTokens, `${family}: maxInputTokens`).toBe(expected.maxInputTokens) + expect(model.contextWindow, `${family}: contextWindow`).toBe(expected.contextWindow) + expect(model.supportsImages, `${family}: supportsImages`).toBe(expected.supportsImages) + + // Table-wide conventions: prices are 0 because these rows carry no per-token accounting, + // and supportsPromptCache is false because no cache is modelled here (not a claim about + // the backend). Tool calling is required for Roo to function. + expect(model.supportsPromptCache, `${family}: supportsPromptCache`).toBe(false) + expect(model.inputPrice, `${family}: inputPrice`).toBe(0) + expect(model.outputPrice, `${family}: outputPrice`).toBe(0) + expect(model.supportsToolCalling, `${family}: supportsToolCalling`).toBe(true) + + // Provider and gauge look rows up by the live client's family string, so the key, family + // and version must agree or lookups silently degrade to the default row. + expect(model.family, `${family}: family`).toBe(family) + expect(model.version, `${family}: version`).toBe(family) + } + }) + it("keeps both window fields populated and positive for every row", () => { - // NOTE: contextWindow and maxInputTokens are intentionally ALLOWED to differ (claude-opus-4.8 - // diverges: 679560 vs 197897). The UI reads maxInputTokens, and that divergence is a deliberate - // tripwire, so we do NOT assert contextWindow === maxInputTokens here (see simurg79/Roo-Code#10). - // The meaningful invariant is that every row carries positive integers for both fields; a - // missing/zero value would point to hand-authored drift rather than a real captured row. + // The two fields are ALLOWED to differ (claude-opus-4.8: 679560 vs 197897), so equality is + // deliberately NOT asserted. The invariant is positive integers on both fields; a missing or + // zero value indicates hand-authored drift, not a captured row. for (const [family, model] of Object.entries(vscodeLlmModels)) { expect(model.contextWindow, `${family}: contextWindow must be a positive integer`).toBeGreaterThan(0) expect(model.maxInputTokens, `${family}: maxInputTokens must be a positive integer`).toBeGreaterThan(0) @@ -57,8 +97,8 @@ describe("vscodeLlmModels", () => { }) it("excludes fabricated/internal/alias families and the dropped legacy rows", () => { - // Integrity guards: these were never part of the authoritative live capture, or were - // removed by the full table REPLACE. Their presence would signal hand-authored drift. + // These were never in the live capture, or were removed by the full table REPLACE; their + // presence would signal hand-authored drift. expect(vscodeLlmModels).not.toHaveProperty("claude-opus-4.7-high") expect(vscodeLlmModels).not.toHaveProperty("claude-3.5-sonnet") expect(vscodeLlmModels).not.toHaveProperty("claude-4-sonnet") diff --git a/packages/types/src/providers/vscode-llm.ts b/packages/types/src/providers/vscode-llm.ts index d406f8983dd..99a737efe0f 100644 --- a/packages/types/src/providers/vscode-llm.ts +++ b/packages/types/src/providers/vscode-llm.ts @@ -4,34 +4,11 @@ export type VscodeLlmModelId = keyof typeof vscodeLlmModels export const vscodeLlmDefaultModelId: VscodeLlmModelId = "claude-sonnet-4.5" -// Rows below were originally enumerated from `vscode.lm.selectChatModels({ vendor: "copilot" })`. -// The VS Code LM API exposes ONLY `maxInputTokens` (there is no separate context-window field), and -// that is the single value the runtime/condense gate enforces: getModel() sets -// contextWindow = Math.max(0, client.maxInputTokens) in src/api/providers/vscode-lm.ts. So for every -// row `maxInputTokens` IS the enforced context window, and `contextWindow` normally mirrors it -// (the UI reads maxInputTokens via useSelectedModel.ts). Rows whose advertised window is known to -// exceed the measured ceiling deliberately diverge: contextWindow records what Copilot advertises -// while maxInputTokens carries the value the backend actually accepts. -// These ceilings were measured empirically on 2026-06-18 (VS Code 1.125.0) by binary-searching the -// single-message "Message exceeds token limit" threshold per model — they are the largest input the -// backend actually accepts, which for several models is well below the value Copilot advertises: -// - claude-opus-4.8: enforced 679560 -// - claude-opus-4.7 / 4.6, claude-sonnet-4.6, -// gemini-3.1-pro-preview, gemini-3.5-flash: enforced ~197.9K -// - gpt-5.5 / gpt-5.4: enforced ~268.4K -// Additions measured 2026-07-14 (VS Code 1.126.0) with the same single-message binary search: -// - claude-sonnet-5: enforced 925449 (near-full window, unlike the older ~197.9K claude rows) -// - gpt-5.6-luna: enforced 199753 -// - gpt-5.6-sol / gpt-5.6-terra: enforced 271785 -// Additions sourced 2026-07-26 from the Copilot model-picker cache (`chat.cachedLanguageModels` in -// VS Code's User/globalStorage/state.vscdb), which persists the metadata `selectChatModels` returns: -// - claude-opus-5 / gemini-3.6-flash: advertised 935793, NOT yet binary-searched -// Those two rows keep 935793 in contextWindow as the advertised window, but pin maxInputTokens to -// 197897 — the lowest enforced ceiling measured on any row here. Trusting an unverified advertised -// window would overflow the request and hard-fail mid-task, so the conservative floor is used until -// a binary search establishes the real ceiling (mirrors the claude-opus-4.8 divergence above). -// Guardrail: these are empirically measured — re-measure (do not hand-tune) if the models change. -// See GitHub issue simurg79/Roo-Code#10 and myplans/VSCode LM Model Table Integrity/vscode_lm_opus_data_integrity_design.md. +// contextWindow = advertised window; maxInputTokens = measured single-message accepted ceiling, and +// it is what the condense gate and the context gauge enforce — so the two deliberately diverge where +// advertised exceeds measured. Values MUST be re-measured, never hand-tuned or borrowed from a +// sibling row. Per-row evidence, lower-bound and vendor caveats: +// myplans/vscode-lm-model-table-integrity/vscode-lm-model-table-integrity-design.md export const vscodeLlmModels = { "claude-opus-5": { contextWindow: 935793, @@ -43,7 +20,7 @@ export const vscodeLlmModels = { version: "claude-opus-5", name: "Claude Opus 5", supportsToolCalling: true, - maxInputTokens: 197897, + maxInputTokens: 680456, }, "claude-opus-4.8": { contextWindow: 679560, @@ -141,6 +118,18 @@ export const vscodeLlmModels = { supportsToolCalling: true, maxInputTokens: 135790, }, + "gpt-6-astra": { + contextWindow: 871793, + supportsImages: true, + supportsPromptCache: false, + inputPrice: 0, + outputPrice: 0, + family: "gpt-6-astra", + version: "gpt-6-astra", + name: "GPT-6 Astra", + supportsToolCalling: true, + maxInputTokens: 271783, + }, "gpt-5.6-luna": { contextWindow: 199753, supportsImages: true, @@ -249,6 +238,58 @@ export const vscodeLlmModels = { supportsToolCalling: true, maxInputTokens: 12078, }, + "grok-4.6": { + contextWindow: 424794, + // Grok image requests failed with a nondiagnostic HTTP 400, so support is unknown; false is a + // deliberate disable pending evidence, not a measured absence of capability. + supportsImages: false, + supportsPromptCache: false, + inputPrice: 0, + outputPrice: 0, + family: "grok-4.6", + version: "grok-4.6", + name: "Grok 4.6", + supportsToolCalling: true, + maxInputTokens: 199784, + }, + "grok-4.5": { + contextWindow: 424794, + // See grok-4.6 above: images disabled pending evidence, not proven unsupported. + supportsImages: false, + supportsPromptCache: false, + inputPrice: 0, + outputPrice: 0, + family: "grok-4.5", + version: "grok-4.5", + name: "Grok 4.5", + supportsToolCalling: true, + maxInputTokens: 199783, + }, + "gemini-3.8-flash": { + contextWindow: 982833, + supportsImages: true, + supportsPromptCache: false, + inputPrice: 0, + outputPrice: 0, + family: "gemini-3.8-flash", + version: "gemini-3.8-flash", + name: "Gemini 3.8 Flash", + supportsToolCalling: true, + // Largest accepted trial with no rejection observed: a lower bound, not a measured ceiling. + maxInputTokens: 955113, + }, + "gemini-3.7-flash": { + contextWindow: 935793, + supportsImages: true, + supportsPromptCache: false, + inputPrice: 0, + outputPrice: 0, + family: "gemini-3.7-flash", + version: "gemini-3.7-flash", + name: "Gemini 3.7 Flash", + supportsToolCalling: true, + maxInputTokens: 935783, + }, "gemini-3.6-flash": { contextWindow: 935793, supportsImages: true, @@ -259,7 +300,7 @@ export const vscodeLlmModels = { version: "gemini-3.6-flash", name: "Gemini 3.6 Flash", supportsToolCalling: true, - maxInputTokens: 197897, + maxInputTokens: 935785, }, "gemini-3.1-pro-preview": { contextWindow: 197897, diff --git a/src/api/providers/__tests__/vscode-lm.spec.ts b/src/api/providers/__tests__/vscode-lm.spec.ts index fccbad330ef..8fc38ef8d1d 100644 --- a/src/api/providers/__tests__/vscode-lm.spec.ts +++ b/src/api/providers/__tests__/vscode-lm.spec.ts @@ -141,6 +141,85 @@ describe("VsCodeLmHandler", () => { noFamilyHandler.dispose() }) + it.each([ + ["gpt-6-astra", 271783], + ["grok-4.5", 199783], + ["grok-4.6", 199784], + ["gemini-3.7-flash", 935783], + ["gemini-3.8-flash", 955113], + ["claude-opus-5", 680456], + ["gemini-3.6-flash", 935785], + ])("resolves %s to its measured static ceiling when the live client reports it", (family, expected) => { + // The live client wins over the selector, and its inflated maxInputTokens (4096 here stands in + // for any advertised value) must be ignored in favour of the curated row. + const familyHandler = new VsCodeLmHandler({ + vsCodeLmModelSelector: { vendor: "copilot", family: "claude-sonnet-4.5" }, + }) + familyHandler["client"] = { + ...mockLanguageModelChat, + family, + } as unknown as vscode.LanguageModelChat + + expect(familyHandler.getCondenseContextWindow()).toBe(expected) + expect(familyHandler.getCondenseContextWindow()).toBe( + vscodeLlmModels[family as keyof typeof vscodeLlmModels].maxInputTokens, + ) + + familyHandler.dispose() + }) + + it("falls back to the claude-sonnet-4.5 row (167790) for a family absent from the table", () => { + // Pins the literal so a change to the default row's own window is caught here rather than + // silently redefining what "fallback" means for every drifted selector. + const unknownHandler = new VsCodeLmHandler({ + vsCodeLmModelSelector: { vendor: "copilot", family: "gpt-9-nonexistent" }, + }) + unknownHandler["client"] = null + + expect(vscodeLlmDefaultModelId).toBe("claude-sonnet-4.5") + expect(unknownHandler.getCondenseContextWindow()).toBe(167790) + + unknownHandler.dispose() + }) + + it("resolves by family alone, independent of the user-selected vendor", () => { + // Vendor is a selection/routing concern only; the window lookup is family-keyed, so two + // different vendors selecting the same family MUST agree on the condense budget. + const copilotHandler = new VsCodeLmHandler({ + vsCodeLmModelSelector: { vendor: "copilot", family: "gpt-6-astra" }, + }) + const otherVendorHandler = new VsCodeLmHandler({ + vsCodeLmModelSelector: { vendor: "some-other-vendor", family: "gpt-6-astra" }, + }) + + expect(copilotHandler.getCondenseContextWindow()).toBe(271783) + expect(otherVendorHandler.getCondenseContextWindow()).toBe(copilotHandler.getCondenseContextWindow()) + + copilotHandler.dispose() + otherVendorHandler.dispose() + }) + + it("leaves getModel().contextWindow on the live client window for the new families", () => { + // Regression guard for the split: only the condense gate reads the static table. getModel() + // must keep reporting the LIVE window, so the refresh is provably behavior-neutral here. + const liveWindow = 123456 + const astraHandler = new VsCodeLmHandler({ + vsCodeLmModelSelector: { vendor: "copilot", family: "gpt-6-astra" }, + }) + astraHandler["client"] = { + ...mockLanguageModelChat, + family: "gpt-6-astra", + maxInputTokens: liveWindow, + } as unknown as vscode.LanguageModelChat + + expect(astraHandler.getModel().info.contextWindow).toBe(liveWindow) + expect(astraHandler.getModel().info.contextWindow).not.toBe(vscodeLlmModels["gpt-6-astra"].maxInputTokens) + // supportsImages still comes from the static row. + expect(astraHandler.getModel().info.supportsImages).toBe(true) + + astraHandler.dispose() + }) + it("falls back to the live window when the static row's maxInputTokens is non-positive", () => { const family = "claude-opus-4.8" const original = vscodeLlmModels[family].maxInputTokens diff --git a/webview-ui/src/components/ui/hooks/__tests__/useSelectedModel.spec.ts b/webview-ui/src/components/ui/hooks/__tests__/useSelectedModel.spec.ts index 52c6b9fc422..dab5a31bbf2 100644 --- a/webview-ui/src/components/ui/hooks/__tests__/useSelectedModel.spec.ts +++ b/webview-ui/src/components/ui/hooks/__tests__/useSelectedModel.spec.ts @@ -809,6 +809,50 @@ describe("useSelectedModel", () => { expect(result.current.info?.supportsImages).toBe(vscodeLlmModels[listedFamily].supportsImages) }) + it.each([ + ["gpt-6-astra", 271783, true], + ["grok-4.5", 199783, false], + ["grok-4.6", 199784, false], + ["gemini-3.7-flash", 935783, true], + ["gemini-3.8-flash", 955113, true], + ["claude-opus-5", 680456, true], + ["gemini-3.6-flash", 935785, true], + ])("feeds the gauge %s's static maxInputTokens", (family, expected, supportsImages) => { + const apiConfiguration: ProviderSettings = { + apiProvider: "vscode-lm", + vsCodeLmModelSelector: { vendor: "copilot", family }, + } + + const wrapper = createWrapper() + const { result } = renderHook(() => useSelectedModel(apiConfiguration), { wrapper }) + + // The gauge denominator and the condense gate must read the SAME static row, otherwise the + // bar fills at a different point than the one that triggers condensing. + expect(result.current.info?.contextWindow).toBe(expected) + expect(result.current.info?.contextWindow).toBe( + vscodeLlmModels[family as keyof typeof vscodeLlmModels].maxInputTokens, + ) + expect(result.current.info?.supportsImages).toBe(supportsImages) + }) + + it("reads the same row regardless of the user-selected vendor", () => { + const wrapper = createWrapper() + const forVendor = (vendor: string) => + renderHook( + () => + useSelectedModel({ + apiProvider: "vscode-lm", + vsCodeLmModelSelector: { vendor, family: "gemini-3.8-flash" }, + } as ProviderSettings), + { wrapper }, + ).result.current + + // Vendor only affects the display id; the window lookup is family-keyed. + expect(forVendor("copilot").info?.contextWindow).toBe(955113) + expect(forVendor("some-other-vendor").info?.contextWindow).toBe(955113) + expect(forVendor("some-other-vendor").id).toBe("some-other-vendor/gemini-3.8-flash") + }) + it("surfaces supportsImages true for an image-capable family", () => { const apiConfiguration: ProviderSettings = { apiProvider: "vscode-lm", From 5bcaa0df37cf6baf1c4a49f8e9469b3b75c53e3d Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Thu, 10 Sep 2026 16:00:01 -0700 Subject: [PATCH 2/2] chore: bump extension version to 3.53.6 --- src/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/package.json b/src/package.json index f8adf9ee8dc..9dd1581f538 100644 --- a/src/package.json +++ b/src/package.json @@ -3,7 +3,7 @@ "displayName": "%extension.displayName%", "description": "%extension.description%", "publisher": "RooVeterinaryInc", - "version": "3.53.5", + "version": "3.53.6", "icon": "assets/icons/icon.png", "galleryBanner": { "color": "#617A91",