diff --git a/cmd/codeaf/carried.go b/cmd/codeaf/carried.go index bc8d16ed1b..e04d20c3b6 100644 --- a/cmd/codeaf/carried.go +++ b/cmd/codeaf/carried.go @@ -131,9 +131,9 @@ type carriedRoad struct { seat string defaultSeat func() (string, error) signNamed bool - // warmCatalog waits, within ctx, for the profile's model catalog to be - // resolved and written down. Nil for a road with no catalog. - warmCatalog func(ctx context.Context) bool + // warmCatalog waits, within ctx, for the named models' service catalogs + // to be resolved and written down. Nil for a road with no catalog. + warmCatalog func(ctx context.Context, models []string) bool } // carriedModels resolves a shell run's road. It is the person's own profile, @@ -181,8 +181,34 @@ func profileRoad() (carriedRoad, error) { } return seats.Work.Model, nil }, - signNamed: config.AttributionModelAt(settings.ProfileDir), - warmCatalog: settings.Models.Warmed, + signNamed: config.AttributionModelAt(settings.ProfileDir), + warmCatalog: func(ctx context.Context, models []string) bool { + // The child reads each service's own cache without provider keys. + // Waiting only for the default cache leaves a fresh custom service's + // known window invisible to senior-dev. + catalogs := map[string]*catalog.Catalog{} + for _, model := range models { + model = strings.TrimPrefix(strings.TrimSpace(model), modelsource.DefaultID+"/") + if model == "" { + continue + } + service, _ := sources.For(model) + if service.Source.ID == modelsource.DefaultID { + catalogs[modelsource.DefaultID] = settings.Models + continue + } + options := config.CatalogOptionsFor(service, settings.ProfileDir) + key := catalog.CacheKey(options.Source, options.BaseURL) + if catalogs[key] == nil { + catalogs[key] = catalog.LoadLazy(ctx, options) + } + } + ready := true + for _, models := range catalogs { + ready = models.Warmed(ctx) && ready + } + return ready + }, }, nil } @@ -260,11 +286,16 @@ func runCarriedHost(ctx context.Context, inv *delegate.Invocation) error { } return err } + var modelsToWarm []string + if asked != "" { + modelsToWarm = strings.Split(asked, ",") + } if word := asked; word != "" && road.resolveModel != nil { resolved, err := road.resolveModel(word) if err != nil { return err } + modelsToWarm = strings.Split(resolved, ",") if inv.Program.Name == "senior-dev" { // Its pool is addressed through the loopback API's OpenRouter // dialect, just as the chat road's crewFlags spells every seat. @@ -283,6 +314,7 @@ func runCarriedHost(ctx context.Context, inv *delegate.Invocation) error { if err != nil { return err } + modelsToWarm = append(modelsToWarm, road.seat) // AND THE SEAT IS WHAT THE PROGRAM WORKS ON, as a conversation's crew is // ([carriedSeatedLine]): a shell run and a chat run of one program on // one profile route on the same model. @@ -344,7 +376,12 @@ func runCarriedHost(ctx context.Context, inv *delegate.Invocation) error { // it guessed. if inv.Program.Name == "senior-dev" && road.warmCatalog != nil { warmCtx, warmed := context.WithTimeout(runCtx, carriedCatalogWarm) - road.warmCatalog(warmCtx) + for _, flag := range []string{"low", "frontier"} { + if pool := strings.TrimSpace(inv.ExplicitFlags[flag]); pool != "" { + modelsToWarm = append(modelsToWarm, strings.Split(pool, ",")...) + } + } + road.warmCatalog(warmCtx, modelsToWarm) warmed() } ledger := session.UsageLedgerPath() diff --git a/cmd/codeaf/carried_fresh_profile_test.go b/cmd/codeaf/carried_fresh_profile_test.go index 4b7be9247a..cb497aa8af 100644 --- a/cmd/codeaf/carried_fresh_profile_test.go +++ b/cmd/codeaf/carried_fresh_profile_test.go @@ -15,6 +15,7 @@ import ( "github.com/Agent-Field/codeaf/internal/config" "github.com/Agent-Field/codeaf/internal/delegate/builtin" + "github.com/Agent-Field/codeaf/internal/modelsource" ) // The real shell parser must let a model named by --high start on a fresh @@ -46,12 +47,38 @@ func TestSeniorDevShellRefusesAServiceThatCannotServeItsModel(t *testing.T) { testSeniorDevExplicitShellModel(t, shellModelCase{word: "openrouter/fixture/vendor-model", noKey: true, refused: true}) } +// A custom service's /models row can identify a working model without saying +// how much history it accepts. A real child must either use the service's +// known window or the project's explicit limit, or refuse before any turn. +func TestSeniorDevCustomShellModelNeedsKnownOrConfiguredCapacity(t *testing.T) { + for _, tc := range []shellModelCase{ + {word: "fixture-custom/custom-model", custom: true, window: 200_000}, + {word: "fixture-custom/custom-model", custom: true, configuredWindow: 64_000}, + {word: "fixture-custom/custom-model", custom: true, capacityRefused: true}, + {word: "fixture-custom/custom-model", custom: true, window: 16_384, capacityRefused: true}, + } { + name := "listed-window" + if tc.capacityRefused && tc.window > 0 { + name = "tiny-listed-window" + } else if tc.configuredWindow > 0 { + name = "explicit-config" + } else if tc.capacityRefused { + name = "missing-window" + } + t.Run(name, func(t *testing.T) { testSeniorDevExplicitShellModel(t, tc) }) + } +} + type shellModelCase struct { - word string - plain bool - asked bool - refused bool - noKey bool + word string + plain bool + asked bool + refused bool + noKey bool + custom bool + window int + configuredWindow int + capacityRefused bool } func testSeniorDevExplicitShellModel(t *testing.T, tc shellModelCase) { @@ -80,11 +107,25 @@ func testSeniorDevExplicitShellModel(t *testing.T, tc shellModelCase) { t.Setenv(carriedChildEnv, "real") t.Setenv("DO_NOT_TRACK", "1") t.Setenv("CODEAF_NO_UPDATE_CHECK", "1") + modelID := "fixture/vendor-model" + window := 200_000 + if tc.custom { + modelID, window = "custom-model", tc.window + } var calls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/models") { w.Header().Set("Content-Type", "application/json") - fmt.Fprint(w, `{"data":[{"id":"fixture/vendor-model","canonical_slug":"fixture/vendor-model","name":"Fixture model","context_length":200000,"architecture":{"input_modalities":["text"],"output_modalities":["text"]},"pricing":{"prompt":"0","completion":"0","request":"0"},"supported_parameters":["tools","tool_choice","max_tokens"]}]}`) + row := map[string]any{ + "id": modelID, "canonical_slug": modelID, "name": "Fixture model", + "architecture": map[string]any{"input_modalities": []string{"text"}, "output_modalities": []string{"text"}}, + "pricing": map[string]string{"prompt": "0", "completion": "0", "request": "0"}, + "supported_parameters": []string{"tools", "tool_choice", "max_tokens"}, + } + if window > 0 { + row["context_length"] = window + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": []any{row}}) return } if r.Method != http.MethodPost || !strings.HasSuffix(r.URL.Path, "/chat/completions") { @@ -98,7 +139,7 @@ func testSeniorDevExplicitShellModel(t *testing.T, tc shellModelCase) { http.Error(w, err.Error(), 400) return } - if request.Model != "fixture/vendor-model" { + if request.Model != modelID { http.Error(w, "wrong model: "+request.Model, 400) return } @@ -115,13 +156,29 @@ func testSeniorDevExplicitShellModel(t *testing.T, tc shellModelCase) { } if tool != "" { argumentJSON, _ := json.Marshal(arguments) - fmt.Fprintf(w, "data: {\"id\":\"fixture-%d\",\"model\":\"fixture/vendor-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call-%d\",\"type\":\"function\",\"function\":{\"name\":%q,\"arguments\":%s}}]},\"finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":100,\"completion_tokens\":20,\"total_tokens\":120,\"cost\":0}}\n\ndata: [DONE]\n\n", call, call, tool, argumentJSON) + fmt.Fprintf(w, "data: {\"id\":\"fixture-%d\",\"model\":%q,\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call-%d\",\"type\":\"function\",\"function\":{\"name\":%q,\"arguments\":%s}}]},\"finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":100,\"completion_tokens\":20,\"total_tokens\":120,\"cost\":0}}\n\ndata: [DONE]\n\n", call, modelID, call, tool, argumentJSON) } else { - fmt.Fprintf(w, "data: {\"id\":\"fixture-%d\",\"model\":\"fixture/vendor-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":100,\"completion_tokens\":20,\"total_tokens\":120,\"cost\":0}}\n\ndata: [DONE]\n\n", call) + fmt.Fprintf(w, "data: {\"id\":\"fixture-%d\",\"model\":%q,\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"done\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":100,\"completion_tokens\":20,\"total_tokens\":120,\"cost\":0}}\n\ndata: [DONE]\n\n", call, modelID) } })) defer server.Close() t.Setenv("CODEAF_BASE_URL", server.URL+"/api/v1") + if tc.custom { + t.Setenv("OPENROUTER_API_KEY", "") + profile := os.Getenv("CODEAF_HOME") + t.Setenv(config.ProfileDirEnv, profile) + if err := config.WriteSources(profile, []config.PersistedSource{{ + ID: modelsource.CustomID, Written: "fixture-custom", Address: server.URL + "/api/v1", Key: "fixture-key", Order: 1, + }}); err != nil { + t.Fatal(err) + } + } + if tc.configuredWindow > 0 { + text := fmt.Sprintf(`{"provider":{"openrouter":{"models":{"fixture-custom/custom-model":{"limit":{"context":%d,"output":2048}}}}}}`, tc.configuredWindow) + if err := os.WriteFile(filepath.Join(workspace, "senior-dev.json"), []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } previous := carriedStdout output := &lockedBuffer{} carriedStdout = output @@ -132,6 +189,17 @@ func testSeniorDevExplicitShellModel(t *testing.T, tc shellModelCase) { } line = append(line, "--dir", workspace, "--", "Add", "the", "feature.") err := runCarried(program, line) + if tc.capacityRefused { + printed := output.String() + if exitCodeOf(err) == 0 || calls.Load() != 0 || + !strings.Contains(printed, "senior-dev cannot work with "+tc.word) || + !strings.Contains(printed, "nothing was started") || + (tc.window == 0 && !strings.Contains(printed, "senior-dev.json")) || + strings.Contains(printed, "implement ·") || strings.Contains(printed, "bootstrap ·") { + t.Fatalf("unsized custom model exited %d after %d calls: %s", exitCodeOf(err), calls.Load(), printed) + } + return + } if tc.refused { if err == nil || calls.Load() != 0 || !strings.Contains(err.Error(), tc.word) || !strings.Contains(err.Error(), "/crew") || !strings.Contains(err.Error(), "codeaf connect") || output.String() != "" { diff --git a/cmd/codeaf/carried_seniordev_test.go b/cmd/codeaf/carried_seniordev_test.go index 6619851d78..9a66a7729b 100644 --- a/cmd/codeaf/carried_seniordev_test.go +++ b/cmd/codeaf/carried_seniordev_test.go @@ -100,10 +100,12 @@ func TestSeniorDevWorksATaskThroughTheShellHostsModelAPI(t *testing.T) { // within one catalog fetch, so the window senior-dev reads from it is on // disk before its first call (carried.go's carriedCatalogWarm). var warms, callsAtWarm int + var warmedModels []string var warmBound time.Duration carriedModels = func() (carriedRoad, error) { - return carriedRoad{completerFor: model.completerFor, warmCatalog: func(ctx context.Context) bool { + return carriedRoad{completerFor: model.completerFor, warmCatalog: func(ctx context.Context, models []string) bool { warms++ + warmedModels = append([]string(nil), models...) if deadline, ok := ctx.Deadline(); ok { warmBound = time.Until(deadline) } @@ -132,6 +134,9 @@ func TestSeniorDevWorksATaskThroughTheShellHostsModelAPI(t *testing.T) { t.Fatalf("catalog waited for %d times, after %d calls, bounded by %v; want once, before any call, within %v", warms, callsAtWarm, warmBound, catalog.FetchTimeout) } + if strings.Join(warmedModels, ",") != "openrouter/fixture/vendor-model" { + t.Fatalf("catalog warm models = %q, want the selected working model", warmedModels) + } for _, want := range []string{"senior-dev · working in " + workspace, "senior-dev finished", "senior-dev's model said: feature.txt now holds the feature", " · 300 in · 20 out · $0.0020"} { if !strings.Contains(out, want) { t.Fatalf("the shell run never printed %q:\n%s", want, out) diff --git a/docs/changes/unreleased/1811-senior-dev-model-capacity.md b/docs/changes/unreleased/1811-senior-dev-model-capacity.md new file mode 100644 index 0000000000..8337b0f3d5 --- /dev/null +++ b/docs/changes/unreleased/1811-senior-dev-model-capacity.md @@ -0,0 +1,10 @@ +--- +kind: fixed +title: senior-dev checks selected-service model capacity before starting +pr: 1811 +surface: [engine, docs] +invalidates: + - "A shell run warmed only the default service catalog, so a model from another connected service could fail sizing despite that service publishing its window. It now warms the catalogs for the selected high, low and frontier models." + - "A missing or zero window could reach senior-dev's pipeline and fail during work. All remaining pools now refuse before stages or model calls, and provider-supplied project limits remain supported." +--- + diff --git a/internal/manual/chat/senior-dev.md b/internal/manual/chat/senior-dev.md index 14fd695626..0f2c2c6033 100644 --- a/internal/manual/chat/senior-dev.md +++ b/internal/manual/chat/senior-dev.md @@ -1030,8 +1030,9 @@ with exactly those, routing among them call by call when there are several; the the task's first line name them. A name that fits more than one model is put to you to settle. A model none of your connected services can serve is refused before the card, by name, rather than swapped for another. When a catalog was loaded, a model neither it nor -codeaf's own model catalog can size cannot be used: the run ends before its first call with -`senior-dev cannot work with : …`, and nothing is spent. A model you asked for +codeaf's own model catalog or an explicit model configuration can size cannot be used: the run ends before its first call with +`senior-dev cannot work with : …`, and nothing is spent. This applies to every +pool, including `--low` and `--frontier`, even on a shell run without `--asked`. A model you asked for that is known to hold 32,768 tokens or fewer is refused the same way (see the section on a model too small for senior-dev). If models.dev is unavailable and codeaf's catalog does not know the model either, the run starts on an assumed 16,384 tokens and says so. The models are fixed when the run starts; changing the crew later does not move @@ -1052,6 +1053,26 @@ A crew model senior-dev's model catalog cannot size is left out, and its log say and so is one known to hold 32,768 tokens or fewer; if that leaves no working model, it uses its own list instead. +## senior-dev cannot work with my custom model — unknown window, context length, senior-dev.json + +A connected custom service can list a model that works in chat without publishing its +context-window size. senior-dev needs that size to keep a long run's history in it. +A shell run warms the catalogs of the services named in its `--high`, `--low` and +`--frontier` pools, not just the default service's catalog. When a populated catalog +cannot size any remaining pool, the run refuses before its first stage or model call, +names the model and says nothing was started. A row with no window is not a zero-sized +model, and no capacity is invented for it. + +Use a model whose service lists its window, or supply **limits from the provider** in +`senior-dev.json` in the project. Set `provider.openrouter.models["connection/model"].limit.context` +to the provider's context-token limit, and `.limit.output` to its output limit or +a smaller requested output ceiling. `connection/model` is the exact model spelling +codeaf accepts, with no leading `openrouter/`. Here `openrouter` names codeaf's loopback +API format; calls still use the selected connection and codeaf's shared model funnel. +Do not enter an assumed large window just to pass the check. Known windows of 32,768 +tokens or fewer are still refused; the existing conservative guess when no catalog +can be read is unchanged. + ## senior-dev keeps compacting, or refused a model as too small — how much its model can hold, a 32K model, could not learn how much its model holds senior-dev keeps its brief, its tools, its checklist and its progress in its model's diff --git a/internal/seniordev/app/run.go b/internal/seniordev/app/run.go index 82f41ef408..7d8454aa49 100644 --- a/internal/seniordev/app/run.go +++ b/internal/seniordev/app/run.go @@ -292,8 +292,14 @@ func askedRefusal(pool string, known func(string) bool) string { if len(unknown) == 0 { return "" } + return capacityRefusal(unknown) +} + +// capacityRefusal names models whose metadata cannot size a run, and the two +// supported ways to supply it. A missing window is never made up here. +func capacityRefusal(unknown []string) string { return "senior-dev cannot work with " + strings.Join(unknown, ", ") + - ": its model catalog does not know how much it can hold, so nothing was started; ask for a model it knows" + ": its model catalog does not know how much it can hold, so nothing was started; ask for a model it knows or set its verified limits in senior-dev.json" } // refused is the ending of a run that could not start: its brief, its diff --git a/internal/seniordev/app/window_check.go b/internal/seniordev/app/window_check.go index 21dd98acbb..e90bf5d969 100644 --- a/internal/seniordev/app/window_check.go +++ b/internal/seniordev/app/window_check.go @@ -93,7 +93,7 @@ func leaveOutTinyCrewSeats(args cliArgs, keepHigh bool, models seniorDevModels, // compacts every few steps says why at its start. func windowCheck(args cliArgs, models seniorDevModels) (refusal string, guessed []string) { seen := map[string]bool{} - var tiny []string + var tiny, unknown []string for _, pool := range []string{args.High, args.Low, args.Frontier} { for _, ref := range splitPool(pool) { name := strings.TrimPrefix(ref, orclient.Service+"/") @@ -101,17 +101,21 @@ func windowCheck(args cliArgs, models seniorDevModels) (refusal string, guessed continue } seen[name] = true - _, source, err := models.windowOf(ref) - if err != nil { + window, source, err := models.windowOf(ref) + if err != nil || (source != sizedByGuess && window <= 0) { + unknown = append(unknown, name) continue } if source == sizedByGuess { guessed = append(guessed, name) - } else if window, small := models.tinyWindow(ref); small { + } else if window > 0 && window <= tinyWindowTokens { tiny = append(tiny, fmt.Sprintf("%s (%s tokens)", name, groupedTokens(window))) } } } + if len(unknown) > 0 { + return capacityRefusal(unknown), nil + } if len(tiny) > 0 { refusal = "senior-dev cannot work with " + strings.Join(tiny, ", ") + ": a run needs a model that holds more than " + groupedTokens(tinyWindowTokens) + diff --git a/internal/seniordev/app/window_check_test.go b/internal/seniordev/app/window_check_test.go index 562b563692..a35cdbd5a4 100644 --- a/internal/seniordev/app/window_check_test.go +++ b/internal/seniordev/app/window_check_test.go @@ -116,6 +116,48 @@ func TestATinyKnownWindowIsRefusedAndAGuessIsOnlySaid(t *testing.T) { } } +// Every pool can be selected by the runtime. An unsized low or frontier seat +// must be refused before work just as an unsized working seat is, even when +// no --asked flag accompanied a shell invocation. +func TestEveryUnsizedPoolIsRefusedBeforeStarting(t *testing.T) { + for _, pool := range []string{"high", "low", "frontier"} { + t.Run(pool, func(t *testing.T) { + backend := newModelAPIBackend(testModelAPI, "") + backend.catalog = fixtureModelsDev(t) + args := cliArgs{High: "openrouter/fixture/vendor-model"} + switch pool { + case "high": + args.High = "openrouter/custom/no-window" + case "low": + args.Low = "openrouter/custom/no-window" + case "frontier": + args.Frontier = "openrouter/custom/no-window" + } + refusal, guessed := windowCheck(args, seniorDevModels{backend: backend}) + if !strings.HasPrefix(refusal, "senior-dev cannot work with custom/no-window:") || + !strings.Contains(refusal, "nothing was started") || + !strings.Contains(refusal, "senior-dev.json") || len(guessed) != 0 { + t.Fatalf("unsized %s: refusal=%q, guessed=%v", pool, refusal, guessed) + } + }) + } +} + +// A catalog naming a model without its window is not a proof of capacity. +// It is refused just like a row absent from the catalog, never sized at zero. +func TestACatalogRowWithoutCapacityIsRefusedBeforeStarting(t *testing.T) { + backend := newModelAPIBackend(testModelAPI, "") + backend.catalog = fixtureModelsDev(t) + provider := backend.catalog["openrouter"] + model := provider.Models["fixture/vendor-model"] + model.Limit.Context = 0 + provider.Models["fixture/vendor-model"] = model + refusal, guessed := windowCheck(cliArgs{High: "openrouter/fixture/vendor-model"}, seniorDevModels{backend: backend}) + if !strings.Contains(refusal, "nothing was started") || len(guessed) != 0 { + t.Fatalf("zero-capacity catalog: refusal=%q, guessed=%v", refusal, guessed) + } +} + // THE REFUSAL HAPPENS BEFORE ANYTHING IS SPENT. A run asked to work on a model // codeaf's catalog lists at 16,384 tokens ends with the sentence that names it, // and never reaches its first stage, let alone a model call.