Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 89 additions & 0 deletions packages/runtime/src/__tests__/opencode-free-anonymous.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import assert from 'node:assert/strict';
import { describe, test } from 'node:test';
import { generateText } from 'ai';
import type { LlmConnection } from '@maka/core/llm-connections';
import { PROVIDER_REGISTRY } from '@maka/core/provider-registry';
import { getAIModel } from '@maka/runtime/model-factory';

import { testConnection } from '@maka/runtime/test-connection';
Expand Down Expand Up @@ -169,6 +170,94 @@ describe('opencode-free anonymous runtime', () => {
assert.deepEqual(requestedModels, ['custom-broken-free', 'nemotron-3-ultra-free']);
});

test('does not probe a quarantined enabled model', async () => {
const requestedModels: string[] = [];
const connection: LlmConnection = {
slug: 'opencode-free',
name: 'OpenCode Free',
providerType: 'opencode-free',
defaultModel: 'muse-spark-1.2-contributor-free',
enabledModelIds: ['muse-spark-1.2-contributor-free'],
enabled: true,
createdAt: 0,
updatedAt: 0,
};
const fakeFetch: typeof globalThis.fetch = async (_input, init) => {
const body = JSON.parse(String(init?.body)) as { model: string };
requestedModels.push(body.model);
return new Response(
JSON.stringify({ choices: [{ message: { role: 'assistant', content: 'ok' } }] }),
{ status: 200, headers: { 'content-type': 'application/json' } },
);
};

const result = await testConnection(connection, '', undefined, { fetch: fakeFetch });

assert.equal(result.ok, true);
assert.equal(result.modelTested, 'nemotron-3-ultra-free');
assert.deepEqual(requestedModels, ['nemotron-3-ultra-free']);
});

test('keeps a quarantined default from blocking the healthy fallback', async () => {
const requestedModels: string[] = [];
const connection: LlmConnection = {
slug: 'opencode-free',
name: 'OpenCode Free',
providerType: 'opencode-free',
defaultModel: 'x-preview-f-free',
enabledModelIds: ['x-preview-f-free'],
enabled: true,
createdAt: 0,
updatedAt: 0,
};
const fakeFetch: typeof globalThis.fetch = async (_input, init) => {
const body = JSON.parse(String(init?.body)) as { model: string };
requestedModels.push(body.model);
return new Response(
JSON.stringify({ choices: [{ message: { role: 'assistant', content: 'ok' } }] }),
{ status: 200, headers: { 'content-type': 'application/json' } },
);
};

const result = await testConnection(connection, '', undefined, { fetch: fakeFetch });

assert.equal(result.ok, true);
assert.equal(result.modelTested, 'nemotron-3-ultra-free');
assert.deepEqual(requestedModels, ['nemotron-3-ultra-free']);
});

test('returns no model to test when every candidate is quarantined', async () => {
const defaults = PROVIDER_REGISTRY['opencode-free'];
const originalFallbackModels = defaults.fallbackModels;
const originalBrokenModelIds = defaults.brokenModelIds;
defaults.fallbackModels = [];
defaults.brokenModelIds = ['muse-spark-1.2-contributor-free'];

try {
const connection: LlmConnection = {
slug: 'opencode-free',
name: 'OpenCode Free',
providerType: 'opencode-free',
defaultModel: 'muse-spark-1.2-contributor-free',
enabledModelIds: ['muse-spark-1.2-contributor-free'],
enabled: true,
createdAt: 0,
updatedAt: 0,
};
const fakeFetch: typeof globalThis.fetch = async () => {
throw new Error('no request expected');
};

const result = await testConnection(connection, '', undefined, { fetch: fakeFetch });

assert.equal(result.ok, false);
assert.equal(result.errorMessage, 'No model to test');
} finally {
defaults.fallbackModels = originalFallbackModels;
defaults.brokenModelIds = originalBrokenModelIds;
}
});

test('tests an explicit model once without probing fallbacks', async () => {
const requestedModels: string[] = [];
const connection: LlmConnection = {
Expand Down
9 changes: 8 additions & 1 deletion packages/runtime/src/test-connection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,16 @@ async function testConnectionStrict(
return { ok: false, errorMessage: 'No model to test' };
}
if (connection.providerType === 'opencode-free' && !model?.trim()) {
const brokenModelIds = new Set(defaults.brokenModelIds ?? []);
const candidates = [
...new Set([...connectionEnabledModelIds(connection), ...providerFallbackModelIds(defaults)]),
...new Set([
...connectionEnabledModelIds(connection).filter((id) => !brokenModelIds.has(id)),
...providerFallbackModelIds(defaults),
]),
];
if (candidates.length === 0) {
return { ok: false, errorMessage: 'No model to test' };
}
let lastFailure: ConnectionTestResult | undefined;
for (let index = 0; index < candidates.length; index += 1) {
const candidate = candidates[index]!;
Expand Down