[CI only] feat(web-fetch): AI/ML API Toolbox scraping backend

.env.example

@@ -573,6 +573,13 @@ ANTHROPIC_API_KEY=sk-ant-your-key-here
 # Self-hosted Firecrawl endpoint (optional — omit to use cloud API)
 # FIRECRAWL_API_URL=https://your-firecrawl-instance.com
 
+# WebFetch page scraping through the AI/ML API Toolbox (opt-in, paid per page:
+# about $0.0013 a run; each run requests a $0.005 cost limit). Uses
+# AIMLAPI_API_KEY (the first key of a comma-separated pool) and takes
+# precedence over a Firecrawl key for WebFetch. AIMLAPI_API_KEY alone does not
+# turn it on.
+# WEB_FETCH_PROVIDER=aimlapi
+
 # Ollama Web Search. A signed-in local Ollama route needs no key; the key
 # enables https://ollama.com/api/web_search as a hosted fallback.
 # OLLAMA_BASE_URL=http://localhost:11434

README.md

@@ -480,6 +480,15 @@ With Firecrawl enabled:
 
 Free tier at [firecrawl.dev](https://firecrawl.dev) includes 500 credits. The key is optional.
 
+`WebFetch` can also scrape pages through the [AI/ML API](https://aimlapi.com) Toolbox with your AI/ML API key. Each fetched page is one paid Toolbox run (about $0.0013; each run requests a $0.005 cost limit), so it is used only when you choose it:
+
+```bash
+export AIMLAPI_API_KEY=your-aimlapi-key
+export WEB_FETCH_PROVIDER=aimlapi
+```
+
+`WEB_FETCH_PROVIDER=aimlapi` takes precedence over a Firecrawl key for `WebFetch`. Having `AIMLAPI_API_KEY` set without it does not change how pages are fetched. See [AI/ML API Setup](docs/aimlapi-setup.md#webfetch-through-the-aiml-api-toolbox).
+
 ## Headless gRPC Server
 
 OpenClaude can run as a headless gRPC service with bidirectional streaming —

docs/aimlapi-setup.md

@@ -73,6 +73,38 @@ export OPENAI_MODEL="gpt-4o"
 - `/model` lists chat-capable models discovered from the catalog.
 - Send any prompt to confirm responses come back from the selected model.
 
+## WebFetch through the AI/ML API Toolbox
+
+`WebFetch` normally fetches pages over plain HTTP and converts the HTML to
+Markdown, which can miss JavaScript-rendered content. With an AI/ML API key you
+can have it scrape pages through the
+[AI/ML API Toolbox](https://docs.aimlapi.com/quickstart/toolbox) instead:
+
+```bash
+export AIMLAPI_API_KEY="your-aimlapi-key"
+export WEB_FETCH_PROVIDER=aimlapi
+```
+
+- **Opt-in.** Each fetched page is one Toolbox run of `search1api/crawl`, billed
+  to your AI/ML API balance (about $0.0013 per page on 2026-10-19). Every run
+  requests a cost limit with `max_cost_usd: 0.005`. An AI/ML API key on its own
+  never turns this on.
+- `WEB_FETCH_PROVIDER=aimlapi` takes precedence over `FIRECRAWL_API_KEY` /
+  `FIRECRAWL_API_URL` for `WebFetch`. `WebSearch` is not affected.
+- Like the Firecrawl backend, a failed scrape (a page that does not exist, a
+  refused or failed run) is reported as a `WebFetch` error instead of being
+  retried over plain HTTP. Rate-limited requests are retried with the same
+  idempotency key, so a retry is never charged twice.
+- The key and the attribution headers below are sent only to
+  `https://tools.aimlapi.com`, which is fixed and cannot be overridden. When
+  the active aimlapi profile points at your own proxy (a non-canonical
+  `OPENAI_BASE_URL` or `OPENAI_API_BASE`), its key is treated as the proxy's
+  and is not sent; set up a canonical AI/ML API key to use this backend. When
+  `AIMLAPI_API_KEY` holds a comma-separated pool, only its first usable key is
+  sent, with no rotation across keys. Requests never follow redirects. A
+  scraped page is untrusted web content and goes through the same `WebFetch`
+  processing as a page fetched over plain HTTP.
+
 ## Notes
 
 - Model discovery uses the public, unauthenticated `GET /models` endpoint and surfaces only chat-completions models; image, audio, embeddings, and other modalities are intentionally not routed through the coding workflow.

src/tools/WebFetchTool/WebFetchTool.backend.test.ts

@@ -0,0 +1,122 @@
+import { afterEach, beforeEach, expect, mock, test } from 'bun:test'
+
+import {
+  acquireSharedMutationLock,
+  releaseSharedMutationLock,
+} from '../../test/sharedMutationLock.js'
+import { asMockFetch } from '../../test/typedMocks.js'
+
+// WebFetch hands scraped markdown to the secondary model. Mock it so the test
+// exercises only backend selection.
+const haikuMock = mock()
+const fetchMock = mock()
+const originalFetch = globalThis.fetch
+const ENV_KEYS = ['AIMLAPI_API_KEY', 'FIRECRAWL_API_KEY', 'FIRECRAWL_API_URL', 'WEB_FETCH_PROVIDER'] as const
+const originalEnv = Object.fromEntries(ENV_KEYS.map(key => [key, process.env[key]]))
+
+beforeEach(async () => {
+  await acquireSharedMutationLock('tools/WebFetchTool/WebFetchTool.backend.test.ts')
+  haikuMock.mockReset()
+  haikuMock.mockImplementation(async ({ userPrompt }: { userPrompt: string }) => ({
+    message: { content: [{ type: 'text', text: `summary of: ${userPrompt.includes('# Scraped page') ? 'scraped page' : 'other'}` }] },
+  }))
+  fetchMock.mockReset()
+  fetchMock.mockImplementation(async (input: unknown) => {
+    const host = new URL(String(input)).hostname
+    if (host === 'tools.aimlapi.com') {
+      return Response.json({
+        id: 'run-1',
+        status: 'completed',
+        is_error: false,
+        output: { results: { content: '# Scraped page' } },
+        cost: { quoted_usd: 0.0013, actual_usd: 0.0013, over_max: false },
+      })
+    }
+    if (host === 'api.firecrawl.dev') {
+      return Response.json({ success: true, data: { markdown: '# Scraped page' } })
+    }
+    throw new Error(`unexpected request to ${host}`)
+  })
+  globalThis.fetch = asMockFetch(fetchMock)
+  const actual = await import('../../services/api/claude.js')
+  mock.module('../../services/api/claude.js', () => ({
+    ...actual,
+    queryHaiku: haikuMock,
+  }))
+})
+
+afterEach(() => {
+  try {
+    globalThis.fetch = originalFetch
+    for (const key of ENV_KEYS) {
+      const value = originalEnv[key]
+      if (value === undefined) delete process.env[key]
+      else process.env[key] = value
+    }
+    mock.restore()
+  } finally {
+    releaseSharedMutationLock()
+  }
+})
+
+async function runWebFetch(url = 'https://example.com/page') {
+  const nonce = `${Date.now()}-${Math.random()}`
+  const { WebFetchTool } = await import(`./WebFetchTool.js?ts=${nonce}`)
+  return WebFetchTool.call(
+    { url, prompt: 'summarize' },
+    {
+      abortController: new AbortController(),
+      options: { isNonInteractiveSession: true },
+    } as Parameters<typeof WebFetchTool.call>[1],
+  )
+}
+
+function requestedHosts(): string[] {
+  return fetchMock.mock.calls.map(call => new URL(String(call[0])).hostname)
+}
+
+test('an AI/ML API key alone keeps WebFetch on the Firecrawl backend', async () => {
+  process.env.AIMLAPI_API_KEY = 'aiml-test-key'
+  process.env.FIRECRAWL_API_KEY = 'fc-test-key'
+  delete process.env.FIRECRAWL_API_URL
+  delete process.env.WEB_FETCH_PROVIDER
+
+  const { data } = await runWebFetch()
+
+  expect(requestedHosts()).toEqual(['api.firecrawl.dev'])
+  expect(data.result).toBe('summary of: scraped page')
+})
+
+test('WEB_FETCH_PROVIDER=aimlapi scrapes through the AI/ML API Toolbox, ahead of Firecrawl', async () => {
+  process.env.AIMLAPI_API_KEY = 'aiml-test-key'
+  process.env.FIRECRAWL_API_KEY = 'fc-test-key'
+  process.env.WEB_FETCH_PROVIDER = 'aimlapi'
+
+  const { data } = await runWebFetch()
+
+  expect(requestedHosts()).toEqual(['tools.aimlapi.com'])
+  expect(data).toMatchObject({
+    bytes: Buffer.byteLength('# Scraped page'),
+    code: 200,
+    result: 'summary of: scraped page',
+    url: 'https://example.com/page',
+  })
+})
+
+test('an AI/ML API failure is reported, not retried through another backend', async () => {
+  process.env.AIMLAPI_API_KEY = 'aiml-test-key'
+  process.env.FIRECRAWL_API_KEY = 'fc-test-key'
+  process.env.WEB_FETCH_PROVIDER = 'aimlapi'
+  fetchMock.mockImplementation(async () =>
+    Response.json({
+      id: 'run-2',
+      status: 'completed',
+      is_error: true,
+      error: { kind: 'provider_error', retryable: false, message: 'The provider answered HTTP 400.' },
+    }),
+  )
+
+  await expect(runWebFetch()).rejects.toThrow('The provider answered HTTP 400.')
+  expect(requestedHosts()).toEqual(['tools.aimlapi.com'])
+  expect(haikuMock).not.toHaveBeenCalled()
+})

src/tools/WebFetchTool/WebFetchTool.ts

@@ -22,6 +22,7 @@ import {
   MAX_MARKDOWN_LENGTH,
 } from './utils.js'
 import { firecrawlScrape } from '../firecrawl/client.js'
+import { isAimlapiWebFetchSelected, scrapeWithAimlapi } from './aimlapiScrape.js'
 
 function isFirecrawlEnabled(): boolean {
   return Boolean(process.env.FIRECRAWL_API_KEY) || Boolean(process.env.FIRECRAWL_API_URL)
@@ -41,6 +42,23 @@ async function scrapeWithFirecrawl(
   return { markdown, bytes: Buffer.byteLength(markdown) }
 }
 
+type PageScraper = (
+  url: string,
+  signal?: AbortSignal,
+) => Promise<{ markdown: string; bytes: number }>
+
+/**
+ * The scraping backend that replaces raw HTTP, if any. AI/ML API runs are
+ * billed per page, so that backend needs the explicit WEB_FETCH_PROVIDER=aimlapi
+ * choice and then wins over a configured Firecrawl key.
+ */
+function getPageScraper(): PageScraper | null {
+  if (isAimlapiWebFetchSelected()) {
+    return (url, signal) => scrapeWithAimlapi(url, { signal })
+  }
+  return isFirecrawlEnabled() ? scrapeWithFirecrawl : null
+}
+
 const inputSchema = lazySchema(() =>
   z.strictObject({
     url: z.url().describe('The URL to fetch content from'),
@@ -231,8 +249,9 @@ ${DESCRIPTION}`
   ) {
     const start = Date.now()
 
-    if (isFirecrawlEnabled()) {
-      const { markdown, bytes } = await scrapeWithFirecrawl(url, abortController.signal)
+    const scrapePage = getPageScraper()
+    if (scrapePage) {
+      const { markdown, bytes } = await scrapePage(url, abortController.signal)
       const result = await applyPromptToMarkdown(
         prompt,
         markdown,