diff --git a/README.md b/README.md index 20170e2d68..26daa1ef28 100644 --- a/README.md +++ b/README.md @@ -215,29 +215,31 @@ firecrawl scrape https://firecrawl.dev https://firecrawl.dev/blog https://docs.f #### Scrape Options -| Option | Description | -| -------------------------- | ------------------------------------------------------- | -| `-f, --format ` | Output format(s), comma-separated | -| `-H, --html` | Shortcut for `--format html` | -| `-S, --summary` | Shortcut for `--format summary` | -| `--only-main-content` | Extract only main content (removes navs, footers, etc.) | -| `--wait-for ` | Wait time before scraping (for JS-rendered content) | -| `--screenshot` | Take a screenshot | -| `--full-page-screenshot` | Take a full page screenshot | -| `--include-tags ` | Only include specific HTML tags | -| `--exclude-tags ` | Exclude specific HTML tags | -| `--max-age ` | Maximum age of cached content in milliseconds | -| `--lockdown` | Enable lockdown mode for the scrape | -| `--redact-pii` | Redact personally identifiable information from output | -| `--schema ` | JSON schema for structured extraction | -| `--schema-file ` | Path to JSON schema file for structured extraction | -| `--actions ` | JSON actions array to run during scrape | -| `--actions-file ` | Path to JSON actions file | -| `--proxy ` | Proxy mode for scraping (for example, `auto`, `basic`) | -| `-o, --output ` | Save output to file | -| `--json` | Output as JSON format | -| `--pretty` | Pretty print JSON output | -| `--timing` | Show request timing info | +| Option | Description | +| -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `-f, --format ` | Output format(s), comma-separated | +| `-H, --html` | Shortcut for `--format html` | +| `-S, --summary` | Shortcut for `--format summary` | +| `--only-main-content` | Extract only main content (removes navs, footers, etc.) | +| `--wait-for ` | Wait time before scraping (for JS-rendered content) | +| `--screenshot` | Take a screenshot | +| `--full-page-screenshot` | Take a full page screenshot | +| `--include-tags ` | Only include specific HTML tags | +| `--exclude-tags ` | Exclude specific HTML tags | +| `--max-age ` | Maximum age of cached content in milliseconds | +| `--lockdown` | Enable lockdown mode for the scrape | +| `--redact-pii` | Redact personally identifiable information from output | +| `--schema ` | JSON schema for structured extraction | +| `--schema-file ` | Path to JSON schema file for structured extraction | +| `--actions ` | JSON actions array to run during scrape | +| `--actions-file ` | Path to JSON actions file | +| `--proxy ` | Proxy mode for scraping (for example, `auto`, `basic`) | +| `--alexandria
` | Execute an Alexandria capability (`provider/capability`) instead of a URL; repeatable, up to 10. See [`alexandria`](#alexandria---data-providers) (`--exchange`: deprecated alias) | +| `--options ` | JSON options for the `--alexandria` address at the same position (repeatable) | +| `-o, --output ` | Save output to file | +| `--json` | Output as JSON format | +| `--pretty` | Pretty print JSON output | +| `--timing` | Show request timing info | #### Available Formats @@ -299,6 +301,11 @@ firecrawl search "landscape photography" --sources images # Multiple sources firecrawl search "machine learning" --sources web,news,images +# Add Exchange capability hits (data providers, not documents) beside web results. +# Free, needs an API key on a team with Exchange access; execute a hit with +# `firecrawl exchange retrieve`. +firecrawl search "nvidia balance sheet" --sources web,alexandria --json + # Filter by category (GitHub, research-affiliated websites, PDFs) firecrawl search "web data python" --categories github firecrawl search "transformer architecture" --categories research @@ -328,23 +335,23 @@ firecrawl search "AI data tools" #### Search Options -| Option | Description | -| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--limit ` | Maximum results (default: 5, max: 100) | -| `--sources ` | Comma-separated: `web`, `images`, `news` (default: web) | -| `--categories ` | Comma-separated: `github`, `research` (research-affiliated websites -- for papers use [`research search-papers`](#research---search-research-papers)), `pdf`, `developer` | -| `--tbs ` | Time filter: `qdr:h` (hour), `qdr:d` (day), `qdr:w` (week), `qdr:m` (month), `qdr:y` (year) | -| `--location ` | Geo-targeting (e.g., "Germany", "San Francisco,California,United States") | -| `--country ` | ISO country code (default: US) | -| `--timeout ` | Timeout in milliseconds (default: 60000) | -| `--highlights` | Return query-relevant highlights for each result | -| `--no-highlights` | Keep the original search snippets | -| `--ignore-invalid-urls` | Exclude URLs invalid for other Firecrawl endpoints | -| `--scrape` | Enable scraping of search results | -| `--scrape-formats ` | Scrape formats when `--scrape` enabled (default: markdown) | -| `--only-main-content` | Include only main content when scraping (default: true) | -| `-o, --output ` | Save to file | -| `--json` | Output as compact JSON | +| Option | Description | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--limit ` | Maximum results (default: 5, max: 100) | +| `--sources ` | Comma-separated names or JSON source objects: `web`, `images`, `news`, `alexandria` (default: web). Alexandria contracts are in `data.tools` -- see [`alexandria`](#alexandria---data-providers) | +| `--categories ` | Comma-separated: `github`, `research` (research-affiliated websites -- for papers use [`research search-papers`](#research---search-research-papers)), `pdf`, `developer` | +| `--tbs ` | Time filter: `qdr:h` (hour), `qdr:d` (day), `qdr:w` (week), `qdr:m` (month), `qdr:y` (year) | +| `--location ` | Geo-targeting (e.g., "Germany", "San Francisco,California,United States") | +| `--country ` | ISO country code (default: US) | +| `--timeout ` | Timeout in milliseconds (default: 60000) | +| `--highlights` | Return query-relevant highlights for each result | +| `--no-highlights` | Keep the original search snippets | +| `--ignore-invalid-urls` | Exclude URLs invalid for other Firecrawl endpoints | +| `--scrape` | Enable scraping of search results | +| `--scrape-formats ` | Scrape formats when `--scrape` enabled (default: markdown) | +| `--only-main-content` | Include only main content when scraping (default: true) | +| `-o, --output ` | Save to file | +| `--json` | Output as compact JSON | #### Examples @@ -376,6 +383,140 @@ firecrawl search "AI startups funding" --sources news --tbs qdr:w --limit 15 --- +### Alexandria discovery + +```bash +# Semantic discovery alongside web results +firecrawl search "podcast transcripts" --sources web,alexandria --limit 2 --json + +# Include tools matched to query mentions and result domains +firecrawl search "podcast transcripts" --sources web,alexandria --domain-tools --json + +# Explore providers without a search query +firecrawl find-tools --categories finance --limit 2 --json +firecrawl find-tools https://podcasts.apple.com --json + +# Get a specific tool's inputs, response, and examples +firecrawl find-tools --providers particle \ + --capabilities podcasts/episodes/search \ + --expand options,response,examples --limit 2 --json + +# Follow an item or pagination request returned by Find Tools +firecrawl find-tools --request '' --json +``` + +Search requires a query. `sources: ["alexandria"]` finds tools by meaning; +`--domain-tools` adds contextual matches to the same `data.tools` array. Each contract +includes inputs, response fields, examples, price, `matchedBy`, and `matchedUrls`. +Check the response's `warning` if discovery is unavailable. The legacy `exchange` +source name is normalized to `alexandria`. + +`find-tools` uses the zero-credit Find Tools capability through `/v2/scrape`. +Filter with `--providers`, `--categories`, `--groups`, `--capabilities`, or page URLs. +`--level providers|groups|tools` is inferred when omitted; `--expand` selects +contract sections at the tools level. Lists use commas. An item's `next` reveals +more detail; the result's top-level `next` fetches another page. Both are complete +Exchange calls accepted by `--request`. `exchange tools` is an alias. + +These integrations require the matching API deployment and authenticated team +access. Discovery does not execute the tools it returns. Web search and selected +tool execution retain their own charges. The older `exchange discover` and +`exchange skill` commands remain available for existing proxy integrations. + +### `alexandria` - Data providers + +Discover and call Firecrawl Alexandria data providers (FRED, financial datasets, +and more). Discovery is free; each `retrieve` spends the credits its contract +declares. Alexandria needs an API key on a team with Alexandria access -- there is +no keyless fallback, and the CLI refuses before sending anything. `firecrawl +exchange` remains a hidden alias for existing scripts. + +```bash +# Walk the catalogue: cohorts -> providers -> capabilities -> one contract +firecrawl alexandria discover +firecrawl alexandria discover finance +firecrawl alexandria discover finance fred +firecrawl alexandria discover finance fred series/observations + +# Inline the whole tree for a cohort +firecrawl alexandria discover finance --expand all --json + +# Semantic lookup across the whole catalogue +firecrawl alexandria discover --query "balance sheet" --limit 8 + +# Execute a capability (reads the contract first to learn its options) +firecrawl alexandria retrieve fred/series/observations --options '{"series_id":"CPIAUCSL"}' + +# Batch up to 10; each --options pairs with the address at the same position +firecrawl alexandria retrieve fred/series/observations fred/series/search \ + --options '{"series_id":"CPIAUCSL"}' --options '{"q":"inflation"}' --json + +# Same execution through scrape (url-less) +firecrawl scrape --alexandria fred/series/observations --options '{"series_id":"CPIAUCSL"}' + +# Read a paid provider's terms, then accept them (organization admin, interactive) +firecrawl alexandria terms benzinga +firecrawl alexandria terms accept benzinga + +# Find capabilities beside web results +firecrawl search "nvidia balance sheet" --sources web,alexandria --json +``` + +`retrieve` posts to `/v2/scrape` with an `alexandria` array and prints each +item with its `creditsCost` (`--json` mirrors the API envelope: +`{ success, scrape_id, requestId, data: { alexandria: [...], creditsCost } }`). A provider +error inside the batch is reported per item and does not fail the request; the +exit code is 1 only when every item failed, when the request itself was +rejected (403 team not enabled, 402 insufficient credits, 409 duplicate +request), or in keyless mode. With `--json`, a rejected request writes +`{ success: false, requestId, code, error, requiresAction? }` to stdout. + +Paid providers require an organization admin to accept their terms once. Until +then the request is rejected with `THIRD_PARTY_DATA_TERMS_REQUIRED`, no credits +are charged, and the CLI prints the provider, version and dashboard URL (in +`--json`, `requiresAction: { type: "accept_terms", terms, version, url }`). An +agent should show that URL and stop. A human admin accepts in the dashboard or +runs `firecrawl alexandria terms accept `, which shows the document +and asks them to type the provider slug; it refuses without an interactive +terminal and has no `--yes`. Afterwards, rerun with the same `--request-id`. + +Execution generates a request ID once and sends `x-request-id`. It returns the ID +and prints it on stderr, including after a failure. Retry the identical payload +with `--request-id `; never use a new ID to bypass a pending or uncertain 409. +The API reserves the maximum cost before execution and rejects insufficient +credits before calling a provider. `perRecord` prices depend on returned records. + +#### `discover` Options + +| Option | Description | +| -------------------- | --------------------------------------------------------------------------------- | +| `[cohort]` | Cohort slug (first rung), e.g. `finance` | +| `[provider]` | Provider slug (second rung), e.g. `fred` | +| `[capability]` | Capability address (third rung), e.g. `series/observations` | +| `-q, --query ` | Semantic lookup across the whole catalogue (root only) | +| `--limit ` | Maximum semantic hits (1-24, default: 8) | +| `--expand ` | Inline more of the tree on a walk: `capabilities`, `contracts`, `examples`, `all` | +| `--json`, `--pretty` | Output the API payload as JSON | + +#### `retrieve` Options + +| Option | Description | +| -------------------- | -------------------------------------------------------------- | +| `` | One to ten `provider/capability` addresses | +| `--options ` | JSON options for the address at the same position (repeatable) | +| `--timeout ` | Timeout in milliseconds | +| `--json`, `--pretty` | Output the `/v2/scrape` envelope as JSON | + +#### `terms` Options + +| Option | Description | +| -------------------- | --------------------------------------------------------------------------- | +| `` | Provider slug, e.g. `benzinga` | +| `accept ` | Accept the displayed version for your organization (interactive, admin key) | +| `--json`, `--pretty` | Output the provider's terms entry as JSON (show only) | + +--- + ### `developer` - Search developer sources Search an index built for coding agents: GitHub issues, merged pull requests, repository READMEs, and curated documentation sites. Use it for a programming question: code behaviour, a library or framework, an API contract, an error message, or a known bug. diff --git a/skills/firecrawl-scrape/SKILL.md b/skills/firecrawl-scrape/SKILL.md index 7211eebb1b..73cb69da28 100644 --- a/skills/firecrawl-scrape/SKILL.md +++ b/skills/firecrawl-scrape/SKILL.md @@ -1,7 +1,7 @@ --- name: firecrawl-scrape description: | - Extract a URL's content as clean markdown, including JS-rendered pages. Use whenever the user provides a URL and wants its content; prefer over WebFetch. + Extract a URL's content as clean markdown, including JS-rendered pages, or execute a known Alexandria data-provider capability. Use when the user supplies a page URL or has selected a provider tool and wants its data. allowed-tools: - Bash(firecrawl *) - Bash(npx firecrawl-cli *) @@ -9,7 +9,7 @@ allowed-tools: # firecrawl scrape -Scrape one or more URLs. Returns clean, LLM-optimized markdown. Multiple URLs are scraped concurrently. +Scrape one or more URLs, or execute a selected Alexandria capability with `--alexandria`. Page scraping returns content; Alexandria execution returns structured provider results. ## Quick start @@ -35,7 +35,13 @@ firecrawl scrape "https://example.com/pricing" --query "What is the enterprise p Run `firecrawl scrape --help` for the full option list. -**Done when:** you have the scraped content — on stdout, in your `-o` file, or under `.firecrawl/` for multi-URL scrapes — and have inspected it with bounded reads (`head`, `grep`) to answer the request. +## Alexandria provider execution + +Use `firecrawl scrape --alexandria / --options ''` after reading the tool's contract. This is a URL-less request; do not combine it with page URLs or page-scraping options. Inspect each result in `data.alexandria`, including per-item errors and `data.creditsCost` for the total charge. + +Keep the returned request ID. An identical retry uses `--request-id `; a pending or uncertain execution must not be retried under a fresh ID. See the [Alexandria workflow](../firecrawl/rules/alexandria.md) for discovery, required inputs, and a complete example. + +**Done when:** you have inspected the page content or provider results with bounded reads and used them to answer the request. Report per-item provider failures instead of treating the outer response as proof of success. ## Tips diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index c3b4334878..0e15f6fc7b 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -1,7 +1,7 @@ --- name: firecrawl-search description: | - Web search with full page content. Use when no URL is known: finding sources, articles, or news. For papers use firecrawl-research-index; for library, API, error, or bug questions use firecrawl-developer-index. + Web search with optional page content and Alexandria tool discovery. Use when no URL is known: finding sources, articles, news, or data-provider tools by intent. For papers use firecrawl-research-index; for library, API, error, or bug questions use firecrawl-developer-index. allowed-tools: - Bash(firecrawl *) - Bash(npx firecrawl-cli *) @@ -9,7 +9,7 @@ allowed-tools: # firecrawl search -Web search with optional content scraping. Returns search results as JSON, optionally with full page content. +Web search with optional content scraping and Alexandria tool discovery. JSON groups web results under `data.web` and tool contracts under `data.tools`. ## Quick start @@ -28,7 +28,23 @@ Run `firecrawl search --help` for the full option list. `--categories developer` weighs the developer index beside ordinary web results in this same call (no passage control, no index filters). `--categories research` is a website filter, not the paper index. Dedicated skills: [firecrawl-developer-index](../firecrawl-developer-index/SKILL.md) and [firecrawl-research-index](../firecrawl-research-index/SKILL.md). -**Done when:** results are saved under `.firecrawl/`, verified non-empty, processed for the request, and one feedback event is sent within the time window (unless opted out). +## Alexandria tools + +```bash +# Find tool contracts by meaning +firecrawl search "podcast conversations about AI agents" \ + --sources alexandria --limit 2 --json -o .firecrawl/tools.json + +# Web pages and semantic tools, with optional domain matches +firecrawl search "podcast conversations about AI agents" \ + --sources web,alexandria --domain-tools --limit 2 --json -o .firecrawl/web-and-tools.json +``` + +Search always needs a non-empty query. `--sources alexandria` searches tools semantically; `--domain-tools` opts into additional contextual matches. Both return contracts in `data.tools`, with `matchedBy` and `matchedUrls` explaining relevance. Omit `--domain-tools` when semantic matches are enough. Discovery does not execute the returned tools. + +For a known URL or provider, use `firecrawl find-tools`; catalogue filters belong there. Follow the [Alexandria workflow](../firecrawl/rules/alexandria.md) to read contracts, follow `next`, and execute a selected tool through `scrape --alexandria`. + +**Done when:** fresh results have been inspected and used for the request, or no matches have been reported. Apply the feedback guidance below when web results were used; free tool discovery does not require a web-search feedback call. ## Tips @@ -38,9 +54,9 @@ Run `firecrawl search --help` for the full option list. - Use `jq` to extract URLs or titles: `jq -r '.data.web[].url' .firecrawl/search.json` - Naming convention: `.firecrawl/search-{query}.json` or `.firecrawl/search-{query}-scraped.json` -## Send feedback after every search (refunds 1 credit) +## Send feedback after using web search results -Search costs 2 credits. After you've actually used the results (or decided they were useless), send structured feedback with `firecrawl search-feedback `. The first feedback per search refunds 1 credit and helps us improve search quality. **Do this once per search**, in the background, after you finish processing the results — it does not block your main task. +For ordinary web search, after you've actually used the results (or decided they were useless), send structured feedback with `firecrawl search-feedback `. Eligible feedback can refund 1 credit and helps us improve search quality. **Do this once per search**, in the background, after you finish processing the results. Alexandria-only discovery and Find Tools cost 0 credits; do not apply this refund workflow to them. For mixed results, feedback should describe the web results you used. **Opt out:** if `FIRECRAWL_NO_SEARCH_FEEDBACK=1` (or `FIRECRAWL_DISABLE_SEARCH_FEEDBACK=1`) is set, the CLI silently skips the call and never sends anything. Respect that — do not try to work around it. (Team admins can also disable this server-side; the API will return `feedbackErrorCode: "TEAM_OPTED_OUT"` and the CLI will exit 0 silently.) @@ -62,7 +78,7 @@ Verify the search returned results before reading its `id`. Zero-result searches # Send once per search. Rate honestly and replace the placeholder with the # rating that matches what actually happened. The two fields shown # satisfy the substantive-content rule for every rating. -if SEARCH_ID=$(jq -er 'select(any(.data[]; length > 0)) | .id' .firecrawl/search-react-hooks.json); then +if SEARCH_ID=$(jq -er 'select(any(.data | .web, .images, .news, .developer; length > 0)) | .id // empty' .firecrawl/search-react-hooks.json); then firecrawl search-feedback "$SEARCH_ID" \ --rating "" \ --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Most authoritative"}]' \ diff --git a/skills/firecrawl/SKILL.md b/skills/firecrawl/SKILL.md index e38876f267..d18b3465e0 100644 --- a/skills/firecrawl/SKILL.md +++ b/skills/firecrawl/SKILL.md @@ -1,7 +1,7 @@ --- name: firecrawl description: | - Any live-web task via the Firecrawl CLI — including ordinary web research: searching the web, reading or extracting pages, gathering sources, discovering site URLs, bulk extraction, downloading a site, change alerts, or pages needing clicks/login — web only; local files route to firecrawl-parse. For papers use firecrawl-research-index; for library, API, error, or bug questions use firecrawl-developer-index. + Live web data and Alexandria tools via the Firecrawl CLI: web research, reading or extracting pages, discovering data-provider tools, retrieving provider data, gathering sources, discovering site URLs, bulk extraction, downloading a site, change alerts, or pages needing clicks/login. Local files route to firecrawl-parse. For papers use firecrawl-research-index; for library, API, error, or bug questions use firecrawl-developer-index. allowed-tools: - Bash(firecrawl *) - Bash(npx firecrawl-cli *) @@ -9,7 +9,7 @@ allowed-tools: # Firecrawl CLI -Search, scrape, and interact with the web. Returns clean markdown optimized for LLM context windows. +Search, scrape, and interact with the web, or discover and execute Alexandria data tools. Returns page content, tool contracts, or structured provider data. Run `firecrawl --help` or `firecrawl --help` for full option details. For app integration or outcome workflows (research briefs, SEO audits, etc.), route to the `firecrawl-build` / `firecrawl-workflows` skills — see [When to Load References](#when-to-load-references). @@ -30,19 +30,22 @@ Follow this escalation pattern: 5. **Monitor** - Need recurring checks or ongoing alerts. Prefer setting a monitor with `--page` plus `--goal` instead of doing repeated one-off scrapes. 6. **Interact** - Scrape first, then interact with the page (pagination, modals, form submissions, multi-step navigation). -| Need | Command | When | -| --------------------------- | --------------------- | --------------------------------------------------------------- | -| Find pages on a topic | `search` | No specific URL yet | -| Find research papers | `research` | Biomedical/clinical/scientific literature — use the paper index | -| Answer a coding question | `developer` | Issues, merged PRs, READMEs, and docs — not a general web page | -| Get a page's content | `scrape` | Have a URL, page is static or JS-rendered | -| Find URLs within a site | `map` | Need to locate a specific subpage | -| Bulk extract a site section | `crawl` | Need many pages (e.g., all /docs/) | -| AI-powered data extraction | `agent` | Need structured data from complex sites | -| Interact with a page | `scrape` + `interact` | Content requires clicks, form fills, pagination, or login | -| Download a site to files | `x download` | Save an entire site as local files | -| Parse a local file | `parse` | File on disk (PDF, DOCX, XLSX, etc.) — not a URL | -| Watch pages for changes | `monitor` | Schedule recurring scrapes/crawls, diff against snapshots | +| Need | Command | When | +| -------------------------------- | ----------------------------- | --------------------------------------------------------------- | +| Find pages on a topic | `search` | No specific URL yet | +| Find data tools by intent | `search --sources alexandria` | Describe the data needed; returns tool contracts | +| Explore tools by URL or provider | `find-tools` | Contextual lookup and progressive disclosure | +| Execute a known data tool | `scrape --alexandria` | Have a provider, capability, and its required inputs | +| Find research papers | `research` | Biomedical/clinical/scientific literature — use the paper index | +| Answer a coding question | `developer` | Issues, merged PRs, READMEs, and docs — not a general web page | +| Get a page's content | `scrape` | Have a URL, page is static or JS-rendered | +| Find URLs within a site | `map` | Need to locate a specific subpage | +| Bulk extract a site section | `crawl` | Need many pages (e.g., all /docs/) | +| AI-powered data extraction | `agent` | Need structured data from complex sites | +| Interact with a page | `scrape` + `interact` | Content requires clicks, form fills, pagination, or login | +| Download a site to files | `x download` | Save an entire site as local files | +| Parse a local file | `parse` | File on disk (PDF, DOCX, XLSX, etc.) — not a URL | +| Watch pages for changes | `monitor` | Schedule recurring scrapes/crawls, diff against snapshots | For detailed command reference, run `firecrawl --help`. @@ -64,6 +67,7 @@ For detailed command reference, run `firecrawl --help`. ## When to Load References - **Searching the web or finding sources first** -> [firecrawl-search](../firecrawl-search/SKILL.md) +- **Discovering Alexandria tools, walking providers and capabilities, or executing provider data requests** -> [rules/alexandria.md](rules/alexandria.md). Semantic search and optional domain matches share `data.tools`; Find Tools reveals contracts without executing the selected provider. - **Finding research papers (biomedical, clinical, or scientific literature; PubMed, bioRxiv, medRxiv, arXiv)** -> [firecrawl-research-index](../firecrawl-research-index/SKILL.md). Use the paper index instead of scraping PubMed or Google Scholar by hand; `search --categories research` is a website filter, not the paper index. - **Answering a library, API, error, or known-bug question from issues, merged PRs, READMEs, or docs** -> [firecrawl-developer-index](../firecrawl-developer-index/SKILL.md) - **Scraping a known URL** -> [firecrawl-scrape](../firecrawl-scrape/SKILL.md) @@ -107,7 +111,7 @@ Single format outputs raw content. Multiple formats (e.g., `--format markdown,li ## Feedback -After using search results, send `firecrawl search-feedback` (the first feedback per search refunds 1 credit). The full pattern, guard, and rules live in [firecrawl-search](../firecrawl-search/SKILL.md). +After using web search results, follow the feedback guidance in [firecrawl-search](../firecrawl-search/SKILL.md). Free Alexandria-only discovery and Find Tools do not earn a web-search credit refund. For non-search endpoint jobs, use `firecrawl feedback ` to send concise job-level feedback through `/v2/feedback`. Supported endpoints are `search`, `scrape`, `parse`, and `map`. diff --git a/skills/firecrawl/rules/alexandria.md b/skills/firecrawl/rules/alexandria.md new file mode 100644 index 0000000000..b8a2171bec --- /dev/null +++ b/skills/firecrawl/rules/alexandria.md @@ -0,0 +1,62 @@ +# Alexandria: discover a tool, read its contract, get its data + +Use Alexandria when the task needs a provider's structured data. These commands require a CLI and API deployment with Alexandria support and an authenticated team with access. Check `firecrawl find-tools --help` for the installed command surface. + +## Choose the discovery request + +- **Describe the data needed:** `firecrawl search "your intent" --sources alexandria --limit 2 --json`. Add `web` to the sources to also retrieve web results. Search requires a query. +- **Match tools to result domains:** add `--domain-tools` to Search. This is optional and can add lookup latency. Semantic and contextual matches share `data.tools`. +- **Already have a URL or provider:** use `find-tools`, without a search query. It looks up the catalogue; it does not fetch the supplied page or execute the tools it finds. + +Inspect a Search tool's `provider`, `capability`, `options`, `requiresOneOf`, `response`, `examples`, `creditsCost`, and `perRecord`. `matchedBy` and `matchedUrls` explain why it appeared. A discovery `warning` means lookup was unavailable, rather than proving no tools match. A zero-result CLI search may leave no new output file; do not read a stale file from a previous run. + +## Reveal only the detail needed + +```bash +# Start with a provider; lookup depth is inferred +firecrawl find-tools --providers particle --limit 2 --json \ + -o .firecrawl/find-tools.json + +# Or start from a page returned by search or scrape +firecrawl find-tools "https://podcasts.apple.com" --limit 2 --json \ + -o .firecrawl/page-tools.json + +# Read the inputs, response, and examples for a known capability +firecrawl find-tools --providers particle \ + --capabilities podcasts/episodes/search \ + --expand options,response,examples --limit 2 --json \ + -o .firecrawl/episode-contract.json +``` + +Find Tools returns its catalogue page inside `data.alexandria[0].data`: + +```bash +jq '.data.alexandria[0].data | {level, items, next}' .firecrawl/find-tools.json +``` + +Follow the selected item's `next` to reveal more detail. Follow the catalogue page's top-level `next` for another page. Pass the complete returned request unchanged: + +```bash +firecrawl find-tools --request '' --json \ + -o .firecrawl/find-tools-next.json +``` + +`--request` cannot be combined with URLs or lookup filters. Provider, category, group, and capability selectors are optional; add only those needed to narrow the lookup. If necessary, `--level providers|groups|tools` overrides the inferred depth. Reuse contracts already present in Search results instead of fetching them again. + +## Execute the selected capability + +Use the inputs from the current contract. For Particle episode search, `keyword_search` supplies the exact words to find: + +```bash +firecrawl scrape --alexandria particle/podcasts/episodes/search \ + --options '{"keyword_search":"AI agents","limit":2}' \ + --json -o .firecrawl/episodes.json +``` + +Inspect `data.alexandria` for provider results and per-item errors; an overall successful response can contain a failed item. `data.creditsCost` is the total charge. Tool discovery and Find Tools are free. Web search retains its own cost, and executing a selected provider uses its published price, including per-record pricing where specified. Browsing a contract does not accept provider terms or execute the provider. + +The CLI generates a request ID and prints it on stderr, including on failure; successful JSON output also includes `requestId`. Retry an identical payload with `--request-id `. If execution is pending or uncertain, report that state rather than creating a new ID to trigger another execution. Credit and provider-terms rejections must be resolved before execution can proceed. + +## Provider terms (THIRD_PARTY_DATA_TERMS_REQUIRED) + +Paid providers need an organization admin to accept their terms once. Until then, `--json` execution writes `{"success":false,"code":"THIRD_PARTY_DATA_TERMS_REQUIRED","requiresAction":{"type":"accept_terms","terms":"","version":"...","url":"..."}}` to stdout and exits 1; no credits are charged. When this happens, show the user `requiresAction.url` and stop. Do not retry, and do not try to accept: acceptance is a legal act by a human. A human admin accepts in the dashboard at that URL, or reads and accepts from their own terminal with `firecrawl alexandria terms ` and `firecrawl alexandria terms accept ` (interactive only, needs the admin's own API key). Once they confirm, rerun the identical command with the same `--request-id`. diff --git a/src/__tests__/commands/exchange.test.ts b/src/__tests__/commands/exchange.test.ts new file mode 100644 index 0000000000..411ebb0f5c --- /dev/null +++ b/src/__tests__/commands/exchange.test.ts @@ -0,0 +1,1033 @@ +/** + * Tests for exchange command + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { + EXCHANGE_KEY_REQUIRED, + buildDiscoverPath, + buildExchangeCalls, + exchangeErrorMessage, + executeExchangeDiscover, + executeExchangeRetrieve, + handleExchangeDiscoverCommand, + handleExchangeRetrieveCommand, + parseExchangeAddress, +} from '../../commands/exchange'; +import { + handleExchangeTermsAcceptCommand, + handleExchangeTermsCommand, +} from '../../commands/terms'; +import { getClient, isKeylessMode } from '../../utils/client'; +import { initializeConfig } from '../../utils/config'; +import { writeOutput } from '../../utils/output'; +import { setupTest, teardownTest } from '../utils/mock-client'; +import { input } from '@inquirer/prompts'; + +vi.mock('../../utils/output', () => ({ writeOutput: vi.fn() })); +vi.mock('@inquirer/prompts', () => ({ input: vi.fn() })); + +vi.mock('../../utils/client', async () => { + const actual = await vi.importActual('../../utils/client'); + return { + ...actual, + getClient: vi.fn(), + isKeylessMode: vi.fn(() => false), + }; +}); + +// The SDK's HTTP layer rejects non-2xx with an axios-shaped error. +const axiosError = (status: number, body: Record) => + Object.assign(new Error(`Request failed with status code ${status}`), { + response: { status, data: body }, + }); + +const successItem = { + provider: 'fred', + capability: 'series/observations', + creditsCost: 1, + data: { observations: [{ date: '2024-01-01', value: '308.4' }] }, + records: 1, + upstreamStatus: 200, +}; + +const failedItem = { + provider: 'fred', + capability: 'series/search', + error: { + code: 'credential_missing', + message: 'FRED credential is not configured.', + status: 503, + }, +}; + +const termsRequiredBody = { + success: false, + code: 'THIRD_PARTY_DATA_TERMS_REQUIRED', + error: + "An organization admin must accept the benzinga provider's terms (version 2026-09-12-placeholder) before this request can run. Accept them at https://www.firecrawl.dev/app/alexandria/benzinga", + requiresAction: { + type: 'accept_terms', + terms: 'benzinga', + version: '2026-09-12-placeholder', + url: 'https://www.firecrawl.dev/app/alexandria/benzinga', + }, +}; + +const fiscalTerms = { + provider: 'fiscal-ai', + name: 'Fiscal.ai', + website: 'fiscal.ai', + required: true, + terms: { + key: 'fiscal-ai', + version: '2026-09-12-placeholder', + effective: '2026-09-12', + publisher: 'Fiscal.ai', + body: ['Hello world.'], + document: '---\npublisher: Fiscal.ai\n---\n\nHello world.\n', + digest: 'sha256:abc', + }, +}; + +describe('exchange helpers', () => { + describe('parseExchangeAddress', () => { + it('splits on the first slash so capability addresses keep theirs', () => { + expect(parseExchangeAddress('fred/series/observations')).toEqual({ + provider: 'fred', + capability: 'series/observations', + }); + }); + + it('rejects addresses without a provider or capability', () => { + expect(() => parseExchangeAddress('fred')).toThrow( + /provider\/capability/ + ); + expect(() => parseExchangeAddress('/finance')).toThrow( + /provider\/capability/ + ); + expect(() => parseExchangeAddress('fred/')).toThrow( + /provider\/capability/ + ); + }); + }); + + describe('buildExchangeCalls', () => { + it('pairs each --options value with the address at the same position', () => { + expect( + buildExchangeCalls( + ['fred/series/observations', 'fred/series/search'], + ['{"series_id":"CPIAUCSL"}'] + ) + ).toEqual([ + { + provider: 'fred', + capability: 'series/observations', + options: { series_id: 'CPIAUCSL' }, + }, + { provider: 'fred', capability: 'series/search' }, + ]); + }); + + it('caps a request at 10 capabilities', () => { + const addresses = Array.from({ length: 11 }, (_, i) => `p/cap${i}`); + expect(() => buildExchangeCalls(addresses)).toThrow(/at most 10/); + }); + + it('rejects more --options values than addresses', () => { + expect(() => + buildExchangeCalls(['fred/series/observations'], ['{}', '{}']) + ).toThrow(/More --options values/); + }); + + it('rejects options that are not a JSON object', () => { + expect(() => + buildExchangeCalls(['fred/series/observations'], ['[1]']) + ).toThrow(/expected a JSON object/); + expect(() => + buildExchangeCalls(['fred/series/observations'], ['{oops']) + ).toThrow(/Invalid JSON in --options/); + }); + + it('requires at least one address', () => { + expect(() => buildExchangeCalls([])).toThrow(/At least one/); + }); + }); + + describe('buildDiscoverPath', () => { + it('walks the catalogue one rung at a time', () => { + expect(buildDiscoverPath({})).toBe('/exchange/discover'); + expect(buildDiscoverPath({ cohort: 'finance' })).toBe( + '/exchange/discover/finance' + ); + expect(buildDiscoverPath({ cohort: 'finance', provider: 'fred' })).toBe( + '/exchange/discover/finance/fred' + ); + expect( + buildDiscoverPath({ + cohort: 'finance', + provider: 'fred', + capability: 'series/observations', + }) + ).toBe('/exchange/discover/finance/fred/series/observations'); + }); + + it('encodes each segment without touching the slashes in an address', () => { + expect( + buildDiscoverPath({ + cohort: 'finance', + provider: 'fred', + capability: 'series/obs ervations', + }) + ).toBe('/exchange/discover/finance/fred/series/obs%20ervations'); + }); + + it('sends a semantic lookup as ?q= with its limit', () => { + expect(buildDiscoverPath({ query: 'balance sheet', limit: 8 })).toBe( + '/exchange/discover?q=balance+sheet&limit=8' + ); + }); + + it('forwards expand on a walk', () => { + expect(buildDiscoverPath({ cohort: 'finance', expand: 'all' })).toBe( + '/exchange/discover/finance?expand=all' + ); + }); + + it('refuses --query combined with path arguments', () => { + expect(() => + buildDiscoverPath({ cohort: 'finance', query: 'balance sheet' }) + ).toThrow(/--query/); + }); + }); + + describe('exchangeErrorMessage', () => { + it('prefers the API body error and code', () => { + expect( + exchangeErrorMessage( + axiosError(403, { + success: false, + error: 'Exchange is not enabled for this team.', + }) + ) + ).toBe('Exchange is not enabled for this team.'); + expect( + exchangeErrorMessage( + axiosError(409, { + success: false, + code: 'duplicate_request', + error: 'This request id was already charged.', + chargeId: 'c1', + }) + ) + ).toBe( + 'This request id was already charged. (duplicate_request) — chargeId: c1 — Use a new --request-id for a new payload.' + ); + }); + + it('falls back to the status and then the error message', () => { + expect(exchangeErrorMessage(axiosError(402, {}))).toBe( + 'Firecrawl request failed (HTTP 402)' + ); + expect(exchangeErrorMessage(new Error('boom'))).toBe('boom'); + expect(exchangeErrorMessage('nope')).toBe('Unknown error occurred'); + }); + }); +}); + +describe('executeExchangeDiscover / executeExchangeRetrieve', () => { + let mockHttpGet: ReturnType; + let mockHttpPost: ReturnType; + + beforeEach(() => { + setupTest(); + initializeConfig({ + apiKey: 'test-api-key', + apiUrl: 'https://api.firecrawl.dev', + }); + mockHttpGet = vi.fn(); + mockHttpPost = vi.fn(); + vi.mocked(getClient).mockReturnValue({ + http: { get: mockHttpGet, post: mockHttpPost }, + } as any); + vi.mocked(isKeylessMode).mockReturnValue(false); + }); + + afterEach(() => { + teardownTest(); + vi.clearAllMocks(); + }); + + it('GETs the proxied discover path and returns the payload verbatim', async () => { + const payload = { + capabilities: [ + { + address: 'series/observations', + cohorts: ['finance'], + concept: 'series/observations', + creditsCost: 1, + provider: 'fred', + similarity: 0.8123, + }, + ], + query: 'cpi', + searched: 54, + source: 'process', + }; + mockHttpGet.mockResolvedValue({ data: payload }); + + const result = await executeExchangeDiscover({ + query: 'cpi', + apiKey: 'fc-key', + apiUrl: 'http://localhost:3002', + }); + + expect(getClient).toHaveBeenCalledWith({ + apiKey: 'fc-key', + apiUrl: 'http://localhost:3002', + }); + expect(mockHttpGet).toHaveBeenCalledWith('/exchange/discover?q=cpi'); + expect(result).toEqual({ success: true, data: payload }); + }); + + it('relays the API error body from discover', async () => { + mockHttpGet.mockRejectedValue( + axiosError(501, { + code: 'semantic_not_configured', + error: 'Semantic lookup needs an embedding key.', + }) + ); + + const result = await executeExchangeDiscover({ query: 'cpi' }); + + expect(result).toEqual({ + success: false, + error: + 'Semantic lookup needs an embedding key. (semantic_not_configured)', + }); + }); + + it('refuses discover in keyless mode without calling the API', async () => { + vi.mocked(isKeylessMode).mockReturnValue(true); + + const result = await executeExchangeDiscover({ cohort: 'finance' }); + + expect(result).toEqual({ success: false, error: EXCHANGE_KEY_REQUIRED }); + expect(getClient).not.toHaveBeenCalled(); + expect(mockHttpGet).not.toHaveBeenCalled(); + }); + + it('POSTs the url-less /v2/scrape shape and passes creditsCost through untouched', async () => { + mockHttpPost.mockResolvedValue({ + data: { + success: true, + scrape_id: 'scrape-1', + data: { alexandria: [successItem, failedItem], creditsCost: 1 }, + }, + }); + + const result = await executeExchangeRetrieve({ + calls: [ + { + provider: 'fred', + capability: 'series/observations', + options: { series_id: 'CPIAUCSL' }, + }, + { provider: 'fred', capability: 'series/search' }, + ], + timeout: 30000, + }); + + expect(mockHttpPost).toHaveBeenCalledTimes(1); + expect(mockHttpPost).toHaveBeenCalledWith( + '/v2/scrape', + { + alexandria: [ + { + provider: 'fred', + capability: 'series/observations', + options: { series_id: 'CPIAUCSL' }, + }, + { provider: 'fred', capability: 'series/search' }, + ], + integration: 'cli', + timeout: 30000, + }, + { headers: { 'x-request-id': result.requestId } } + ); + expect(result).toEqual({ + success: true, + scrapeId: 'scrape-1', + requestId: expect.any(String), + exchange: [successItem, failedItem], + creditsCost: 1, + }); + }); + + it('reuses the same execution ID on a manual retry after an uncertain failure', async () => { + const calls = [{ provider: 'fred', capability: 'series/observations' }]; + mockHttpPost + .mockRejectedValueOnce(new Error('Connection reset')) + .mockResolvedValueOnce({ + data: { success: true, data: { alexandria: [] } }, + }); + const first = await executeExchangeRetrieve({ calls }); + expect(first.success).toBe(false); + const retry = await executeExchangeRetrieve({ + calls, + requestId: first.requestId, + }); + expect(retry.requestId).toBe(first.requestId); + expect(mockHttpPost.mock.calls[0]).toEqual(mockHttpPost.mock.calls[1]); + expect(mockHttpPost.mock.calls[0][2]).toEqual({ + headers: { 'x-request-id': first.requestId }, + }); + }); + + it('omits timeout when not provided', async () => { + mockHttpPost.mockResolvedValue({ + data: { + success: true, + scrape_id: 'scrape-2', + data: { alexandria: [successItem], creditsCost: 1 }, + }, + }); + + await executeExchangeRetrieve({ + calls: [{ provider: 'fred', capability: 'series/observations' }], + }); + + expect(mockHttpPost).toHaveBeenCalledWith( + '/v2/scrape', + { + alexandria: [{ provider: 'fred', capability: 'series/observations' }], + integration: 'cli', + }, + { headers: { 'x-request-id': expect.any(String) } } + ); + }); + + it('relays the 403 body when the team has no Exchange flag', async () => { + mockHttpPost.mockRejectedValue( + axiosError(403, { + success: false, + error: 'Exchange is not enabled for this team.', + }) + ); + + const result = await executeExchangeRetrieve({ + calls: [{ provider: 'fred', capability: 'series/observations' }], + }); + + expect(result).toEqual({ + success: false, + requestId: expect.any(String), + error: 'Exchange is not enabled for this team.', + }); + }); + + it('carries requiresAction through a 403 terms failure', async () => { + mockHttpPost.mockRejectedValue(axiosError(403, termsRequiredBody)); + + const result = await executeExchangeRetrieve({ + calls: [{ provider: 'benzinga', capability: 'news' }], + }); + + expect(result).toEqual({ + success: false, + requestId: expect.any(String), + error: termsRequiredBody.error, + code: 'THIRD_PARTY_DATA_TERMS_REQUIRED', + requiresAction: termsRequiredBody.requiresAction, + }); + }); + + it('drops a malformed requiresAction but keeps the code', async () => { + mockHttpPost.mockRejectedValue( + axiosError(403, { + ...termsRequiredBody, + requiresAction: { type: 'accept_terms', terms: 'benzinga' }, + }) + ); + + const result = await executeExchangeRetrieve({ + calls: [{ provider: 'benzinga', capability: 'news' }], + }); + + expect(result.code).toBe('THIRD_PARTY_DATA_TERMS_REQUIRED'); + expect(result.requiresAction).toBeUndefined(); + }); + + it('refuses retrieve in keyless mode without calling the API', async () => { + vi.mocked(isKeylessMode).mockReturnValue(true); + + const result = await executeExchangeRetrieve({ + calls: [{ provider: 'fred', capability: 'series/observations' }], + }); + + expect(result).toEqual({ + success: false, + requestId: expect.any(String), + error: EXCHANGE_KEY_REQUIRED, + }); + expect(mockHttpPost).not.toHaveBeenCalled(); + }); + + it('rejects an empty or oversized batch before calling the API', async () => { + const empty = await executeExchangeRetrieve({ calls: [] }); + expect(empty.success).toBe(false); + expect(empty.error).toMatch(/At least one/); + + const oversized = await executeExchangeRetrieve({ + calls: Array.from({ length: 11 }, (_, i) => ({ + provider: 'p', + capability: `cap${i}`, + })), + }); + expect(oversized.success).toBe(false); + expect(oversized.error).toMatch(/at most 10/); + expect(mockHttpPost).not.toHaveBeenCalled(); + }); +}); + +describe('handleExchangeDiscoverCommand / handleExchangeRetrieveCommand', () => { + let mockHttpGet: ReturnType; + let mockHttpPost: ReturnType; + let exitSpy: ReturnType; + let errorSpy: ReturnType; + let stderrSpy: ReturnType; + + const writtenOutput = () => + vi.mocked(writeOutput).mock.calls.at(-1)?.[0] as string; + + beforeEach(() => { + setupTest(); + initializeConfig({ + apiKey: 'test-api-key', + apiUrl: 'https://api.firecrawl.dev', + }); + mockHttpGet = vi.fn(); + mockHttpPost = vi.fn(); + vi.mocked(getClient).mockReturnValue({ + http: { get: mockHttpGet, post: mockHttpPost }, + } as any); + vi.mocked(isKeylessMode).mockReturnValue(false); + exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code: number) => { + throw new Error(`exit ${code}`); + }) as never); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + stderrSpy = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + }); + + afterEach(() => { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + stderrSpy.mockRestore(); + teardownTest(); + vi.clearAllMocks(); + }); + + it('prints cohorts in a readable list', async () => { + mockHttpGet.mockResolvedValue({ + data: { + cohorts: [ + { cohort: 'finance', about: 'Markets and macro.', providers: 3 }, + ], + }, + }); + + await handleExchangeDiscoverCommand({}); + + const output = writtenOutput(); + expect(output).toContain('=== Cohorts ==='); + expect(output).toContain('finance (3 providers)'); + expect(output).toContain('Markets and macro.'); + }); + + it('prints semantic hits with their address, cost and similarity', async () => { + mockHttpGet.mockResolvedValue({ + data: { + capabilities: [ + { + address: 'series/observations', + cohorts: ['finance'], + concept: 'series/observations', + creditsCost: 1, + provider: 'fred', + similarity: 0.8123, + }, + ], + query: 'cpi', + searched: 54, + source: 'process', + }, + }); + + await handleExchangeDiscoverCommand({ query: 'cpi' }); + + const output = writtenOutput(); + expect(output).toContain('=== Capabilities matching "cpi" ==='); + expect(output).toContain('fred/series/observations (1 credits)'); + expect(output).toContain('Similarity: 0.8123'); + expect(output).toContain('Searched 54 capabilities (process).'); + }); + + it('prints a contract with its options and a ready-to-run retrieve line', async () => { + mockHttpGet.mockResolvedValue({ + data: { + capability: 'series/observations', + provider: 'fred', + creditsCost: 1, + label: 'FRED series observations', + options: [ + { + name: 'series_id', + required: true, + description: 'FRED series id, e.g. CPIAUCSL', + }, + ], + returns: { key: 'observations' }, + example: { options: { series_id: 'CPIAUCSL' } }, + }, + }); + + await handleExchangeDiscoverCommand({ + cohort: 'finance', + provider: 'fred', + capability: 'series/observations', + }); + + expect(mockHttpGet).toHaveBeenCalledWith( + '/exchange/discover/finance/fred/series/observations' + ); + const output = writtenOutput(); + expect(output).toContain('=== fred/series/observations ==='); + expect(output).toContain('Credits cost: 1'); + expect(output).toContain('series_id (required: true)'); + expect(output).toContain('FRED series id, e.g. CPIAUCSL'); + expect(output).toContain( + `Retrieve: firecrawl exchange retrieve fred/series/observations --options '{"series_id":"CPIAUCSL"}'` + ); + }); + + it('writes the discover payload verbatim with --json', async () => { + const payload = { cohort: 'finance', providers: [] }; + mockHttpGet.mockResolvedValue({ data: payload }); + + await handleExchangeDiscoverCommand({ cohort: 'finance', json: true }); + + expect(JSON.parse(writtenOutput())).toEqual(payload); + }); + + it('exits 1 with the explanatory error in keyless mode', async () => { + vi.mocked(isKeylessMode).mockReturnValue(true); + + await expect(handleExchangeDiscoverCommand({})).rejects.toThrow('exit 1'); + + expect(errorSpy).toHaveBeenCalledWith('Error:', EXCHANGE_KEY_REQUIRED); + expect(mockHttpGet).not.toHaveBeenCalled(); + expect(writeOutput).not.toHaveBeenCalled(); + }); + + it('prints each retrieve item with its creditsCost and the total', async () => { + mockHttpPost.mockResolvedValue({ + data: { + success: true, + scrape_id: 'scrape-1', + data: { alexandria: [successItem, failedItem], creditsCost: 1 }, + }, + }); + + await handleExchangeRetrieveCommand({ + calls: [ + { provider: 'fred', capability: 'series/observations' }, + { provider: 'fred', capability: 'series/search' }, + ], + }); + + const output = writtenOutput(); + expect(output).toContain('fred/series/observations'); + expect(output).toContain('Credits cost: 1'); + expect(output).toContain('Records: 1'); + expect(output).toContain('"value": "308.4"'); + expect(output).toContain( + 'Error [credential_missing]: FRED credential is not configured. (HTTP 503)' + ); + expect(output).toContain('Total credits cost: 1'); + expect(stderrSpy).toHaveBeenCalledWith('Scrape ID: scrape-1\n'); + expect(exitSpy).not.toHaveBeenCalled(); + }); + + it('mirrors the /v2/scrape envelope with --json', async () => { + mockHttpPost.mockResolvedValue({ + data: { + success: true, + scrape_id: 'scrape-1', + data: { alexandria: [successItem], creditsCost: 1 }, + }, + }); + + await handleExchangeRetrieveCommand({ + calls: [{ provider: 'fred', capability: 'series/observations' }], + json: true, + }); + + expect(JSON.parse(writtenOutput())).toEqual({ + success: true, + requestId: expect.any(String), + scrape_id: 'scrape-1', + data: { alexandria: [successItem], creditsCost: 1 }, + }); + }); + + it('exits 1 only when every item in the batch failed', async () => { + mockHttpPost.mockResolvedValue({ + data: { + success: true, + scrape_id: 'scrape-3', + data: { alexandria: [failedItem], creditsCost: 0 }, + }, + }); + + await expect( + handleExchangeRetrieveCommand({ + calls: [{ provider: 'fred', capability: 'series/search' }], + }) + ).rejects.toThrow('exit 1'); + + expect(writtenOutput()).toContain('Error [credential_missing]'); + }); + + it('exits 1 with the API error when the request itself fails', async () => { + mockHttpPost.mockRejectedValue( + axiosError(403, { + success: false, + error: 'Exchange is not enabled for this team.', + }) + ); + + await expect( + handleExchangeRetrieveCommand({ + calls: [{ provider: 'fred', capability: 'series/observations' }], + }) + ).rejects.toThrow('exit 1'); + + expect(errorSpy).toHaveBeenCalledWith( + 'Error:', + 'Exchange is not enabled for this team.' + ); + }); + + it('exits 1 with the explanatory error for retrieve in keyless mode', async () => { + vi.mocked(isKeylessMode).mockReturnValue(true); + + await expect( + handleExchangeRetrieveCommand({ + calls: [{ provider: 'fred', capability: 'series/observations' }], + }) + ).rejects.toThrow('exit 1'); + + expect(errorSpy).toHaveBeenCalledWith('Error:', EXCHANGE_KEY_REQUIRED); + expect(mockHttpPost).not.toHaveBeenCalled(); + }); + + it('prints the terms block on stderr for a 403 terms failure', async () => { + mockHttpPost.mockRejectedValue(axiosError(403, termsRequiredBody)); + + await expect( + handleExchangeRetrieveCommand({ + calls: [{ provider: 'benzinga', capability: 'news' }], + }) + ).rejects.toThrow('exit 1'); + + const stderr = stderrSpy.mock.calls + .map((call: unknown[]) => call[0]) + .join(''); + expect(stderr).toMatch(/^Request ID: /); + expect(stderr).toContain('Alexandria provider terms required'); + expect(stderr).toContain('Provider: benzinga'); + expect(stderr).toContain('Version: 2026-09-12-placeholder'); + expect(stderr).toContain( + 'Accept at: https://www.firecrawl.dev/app/alexandria/benzinga' + ); + expect(stderr).toContain('firecrawl alexandria terms accept benzinga'); + expect(stderr).toContain('No credits were charged.'); + expect(stderr).toContain('reuse --request-id'); + expect(errorSpy).not.toHaveBeenCalled(); + expect(writeOutput).not.toHaveBeenCalled(); + }); + + it('writes a failure envelope to stdout with --json', async () => { + mockHttpPost.mockRejectedValue(axiosError(403, termsRequiredBody)); + + await expect( + handleExchangeRetrieveCommand({ + calls: [{ provider: 'benzinga', capability: 'news' }], + requestId: 'req-1', + json: true, + }) + ).rejects.toThrow('exit 1'); + + expect(vi.mocked(writeOutput).mock.calls.at(-1)).toEqual([ + expect.any(String), + ]); + expect(JSON.parse(writtenOutput())).toEqual({ + success: false, + requestId: 'req-1', + code: 'THIRD_PARTY_DATA_TERMS_REQUIRED', + error: termsRequiredBody.error, + requiresAction: termsRequiredBody.requiresAction, + }); + expect(errorSpy).not.toHaveBeenCalled(); + }); +}); + +describe('handleExchangeTermsCommand / handleExchangeTermsAcceptCommand', () => { + let mockHttpGet: ReturnType; + let mockHttpPost: ReturnType; + let exitSpy: ReturnType; + let errorSpy: ReturnType; + let stderrSpy: ReturnType; + const tty = { + stdin: process.stdin.isTTY, + stdout: process.stdout.isTTY, + }; + + const setTTY = (value: boolean) => { + Object.defineProperty(process.stdin, 'isTTY', { + value, + configurable: true, + }); + Object.defineProperty(process.stdout, 'isTTY', { + value, + configurable: true, + }); + }; + + beforeEach(() => { + setupTest(); + initializeConfig({ + apiKey: 'test-api-key', + apiUrl: 'https://api.firecrawl.dev', + }); + mockHttpGet = vi.fn().mockResolvedValue({ + data: { providers: [{ provider: 'fred', terms: null }, fiscalTerms] }, + }); + mockHttpPost = vi.fn(); + vi.mocked(getClient).mockReturnValue({ + http: { get: mockHttpGet, post: mockHttpPost }, + } as any); + vi.mocked(isKeylessMode).mockReturnValue(false); + exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code: number) => { + throw new Error(`exit ${code}`); + }) as never); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + stderrSpy = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + }); + + afterEach(() => { + setTTY(tty.stdin as boolean); + Object.defineProperty(process.stdout, 'isTTY', { + value: tty.stdout, + configurable: true, + }); + exitSpy.mockRestore(); + errorSpy.mockRestore(); + stderrSpy.mockRestore(); + vi.unstubAllGlobals(); + teardownTest(); + vi.clearAllMocks(); + }); + + it('shows the provider terms with their version and document', async () => { + await handleExchangeTermsCommand({ provider: 'fiscal-ai' }); + + expect(mockHttpGet).toHaveBeenCalledWith( + '/exchange/provider-terms?surface=web' + ); + const output = vi.mocked(writeOutput).mock.calls.at(-1)?.[0] as string; + expect(output).toContain('Fiscal.ai (fiscal-ai) Alexandria provider terms'); + expect(output).toContain('Publisher: Fiscal.ai'); + expect(output).toContain('Version: 2026-09-12-placeholder'); + expect(output).toContain('Effective: 2026-09-12'); + expect(output).toContain(fiscalTerms.terms.document.trimEnd()); + expect(exitSpy).not.toHaveBeenCalled(); + }); + + it('refuses to accept without an interactive terminal', async () => { + setTTY(false); + process.env.FIRECRAWL_DASHBOARD_URL = 'http://localhost:3001'; + + await expect( + handleExchangeTermsAcceptCommand({ provider: 'fiscal-ai' }) + ).rejects.toThrow('exit 2'); + + expect(errorSpy).toHaveBeenCalledWith( + 'Error: Accepting provider terms needs an interactive terminal. Accept in the dashboard at http://localhost:3001/app/alexandria/fiscal-ai instead.' + ); + expect(mockHttpGet).not.toHaveBeenCalled(); + delete process.env.FIRECRAWL_DASHBOARD_URL; + }); + + it('accepts after the slug is typed back and posts the displayed version', async () => { + setTTY(true); + vi.mocked(input).mockResolvedValue('fiscal-ai'); + const fetchMock = vi.fn().mockResolvedValue({ + status: 200, + json: async () => ({ + success: true, + provider: 'fiscal-ai', + version: '2026-09-12-placeholder', + digest: 'sha256:abc', + }), + }); + vi.stubGlobal('fetch', fetchMock); + + await handleExchangeTermsAcceptCommand({ provider: 'fiscal-ai' }); + + expect(fetchMock).toHaveBeenCalledWith( + 'https://www.firecrawl.dev/api/exchange/provider-access/accept', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ + Authorization: 'Bearer test-api-key', + }), + body: JSON.stringify({ + provider: 'fiscal-ai', + version: '2026-09-12-placeholder', + confirmed: true, + }), + }) + ); + expect(stderrSpy).toHaveBeenCalledWith( + expect.stringContaining( + 'Accepted fiscal-ai provider terms version 2026-09-12-placeholder (digest sha256:abc)' + ) + ); + expect(exitSpy).not.toHaveBeenCalled(); + }); + + it('falls back to Find Tools for a provider absent from the terms list and reports no terms', async () => { + mockHttpPost.mockResolvedValue({ + data: { + success: true, + data: { + alexandria: [ + { + provider: 'firecrawl', + capability: 'find-tools', + creditsCost: 0, + data: { items: [{ provider: 'sec' }] }, + }, + ], + creditsCost: 0, + }, + }, + }); + + await handleExchangeTermsCommand({ provider: 'sec' }); + + expect(mockHttpPost).toHaveBeenCalledWith( + '/v2/scrape', + expect.objectContaining({ + alexandria: [ + { + provider: 'firecrawl', + capability: 'find-tools', + options: { providers: ['sec'], level: 'providers', limit: 1 }, + }, + ], + }), + expect.any(Object) + ); + expect(stderrSpy).toHaveBeenCalledWith( + 'sec has no provider terms to accept.\n' + ); + expect(exitSpy).not.toHaveBeenCalled(); + }); + + it('writes {provider, terms: null} to stdout in --json mode for the Find Tools fallback', async () => { + mockHttpPost.mockResolvedValue({ + data: { + success: true, + data: { + alexandria: [ + { + provider: 'firecrawl', + capability: 'find-tools', + creditsCost: 0, + data: { items: [{ provider: 'sec' }] }, + }, + ], + creditsCost: 0, + }, + }, + }); + + await handleExchangeTermsCommand({ provider: 'sec', json: true }); + + const written = vi.mocked(writeOutput).mock.calls.at(-1)?.[0] as string; + expect(JSON.parse(written)).toEqual({ provider: 'sec', terms: null }); + expect(exitSpy).not.toHaveBeenCalled(); + }); + + it('refuses cleanly through terms accept when Find Tools finds the provider but no terms exist', async () => { + setTTY(true); + mockHttpPost.mockResolvedValue({ + data: { + success: true, + data: { + alexandria: [ + { + provider: 'firecrawl', + capability: 'find-tools', + creditsCost: 0, + data: { items: [{ provider: 'sec' }] }, + }, + ], + creditsCost: 0, + }, + }, + }); + + await handleExchangeTermsAcceptCommand({ provider: 'sec' }); + + expect(stderrSpy).toHaveBeenCalledWith( + 'sec has no provider terms to accept.\n' + ); + expect(input).not.toHaveBeenCalled(); + expect(exitSpy).not.toHaveBeenCalled(); + }); + + it('still throws unknown provider when neither the terms list nor Find Tools know it', async () => { + mockHttpPost.mockResolvedValue({ + data: { + success: true, + data: { + alexandria: [ + { + provider: 'firecrawl', + capability: 'find-tools', + creditsCost: 0, + data: { items: [] }, + }, + ], + creditsCost: 0, + }, + }, + }); + + await expect( + handleExchangeTermsCommand({ provider: 'nonexistent' }) + ).rejects.toThrow('exit 1'); + + expect(errorSpy).toHaveBeenCalledWith( + 'Error:', + expect.stringContaining('Unknown Alexandria provider "nonexistent"') + ); + }); +}); diff --git a/src/__tests__/commands/find-tools.test.ts b/src/__tests__/commands/find-tools.test.ts new file mode 100644 index 0000000000..25aa2a3384 --- /dev/null +++ b/src/__tests__/commands/find-tools.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import { buildFindToolsCall } from '../../commands/find-tools'; + +describe('Find Tools', () => { + it('builds an inferred lookup and preserves next requests', () => { + const call = buildFindToolsCall([], { + providers: 'particle', + capabilities: 'podcasts/episodes/search', + expand: 'options,response', + limit: 2, + }); + expect(call).toEqual({ + provider: 'firecrawl', + capability: 'find-tools', + options: { + providers: ['particle'], + capabilities: ['podcasts/episodes/search'], + expand: ['options', 'response'], + limit: 2, + }, + }); + const next = { ...call, options: { ...call.options, offset: 2 } }; + expect(buildFindToolsCall([], { request: JSON.stringify(next) })).toEqual( + next + ); + }); + it('does not turn a discovery next request into arbitrary paid execution', () => { + expect(() => + buildFindToolsCall([], { + request: JSON.stringify({ + provider: 'particle', + capability: 'podcasts/episodes/search', + }), + }) + ).toThrow(); + expect(() => buildFindToolsCall([], { limit: 0 })).toThrow(); + expect(() => buildFindToolsCall([], { offset: -1 })).toThrow(); + }); +}); diff --git a/src/__tests__/commands/scrape.test.ts b/src/__tests__/commands/scrape.test.ts index aaeae6b967..0106ffb5eb 100644 --- a/src/__tests__/commands/scrape.test.ts +++ b/src/__tests__/commands/scrape.test.ts @@ -3,7 +3,10 @@ */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { executeScrape } from '../../commands/scrape'; +import { + executeScrape, + handleScrapeExchangeCommand, +} from '../../commands/scrape'; import { getClient } from '../../utils/client'; import { initializeConfig } from '../../utils/config'; import { setupTest, teardownTest } from '../utils/mock-client'; @@ -366,6 +369,39 @@ describe('executeScrape', () => { }); }); + it('should include domainTools when provided', async () => { + const mockResponse = { markdown: '# Test', tools: [{ name: 'tool' }] }; + mockClient.scrape.mockResolvedValue(mockResponse); + + const result = await executeScrape({ + url: 'https://example.com', + domainTools: true, + }); + + expect(mockClient.scrape).toHaveBeenCalledWith('https://example.com', { + formats: ['markdown'], + integration: 'cli', + domainTools: true, + }); + expect(result.data?.tools).toEqual([{ name: 'tool' }]); + }); + + it('maps --domain-tools to domainTools', async () => { + const mockResponse = { markdown: '# Test', tools: [{ name: 'tool' }] }; + mockClient.scrape.mockResolvedValue(mockResponse); + + await executeScrape({ + url: 'https://example.com', + domainTools: true, + }); + + expect(mockClient.scrape).toHaveBeenCalledWith('https://example.com', { + formats: ['markdown'], + integration: 'cli', + domainTools: true, + }); + }); + it('should not include location parameter when not provided', async () => { const mockResponse = { markdown: '# Test' }; mockClient.scrape.mockResolvedValue(mockResponse); @@ -484,3 +520,122 @@ describe('executeScrape', () => { }); }); }); + +describe('handleScrapeExchangeCommand', () => { + let mockHttpPost: ReturnType; + let exitSpy: ReturnType; + let errorSpy: ReturnType; + let stdoutSpy: ReturnType; + let stderrSpy: ReturnType; + + beforeEach(() => { + setupTest(); + initializeConfig({ + apiKey: 'test-api-key', + apiUrl: 'https://api.firecrawl.dev', + }); + mockHttpPost = vi.fn(); + vi.mocked(getClient).mockReturnValue({ + http: { post: mockHttpPost }, + } as any); + exitSpy = vi.spyOn(process, 'exit').mockImplementation(((code: number) => { + throw new Error(`exit ${code}`); + }) as never); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + stdoutSpy = vi + .spyOn(process.stdout, 'write') + .mockImplementation(() => true); + stderrSpy = vi + .spyOn(process.stderr, 'write') + .mockImplementation(() => true); + }); + + afterEach(() => { + exitSpy.mockRestore(); + errorSpy.mockRestore(); + stdoutSpy.mockRestore(); + stderrSpy.mockRestore(); + teardownTest(); + vi.clearAllMocks(); + }); + + it('delegates --alexandria to the url-less /v2/scrape executor', async () => { + mockHttpPost.mockResolvedValue({ + data: { + success: true, + scrape_id: 'scrape-9', + data: { + alexandria: [ + { + provider: 'fred', + capability: 'series/observations', + creditsCost: 1, + data: { observations: [] }, + }, + ], + creditsCost: 1, + }, + }, + }); + + await handleScrapeExchangeCommand( + ['fred/series/observations'], + ['{"series_id":"CPIAUCSL"}'], + { + apiKey: 'fc-key', + apiUrl: 'http://localhost:3002', + json: true, + requestId: 'scrape-retry', + } + ); + + expect(getClient).toHaveBeenCalledWith({ + apiKey: 'fc-key', + apiUrl: 'http://localhost:3002', + }); + expect(mockHttpPost).toHaveBeenCalledWith( + '/v2/scrape', + { + alexandria: [ + { + provider: 'fred', + capability: 'series/observations', + options: { series_id: 'CPIAUCSL' }, + }, + ], + integration: 'cli', + }, + { headers: { 'x-request-id': 'scrape-retry' } } + ); + const written = stdoutSpy.mock.calls.at(-1)?.[0] as string; + expect(JSON.parse(written)).toEqual({ + success: true, + scrape_id: 'scrape-9', + requestId: 'scrape-retry', + data: { + alexandria: [ + { + provider: 'fred', + capability: 'series/observations', + creditsCost: 1, + data: { observations: [] }, + }, + ], + creditsCost: 1, + }, + }); + expect(exitSpy).not.toHaveBeenCalled(); + }); + + it('exits 1 on a malformed address without calling the API', async () => { + await expect(handleScrapeExchangeCommand(['fred'], [], {})).rejects.toThrow( + 'exit 1' + ); + + expect(errorSpy).toHaveBeenCalledWith( + 'Error:', + expect.stringContaining('provider/capability') + ); + expect(mockHttpPost).not.toHaveBeenCalled(); + }); +}); diff --git a/src/__tests__/commands/search.test.ts b/src/__tests__/commands/search.test.ts index aeded653fc..c434aef2f1 100644 --- a/src/__tests__/commands/search.test.ts +++ b/src/__tests__/commands/search.test.ts @@ -4,7 +4,8 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { executeSearch, handleSearchCommand } from '../../commands/search'; -import { getClient } from '../../utils/client'; +import { EXCHANGE_KEY_REQUIRED } from '../../commands/exchange'; +import { getClient, isKeylessMode } from '../../utils/client'; import { initializeConfig } from '../../utils/config'; import { writeOutput } from '../../utils/output'; import { setupTest, teardownTest } from '../utils/mock-client'; @@ -17,6 +18,7 @@ vi.mock('../../utils/client', async () => { return { ...actual, getClient: vi.fn(), + isKeylessMode: vi.fn(() => false), }; }); @@ -54,6 +56,7 @@ describe('executeSearch', () => { }; vi.mocked(getClient).mockReturnValue(mockClient as any); + vi.mocked(isKeylessMode).mockReturnValue(false); }); afterEach(() => { @@ -61,6 +64,177 @@ describe('executeSearch', () => { vi.clearAllMocks(); }); + describe('Exchange source', () => { + const exchangeHit = { + provider: 'fred', + capability: 'series/observations', + concept: 'series/observations', + cohorts: ['finance'], + creditsCost: 1, + similarity: 0.8123, + }; + + it('normalizes the legacy exchange source to alexandria beside web', async () => { + mockHttpPost.mockResolvedValue( + mockSearchResponse({ web: [], exchange: [exchangeHit] }) + ); + + await executeSearch({ + query: 'nvidia balance sheet', + sources: ['web', 'exchange'], + }); + + expect(mockHttpPost).toHaveBeenCalledWith('/v2/search', { + query: 'nvidia balance sheet', + limit: undefined, + integration: 'cli', + sources: [{ type: 'web' }, { type: 'alexandria' }], + }); + }); + + it('preserves unified contracts and rejects catalogue browsing in search', async () => { + const tools = [ + { + id: 'particle/podcasts/episodes/search', + matchedBy: ['semantic', 'domain'], + matchedUrls: ['https://podcasts.apple.com'], + options: [{ name: 'semantic_search', type: 'string' }], + }, + ]; + mockHttpPost.mockResolvedValue(mockSearchResponse({ tools })); + const result = await executeSearch({ + query: 'podcast episodes', + sources: ['alexandria'], + domainTools: true, + }); + expect(result.data?.tools).toEqual(tools); + expect(mockHttpPost.mock.calls[0][1]).toMatchObject({ + domainTools: true, + }); + mockHttpPost.mockClear(); + expect( + (await executeSearch({ query: '', sources: ['alexandria'] })).success + ).toBe(false); + expect( + ( + await executeSearch({ + query: 'podcasts', + sources: [{ type: 'alexandria', mode: 'browse' }], + }) + ).success + ).toBe(false); + expect(mockHttpPost).not.toHaveBeenCalled(); + }); + + it('maps --domain-tools to domainTools', async () => { + mockHttpPost.mockResolvedValue(mockSearchResponse({ tools: [] })); + + await executeSearch({ + query: 'podcast episodes', + sources: ['alexandria'], + domainTools: true, + }); + + expect(mockHttpPost.mock.calls[0][1]).toMatchObject({ + domainTools: true, + }); + }); + + it('passes data.exchange, id and creditsUsed through untouched', async () => { + const web = [{ url: 'https://example.com', title: 'Example' }]; + mockHttpPost.mockResolvedValue( + mockSearchResponse( + { web, exchange: [exchangeHit] }, + { id: 'search-1', creditsUsed: 0 } + ) + ); + + const result = await executeSearch({ + query: 'nvidia balance sheet', + sources: ['web', 'exchange'], + }); + + expect(result.success).toBe(true); + expect(result.data).toEqual({ web, exchange: [exchangeHit] }); + expect(result.id).toBe('search-1'); + expect(result.creditsUsed).toBe(0); + }); + + it('leaves data.exchange absent when the API omitted it', async () => { + mockHttpPost.mockResolvedValue( + mockSearchResponse({ web: [{ url: 'https://example.com' }] }) + ); + + const result = await executeSearch({ + query: 'nvidia balance sheet', + sources: ['web', 'exchange'], + }); + + expect(result.success).toBe(true); + expect(result.data).not.toHaveProperty('exchange'); + }); + + it('refuses the exchange source in keyless mode before calling the API', async () => { + vi.mocked(isKeylessMode).mockReturnValue(true); + + const result = await executeSearch({ + query: 'nvidia balance sheet', + sources: ['web', 'exchange'], + }); + + expect(result).toEqual({ success: false, error: EXCHANGE_KEY_REQUIRED }); + expect(mockHttpPost).not.toHaveBeenCalled(); + }); + + it('prints an Exchange Providers section and labels the web group', async () => { + mockHttpPost.mockResolvedValue( + mockSearchResponse({ + web: [{ url: 'https://example.com', title: 'Example' }], + exchange: [exchangeHit], + }) + ); + + await handleSearchCommand({ + query: 'nvidia balance sheet', + sources: ['web', 'exchange'], + }); + + const output = vi.mocked(writeOutput).mock.calls.at(-1)?.[0] as string; + expect(output).toContain('=== Web Results ==='); + expect(output).toContain('=== Exchange Providers ==='); + expect(output).toContain('fred/series/observations'); + expect(output).toContain('Concept: series/observations'); + expect(output).toContain('Cohorts: finance'); + expect(output).toContain('Credits per call: 1'); + expect(output).toContain('Similarity: 0.8123'); + expect(output).toContain( + 'Contract: firecrawl exchange discover finance fred series/observations' + ); + }); + + it('treats exchange-only hits as results', async () => { + mockHttpPost.mockResolvedValue( + mockSearchResponse({ exchange: [exchangeHit] }, { creditsUsed: 0 }) + ); + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + + await handleSearchCommand({ + query: 'cpi', + sources: ['exchange'], + json: true, + }); + + expect(logSpy).not.toHaveBeenCalledWith('No results found.'); + const output = vi.mocked(writeOutput).mock.calls.at(-1)?.[0] as string; + expect(JSON.parse(output)).toEqual({ + success: true, + data: { exchange: [exchangeHit] }, + creditsUsed: 0, + }); + logSpy.mockRestore(); + }); + }); + describe('API call generation', () => { it('should call /v2/search with correct query and default options', async () => { mockHttpPost.mockResolvedValue( diff --git a/src/commands/exchange.ts b/src/commands/exchange.ts new file mode 100644 index 0000000000..80e551025a --- /dev/null +++ b/src/commands/exchange.ts @@ -0,0 +1,683 @@ +/** + * Exchange command implementation + * + * `firecrawl exchange discover` walks or searches the capability catalogue + * (GET /exchange/discover) and `firecrawl exchange retrieve` executes + * capabilities through the url-less /v2/scrape shape. Both need a Firecrawl + * API key on a team with Exchange access: the keyless free tier has no team + * flag, so we refuse locally instead of surfacing an opaque 403. + */ + +import type { + ExchangeCall, + ExchangeDiscoverOptions, + ExchangeDiscoverResult, + ExchangeRequiresAction, + ExchangeRetrieveOptions, + ExchangeRetrieveResult, + ExchangeScrapeFailure, + ExchangeScrapeResponse, + ExchangeScrapeResult, +} from '../types/exchange'; +import { getClient, isKeylessMode } from '../utils/client'; +import { writeOutput } from '../utils/output'; +import { + TERMS_REQUIRED_CODE, + formatTermsRequired, + parseRequiresAction, +} from '../utils/terms'; +import { randomUUID } from 'node:crypto'; + +export const EXCHANGE_KEY_REQUIRED = + 'Alexandria requires a Firecrawl API key on a team with Alexandria access. ' + + 'Set FIRECRAWL_API_KEY, pass --api-key, or run "firecrawl login".'; + +const MAX_EXCHANGE_CALLS = 10; + +/** + * Refuse before the request leaves the machine when no key would be sent. + */ +export function assertExchangeKeyed(apiKey?: string, apiUrl?: string): void { + if (isKeylessMode(apiKey, apiUrl)) { + throw new Error(EXCHANGE_KEY_REQUIRED); + } +} + +/** + * Split `provider/capability` on the first slash. Capability addresses carry + * slashes of their own (`series/observations`), so only the first one + * separates the provider. + */ +export function parseExchangeAddress(address: string): ExchangeCall { + const trimmed = address.trim(); + const slash = trimmed.indexOf('/'); + if (slash <= 0 || slash === trimmed.length - 1) { + throw new Error( + `Invalid exchange address "${address}": expected provider/capability (e.g. fred/series/observations)` + ); + } + return { + provider: trimmed.slice(0, slash), + capability: trimmed.slice(slash + 1), + }; +} + +/** + * Parse one `--options` value into the capability's options object. + */ +export function parseExchangeOptions( + raw: string, + label: string = '--options' +): Record { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (error) { + throw new Error( + `Invalid JSON in ${label}: ${error instanceof Error ? error.message : 'Unable to parse JSON'}` + ); + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error(`Invalid ${label}: expected a JSON object`); + } + return parsed as Record; +} + +/** + * Pair each address with the `--options` value at the same position. A single + * `--options` therefore applies to the first address only. + */ +export function buildExchangeCalls( + addresses: string[], + optionsJson: string[] = [] +): ExchangeCall[] { + if (addresses.length === 0) { + throw new Error('At least one provider/capability address is required.'); + } + if (addresses.length > MAX_EXCHANGE_CALLS) { + throw new Error( + `Exchange accepts at most ${MAX_EXCHANGE_CALLS} capabilities per request (got ${addresses.length}).` + ); + } + if (optionsJson.length > addresses.length) { + throw new Error( + `More --options values (${optionsJson.length}) than capabilities (${addresses.length}); each --options pairs with the address at the same position.` + ); + } + return addresses.map((address, index) => { + const call = parseExchangeAddress(address); + const raw = optionsJson[index]; + if (raw !== undefined) { + call.options = parseExchangeOptions(raw); + } + return call; + }); +} + +/** + * Compose the proxied discover path. Segments are encoded one at a time so a + * capability address keeps its slashes. + */ +export function buildDiscoverPath(options: ExchangeDiscoverOptions): string { + const segments = [options.cohort, options.provider, options.capability] + .map((segment) => segment?.trim() ?? '') + .filter((segment) => segment !== ''); + + if (options.query && segments.length > 0) { + throw new Error( + '--query searches the whole catalogue; drop the cohort/provider/capability arguments or the --query.' + ); + } + + const path = segments + .map((segment) => segment.split('/').map(encodeURIComponent).join('/')) + .join('/'); + + const params = new URLSearchParams(); + if (options.query) params.set('q', options.query); + if (options.limit !== undefined) params.set('limit', String(options.limit)); + if (options.expand) params.set('expand', options.expand); + const queryString = params.toString(); + + return `/exchange/discover${path ? `/${path}` : ''}${queryString ? `?${queryString}` : ''}`; +} + +/** + * One-line, per-code hint for the request-level failure codes the API + * returns from `{ success: false, error, code, chargeId? }` bodies: + * `duplicate_request` (409), `request_in_flight` (409), + * `request_unresolved` (503), `unknown_provider` (404), + * `insufficient_credits` (402), `billing_unavailable` (503), + * `THIRD_PARTY_DATA_TERMS_REQUIRED` (403). + */ +function exchangeErrorHint(code?: string): string | undefined { + switch (code) { + case TERMS_REQUIRED_CODE: + return 'An organization admin must accept the provider terms in the dashboard or with "firecrawl alexandria terms accept ", then rerun.'; + case 'request_in_flight': + return 'Retry with the same --request-id once the in-flight request finishes.'; + case 'request_unresolved': + return 'Keep using this --request-id; do not create a new one until it resolves.'; + case 'duplicate_request': + return 'Use a new --request-id for a new payload.'; + default: + return undefined; + } +} + +interface ExchangeErrorDetails { + message?: string; + code?: string; + chargeId?: string; + requiresAction?: ExchangeRequiresAction; +} + +/** + * Pull `{error, code, chargeId, requiresAction}` off an axios error's response + * body, when present. + */ +function exchangeErrorDetails(error: unknown): ExchangeErrorDetails { + const response = (error as any)?.response; + const body = response?.data; + if (body && typeof body === 'object') { + return { + message: typeof body.error === 'string' ? body.error : undefined, + code: typeof body.code === 'string' ? body.code : undefined, + chargeId: typeof body.chargeId === 'string' ? body.chargeId : undefined, + requiresAction: parseRequiresAction(body.requiresAction), + }; + } + return {}; +} + +/** + * One line for a request-level failure: the API message, its code, the + * chargeId when a charge was created, and the per-code hint. + */ +export function formatExchangeFailure(failure: { + error?: string; + code?: string; + chargeId?: string; +}): string { + const message = failure.error ?? 'Unknown error occurred'; + const parts = [failure.code ? `${message} (${failure.code})` : message]; + if (failure.chargeId) parts.push(`chargeId: ${failure.chargeId}`); + const hint = exchangeErrorHint(failure.code); + if (hint) parts.push(hint); + return parts.join(' — '); +} + +/** + * The SDK's HTTP layer throws an axios error on non-2xx. Prefer the API's own + * `{error, code}` body (403 not enabled, 402 insufficient credits, 409 + * duplicate request) over "Request failed with status code N". + */ +export function exchangeErrorMessage(error: unknown): string { + const { message, code, chargeId } = exchangeErrorDetails(error); + if (message) { + return formatExchangeFailure({ error: message, code, chargeId }); + } + const response = (error as any)?.response; + if (typeof response?.status === 'number') { + return `Firecrawl request failed (HTTP ${response.status})`; + } + return error instanceof Error ? error.message : 'Unknown error occurred'; +} + +/** + * Execute exchange discover + */ +export async function executeExchangeDiscover( + options: ExchangeDiscoverOptions +): Promise { + try { + assertExchangeKeyed(options.apiKey, options.apiUrl); + const path = buildDiscoverPath(options); + const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); + const response = await (app as any).http.get(path); + return { + success: true, + data: (response?.data ?? {}) as Record, + }; + } catch (error) { + return { success: false, error: exchangeErrorMessage(error) }; + } +} + +/** + * Execute exchange retrieve through the url-less /v2/scrape shape + */ +export async function executeExchangeRetrieve( + options: ExchangeRetrieveOptions +): Promise { + const requestId = options.requestId ?? randomUUID(); + try { + if (!/^[A-Za-z0-9._:-]{1,128}$/.test(requestId)) + throw new Error( + 'Invalid --request-id. Use 1–128 letters, digits, dots, underscores, colons or hyphens.' + ); + assertExchangeKeyed(options.apiKey, options.apiUrl); + if (!options.calls || options.calls.length === 0) { + throw new Error('At least one provider/capability address is required.'); + } + if (options.calls.length > MAX_EXCHANGE_CALLS) { + throw new Error( + `Exchange accepts at most ${MAX_EXCHANGE_CALLS} capabilities per request (got ${options.calls.length}).` + ); + } + + const body: Record = { + alexandria: options.calls, + integration: 'cli', + }; + if (options.timeout !== undefined) { + body.timeout = options.timeout; + } + + const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); + const response = await (app as any).http.post('/v2/scrape', body, { + headers: { 'x-request-id': requestId }, + }); + const envelope = (response?.data ?? {}) as ExchangeScrapeResponse; + + if (envelope.success === false) { + throw new Error(envelope.error || 'Exchange request failed'); + } + + return { + success: true, + scrapeId: envelope.scrape_id, + requestId, + exchange: envelope.data?.alexandria ?? [], + creditsCost: envelope.data?.creditsCost, + }; + } catch (error) { + const { message, code, chargeId, requiresAction } = + exchangeErrorDetails(error); + return { + success: false, + requestId, + error: message ?? exchangeErrorMessage(error), + code, + chargeId, + requiresAction, + }; + } +} + +function isFailure(item: ExchangeScrapeResult): item is ExchangeScrapeFailure { + return ( + typeof (item as ExchangeScrapeFailure).error === 'object' && + (item as ExchangeScrapeFailure).error !== null + ); +} + +function indent(text: string, prefix: string = ' '): string { + return text + .split('\n') + .map((line) => `${prefix}${line}`) + .join('\n'); +} + +function stringifyValue(value: unknown): string { + if (typeof value === 'string') return value; + return JSON.stringify(value, null, 2) ?? String(value); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() !== '' ? value : undefined; +} + +function formatRetrieveReadable(result: ExchangeRetrieveResult): string { + const lines: string[] = []; + for (const item of result.exchange ?? []) { + const address = `${item.provider ?? '?'}/${item.capability ?? '?'}`; + lines.push(address); + if (isFailure(item)) { + const status = + item.error.status !== undefined ? ` (HTTP ${item.error.status})` : ''; + lines.push( + ` Error [${item.error.code}]: ${item.error.message}${status}` + ); + lines.push(''); + continue; + } + lines.push(` Credits cost: ${item.creditsCost}`); + if (item.records !== undefined) lines.push(` Records: ${item.records}`); + if (item.upstreamStatus !== undefined) { + lines.push(` Upstream status: ${item.upstreamStatus}`); + } + if (item.recordedAt) lines.push(` Recorded at: ${item.recordedAt}`); + lines.push(' --- Data ---'); + lines.push(indent(stringifyValue(item.data))); + lines.push(' --- End Data ---'); + lines.push(''); + } + if (result.creditsCost !== undefined) { + lines.push(`Total credits cost: ${result.creditsCost}`); + } + return lines.join('\n'); +} + +function formatCapabilityLine( + provider: string | undefined, + capability: Record +): string[] { + const address = str(capability.address) ?? str(capability.capability) ?? '?'; + const owner = str(capability.provider) ?? provider; + const cost = + capability.creditsCost !== undefined + ? ` (${capability.creditsCost} credits)` + : ''; + const lines = [`${owner ? `${owner}/` : ''}${address}${cost}`]; + const label = str(capability.label); + if (label) lines.push(` ${label}`); + const whenToUse = str(capability.whenToUse); + if (whenToUse) lines.push(` When to use: ${whenToUse}`); + return lines; +} + +function formatConcepts( + provider: string | undefined, + concepts: unknown[] +): string[] { + const lines: string[] = []; + for (const concept of concepts) { + if (!isRecord(concept)) continue; + const name = str(concept.name) ?? str(concept.id) ?? 'concept'; + lines.push(`## ${name}`); + const about = str(concept.about); + if (about) lines.push(about); + for (const capability of asArray(concept.capabilities)) { + if (typeof capability === 'string') { + lines.push(`${provider ? `${provider}/` : ''}${capability}`); + } else if (isRecord(capability)) { + lines.push(...formatCapabilityLine(provider, capability)); + } + } + lines.push(''); + } + return lines; +} + +function formatProviderSummary(entry: Record): string[] { + const slug = str(entry.provider) ?? '?'; + const name = str(entry.name); + const count = + typeof entry.capabilities === 'number' + ? ` (${entry.capabilities} capabilities)` + : ''; + const lines = [`${slug}${name ? ` - ${name}` : ''}${count}`]; + const description = str(entry.description); + if (description) lines.push(` ${description}`); + const cohorts = asArray(entry.cohorts); + if (cohorts.length > 0) lines.push(` Cohorts: ${cohorts.join(', ')}`); + if (entry.executable === false) { + const reason = str(entry.reason); + lines.push(` Not executable${reason ? `: ${reason}` : ''}`); + } + const concepts = asArray(entry.concepts); + if (concepts.length > 0) { + lines.push(''); + lines.push(...formatConcepts(slug, concepts).map((line) => ` ${line}`)); + } + lines.push(''); + return lines; +} + +function formatDiscoverReadable(data: Record): string { + const lines: string[] = []; + + if (Array.isArray(data.capabilities) && str(data.query)) { + lines.push(`=== Capabilities matching "${data.query}" ===`); + lines.push(''); + for (const hit of data.capabilities) { + if (!isRecord(hit)) continue; + lines.push(...formatCapabilityLine(undefined, hit)); + const concept = str(hit.concept); + if (concept) lines.push(` Concept: ${concept}`); + const cohorts = asArray(hit.cohorts); + if (cohorts.length > 0) lines.push(` Cohorts: ${cohorts.join(', ')}`); + if (typeof hit.similarity === 'number') { + lines.push(` Similarity: ${hit.similarity}`); + } + lines.push(''); + } + if (data.capabilities.length === 0) { + lines.push('No capabilities matched.'); + lines.push(''); + } + if (data.searched !== undefined) { + lines.push( + `Searched ${data.searched} capabilities${str(data.source) ? ` (${data.source})` : ''}.` + ); + } + return lines.join('\n'); + } + + if (Array.isArray(data.cohorts)) { + lines.push('=== Cohorts ==='); + lines.push(''); + for (const cohort of data.cohorts) { + if (!isRecord(cohort)) continue; + const providers = cohort.providers; + const count = + typeof providers === 'number' + ? ` (${providers} providers)` + : Array.isArray(providers) + ? ` (${providers.length} providers)` + : ''; + lines.push(`${str(cohort.cohort) ?? '?'}${count}`); + const about = str(cohort.about); + if (about) lines.push(` ${about}`); + if (Array.isArray(providers)) { + for (const entry of providers) { + if (!isRecord(entry)) continue; + lines.push( + ...formatProviderSummary(entry).map((line) => + line ? ` ${line}` : line + ) + ); + } + } + lines.push(''); + } + lines.push('Next: firecrawl exchange discover '); + return lines.join('\n'); + } + + if (Array.isArray(data.providers) && str(data.cohort)) { + lines.push(`=== Providers in ${data.cohort} ===`); + lines.push(''); + for (const entry of data.providers) { + if (isRecord(entry)) lines.push(...formatProviderSummary(entry)); + } + lines.push(`Next: firecrawl exchange discover ${data.cohort} `); + return lines.join('\n'); + } + + if (Array.isArray(data.concepts) && str(data.provider)) { + const slug = data.provider as string; + const name = str(data.name); + lines.push(`=== ${name ? `${name} (${slug})` : slug} ===`); + const description = str(data.description); + if (description) lines.push(description); + const website = str(data.website); + if (website) lines.push(`Website: ${website}`); + lines.push(''); + lines.push(...formatConcepts(slug, data.concepts)); + lines.push( + `Next: firecrawl exchange discover ${slug} ` + ); + return lines.join('\n'); + } + + if (str(data.capability) && str(data.provider)) { + const address = `${data.provider}/${data.capability}`; + lines.push(`=== ${address} ===`); + const label = str(data.label); + if (label) lines.push(label); + if (data.creditsCost !== undefined) { + lines.push(`Credits cost: ${data.creditsCost}`); + } + if (data.executable === false) { + const reason = str(data.reason); + lines.push(`Not executable${reason ? `: ${reason}` : ''}`); + } + const whenToUse = str(data.whenToUse); + if (whenToUse) lines.push(`When to use: ${whenToUse}`); + lines.push(''); + const options = asArray(data.options); + lines.push('Options:'); + if (options.length === 0) lines.push(' (none)'); + for (const option of options) { + if (!isRecord(option)) continue; + const { name, description, about, ...rest } = option; + const details = Object.entries(rest) + .map(([key, value]) => `${key}: ${JSON.stringify(value)}`) + .join(', '); + lines.push(` ${str(name) ?? '?'}${details ? ` (${details})` : ''}`); + const summary = str(description) ?? str(about); + if (summary) lines.push(` ${summary}`); + } + const requiresOneOf = asArray(data.requiresOneOf); + if (requiresOneOf.length > 0) { + lines.push(`Requires one of: ${requiresOneOf.join(', ')}`); + } + if (data.returns !== undefined) { + lines.push(''); + lines.push('Returns:'); + lines.push(indent(stringifyValue(data.returns))); + } + lines.push(''); + const example = isRecord(data.example) ? data.example : undefined; + const exampleOptions = + example && isRecord(example.options) ? example.options : example; + lines.push( + `Retrieve: firecrawl exchange retrieve ${address}${ + exampleOptions + ? ` --options '${JSON.stringify(exampleOptions)}'` + : " --options ''" + }` + ); + return lines.join('\n'); + } + + return JSON.stringify(data, null, 2); +} + +function stringifyJson(payload: unknown, pretty?: boolean): string { + return pretty ? JSON.stringify(payload, null, 2) : JSON.stringify(payload); +} + +export function writeExchangeOutput( + jsonPayload: unknown, + readable: string, + options: { output?: string; json?: boolean; pretty?: boolean } +): void { + const content = + options.json || options.pretty + ? stringifyJson(jsonPayload, options.pretty) + : readable; + writeOutput(content, options.output, !!options.output); +} + +/** + * Request-level failure: `--json` gets a `{success:false, ...}` envelope on + * stdout; human mode gets the terms block or a one-line error on stderr. + */ +function reportExchangeFailure( + result: ExchangeRetrieveResult, + options: { json?: boolean; pretty?: boolean } +): void { + if (options.json || options.pretty) { + writeOutput( + stringifyJson( + { + success: false, + requestId: result.requestId, + code: result.code, + error: result.error, + requiresAction: result.requiresAction, + }, + options.pretty + ) + ); + } else if (result.requiresAction) { + process.stderr.write(formatTermsRequired(result.requiresAction)); + } else { + console.error('Error:', formatExchangeFailure(result)); + } +} + +/** + * Handle exchange discover command output + */ +export async function handleExchangeDiscoverCommand( + options: ExchangeDiscoverOptions +): Promise { + const result = await executeExchangeDiscover(options); + + if (!result.success || !result.data) { + console.error('Error:', result.error ?? 'Unknown error occurred'); + process.exit(1); + } + + writeExchangeOutput( + result.data, + formatDiscoverReadable(result.data), + options + ); +} + +/** + * Handle exchange retrieve command output. A provider error inside the batch + * does not fail the request; the exit code is 1 only when every item failed. + */ +export async function handleExchangeRetrieveCommand( + options: ExchangeRetrieveOptions +): Promise { + const result = await executeExchangeRetrieve(options); + if (result.requestId) + process.stderr.write(`Request ID: ${result.requestId}\n`); + + if (!result.success) { + reportExchangeFailure(result, options); + if (result.requestId) + process.stderr.write( + `If retrying the identical payload, reuse --request-id ${result.requestId}. Do not replace the ID for pending or uncertain execution.\n` + ); + process.exit(1); + } + + if (result.scrapeId) { + process.stderr.write(`Scrape ID: ${result.scrapeId}\n`); + } + + const jsonPayload: Record = { + success: true, + requestId: result.requestId, + data: { + alexandria: result.exchange ?? [], + creditsCost: result.creditsCost, + }, + }; + if (result.scrapeId) { + jsonPayload.scrape_id = result.scrapeId; + } + + writeExchangeOutput(jsonPayload, formatRetrieveReadable(result), options); + + const items = result.exchange ?? []; + if (items.length > 0 && items.every(isFailure)) { + process.exit(1); + } +} diff --git a/src/commands/find-tools.ts b/src/commands/find-tools.ts new file mode 100644 index 0000000000..160537b049 --- /dev/null +++ b/src/commands/find-tools.ts @@ -0,0 +1,123 @@ +import { Command, Option } from 'commander'; +import { handleExchangeRetrieveCommand } from './exchange'; +import type { ExchangeCall } from '../types/exchange'; + +const provider = 'firecrawl'; +const capability = 'find-tools'; +const selectors = [ + 'providers', + 'categories', + 'groups', + 'capabilities', + 'expand', +]; + +export function buildFindToolsCall( + urls: string[], + input: Record +): ExchangeCall { + if (input.request) { + if ( + urls.length || + [...selectors, 'level', 'limit', 'offset'].some( + (key) => input[key] !== undefined + ) + ) { + throw new Error('--request cannot be combined with lookup filters.'); + } + const call = JSON.parse(input.request); + if ( + call?.provider !== provider || + call?.capability !== capability || + Object.keys(call).some( + (key) => !['provider', 'capability', 'options'].includes(key) + ) + ) { + throw new Error( + '--request must be a next request returned by Find Tools.' + ); + } + if ( + !call.options || + typeof call.options !== 'object' || + Array.isArray(call.options) + ) + throw new Error('Find Tools options must be an object.'); + return call; + } + const options: Record = {}; + if (urls.length) options.urls = urls; + for (const key of selectors) + if (input[key]) + options[key] = input[key] + .split(',') + .map((s: string) => s.trim()) + .filter(Boolean); + for (const key of ['level', 'limit', 'offset']) + if (input[key] !== undefined) options[key] = input[key]; + if ( + input.limit !== undefined && + (!Number.isInteger(input.limit) || input.limit < 1 || input.limit > 100) + ) + throw new Error('--limit must be an integer from 1 to 100.'); + if ( + input.offset !== undefined && + (!Number.isInteger(input.offset) || input.offset < 0) + ) + throw new Error('--offset must be a non-negative integer.'); + return { provider, capability, options }; +} + +export function createFindToolsCommand(name = 'find-tools'): Command { + const command = new Command(name) + .description( + 'Free contextual lookup and progressive disclosure. Follow next with --request; use search --sources alexandria for semantic discovery.' + ) + .argument( + '[urls...]', + 'Page URLs from search or scrape; pages are not fetched' + ) + .option('--providers ', 'Provider IDs, comma-separated') + .option('--categories ', 'Catalogue categories, e.g. finance,podcasts') + .option('--groups ', 'Group IDs returned by Find Tools') + .option('--capabilities ', 'Exact capability addresses') + .addOption( + new Option( + '--level ', + 'Optional depth; inferred from selectors' + ).choices(['providers', 'groups', 'tools']) + ) + .option( + '--expand ', + 'Contract sections: options,response,examples' + ) + .option('--limit ', 'Results per page, 1-100 (default 5)', Number) + .option( + '--offset ', + 'Page offset; prefer --request to preserve scope', + Number + ) + .option( + '--request ', + 'A complete next request from the previous result' + ) + .option('--request-id ', 'Reuse for retrying an identical request') + .option('-k, --api-key ', 'Firecrawl API key') + .option('--api-url ', 'API URL') + .option('--json', 'Output JSON') + .option('--pretty', 'Output formatted JSON') + .option('-o, --output ', 'Output file path') + .action(async (urls: string[], options) => { + try { + const call = buildFindToolsCall(urls, options); + await handleExchangeRetrieveCommand({ ...options, calls: [call] }); + } catch (error) { + console.error( + 'Error:', + error instanceof Error ? error.message : String(error) + ); + process.exitCode = 1; + } + }); + return command; +} diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index c4d86f7c15..5bd0c1ae9d 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -18,6 +18,33 @@ import { import { getOrigin } from '../utils/url'; import { executeMap } from './map'; import { getStatus } from './status'; +import { buildExchangeCalls, handleExchangeRetrieveCommand } from './exchange'; +import type { ExchangeRetrieveOptions } from '../types/exchange'; +import { formatToolsCompact } from '../utils/alexandria'; +import { parseRequiresAction } from '../utils/terms'; + +/** + * `firecrawl scrape --exchange provider/capability --options ''`: + * a url-less scrape that executes Exchange capabilities. Delegates to the + * same executor as `firecrawl exchange retrieve`. + */ +export async function handleScrapeExchangeCommand( + addresses: string[], + optionsJson: string[], + options: Omit +): Promise { + let calls: ExchangeRetrieveOptions['calls']; + try { + calls = buildExchangeCalls(addresses, optionsJson); + } catch (error) { + console.error( + 'Error:', + error instanceof Error ? error.message : 'Unknown error occurred' + ); + process.exit(1); + } + await handleExchangeRetrieveCommand({ ...options, calls }); +} /** * Output timing information if requested @@ -138,6 +165,10 @@ export async function executeScrape( scrapeParams.redactPII = true; } + if (options.domainTools) { + scrapeParams.domainTools = true; + } + // Execute scrape with timing - only wrap the scrape call in try-catch const requestStartTime = Date.now(); @@ -186,10 +217,18 @@ export async function executeScrape( const requestEndTime = Date.now(); outputTiming(options, requestStartTime, requestEndTime, error); - return { + const failure: ScrapeResult = { success: false, error: error instanceof Error ? error.message : 'Unknown error occurred', }; + const code = (error as any)?.code; + if (typeof code === 'string') { + failure.code = code; + failure.requiresAction = parseRequiresAction( + (error as any)?.details?.requiresAction + ); + } + return failure; } } @@ -201,6 +240,19 @@ export async function handleScrapeCommand( ): Promise { const result = await executeScrape(options); + // --domain-tools: print the matched tools alongside the scrape output, + // unless JSON is forced (JSON output already includes data.tools as-is). + if ( + options.domainTools && + result.success && + result.data?.tools && + !options.json && + !options.output + ) { + console.error(formatToolsCompact(result.data.tools)); + console.error(''); + } + // Query mode: output answer directly if (options.query && result.success && result.data?.answer) { writeOutput(result.data.answer, options.output, !!options.output); diff --git a/src/commands/search.ts b/src/commands/search.ts index 1abda30b17..b481658d9e 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -11,9 +11,16 @@ import type { ImageSearchResult, NewsSearchResult, DeveloperSearchResult, + ExchangeSearchResult, } from '../types/search'; import { getClient, isKeylessMode, keylessRequest } from '../utils/client'; import { writeOutput } from '../utils/output'; +import { assertExchangeKeyed, exchangeErrorMessage } from './exchange'; +import { + normalizeSources, + hasAlexandria, + formatTools, +} from '../utils/alexandria'; /** * Execute search command @@ -34,10 +41,23 @@ export async function executeSearch( // Add sources if specified if (options.sources && options.sources.length > 0) { - searchParams.sources = options.sources.map((source) => ({ - type: source, - })); + // The exchange source is gated per team; the keyless tier has no team, + // so refuse here rather than surface an opaque 403. + if (hasAlexandria(options.sources)) { + assertExchangeKeyed(options.apiKey, options.apiUrl); + } + searchParams.sources = normalizeSources(options.sources).map((source) => + typeof source === 'string' ? { type: source } : source + ); } + if (options.domainTools) { + assertExchangeKeyed(options.apiKey, options.apiUrl); + searchParams.domainTools = true; + } + if (!options.query.trim()) + throw new Error( + 'A query is required for search. Use firecrawl find-tools for catalogue lookup.' + ); // Add categories if specified if (options.categories && options.categories.length > 0) { @@ -94,7 +114,7 @@ export async function executeSearch( } const searchBody = { - query: options.query, + ...(options.query.trim() ? { query: options.query } : {}), ...searchParams, }; @@ -121,6 +141,7 @@ export async function executeSearch( const payload = (envelope.data ?? {}) as Record; const data: SearchResultData = {}; + if (payload.tools) data.tools = payload.tools; if (payload.web) data.web = payload.web as WebSearchResult[]; if (payload.images) data.images = payload.images as ImageSearchResult[]; if (payload.news) data.news = payload.news as NewsSearchResult[]; @@ -128,6 +149,11 @@ export async function executeSearch( // results, so the API returns its hits in their own group. if (payload.developer) data.developer = payload.developer as DeveloperSearchResult[]; + // Exchange hits are capability matches (never documents). The API omits + // the group entirely when the Exchange is unreachable, so pass it through + // exactly as received. + if (payload.exchange) + data.exchange = payload.exchange as ExchangeSearchResult[]; return { success: true, @@ -139,7 +165,7 @@ export async function executeSearch( } catch (error) { return { success: false, - error: error instanceof Error ? error.message : 'Unknown error occurred', + error: exchangeErrorMessage(error), }; } } @@ -170,9 +196,11 @@ function formatSearchReadable( // Label the web group whenever another group follows it, so the reader can // tell the groups apart. const hasDeveloperResults = !!data.developer && data.developer.length > 0; + const hasExchangeResults = !!data.exchange && data.exchange.length > 0; if ( (options.sources && options.sources.length > 1) || - hasDeveloperResults + hasDeveloperResults || + hasExchangeResults ) { lines.push('=== Web Results ==='); lines.push(''); @@ -270,6 +298,39 @@ function formatSearchReadable( } } + // Format exchange capability hits + if (data.exchange && data.exchange.length > 0) { + if (lines.length > 0) { + lines.push(''); + } + lines.push('=== Exchange Providers ==='); + lines.push(''); + + for (const hit of data.exchange) { + lines.push(`${hit.provider}/${hit.capability}`); + if (hit.concept) { + lines.push(` Concept: ${hit.concept}`); + } + if (hit.cohorts && hit.cohorts.length > 0) { + lines.push(` Cohorts: ${hit.cohorts.join(', ')}`); + } + if (hit.creditsCost !== undefined) { + lines.push(` Credits per call: ${hit.creditsCost}`); + } + if (hit.similarity !== undefined) { + lines.push(` Similarity: ${hit.similarity}`); + } + const cohort = hit.cohorts?.[0]; + if (cohort) { + lines.push( + ` Contract: firecrawl exchange discover ${cohort} ${hit.provider} ${hit.capability}` + ); + } + lines.push(''); + } + } + + if (data.tools) lines.push(formatTools(data.tools)); return lines.join('\n'); } @@ -295,7 +356,9 @@ export async function handleSearchCommand( (result.data.web && result.data.web.length > 0) || (result.data.images && result.data.images.length > 0) || (result.data.news && result.data.news.length > 0) || - (result.data.developer && result.data.developer.length > 0); + (result.data.developer && result.data.developer.length > 0) || + (result.data.exchange && result.data.exchange.length > 0) || + result.data.tools?.length; if (!hasResults) { console.log('No results found.'); diff --git a/src/commands/skills.ts b/src/commands/skills.ts new file mode 100644 index 0000000000..e97a94e25e --- /dev/null +++ b/src/commands/skills.ts @@ -0,0 +1,45 @@ +import { getClient } from '../utils/client'; +import { writeOutput } from '../utils/output'; +import { assertExchangeKeyed, exchangeErrorMessage } from './exchange'; + +export async function handleSkillCommand(options: { + urls?: string[]; + query?: string; + context?: 'search' | 'scrape'; + id?: string; + apiKey?: string; + apiUrl?: string; + output?: string; +}): Promise { + try { + assertExchangeKeyed(options.apiKey, options.apiUrl); + if (!options.id && !options.urls?.length && !options.query?.trim()) + throw new Error('Provide at least one URL or --query.'); + if (options.id === '.' || options.id === '..') + throw new Error('Invalid skill ID.'); + const app = getClient(options) as any; + const response = options.id + ? await app.http.get( + `/exchange/skills/${encodeURIComponent(options.id)}/SKILL.md` + ) + : await app.http.post( + '/exchange/skills/resolve', + JSON.stringify({ + urls: options.urls ?? [], + query: options.query, + context: options.context, + }), + { headers: { 'Content-Type': 'application/json' } } + ); + writeOutput( + typeof response.data === 'string' + ? response.data + : JSON.stringify(response.data, null, 2), + options.output, + !!options.output + ); + } catch (error) { + console.error('Error:', exchangeErrorMessage(error)); + process.exit(1); + } +} diff --git a/src/commands/terms.ts b/src/commands/terms.ts new file mode 100644 index 0000000000..d8e1df4e28 --- /dev/null +++ b/src/commands/terms.ts @@ -0,0 +1,243 @@ +/** + * `firecrawl alexandria terms ` shows a provider's terms document + * from GET /exchange/provider-terms; `firecrawl alexandria terms accept + * ` shows it, asks the user to type the provider slug, and posts the + * acceptance to the dashboard. Acceptance is interactive only. + */ + +import type { + ExchangeProviderTerms, + ExchangeTermsOptions, +} from '../types/exchange'; +import { getClient } from '../utils/client'; +import { getApiKey, getDashboardUrl } from '../utils/config'; +import { writeOutput } from '../utils/output'; +import { buildFindToolsCall } from './find-tools'; +import { + assertExchangeKeyed, + exchangeErrorMessage, + executeExchangeRetrieve, + writeExchangeOutput, +} from './exchange'; + +async function fetchProviderTermsList( + options: ExchangeTermsOptions +): Promise { + assertExchangeKeyed(options.apiKey, options.apiUrl); + const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); + const response = await (app as any).http.get( + '/exchange/provider-terms?surface=web' + ); + return Array.isArray(response?.data?.providers) + ? response.data.providers + : []; +} + +/** + * The provider-terms list only carries providers that are gated (plus + * firecrawl and fiscal-ai). A provider absent from it may still exist in the + * catalogue with no terms of its own, so fall back to a free Find Tools + * lookup before calling it unknown. + */ +async function providerExistsInCatalogue( + provider: string, + options: ExchangeTermsOptions +): Promise { + const call = buildFindToolsCall([], { + providers: provider, + level: 'providers', + limit: 1, + }); + const result = await executeExchangeRetrieve({ + apiKey: options.apiKey, + apiUrl: options.apiUrl, + calls: [call], + }); + if (!result.success) return false; + const item = (result.exchange ?? [])[0] as any; + if (!item || item.error) return false; + const items = Array.isArray(item.data?.items) ? item.data.items : []; + return items.length > 0; +} + +async function resolveProviderTerms( + options: ExchangeTermsOptions +): Promise { + const providers = await fetchProviderTermsList(options); + const entry = providers.find((p) => p.provider === options.provider); + if (entry) return entry; + if (await providerExistsInCatalogue(options.provider, options)) { + return { provider: options.provider, terms: null }; + } + throw new Error( + `Unknown Alexandria provider "${options.provider}". List providers with "firecrawl alexandria discover".` + ); +} + +function writeNoTermsOutput( + entry: ExchangeProviderTerms, + options: ExchangeTermsOptions +): void { + if (options.json || options.pretty) { + writeExchangeOutput( + { provider: entry.provider, terms: null }, + `${entry.provider} has no provider terms to accept.\n`, + options + ); + return; + } + process.stderr.write(`${entry.provider} has no provider terms to accept.\n`); +} + +function formatTermsReadable(entry: ExchangeProviderTerms): string { + const terms = entry.terms!; + const lines = [ + `${entry.name ?? entry.provider} (${entry.provider}) Alexandria provider terms`, + ]; + if (terms.publisher) lines.push(`Publisher: ${terms.publisher}`); + lines.push(`Version: ${terms.version}`); + if (terms.effective) lines.push(`Effective: ${terms.effective}`); + lines.push('', terms.document.trimEnd(), ''); + return lines.join('\n'); +} + +function dashboardTermsUrl(provider: string): string { + return `${getDashboardUrl()}/app/alexandria/${encodeURIComponent(provider)}`; +} + +/** + * Show a provider's terms. Exits 0 with a note when the provider has none. + */ +export async function handleExchangeTermsCommand( + options: ExchangeTermsOptions +): Promise { + let entry: ExchangeProviderTerms; + try { + entry = await resolveProviderTerms(options); + } catch (error) { + console.error('Error:', exchangeErrorMessage(error)); + process.exit(1); + } + if (!entry.terms) { + writeNoTermsOutput(entry, options); + return; + } + writeExchangeOutput(entry, formatTermsReadable(entry), options); +} + +async function postAcceptance( + provider: string, + version: string, + apiKey: string +): Promise<{ status: number; body: Record }> { + const response = await fetch( + `${getDashboardUrl()}/api/exchange/provider-access/accept`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ provider, version, confirmed: true }), + } + ); + const body = await response.json().catch(() => ({})); + return { status: response.status, body: body ?? {} }; +} + +function acceptFailureMessage( + provider: string, + status: number, + body: Record +): string { + switch (status) { + case 401: + return 'The dashboard rejected this API key. Check FIRECRAWL_API_KEY or run "firecrawl login".'; + case 403: + return `Only a team admin's own API key can accept provider terms. Ask an organization admin to accept at ${dashboardTermsUrl(provider)}.`; + case 404: + return `The dashboard does not know the provider "${provider}".`; + case 409: + return `The ${provider} terms changed${ + typeof body.version === 'string' + ? ` (current version ${body.version})` + : '' + } since they were displayed. Nothing was accepted; rerun "firecrawl alexandria terms accept ${provider}" to review the current version.`; + default: + return typeof body.error === 'string' + ? body.error + : `Accepting provider terms failed (HTTP ${status})`; + } +} + +/** + * Interactive acceptance: refuse without a TTY, show the document, require + * the provider slug typed back, then post the displayed version. + */ +export async function handleExchangeTermsAcceptCommand( + options: ExchangeTermsOptions +): Promise { + if (!process.stdin.isTTY || !process.stdout.isTTY) { + console.error( + `Error: Accepting provider terms needs an interactive terminal. Accept in the dashboard at ${dashboardTermsUrl(options.provider)} instead.` + ); + process.exit(2); + } + + let entry: ExchangeProviderTerms; + try { + entry = await resolveProviderTerms(options); + } catch (error) { + console.error('Error:', exchangeErrorMessage(error)); + process.exit(1); + } + if (!entry.terms) { + writeNoTermsOutput(entry, options); + return; + } + const { provider } = entry; + const { version } = entry.terms; + writeOutput(formatTermsReadable(entry)); + + const { input } = await import('@inquirer/prompts'); + const typed = await input({ + message: `Accept ${provider} terms version ${version} for your organization? Type "${provider}" to confirm:`, + }); + if (typed.trim() !== provider) { + console.error('Error: Confirmation did not match. Nothing was accepted.'); + process.exit(1); + } + + const apiKey = getApiKey(options.apiKey); + if (!apiKey) { + console.error( + 'Error: An API key is required to accept provider terms. Set FIRECRAWL_API_KEY or run "firecrawl login".' + ); + process.exit(1); + } + + let outcome: Awaited>; + try { + outcome = await postAcceptance(provider, version, apiKey); + } catch (error) { + console.error( + 'Error:', + error instanceof Error ? error.message : 'Unknown error occurred' + ); + process.exit(1); + } + if (outcome.status !== 200 || outcome.body.success !== true) { + console.error( + 'Error:', + acceptFailureMessage(provider, outcome.status, outcome.body) + ); + process.exit(1); + } + process.stderr.write( + `Accepted ${provider} provider terms version ${outcome.body.version ?? version}${ + typeof outcome.body.digest === 'string' + ? ` (digest ${outcome.body.digest})` + : '' + }. Rerun the request with the same --request-id.\n` + ); +} diff --git a/src/index.ts b/src/index.ts index 9bef16f0fd..e195670e46 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,7 +11,13 @@ import { handleScrapeCommand, handleMultiScrapeCommand, handleAllScrapeCommand, + handleScrapeExchangeCommand, } from './commands/scrape'; +import { + buildExchangeCalls, + handleExchangeDiscoverCommand, + handleExchangeRetrieveCommand, +} from './commands/exchange'; import { initializeConfig, updateConfig } from './utils/config'; import { configure, viewConfig } from './commands/config'; import { handleCreditUsageCommand } from './commands/credit-usage'; @@ -20,6 +26,13 @@ import { handleMapCommand } from './commands/map'; import { handleParseCommand } from './commands/parse'; import { createMonitorCommand } from './commands/monitor'; import { handleSearchCommand } from './commands/search'; +import { addAlexandriaOptions, parseSearchSources } from './utils/alexandria'; +import { handleSkillCommand } from './commands/skills'; +import { + handleExchangeTermsAcceptCommand, + handleExchangeTermsCommand, +} from './commands/terms'; +import { createFindToolsCommand } from './commands/find-tools'; import { handleDeveloperSearchCommand } from './commands/developer'; import { handleInspectPaperCommand, @@ -210,6 +223,13 @@ function parseWebhookOption( return trimmed; } +function collectRepeatable( + value: string, + previous: string[] | undefined +): string[] { + return [...(previous ?? []), value]; +} + function parseCommaList(raw: string | undefined): string[] | undefined { if (!raw) return undefined; const values = raw @@ -411,11 +431,37 @@ function createScrapeCommand(): Command { 'Redact personally identifiable information from returned content', false ) + .option( + '--domain-tools', + 'Include domain-matched tool contracts alongside the scrape result' + ) .option('--schema ', 'JSON schema for structured extraction') .option('--schema-file ', 'Path to JSON schema file') .option('--actions ', 'JSON actions array to run during scrape') .option('--actions-file ', 'Path to JSON actions file') .option('--proxy ', 'Proxy mode for scraping (e.g., auto, basic)') + .option( + '--alexandria ', + 'Execute an Alexandria capability instead of scraping a URL (repeatable, up to 10). Requires an API key on a team with Alexandria access.', + collectRepeatable + ) + .addOption( + new Option( + '--exchange ', + 'Deprecated alias for --alexandria' + ) + .argParser(collectRepeatable) + .hideHelp() + ) + .option( + '--options ', + 'JSON options for the --alexandria capability at the same position (repeatable)', + collectRepeatable + ) + .option( + '--request-id ', + 'Execution ID; reuse for retries of the same Exchange payload' + ) .action(async (positionalArgs, options) => { // Collect URLs from positional args and --url option @@ -436,6 +482,33 @@ function createScrapeCommand(): Command { // Remove duplicates urls = [...new Set(urls)]; + // Alexandria execution is url-less; the API rejects url + alexandria together. + const alexandriaAddresses = [ + ...(options.alexandria ?? []), + ...(options.exchange ?? []), + ]; + if (alexandriaAddresses.length > 0) { + if (urls.length > 0) { + console.error( + 'Error: --alexandria cannot be combined with a URL. Scrape a URL or execute Alexandria capabilities, not both.' + ); + process.exit(1); + } + await handleScrapeExchangeCommand( + alexandriaAddresses, + options.options ?? [], + { + apiKey: options.apiKey, + apiUrl: options.apiUrl, + requestId: options.requestId, + output: options.output, + json: options.json, + pretty: options.pretty, + } + ); + return; + } + if (urls.length === 0) { console.error( 'Error: URL is required. Provide it as argument or use --url option.' @@ -907,8 +980,8 @@ Max upload size: 50 MB */ function createSearchCommand(): Command { const searchCmd = new Command('search') - .description('Search the web using Firecrawl') - .argument('', 'Search query') + .description('Search the web or discover Alexandria tools') + .argument('', 'Semantic search query') .option( '--limit ', 'Maximum number of results (default: 5, max: 100)', @@ -916,7 +989,7 @@ function createSearchCommand(): Command { ) .option( '--sources ', - 'Comma-separated sources to search: web, images, news (default: web)' + 'Sources as comma-separated names or a JSON array: web, images, news, alexandria (default: web). Alexandria discovers accessible tools for free; exchange is a compatibility alias.' ) .option( '--categories ', @@ -975,23 +1048,15 @@ function createSearchCommand(): Command { // ) .option('--json', 'Output as compact JSON', false) .action(async (query, options) => { - // Parse sources let sources: SearchSource[] | undefined; - if (options.sources) { - sources = options.sources - .split(',') - .map((s: string) => s.trim().toLowerCase()) as SearchSource[]; - - // Validate sources - const validSources = ['web', 'images', 'news']; - for (const source of sources) { - if (!validSources.includes(source)) { - console.error( - `Error: Invalid source "${source}". Valid sources: ${validSources.join(', ')}` - ); - process.exit(1); - } - } + try { + sources = parseSearchSources(options.sources); + } catch (error) { + console.error( + 'Error:', + error instanceof Error ? error.message : String(error) + ); + process.exit(1); } // Parse categories @@ -1022,9 +1087,10 @@ function createSearchCommand(): Command { } const searchOptions = { - query, + query: query ?? '', limit: options.limit, sources, + domainTools: options.domainTools, categories, tbs: options.tbs, location: options.location, @@ -1045,7 +1111,172 @@ function createSearchCommand(): Command { await handleSearchCommand(searchOptions); }); - return searchCmd; + return addAlexandriaOptions(searchCmd); +} + +/** + * Create and configure the alexandria command group. Registered twice: as + * `alexandria` and as the hidden legacy alias `exchange`. + */ +function createExchangeCommand(name: string = 'alexandria'): Command { + const exchangeCmd = new Command(name) + .description( + 'Discover and call Firecrawl Alexandria data providers. Requires an API key on a team with Alexandria access (no keyless fallback).' + ) + .addHelpText( + 'after', + ` +Examples: + $ firecrawl ${name} discover # cohorts + $ firecrawl ${name} discover finance # providers in a cohort + $ firecrawl ${name} discover finance fred # what a provider can do + $ firecrawl ${name} discover finance fred series/observations # full contract + $ firecrawl ${name} discover --query "balance sheet" --limit 8 # semantic lookup + $ firecrawl ${name} retrieve fred/series/observations --options '{"series_id":"CPIAUCSL"}' + $ firecrawl ${name} terms benzinga # read a provider's terms + $ firecrawl ${name} terms accept benzinga # accept them (admin, interactive) + $ firecrawl search "nvidia balance sheet" --sources web,alexandria --json` + ); + + exchangeCmd + .command('discover') + .description( + 'Browse the Exchange catalogue. No arguments lists cohorts; add a cohort, then a provider, then a capability address to read its full contract (options, returns, creditsCost). --query ranks capabilities semantically across the whole catalogue.' + ) + .argument('[cohort]', 'Cohort slug, e.g. finance') + .argument('[provider]', 'Provider slug, e.g. fred') + .argument('[capability]', 'Capability address, e.g. series/observations') + .option( + '-q, --query ', + 'Semantic lookup across the whole catalogue (cannot be combined with path arguments)' + ) + .option( + '--limit ', + 'Maximum semantic hits (1-24, default: 8)', + parseInt + ) + .option( + '--expand ', + 'Inline more of the tree on a walk: capabilities, contracts, examples, all' + ) + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) + .option('--api-url ', 'API URL (overrides global --api-url)') + .option('-o, --output ', 'Output file path (default: stdout)') + .option('--json', 'Output as compact JSON', false) + .option('--pretty', 'Pretty print JSON output', false) + .action(async (cohort, provider, capability, options) => { + await handleExchangeDiscoverCommand({ + cohort, + provider, + capability, + query: options.query, + limit: options.limit, + expand: options.expand, + apiKey: options.apiKey, + apiUrl: options.apiUrl, + output: options.output, + json: options.json, + pretty: options.pretty, + }); + }); + + exchangeCmd + .command('retrieve') + .description( + 'Execute one or more Exchange capabilities (up to 10) through /v2/scrape and print each result with its creditsCost. Read the contract first with `exchange discover`.' + ) + .argument( + '', + 'provider/capability addresses, e.g. fred/series/observations' + ) + .option( + '--options ', + 'JSON options for the address at the same position (repeatable)', + collectRepeatable + ) + .option( + '--request-id ', + 'Execution ID; reuse for retries of the same payload' + ) + .option('--timeout ', 'Timeout in milliseconds', parseInt) + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) + .option('--api-url ', 'API URL (overrides global --api-url)') + .option('-o, --output ', 'Output file path (default: stdout)') + .option('--json', 'Output as compact JSON', false) + .option('--pretty', 'Pretty print JSON output', false) + .action(async (addresses: string[], options) => { + let calls; + try { + calls = buildExchangeCalls(addresses, options.options); + } catch (error) { + console.error( + 'Error:', + error instanceof Error ? error.message : 'Unknown error occurred' + ); + process.exit(1); + } + await handleExchangeRetrieveCommand({ + calls, + requestId: options.requestId, + timeout: options.timeout, + apiKey: options.apiKey, + apiUrl: options.apiUrl, + output: options.output, + json: options.json, + pretty: options.pretty, + }); + }); + + const termsCmd = exchangeCmd + .command('terms') + .description( + "Show an Alexandria provider's terms document. Paid providers need an organization admin to accept their terms before retrieve runs." + ) + .argument('', 'Provider slug, e.g. benzinga') + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) + .option('--api-url ', 'API URL (overrides global --api-url)') + .option('-o, --output ', 'Output file path (default: stdout)') + .option('--json', 'Output the provider entry as compact JSON', false) + .option('--pretty', 'Pretty print JSON output', false) + .action(async (provider: string, options) => { + await handleExchangeTermsCommand({ ...options, provider }); + }); + termsCmd + .command('accept') + .description( + "Accept a provider's current terms for your organization. Interactive only: shows the document and asks you to type the provider slug. Needs a team admin's own API key." + ) + .argument('', 'Provider slug, e.g. benzinga') + .option( + '-k, --api-key ', + 'Firecrawl API key (overrides global --api-key)' + ) + .option('--api-url ', 'API URL (overrides global --api-url)') + .action(async (provider: string, options) => { + await handleExchangeTermsAcceptCommand({ ...options, provider }); + }); + + exchangeCmd.addCommand(createFindToolsCommand('tools')); + exchangeCmd + .command('skill') + .description('Read a resolved tool skill document as Markdown') + .argument('', 'Skill ID returned by contextual tool lookup') + .option('-k, --api-key ', 'Firecrawl API key') + .option('--api-url ', 'API URL') + .option('-o, --output ', 'Output file path') + .action(async (id: string, options) => + handleSkillCommand({ ...options, id }) + ); + return exchangeCmd; } /** @@ -2092,6 +2323,9 @@ program.addCommand(createMapCommand()); program.addCommand(createParseCommand()); program.addCommand(createMonitorCommand()); program.addCommand(createSearchCommand()); +program.addCommand(createExchangeCommand()); +program.addCommand(createExchangeCommand('exchange'), { hidden: true }); +program.addCommand(createFindToolsCommand()); program.addCommand(createDeveloperCommand()); program.addCommand(createResearchCommand()); program.addCommand(createFeedbackCommand()); diff --git a/src/types/exchange.ts b/src/types/exchange.ts new file mode 100644 index 0000000000..ef99b75384 --- /dev/null +++ b/src/types/exchange.ts @@ -0,0 +1,137 @@ +/** + * Types for the exchange commands + * + * Mirrors the frozen Firecrawl Exchange interface: discovery goes through + * GET /exchange/discover and execution goes through POST /v2/scrape with an + * `alexandria` array (url-less scrape). + */ + +/** One capability to execute: `provider/capability` plus its options. */ +export interface ExchangeCall { + provider: string; + capability: string; + options?: Record; +} + +interface ExchangeCommonOptions { + /** API key for Firecrawl */ + apiKey?: string; + /** API URL for Firecrawl */ + apiUrl?: string; + /** Output file path */ + output?: string; + /** Output as JSON format */ + json?: boolean; + /** Pretty print JSON output */ + pretty?: boolean; +} + +export interface ExchangeDiscoverOptions extends ExchangeCommonOptions { + /** Cohort slug (first rung of the walk), e.g. `finance` */ + cohort?: string; + /** Provider slug (second rung), e.g. `fred` */ + provider?: string; + /** Capability address (third rung), e.g. `series/observations` */ + capability?: string; + /** Semantic lookup across the whole catalogue (root only) */ + query?: string; + /** Result cap for semantic lookup (1..24) */ + limit?: number; + /** How much of the tree to inline on a walk: capabilities, contracts, examples, all */ + expand?: string; +} + +/** The API's `requiresAction` payload on a THIRD_PARTY_DATA_TERMS_REQUIRED failure. */ +export interface ExchangeRequiresAction { + type: 'accept_terms'; + terms: string; + version: string; + url: string; +} + +export interface ExchangeTermsOptions extends ExchangeCommonOptions { + /** Provider slug, e.g. `benzinga` */ + provider: string; +} + +/** One entry of GET /exchange/provider-terms. `terms` is null when none apply. */ +export interface ExchangeProviderTerms { + provider: string; + name?: string; + website?: string; + required?: boolean; + terms: { + key?: string; + version: string; + effective?: string; + publisher?: string; + body?: string[]; + document: string; + digest?: string; + } | null; +} + +export interface ExchangeRetrieveOptions extends ExchangeCommonOptions { + requestId?: string; + /** 1..10 capabilities to execute in one request */ + calls: ExchangeCall[]; + /** Timeout in milliseconds forwarded to /v2/scrape */ + timeout?: number; +} + +/** A successful per-item execution result. */ +export interface ExchangeScrapeSuccess { + provider: string; + capability: string; + creditsCost: number; + data: unknown; + records?: number; + upstreamStatus?: number; + recordedAt?: string; + [key: string]: unknown; +} + +/** A failed per-item execution result. The batch itself still returns 200. */ +export interface ExchangeScrapeFailure { + provider?: string; + capability?: string; + error: { code: string; message: string; status?: number }; + [key: string]: unknown; +} + +export type ExchangeScrapeResult = + | ExchangeScrapeSuccess + | ExchangeScrapeFailure; + +export interface ExchangeScrapeResponse { + success: boolean; + scrape_id?: string; + data?: { + alexandria: ExchangeScrapeResult[]; + creditsCost: number; + }; + error?: string; +} + +export interface ExchangeDiscoverResult { + success: boolean; + /** The discover payload exactly as returned by the API */ + data?: Record; + error?: string; +} + +export interface ExchangeRetrieveResult { + requestId?: string; + success: boolean; + scrapeId?: string; + exchange?: ExchangeScrapeResult[]; + /** Sum of the successful items' creditsCost, passed through untouched */ + creditsCost?: number; + error?: string; + /** Error code from the request-level failure body, e.g. `duplicate_request` */ + code?: string; + /** Charge id from the request-level failure body, when a charge was created */ + chargeId?: string; + /** Present when an organization admin must accept provider terms first */ + requiresAction?: ExchangeRequiresAction; +} diff --git a/src/types/scrape.ts b/src/types/scrape.ts index 6cdbe89e8c..6feb11342a 100644 --- a/src/types/scrape.ts +++ b/src/types/scrape.ts @@ -2,6 +2,8 @@ * Types and interfaces for the scrape command */ +import type { ExchangeRequiresAction } from './exchange'; + export type ScrapeFormat = | 'markdown' | 'html' @@ -72,10 +74,16 @@ export interface ScrapeOptions { lockdown?: boolean; /** Redact personally identifiable information from returned content */ redactPII?: boolean; + /** Include domain-matched contracts in tools alongside semantic matches */ + domainTools?: boolean; } export interface ScrapeResult { success: boolean; data?: any; error?: string; + /** Error code from the API failure body, when the SDK error carried one */ + code?: string; + /** Present when an organization admin must accept provider terms first */ + requiresAction?: ExchangeRequiresAction; } diff --git a/src/types/search.ts b/src/types/search.ts index 04486bf543..44482c3dc1 100644 --- a/src/types/search.ts +++ b/src/types/search.ts @@ -3,8 +3,9 @@ */ import type { ScrapeFormat } from './scrape'; +import type { SearchSourceInput } from '../utils/alexandria'; -export type SearchSource = 'web' | 'images' | 'news'; +export type SearchSource = SearchSourceInput; export type SearchCategory = 'github' | 'research' | 'pdf' | 'developer'; export interface SearchOptions { @@ -16,8 +17,9 @@ export interface SearchOptions { apiUrl?: string; /** Maximum number of results (default: 5, max: 100) */ limit?: number; - /** Sources to search: web, images, news (default: web) */ + /** Sources to search: web, images, news, exchange (default: web) */ sources?: SearchSource[]; + domainTools?: boolean; /** Categories to filter results: github, research, pdf, developer */ categories?: SearchCategory[]; /** Time-based search parameter (e.g., qdr:h, qdr:d, qdr:w, qdr:m, qdr:y) */ @@ -111,12 +113,29 @@ export interface DeveloperSearchResult { category?: string; } +/** + * One capability hit from the `exchange` source. These are catalogue matches, + * never documents: execute one with `firecrawl exchange retrieve + * /` after reading its contract via `exchange discover`. + */ +export interface ExchangeSearchResult { + provider: string; + capability: string; + concept?: string; + cohorts?: string[]; + creditsCost?: number; + similarity?: number; +} + export interface SearchResultData { + tools?: Array>; web?: WebSearchResult[]; images?: ImageSearchResult[]; news?: NewsSearchResult[]; /** Present when the `developer` category is requested. */ developer?: DeveloperSearchResult[]; + /** Present when the `exchange` source is requested and the Exchange answered. */ + exchange?: ExchangeSearchResult[]; } export interface SearchResult { diff --git a/src/utils/alexandria.ts b/src/utils/alexandria.ts new file mode 100644 index 0000000000..aa719b7112 --- /dev/null +++ b/src/utils/alexandria.ts @@ -0,0 +1,78 @@ +import type { Command } from 'commander'; + +export type SearchSourceInput = + | string + | ({ type: string } & Record); +const sourceNames = ['web', 'images', 'news', 'alexandria']; +export function normalizeSources( + sources: SearchSourceInput[] +): SearchSourceInput[] { + return sources.map((source) => { + const name = typeof source === 'string' ? source : source?.type; + const type = name === 'exchange' ? 'alexandria' : name; + if (!sourceNames.includes(type)) + throw new Error(`Invalid source: ${String(name)}`); + if ( + typeof source !== 'string' && + type === 'alexandria' && + Object.keys(source).some((key) => key !== 'type') + ) { + throw new Error( + 'Search accepts only the Alexandria source type. Use firecrawl find-tools for catalogue filters.' + ); + } + return typeof source === 'string' ? type : { ...source, type }; + }); +} +export function hasAlexandria(sources: SearchSourceInput[] = []): boolean { + return sources.some((source) => { + const name = typeof source === 'string' ? source : source?.type; + return name === 'alexandria' || name === 'exchange'; + }); +} +export function addAlexandriaOptions(command: Command): Command { + return command.option( + '--domain-tools', + 'Include domain-matched contracts in tools alongside semantic matches' + ); +} +export function parseSearchSources( + raw: string | undefined +): SearchSourceInput[] | undefined { + if (!raw) return undefined; + const sources = raw.trim().startsWith('[') + ? JSON.parse(raw) + : raw + .split(',') + .map((value) => value.trim()) + .filter(Boolean); + if (!Array.isArray(sources) || !sources.length) + throw new Error('--sources must contain source names or a JSON array.'); + return normalizeSources(sources); +} +export function formatToolsCompact(tools: Array>): string { + const lines = tools.map((tool) => { + const address = `${tool.provider}/${tool.capability}`; + const cost = `${tool.creditsCost} credits`; + const matchedBy = (tool.matchedBy ?? []).join(', '); + return `${address} ${cost} matched by ${matchedBy}`; + }); + lines.push('Full contracts are in the --json output.'); + return lines.join('\n'); +} +export function formatTools(tools: Array>): string { + return tools + .map((tool) => + [ + `${tool.name} (${tool.provider}/${tool.capability})`, + tool.description, + `${tool.creditsCost} credits per ${tool.perRecord ? 'record' : 'call'}`, + `Matched by: ${(tool.matchedBy ?? []).join(', ')}`, + ...(tool.matchedUrls ?? []), + ...['options', 'requiresOneOf', 'response', 'examples'] + .filter((key) => tool[key] !== undefined) + .map((key) => `${key}: ${JSON.stringify(tool[key], null, 2)}`), + ].join('\n') + ) + .join('\n\n'); +} diff --git a/src/utils/config.ts b/src/utils/config.ts index 1374a308fc..afa37deddd 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -73,6 +73,17 @@ export function getApiKey(providedKey?: string): string | undefined { } const DEFAULT_API_URL = 'https://api.firecrawl.dev'; +const DEFAULT_DASHBOARD_URL = 'https://www.firecrawl.dev'; + +/** + * Dashboard base URL: FIRECRAWL_DASHBOARD_URL overrides the cloud default. + */ +export function getDashboardUrl(): string { + return (process.env.FIRECRAWL_DASHBOARD_URL || DEFAULT_DASHBOARD_URL).replace( + /\/$/, + '' + ); +} /** * Check if using a custom (non-cloud) API URL diff --git a/src/utils/options.ts b/src/utils/options.ts index 878e6b5f54..d21e30d2cd 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -123,5 +123,6 @@ export function parseScrapeOptions(options: any): ScrapeOptions { profile, lockdown: options.lockdown, redactPII: options.redactPii ?? options.redactPII, + domainTools: options.domainTools, }; } diff --git a/src/utils/output.ts b/src/utils/output.ts index 4c7842c065..2e34886179 100644 --- a/src/utils/output.ts +++ b/src/utils/output.ts @@ -5,6 +5,7 @@ import * as fs from 'fs'; import * as path from 'path'; import type { ScrapeResult, ScrapeFormat } from '../types/scrape'; +import { formatTermsRequired } from './terms'; /** * Determine if output should be JSON based on flag or file extension @@ -178,8 +179,21 @@ export function handleScrapeOutput( json: boolean = false ): void { if (!result.success) { - // Always use stderr for errors to allow piping - console.error('Error:', result.error); + if (result.code && shouldOutputJson(outputPath, json)) { + const envelope = { + success: false, + code: result.code, + error: result.error, + requiresAction: result.requiresAction, + }; + writeOutput( + pretty ? JSON.stringify(envelope, null, 2) : JSON.stringify(envelope) + ); + } else if (result.requiresAction) { + process.stderr.write(formatTermsRequired(result.requiresAction)); + } else { + console.error('Error:', result.error); + } process.exit(1); } diff --git a/src/utils/terms.ts b/src/utils/terms.ts new file mode 100644 index 0000000000..8a64f2e96a --- /dev/null +++ b/src/utils/terms.ts @@ -0,0 +1,41 @@ +/** + * Provider-terms helpers shared by exchange retrieve, scrape --alexandria and + * plain scrape: parse the API's `requiresAction` payload and render the + * human-mode block for THIRD_PARTY_DATA_TERMS_REQUIRED. + */ + +import type { ExchangeRequiresAction } from '../types/exchange'; + +export const TERMS_REQUIRED_CODE = 'THIRD_PARTY_DATA_TERMS_REQUIRED'; + +/** + * Accept only a well-formed `accept_terms` action; anything else is dropped so + * a malformed body degrades to the plain error message. + */ +export function parseRequiresAction( + value: unknown +): ExchangeRequiresAction | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const { type, terms, version, url } = value as Record; + if ( + type !== 'accept_terms' || + typeof terms !== 'string' || + typeof version !== 'string' || + typeof url !== 'string' + ) { + return undefined; + } + return { type, terms, version, url }; +} + +export function formatTermsRequired(action: ExchangeRequiresAction): string { + return [ + 'Alexandria provider terms required', + ` Provider: ${action.terms}`, + ` Version: ${action.version}`, + ` Accept at: ${action.url}`, + `An organization admin accepts in the dashboard or with \`firecrawl alexandria terms accept ${action.terms}\`.`, + 'No credits were charged.', + '', + ].join('\n'); +}