From cf5a9aceba745996222afb344d9c0aaefde16621 Mon Sep 17 00:00:00 2001 From: Max Loffgren Date: Fri, 11 Sep 2026 15:13:20 -0500 Subject: [PATCH 1/8] feat(cli): submit keyless job feedback --- src/__tests__/cli-argv.test.ts | 25 ++++++++ src/__tests__/commands/feedback.test.ts | 46 +++++++++++++++ .../utils/feedback-invitation.test.ts | 43 ++++++++++++++ src/commands/feedback.ts | 57 ++++++++++++------- src/commands/parse.ts | 11 ++++ src/commands/scrape.ts | 7 +++ src/commands/search.ts | 8 +++ src/index.ts | 45 ++++++++++++++- src/types/search.ts | 1 + src/utils/client.ts | 2 + src/utils/feedback-invitation.ts | 27 +++++++++ src/utils/feedback-settings.ts | 12 ++++ 12 files changed, 263 insertions(+), 21 deletions(-) create mode 100644 src/__tests__/utils/feedback-invitation.test.ts create mode 100644 src/utils/feedback-invitation.ts create mode 100644 src/utils/feedback-settings.ts diff --git a/src/__tests__/cli-argv.test.ts b/src/__tests__/cli-argv.test.ts index fd04e7d3a3..e9e5652114 100644 --- a/src/__tests__/cli-argv.test.ts +++ b/src/__tests__/cli-argv.test.ts @@ -7,6 +7,31 @@ describe('CLI argv parsing', () => { const cliPath = resolve(process.cwd(), 'dist/index.js'); const testWithBuiltCli = existsSync(cliPath) ? it : it.skip; + testWithBuiltCli( + 'describes substantive keyless evidence in feedback help', + () => { + const result = spawnSync( + process.execPath, + [cliPath, 'feedback', '--help'], + { + cwd: process.cwd(), + encoding: 'utf8', + } + ); + expect(result.status).toBe(0); + for (const field of [ + '--task', + '--assessment', + '--observations-file', + 'one-based position', + 'source_comparison', + 'UTC day', + ]) { + expect(result.stdout).toContain(field); + } + } + ); + testWithBuiltCli('lists the developer command in root help output', () => { const result = spawnSync(process.execPath, [cliPath, '--help'], { cwd: process.cwd(), diff --git a/src/__tests__/commands/feedback.test.ts b/src/__tests__/commands/feedback.test.ts index cbb906ace7..9a344778ed 100644 --- a/src/__tests__/commands/feedback.test.ts +++ b/src/__tests__/commands/feedback.test.ts @@ -18,6 +18,10 @@ vi.mock('../../utils/client', async () => { }; }); +vi.mock('../../utils/credentials', () => ({ + loadCredentials: vi.fn(() => null), +})); + describe('executeEndpointFeedback', () => { let mockFetch: ReturnType; @@ -39,6 +43,48 @@ describe('executeEndpointFeedback', () => { delete process.env.FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK; }); + it('submits category evidence without an API key', async () => { + initializeConfig({ + apiKey: undefined, + apiUrl: 'https://api.firecrawl.dev', + }); + delete process.env.FIRECRAWL_API_KEY; + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ + success: true, + feedbackId: 'feedback-1', + creditsRefunded: 0, + }), + }); + const observations = [ + { + kind: 'table', + basis: 'output', + detail: 'The table contains the expected column headings.', + location: 'Page 2', + }, + ]; + const result = await executeEndpointFeedback({ + endpoint: 'parse', + jobId: '00000000-0000-4000-8000-000000000001', + rating: 'good', + task: 'Read the table headings', + assessment: 'The output preserved all table headings.', + observations, + }); + expect(result.success).toBe(true); + const [, init] = mockFetch.mock.calls[0]; + expect(init.headers.Authorization).toBeUndefined(); + expect(JSON.parse(init.body)).toMatchObject({ + endpoint: 'parse', + observations, + origin: 'cli', + integration: 'cli', + }); + }); + it('posts generic endpoint feedback to /v2/feedback', async () => { mockFetch.mockResolvedValue({ ok: true, diff --git a/src/__tests__/utils/feedback-invitation.test.ts b/src/__tests__/utils/feedback-invitation.test.ts new file mode 100644 index 0000000000..9b5276bf71 --- /dev/null +++ b/src/__tests__/utils/feedback-invitation.test.ts @@ -0,0 +1,43 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { reportFeedbackInvitation } from '../../utils/feedback-invitation'; + +describe('feedback invitation output', () => { + afterEach(() => { + vi.restoreAllMocks(); + delete process.env.FIRECRAWL_NO_ENDPOINT_FEEDBACK; + }); + it('keeps content stdout unchanged and writes optional guidance to stderr', () => { + const stdout = vi.spyOn(process.stdout, 'write').mockReturnValue(true); + const stderr = vi.spyOn(process.stderr, 'write').mockReturnValue(true); + reportFeedbackInvitation( + { + jobId: 'job-1', + feedback: { jobId: 'job-1', message: 'Optional feedback.' }, + }, + 'parse' + ); + expect(stdout).not.toHaveBeenCalled(); + expect(stderr.mock.calls.flat().join('')).toContain( + 'firecrawl feedback parse job-1' + ); + }); + it('suppresses invitations when feedback is disabled locally', () => { + process.env.FIRECRAWL_NO_ENDPOINT_FEEDBACK = 'true'; + const stderr = vi.spyOn(process.stderr, 'write').mockReturnValue(true); + reportFeedbackInvitation( + { + jobId: 'job-1', + feedback: { jobId: 'job-1', message: 'Optional feedback.' }, + }, + 'search' + ); + expect(stderr.mock.calls.flat().join('')).not.toContain( + 'Optional feedback' + ); + }); + it('does not invent invitations when metadata is absent', () => { + const stderr = vi.spyOn(process.stderr, 'write').mockReturnValue(true); + reportFeedbackInvitation(undefined, 'scrape'); + expect(stderr).not.toHaveBeenCalled(); + }); +}); diff --git a/src/commands/feedback.ts b/src/commands/feedback.ts index 14318a8c9e..147f60da4a 100644 --- a/src/commands/feedback.ts +++ b/src/commands/feedback.ts @@ -1,6 +1,11 @@ +import { isEndpointFeedbackDisabledLocally } from '../utils/feedback-settings'; +export { + isEndpointFeedbackDisabledLocally, + ENDPOINT_FEEDBACK_OPT_OUT_ENV_VARS, +} from '../utils/feedback-settings'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; import { dirname } from 'path'; -import { getConfig, isCustomApiUrl, validateConfig } from '../utils/config'; +import { getConfig } from '../utils/config'; import { getClient } from '../utils/client'; import { parseMissingContentArg, @@ -16,6 +21,9 @@ export interface EndpointFeedbackOptions { endpoint: EndpointFeedbackEndpoint; jobId: string; rating: SearchFeedbackRating; + task?: string; + assessment?: string; + observations?: Record[]; issues?: string[]; tags?: string[]; note?: string; @@ -34,6 +42,8 @@ export interface EndpointFeedbackOptions { } export type EndpointFeedbackErrorCode = + | 'DAILY_LIMIT_REACHED' + | 'FEEDBACK_UNAVAILABLE' | 'JOB_NOT_FOUND' | 'SEARCH_NOT_FOUND' | 'FEEDBACK_WINDOW_EXPIRED' @@ -60,12 +70,6 @@ export interface EndpointFeedbackResult { disabledSource?: 'env' | 'team'; } -export const ENDPOINT_FEEDBACK_OPT_OUT_ENV_VARS = [ - 'FIRECRAWL_NO_ENDPOINT_FEEDBACK', - 'FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK', -] as const; - -const TRUTHY = new Set(['1', 'true', 'yes', 'on']); const DEFAULT_API_URL = 'https://api.firecrawl.dev'; export const ENDPOINT_FEEDBACK_ENDPOINTS: EndpointFeedbackEndpoint[] = [ @@ -188,16 +192,25 @@ export function parseEndpointFeedbackRating( return rating as SearchFeedbackRating; } -export function isEndpointFeedbackDisabledLocally( - env: NodeJS.ProcessEnv = process.env -): boolean { - for (const key of ENDPOINT_FEEDBACK_OPT_OUT_ENV_VARS) { - const value = env[key]; - if (typeof value === 'string' && TRUTHY.has(value.trim().toLowerCase())) { - return true; - } +export function parseObservations( + raw?: string, + filePath?: string +): Record[] | undefined { + if (raw === undefined && filePath === undefined) return undefined; + if (raw !== undefined && filePath !== undefined) + throw new Error('Provide either --observations or --observations-file.'); + const value: unknown = JSON.parse(raw ?? readFileSync(filePath!, 'utf8')); + if ( + !Array.isArray(value) || + value.length < 1 || + value.length > 20 || + value.some( + (item) => !item || typeof item !== 'object' || Array.isArray(item) + ) + ) { + throw new Error('Observations must be a JSON array of 1-20 objects.'); } - return false; + return value; } export function parseEndpointFeedbackCliOptions(options: { @@ -209,8 +222,14 @@ export function parseEndpointFeedbackCliOptions(options: { valuableSources?: string; missingContent?: string | string[]; rating?: string; + observations?: string; + observationsFile?: string; }) { return { + observations: parseObservations( + options.observations, + options.observationsFile + ), rating: parseEndpointFeedbackRating(String(options.rating || '')), issues: parseFeedbackListArg(options.issues, '--issues'), tags: parseFeedbackListArg(options.tags, '--tags'), @@ -244,9 +263,6 @@ export async function executeEndpointFeedback( /\/$/, '' ); - if (!isCustomApiUrl(apiUrl)) { - validateConfig(apiKey); - } const body: Record = { endpoint: options.endpoint, @@ -260,6 +276,9 @@ export async function executeEndpointFeedback( ['issues', normalizeList(options.issues)], ['tags', normalizeList(options.tags)], ['note', options.note], + ['task', options.task], + ['assessment', options.assessment], + ['observations', options.observations], ['valuableSources', options.valuableSources], ['missingContent', options.missingContent], ['querySuggestions', options.querySuggestions], diff --git a/src/commands/parse.ts b/src/commands/parse.ts index 4aa832fd9f..21c6c9947b 100644 --- a/src/commands/parse.ts +++ b/src/commands/parse.ts @@ -1,3 +1,7 @@ +import { + reportFeedbackInvitation, + filterFeedbackMetadata, +} from '../utils/feedback-invitation'; /** * Parse command implementation * @@ -194,6 +198,13 @@ export async function executeParse( const payload = (await response.json().catch(() => ({}))) as any; + if (keyless && payload?.data?.metadata) + payload.data.metadata = filterFeedbackMetadata(payload.data.metadata); + if (keyless) + reportFeedbackInvitation( + payload?.data?.metadata ?? payload?.metadata, + 'parse' + ); if (!response.ok || payload?.success === false) { const message = payload?.error || diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index c4d86f7c15..09d198eb74 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -1,3 +1,7 @@ +import { + reportFeedbackInvitation, + filterFeedbackMetadata, +} from '../utils/feedback-invitation'; /** * Scrape command implementation */ @@ -151,6 +155,9 @@ export async function executeScrape( ...scrapeParams, }); result = json?.data ?? json; + if (result?.metadata) + result.metadata = filterFeedbackMetadata(result.metadata); + reportFeedbackInvitation(result?.metadata, 'scrape'); } else { const app = getClient({ apiKey: options.apiKey, diff --git a/src/commands/search.ts b/src/commands/search.ts index 1abda30b17..20fd5f6625 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -1,3 +1,7 @@ +import { + reportFeedbackInvitation, + filterFeedbackMetadata, +} from '../utils/feedback-invitation'; /** * Search command implementation */ @@ -118,6 +122,8 @@ export async function executeSearch( ); envelope = (httpResponse?.data ?? {}) as Record; } + envelope.metadata = filterFeedbackMetadata(envelope.metadata); + reportFeedbackInvitation(envelope.metadata, 'search'); const payload = (envelope.data ?? {}) as Record; const data: SearchResultData = {}; @@ -135,6 +141,7 @@ export async function executeSearch( warning: envelope.warning, id: envelope.id, creditsUsed: envelope.creditsUsed, + metadata: envelope.metadata, }; } catch (error) { return { @@ -315,6 +322,7 @@ export async function handleSearchCommand( if (result.warning) { jsonOutput.warning = result.warning; } + if (result.metadata) jsonOutput.metadata = result.metadata; if (result.id) { jsonOutput.id = result.id; } diff --git a/src/index.ts b/src/index.ts index 9bef16f0fd..68cbefaaf3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -87,7 +87,6 @@ const AUTH_REQUIRED_COMMANDS = [ 'download', 'crawl', 'map', - 'feedback', 'search-feedback', 'agent', 'browser', @@ -417,6 +416,10 @@ function createScrapeCommand(): Command { .option('--actions-file ', 'Path to JSON actions file') .option('--proxy ', 'Proxy mode for scraping (e.g., auto, basic)') + .addHelpText( + 'after', + '\nOptional feedback: firecrawl feedback --rating --task --assessment --observations-file . Use only evidence already available. Invitations and job references appear in metadata or stderr. Feedback does not consume operation quota.' + ) .action(async (positionalArgs, options) => { // Collect URLs from positional args and --url option let urls: string[] = []; @@ -866,6 +869,10 @@ Supported file types: .html, .htm, .pdf, .docx, .doc, .odt, .rtf, .xlsx, .xls Max upload size: 50 MB ` ) + .addHelpText( + 'after', + '\nOptional feedback: firecrawl feedback --rating --task --assessment --observations-file . Use only evidence already available. Invitations and job references appear in metadata or stderr. Feedback does not consume operation quota.' + ) .action(async (file: string, options) => { let format: string | undefined; if (options.html) { @@ -974,6 +981,10 @@ function createSearchCommand(): Command { // false // ) .option('--json', 'Output as compact JSON', false) + .addHelpText( + 'after', + '\nOptional feedback: firecrawl feedback --rating --task --assessment --observations-file . Use only evidence already available. Invitations and job references appear in metadata or stderr. Feedback does not consume operation quota.' + ) .action(async (query, options) => { // Parse sources let sources: SearchSource[] | undefined; @@ -1422,7 +1433,9 @@ function createSearchFeedbackCommand(): Command { */ function createFeedbackCommand(): Command { const cmd = new Command('feedback') - .description('Send feedback on a Firecrawl endpoint job.') + .description( + 'Send optional evidence about a job. Keyless Search, Scrape, and Parse accept one submission per category per UTC day without consuming operation quota.' + ) .argument('', 'Endpoint: search | scrape | parse | map') .argument('', 'The job id returned by the endpoint') .requiredOption('--rating ', 'Overall rating: good | bad | partial') @@ -1435,6 +1448,22 @@ function createFeedbackCommand(): Command { 'Comma-separated tags OR JSON array of tags' ) .option('--note ', 'Short note describing the feedback') + .option( + '--task ', + 'Task the output needed to support, required for keyless feedback' + ) + .option( + '--assessment ', + 'Meaningful assessment, required for keyless feedback' + ) + .option( + '--observations ', + 'JSON array of category-specific observations with kind, detail, and basis (output, source_comparison, or expectation)' + ) + .option( + '--observations-file ', + 'Read observations JSON from a file; use only evidence already available' + ) .option( '--valuable-sources ', 'Comma-separated URLs OR JSON array of {url, reason} entries' @@ -1469,6 +1498,15 @@ function createFeedbackCommand(): Command { 'Suppress output; useful when called in the background by another agent', false ) + .addHelpText( + 'after', + '\nKeyless evidence: task, assessment, and each observation detail must contain 10-2000 characters. Submit 1-20 observations.\n' + + 'Search: kind useful or irrelevant, source web/images/news, and one-based position within that delivered group; or kind missing with topic and optional knownSources URLs.\n' + + 'Scrape: kind correct, missing, incorrect, or failure; optional location and already-observed retryOutcome.\n' + + 'Parse: kind correct, text, table, layout, or completeness; optional location.\n' + + 'All observations require detail and basis: output, source_comparison, or expectation. source_comparison also requires comparison: {reference, detail}.\n' + + 'Use only evidence already available. One accepted submission per keyless identity, category, and UTC day, shared across clients.' + ) .action(async (endpointArg: string, jobId: string, options: any) => { let endpoint; try { @@ -1493,6 +1531,9 @@ function createFeedbackCommand(): Command { issues: parsed.issues, tags: parsed.tags, note: options.note, + task: options.task, + assessment: options.assessment, + observations: parsed.observations, valuableSources: parsed.valuableSources, missingContent: parsed.missingContent, querySuggestions: options.querySuggestions, diff --git a/src/types/search.ts b/src/types/search.ts index 04486bf543..1088858ef0 100644 --- a/src/types/search.ts +++ b/src/types/search.ts @@ -120,6 +120,7 @@ export interface SearchResultData { } export interface SearchResult { + metadata?: Record; success: boolean; data?: SearchResultData; warning?: string; diff --git a/src/utils/client.ts b/src/utils/client.ts index 6519495bfc..ee47e7bb47 100644 --- a/src/utils/client.ts +++ b/src/utils/client.ts @@ -1,3 +1,4 @@ +import { reportFeedbackInvitation } from './feedback-invitation'; /** * Firecrawl client utility * Provides a singleton client instance initialized with global configuration @@ -41,6 +42,7 @@ export async function keylessRequest( }); const json: any = await response.json().catch(() => ({})); if (!response.ok) { + reportFeedbackInvitation(json?.metadata, path.split('/').pop()!); throw new Error( json?.error || `Firecrawl request failed (HTTP ${response.status})` ); diff --git a/src/utils/feedback-invitation.ts b/src/utils/feedback-invitation.ts new file mode 100644 index 0000000000..e8c9c87190 --- /dev/null +++ b/src/utils/feedback-invitation.ts @@ -0,0 +1,27 @@ +import { isEndpointFeedbackDisabledLocally } from './feedback-settings'; + +export function filterFeedbackMetadata(metadata: any): any { + if ( + !isEndpointFeedbackDisabledLocally() || + !metadata || + typeof metadata !== 'object' + ) + return metadata; + const { feedback: _feedback, ...rest } = metadata; + return rest; +} + +export function reportFeedbackInvitation( + metadata: any, + endpoint: string +): void { + metadata = filterFeedbackMetadata(metadata); + if (typeof metadata?.jobId === 'string') { + process.stderr.write(`Feedback job (${endpoint}): ${metadata.jobId}\n`); + } + if (typeof metadata?.feedback?.message === 'string') { + process.stderr.write( + `${metadata.feedback.message}\nUse: firecrawl feedback ${endpoint} ${metadata.feedback.jobId} --rating --task --assessment --observations-file \n` + ); + } +} diff --git a/src/utils/feedback-settings.ts b/src/utils/feedback-settings.ts new file mode 100644 index 0000000000..7eca34d9ba --- /dev/null +++ b/src/utils/feedback-settings.ts @@ -0,0 +1,12 @@ +export const ENDPOINT_FEEDBACK_OPT_OUT_ENV_VARS = [ + 'FIRECRAWL_NO_ENDPOINT_FEEDBACK', + 'FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK', +] as const; + +export function isEndpointFeedbackDisabledLocally( + env: NodeJS.ProcessEnv = process.env +): boolean { + return ENDPOINT_FEEDBACK_OPT_OUT_ENV_VARS.some((key) => + /^(1|true|yes|on)$/i.test(env[key]?.trim() ?? '') + ); +} From 521389c6bb838df1b101b6e2730ccb466d7252b4 Mon Sep 17 00:00:00 2001 From: Max Loffgren Date: Fri, 11 Sep 2026 15:23:55 -0500 Subject: [PATCH 2/8] docs(cli): distinguish keyless and authenticated feedback --- README.md | 49 +++++++++++++++++++++----------- skills/firecrawl-search/SKILL.md | 24 ++++++++++++++-- skills/firecrawl/SKILL.md | 8 ++++-- 3 files changed, 59 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 20170e2d68..f5d103f458 100644 --- a/README.md +++ b/README.md @@ -467,9 +467,20 @@ Paper ids accept `pmid:`, `pmcid:`, `doi:`, and `arxiv:` forms, plus canonical ` ### `feedback` - Send endpoint job feedback -Send concise feedback for a completed v2 `search`, `scrape`, `parse`, or `map` -job. For search-result quality, `search-feedback` is still the most guided -command; `feedback` is the generic endpoint/job surface. +Send optional evidence through `/v2/feedback`. Keyless `search`, `scrape`, and +`parse` jobs require `--rating`, `--task`, `--assessment`, and 1-20 observations +provided through `--observations` or `--observations-file`. Use the returned job +reference and evidence already available; no user interview or additional +investigation is required. Run `firecrawl feedback --help` for category fields. + +Keyless feedback accepts one new submission per identity, category, and UTC day +across clients. References expire after 24 hours. Submitting feedback does not +consume or restore operation allowance. Invitations and references appear in +metadata or stderr, preserving ordinary stdout. + +Authenticated callers retain the existing fields. `search-feedback` remains an +authenticated Search command and cannot submit feedback for keyless jobs. The +following example uses the authenticated endpoint feedback contract: ```bash firecrawl feedback scrape 0193f6c5-1234-7890-abcd-1234567890ab \ @@ -489,20 +500,24 @@ endpoint feedback calls silently. #### Feedback Options -| Option | Description | -| -------------------------------- | -------------------------------------------- | -| `--rating ` | Required: `good`, `partial`, or `bad` | -| `--issues ` | Comma-separated issue codes or JSON array | -| `--tags ` | Comma-separated tags or JSON array | -| `--note ` | Short human-readable feedback | -| `--valuable-sources ` | JSON array of `{url, reason}` entries | -| `--missing-content ` | JSON array of `{topic, description}` entries | -| `--query-suggestions ` | Search/query improvement notes | -| `--url ` | Relevant URL for scrape or parse feedback | -| `--page-numbers ` | Comma-separated page numbers or JSON array | -| `--metadata ` | Small JSON object with extra context | -| `--metadata-file ` | Path to small metadata JSON object | -| `--silent` | Suppress output for background agent calls | +| Option | Description | +| -------------------------------- | ---------------------------------------------------- | +| `--rating ` | Required: `good`, `partial`, or `bad` | +| `--task ` | Task intent, required for keyless feedback | +| `--assessment ` | Assessment, required for keyless feedback | +| `--observations ` | JSON array of category-specific keyless observations | +| `--observations-file ` | File containing the observations JSON array | +| `--issues ` | Comma-separated issue codes or JSON array | +| `--tags ` | Comma-separated tags or JSON array | +| `--note ` | Short human-readable feedback | +| `--valuable-sources ` | JSON array of `{url, reason}` entries | +| `--missing-content ` | JSON array of `{topic, description}` entries | +| `--query-suggestions ` | Search/query improvement notes | +| `--url ` | Relevant URL for scrape or parse feedback | +| `--page-numbers ` | Comma-separated page numbers or JSON array | +| `--metadata ` | Small JSON object with extra context | +| `--metadata-file ` | Path to small metadata JSON object | +| `--silent` | Suppress output for background agent calls | --- diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index c3b4334878..798a1e06b4 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -28,7 +28,7 @@ Run `firecrawl search --help` for the full option list. `--categories developer` weighs the developer index beside ordinary web results in this same call (no passage control, no index filters). `--categories research` is a website filter, not the paper index. Dedicated skills: [firecrawl-developer-index](../firecrawl-developer-index/SKILL.md) and [firecrawl-research-index](../firecrawl-research-index/SKILL.md). -**Done when:** results are saved under `.firecrawl/`, verified non-empty, processed for the request, and one feedback event is sent within the time window (unless opted out). +**Done when:** the search response has been checked and processed for the request. Feedback is optional and is not a completion requirement. ## Tips @@ -38,9 +38,27 @@ Run `firecrawl search --help` for the full option list. - Use `jq` to extract URLs or titles: `jq -r '.data.web[].url' .firecrawl/search.json` - Naming convention: `.firecrawl/search-{query}.json` or `.firecrawl/search-{query}-scraped.json` -## Send feedback after every search (refunds 1 credit) +## Optional feedback -Search costs 2 credits. After you've actually used the results (or decided they were useless), send structured feedback with `firecrawl search-feedback `. The first feedback per search refunds 1 credit and helps us improve search quality. **Do this once per search**, in the background, after you finish processing the results — it does not block your main task. +Choose the feedback contract that matches the search job's authentication. A returned `metadata.feedback` invitation identifies the keyless submission endpoint and job ID. Do not send the keyless fields to the authenticated Search feedback route, or add credentials to submit feedback for a keyless job. + +### Keyless Search + +Use `firecrawl feedback search ` with `--rating`, `--task`, `--assessment`, and `--observations-file`. The task describes what the search needed to answer; the assessment describes how well it answered that task. Supply 1-20 observations: + +- Useful or irrelevant results: `kind`, `source` (`web`, `images`, or `news`), and one-based `position` within that delivered group. +- Missing information: `kind: "missing"`, `topic`, and optional `knownSources` URLs, only when already known. +- Every observation requires `detail` and `basis`: `output`, `source_comparison`, or `expectation`. A source comparison also requires `comparison: {reference, detail}`. Use expectations for gaps that have not been verified against a source. + +Task, assessment, and observation detail each require 10-2000 characters. Use only evidence already available. Do not ask the user to complete an interview, investigate further, guess missing content, or diagnose causes merely to submit feedback. An empty result set can support a missing-information observation if the response includes an eligible job reference. + +One new submission is accepted per keyless identity, category, and UTC day across API, MCP, and CLI. Job references expire after 24 hours. Feedback does not consume or restore operation allowance. Do not submit after every search or retry a daily-limit rejection in a loop. `FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` or `FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK=1` disables this path; respect either flag. + +Run `firecrawl feedback --help` for the full evidence contract. Old instructions using `firecrawl search-feedback` require authentication and do not work for keyless jobs. + +### Authenticated Search + +Authenticated callers can continue using `firecrawl search-feedback ` with `--valuable-sources`, `--missing-content`, and `--query-suggestions`. Its existing validation, feedback window, and refund policy are unchanged. If submitting, do so after processing the results, at most once per search, using only evidence already available. **Opt out:** if `FIRECRAWL_NO_SEARCH_FEEDBACK=1` (or `FIRECRAWL_DISABLE_SEARCH_FEEDBACK=1`) is set, the CLI silently skips the call and never sends anything. Respect that — do not try to work around it. (Team admins can also disable this server-side; the API will return `feedbackErrorCode: "TEAM_OPTED_OUT"` and the CLI will exit 0 silently.) diff --git a/skills/firecrawl/SKILL.md b/skills/firecrawl/SKILL.md index e38876f267..4df51009e6 100644 --- a/skills/firecrawl/SKILL.md +++ b/skills/firecrawl/SKILL.md @@ -107,9 +107,13 @@ Single format outputs raw content. Multiple formats (e.g., `--format markdown,li ## Feedback -After using search results, send `firecrawl search-feedback` (the first feedback per search refunds 1 credit). The full pattern, guard, and rules live in [firecrawl-search](../firecrawl-search/SKILL.md). +Feedback is optional and must not interrupt the requested task. For keyless Search, Scrape, or Parse jobs, use `firecrawl feedback ` with `--rating`, `--task`, `--assessment`, and `--observations-file`. Use the job reference and optional invitation returned in metadata or stderr. Run `firecrawl feedback --help` for category-specific observations. -For non-search endpoint jobs, use `firecrawl feedback ` to send concise job-level feedback through `/v2/feedback`. Supported endpoints are `search`, `scrape`, `parse`, and `map`. +Use only evidence already available, without interviewing the user or doing extra investigation. Keyless submissions are limited to one per identity, category, and UTC day across clients; job references expire after 24 hours. Feedback does not consume or restore operation allowance. Do not send legacy issue/note fields as a substitute for keyless observations. + +Authenticated Search callers can continue using `firecrawl search-feedback` with its existing fields and policy. The two authentication modes use different request contracts; do not add credentials to submit feedback for a keyless job. Details live in [firecrawl-search](../firecrawl-search/SKILL.md). + +Authenticated callers can use `firecrawl feedback ` with the existing issue/note fields for `search`, `scrape`, `parse`, and `map`. The following example is for authenticated feedback: ```bash firecrawl feedback scrape "$SCRAPE_ID" \ From cf929f47a5eb52eb071954ec2274ffa1712b004e Mon Sep 17 00:00:00 2001 From: Max Loffgren Date: Fri, 11 Sep 2026 16:09:33 -0500 Subject: [PATCH 3/8] fix(cli): forward feedback invitation preferences --- README.md | 4 ++- src/__tests__/commands/parse.test.ts | 13 +++++++- .../utils/feedback-preference.test.ts | 32 +++++++++++++++++++ src/commands/parse.ts | 7 ++-- src/utils/client.ts | 6 +++- src/utils/feedback-settings.ts | 6 ++++ 6 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 src/__tests__/utils/feedback-preference.test.ts diff --git a/README.md b/README.md index f5d103f458..f5fc0db73f 100644 --- a/README.md +++ b/README.md @@ -496,7 +496,9 @@ Keep notes and metadata small. Do not send raw scrape or parse outputs as feedback. Set `FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` to make `firecrawl feedback` skip -endpoint feedback calls silently. +endpoint feedback calls silently. Search, Scrape, and Parse also send +`x-firecrawl-no-feedback: 1` so the API does not issue or count invitations +that the CLI suppresses. #### Feedback Options diff --git a/src/__tests__/commands/parse.test.ts b/src/__tests__/commands/parse.test.ts index 18b126d664..17c758fc1e 100644 --- a/src/__tests__/commands/parse.test.ts +++ b/src/__tests__/commands/parse.test.ts @@ -38,6 +38,7 @@ describe('executeParse', () => { }); afterEach(() => { + vi.unstubAllEnvs(); vi.unstubAllGlobals(); fs.rmSync(tmpDir, { recursive: true, force: true }); teardownTest(); @@ -61,7 +62,7 @@ describe('executeParse', () => { ]; expect(url).toBe('https://api.firecrawl.dev/v2/parse'); expect(init.method).toBe('POST'); - expect(init.headers).toBeUndefined(); + expect(init.headers).toEqual({}); const options = JSON.parse(init.body.get('options') as string); expect(options).toEqual({ @@ -71,6 +72,16 @@ describe('executeParse', () => { expect(init.body.get('file')).toBeInstanceOf(Blob); }); + it('forwards local invitation opt-out with the uploaded file', async () => { + vi.stubEnv('FIRECRAWL_NO_ENDPOINT_FEEDBACK', 'true'); + initializeConfig({ apiUrl: 'https://api.firecrawl.dev' }); + const result = await executeParse({ file: filePath }); + expect(result.success).toBe(true); + expect(mockFetch.mock.calls[0][1].headers).toEqual({ + 'x-firecrawl-no-feedback': '1', + }); + }); + it('includes the bearer token when an API key is configured', async () => { initializeConfig({ apiKey: 'fc-test-key', diff --git a/src/__tests__/utils/feedback-preference.test.ts b/src/__tests__/utils/feedback-preference.test.ts new file mode 100644 index 0000000000..8db2c01b5e --- /dev/null +++ b/src/__tests__/utils/feedback-preference.test.ts @@ -0,0 +1,32 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { keylessRequest } from '../../utils/client'; + +vi.mock('../../utils/config', () => ({ + getConfig: () => ({ apiUrl: 'https://example.test' }), +})); +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); + +it.each(['/v2/search', '/v2/scrape'])( + 'forwards invitation opt-out for %s without changing the operation', + async (path) => { + vi.stubEnv('FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK', 'true'); + const fetch = vi + .fn() + .mockResolvedValue({ ok: true, json: async () => ({ success: true }) }); + vi.stubGlobal('fetch', fetch); + expect(await keylessRequest(path, { example: 'fixture' })).toEqual({ + success: true, + }); + expect(fetch.mock.calls[0][1]).toMatchObject({ + headers: { + 'Content-Type': 'application/json', + 'x-firecrawl-no-feedback': '1', + }, + body: JSON.stringify({ example: 'fixture' }), + }); + expect(fetch.mock.calls[0][1].headers.Authorization).toBeUndefined(); + } +); diff --git a/src/commands/parse.ts b/src/commands/parse.ts index 21c6c9947b..61d62bfae0 100644 --- a/src/commands/parse.ts +++ b/src/commands/parse.ts @@ -2,6 +2,7 @@ import { reportFeedbackInvitation, filterFeedbackMetadata, } from '../utils/feedback-invitation'; +import { feedbackPreferenceHeaders } from '../utils/feedback-settings'; /** * Parse command implementation * @@ -188,8 +189,10 @@ export async function executeParse( try { const response = await fetch(`${apiUrl}/v2/parse`, { method: 'POST', - headers: - !keyless && apiKey ? { Authorization: `Bearer ${apiKey}` } : undefined, + headers: { + ...(!keyless && apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), + ...feedbackPreferenceHeaders(), + }, body: form, }); diff --git a/src/utils/client.ts b/src/utils/client.ts index ee47e7bb47..f338ff001b 100644 --- a/src/utils/client.ts +++ b/src/utils/client.ts @@ -1,4 +1,5 @@ import { reportFeedbackInvitation } from './feedback-invitation'; +import { feedbackPreferenceHeaders } from './feedback-settings'; /** * Firecrawl client utility * Provides a singleton client instance initialized with global configuration @@ -37,7 +38,10 @@ export async function keylessRequest( const apiUrl = (getConfig().apiUrl || DEFAULT_API_URL).replace(/\/$/, ''); const response = await fetch(`${apiUrl}${path}`, { method: 'POST', - headers: { 'Content-Type': 'application/json' }, + headers: { + 'Content-Type': 'application/json', + ...feedbackPreferenceHeaders(), + }, body: JSON.stringify(body), }); const json: any = await response.json().catch(() => ({})); diff --git a/src/utils/feedback-settings.ts b/src/utils/feedback-settings.ts index 7eca34d9ba..d461e32515 100644 --- a/src/utils/feedback-settings.ts +++ b/src/utils/feedback-settings.ts @@ -10,3 +10,9 @@ export function isEndpointFeedbackDisabledLocally( /^(1|true|yes|on)$/i.test(env[key]?.trim() ?? '') ); } + +export function feedbackPreferenceHeaders(): Record { + return isEndpointFeedbackDisabledLocally() + ? { 'x-firecrawl-no-feedback': '1' } + : {}; +} From 02baf673e21799c54e3ae0afe74c2e3ac70db3c9 Mon Sep 17 00:00:00 2001 From: Max Loffgren Date: Fri, 11 Sep 2026 17:53:11 -0500 Subject: [PATCH 4/8] docs(cli): clarify shared keyless feedback daily limit --- README.md | 8 ++++---- skills/firecrawl-search/SKILL.md | 2 +- skills/firecrawl/SKILL.md | 2 +- src/index.ts | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f5fc0db73f..a2147d1f6c 100644 --- a/README.md +++ b/README.md @@ -473,10 +473,10 @@ provided through `--observations` or `--observations-file`. Use the returned job reference and evidence already available; no user interview or additional investigation is required. Run `firecrawl feedback --help` for category fields. -Keyless feedback accepts one new submission per identity, category, and UTC day -across clients. References expire after 24 hours. Submitting feedback does not -consume or restore operation allowance. Invitations and references appear in -metadata or stderr, preserving ordinary stdout. +Keyless feedback accepts one new submission per identity per UTC day across +Search, Scrape, Parse, and all clients. References expire after 24 hours. +Submitting feedback does not consume or restore operation allowance. Invitations +and references appear in metadata or stderr, preserving ordinary stdout. Authenticated callers retain the existing fields. `search-feedback` remains an authenticated Search command and cannot submit feedback for keyless jobs. The diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index 798a1e06b4..8d9f285f9d 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -52,7 +52,7 @@ Use `firecrawl feedback search ` with `--rating`, `--task`, `--assessment Task, assessment, and observation detail each require 10-2000 characters. Use only evidence already available. Do not ask the user to complete an interview, investigate further, guess missing content, or diagnose causes merely to submit feedback. An empty result set can support a missing-information observation if the response includes an eligible job reference. -One new submission is accepted per keyless identity, category, and UTC day across API, MCP, and CLI. Job references expire after 24 hours. Feedback does not consume or restore operation allowance. Do not submit after every search or retry a daily-limit rejection in a loop. `FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` or `FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK=1` disables this path; respect either flag. +One new submission is accepted per keyless identity per UTC day across Search, Scrape, Parse, API, MCP, and CLI. Job references expire after 24 hours. Feedback does not consume or restore operation allowance. Do not submit after every search or retry a daily-limit rejection in a loop. `FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` or `FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK=1` disables this path; respect either flag. Run `firecrawl feedback --help` for the full evidence contract. Old instructions using `firecrawl search-feedback` require authentication and do not work for keyless jobs. diff --git a/skills/firecrawl/SKILL.md b/skills/firecrawl/SKILL.md index 4df51009e6..25dab2dd7b 100644 --- a/skills/firecrawl/SKILL.md +++ b/skills/firecrawl/SKILL.md @@ -109,7 +109,7 @@ Single format outputs raw content. Multiple formats (e.g., `--format markdown,li Feedback is optional and must not interrupt the requested task. For keyless Search, Scrape, or Parse jobs, use `firecrawl feedback ` with `--rating`, `--task`, `--assessment`, and `--observations-file`. Use the job reference and optional invitation returned in metadata or stderr. Run `firecrawl feedback --help` for category-specific observations. -Use only evidence already available, without interviewing the user or doing extra investigation. Keyless submissions are limited to one per identity, category, and UTC day across clients; job references expire after 24 hours. Feedback does not consume or restore operation allowance. Do not send legacy issue/note fields as a substitute for keyless observations. +Use only evidence already available, without interviewing the user or doing extra investigation. Keyless submissions are limited to one per identity per UTC day across Search, Scrape, Parse, and all clients; job references expire after 24 hours. Feedback does not consume or restore operation allowance. Do not send legacy issue/note fields as a substitute for keyless observations. Authenticated Search callers can continue using `firecrawl search-feedback` with its existing fields and policy. The two authentication modes use different request contracts; do not add credentials to submit feedback for a keyless job. Details live in [firecrawl-search](../firecrawl-search/SKILL.md). diff --git a/src/index.ts b/src/index.ts index 68cbefaaf3..5515b9a781 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1434,7 +1434,7 @@ function createSearchFeedbackCommand(): Command { function createFeedbackCommand(): Command { const cmd = new Command('feedback') .description( - 'Send optional evidence about a job. Keyless Search, Scrape, and Parse accept one submission per category per UTC day without consuming operation quota.' + 'Send optional evidence about a job. Keyless Search, Scrape, and Parse accept one submission per identity per UTC day across all three categories without consuming operation quota.' ) .argument('', 'Endpoint: search | scrape | parse | map') .argument('', 'The job id returned by the endpoint') @@ -1505,7 +1505,7 @@ function createFeedbackCommand(): Command { 'Scrape: kind correct, missing, incorrect, or failure; optional location and already-observed retryOutcome.\n' + 'Parse: kind correct, text, table, layout, or completeness; optional location.\n' + 'All observations require detail and basis: output, source_comparison, or expectation. source_comparison also requires comparison: {reference, detail}.\n' + - 'Use only evidence already available. One accepted submission per keyless identity, category, and UTC day, shared across clients.' + 'Use only evidence already available. One accepted submission per keyless identity per UTC day, shared across Search, Scrape, Parse, and all clients.' ) .action(async (endpointArg: string, jobId: string, options: any) => { let endpoint; From 2f3855c6c3d63497033b4badb08602f8bd5ad084 Mon Sep 17 00:00:00 2001 From: Max Loffgren Date: Fri, 11 Sep 2026 18:15:36 -0500 Subject: [PATCH 5/8] fix(cli): preserve keyless feedback and silence opted-out guidance --- src/__tests__/commands/feedback.test.ts | 102 ++++++++---------- .../utils/feedback-invitation.test.ts | 4 +- src/commands/feedback.ts | 5 - src/utils/feedback-invitation.ts | 1 + 4 files changed, 48 insertions(+), 64 deletions(-) diff --git a/src/__tests__/commands/feedback.test.ts b/src/__tests__/commands/feedback.test.ts index 9a344778ed..119a50c1db 100644 --- a/src/__tests__/commands/feedback.test.ts +++ b/src/__tests__/commands/feedback.test.ts @@ -6,18 +6,9 @@ import { parseFeedbackListArg, parsePageNumbersArg, } from '../../commands/feedback'; -import { getClient } from '../../utils/client'; import { initializeConfig } from '../../utils/config'; import { setupTest, teardownTest } from '../utils/mock-client'; -vi.mock('../../utils/client', async () => { - const actual = await vi.importActual('../../utils/client'); - return { - ...actual, - getClient: vi.fn(), - }; -}); - vi.mock('../../utils/credentials', () => ({ loadCredentials: vi.fn(() => null), })); @@ -39,51 +30,56 @@ describe('executeEndpointFeedback', () => { afterEach(() => { teardownTest(); vi.clearAllMocks(); + vi.unstubAllEnvs(); delete process.env.FIRECRAWL_NO_ENDPOINT_FEEDBACK; delete process.env.FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK; }); - it('submits category evidence without an API key', async () => { - initializeConfig({ - apiKey: undefined, - apiUrl: 'https://api.firecrawl.dev', - }); - delete process.env.FIRECRAWL_API_KEY; - mockFetch.mockResolvedValue({ - ok: true, - status: 200, - json: async () => ({ - success: true, - feedbackId: 'feedback-1', - creditsRefunded: 0, - }), - }); - const observations = [ - { - kind: 'table', - basis: 'output', - detail: 'The table contains the expected column headings.', - location: 'Page 2', - }, - ]; - const result = await executeEndpointFeedback({ - endpoint: 'parse', - jobId: '00000000-0000-4000-8000-000000000001', - rating: 'good', - task: 'Read the table headings', - assessment: 'The output preserved all table headings.', - observations, - }); - expect(result.success).toBe(true); - const [, init] = mockFetch.mock.calls[0]; - expect(init.headers.Authorization).toBeUndefined(); - expect(JSON.parse(init.body)).toMatchObject({ - endpoint: 'parse', - observations, - origin: 'cli', - integration: 'cli', - }); - }); + it.each([undefined, 'https://api.firecrawl.dev'])( + 'submits keyless evidence with API URL %s', + async (apiUrl) => { + vi.stubEnv('FIRECRAWL_API_KEY', ''); + initializeConfig({ + apiKey: undefined, + apiUrl: 'https://api.firecrawl.dev', + }); + mockFetch.mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ + success: true, + feedbackId: 'feedback-1', + creditsRefunded: 0, + }), + }); + const observations = [ + { + kind: 'table', + basis: 'output', + detail: 'The table contains the expected column headings.', + location: 'Page 2', + }, + ]; + const result = await executeEndpointFeedback({ + apiUrl, + endpoint: 'parse', + jobId: '00000000-0000-4000-8000-000000000001', + rating: 'good', + task: 'Read the table headings', + assessment: 'The output preserved all table headings.', + observations, + }); + expect(result.success).toBe(true); + const [, init] = mockFetch.mock.calls[0]; + expect(init.headers.Authorization).toBeUndefined(); + expect(JSON.parse(init.body)).toMatchObject({ + endpoint: 'parse', + observations, + origin: 'cli', + integration: 'cli', + }); + } + ); it('posts generic endpoint feedback to /v2/feedback', async () => { mockFetch.mockResolvedValue({ @@ -110,10 +106,6 @@ describe('executeEndpointFeedback', () => { apiUrl: 'http://localhost:3002', }); - expect(getClient).toHaveBeenCalledWith({ - apiKey: undefined, - apiUrl: 'http://localhost:3002', - }); expect(result).toEqual({ success: true, feedbackId: '0193f6c5-1234-7890-abcd-1234567890ab', @@ -195,7 +187,6 @@ describe('executeEndpointFeedback', () => { creditsRefunded: 0, }); - expect(getClient).not.toHaveBeenCalled(); expect(mockFetch).not.toHaveBeenCalled(); }); @@ -224,7 +215,6 @@ describe('executeEndpointFeedback', () => { expect(stderrSpy).not.toHaveBeenCalled(); expect(stdoutSpy).not.toHaveBeenCalled(); - expect(getClient).not.toHaveBeenCalled(); expect(mockFetch).not.toHaveBeenCalled(); } finally { exitSpy.mockRestore(); diff --git a/src/__tests__/utils/feedback-invitation.test.ts b/src/__tests__/utils/feedback-invitation.test.ts index 9b5276bf71..71ca334b78 100644 --- a/src/__tests__/utils/feedback-invitation.test.ts +++ b/src/__tests__/utils/feedback-invitation.test.ts @@ -31,9 +31,7 @@ describe('feedback invitation output', () => { }, 'search' ); - expect(stderr.mock.calls.flat().join('')).not.toContain( - 'Optional feedback' - ); + expect(stderr).not.toHaveBeenCalled(); }); it('does not invent invitations when metadata is absent', () => { const stderr = vi.spyOn(process.stderr, 'write').mockReturnValue(true); diff --git a/src/commands/feedback.ts b/src/commands/feedback.ts index 147f60da4a..e62cea50e8 100644 --- a/src/commands/feedback.ts +++ b/src/commands/feedback.ts @@ -6,7 +6,6 @@ export { import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; import { dirname } from 'path'; import { getConfig } from '../utils/config'; -import { getClient } from '../utils/client'; import { parseMissingContentArg, parseValuableSourcesArg, @@ -253,10 +252,6 @@ export async function executeEndpointFeedback( } try { - if (options.apiKey || options.apiUrl) { - getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); - } - const config = getConfig(); const apiKey = options.apiKey || config.apiKey; const apiUrl = (options.apiUrl || config.apiUrl || DEFAULT_API_URL).replace( diff --git a/src/utils/feedback-invitation.ts b/src/utils/feedback-invitation.ts index e8c9c87190..c314d24df6 100644 --- a/src/utils/feedback-invitation.ts +++ b/src/utils/feedback-invitation.ts @@ -15,6 +15,7 @@ export function reportFeedbackInvitation( metadata: any, endpoint: string ): void { + if (isEndpointFeedbackDisabledLocally()) return; metadata = filterFeedbackMetadata(metadata); if (typeof metadata?.jobId === 'string') { process.stderr.write(`Feedback job (${endpoint}): ${metadata.jobId}\n`); From aec2df90728bfdd9b2c1ff19fbe1329d0e3b7f59 Mon Sep 17 00:00:00 2001 From: Max Loffgren Date: Fri, 11 Sep 2026 21:53:34 -0500 Subject: [PATCH 6/8] fix(cli): retain keyless invitations and optional submissions --- README.md | 5 +---- skills/firecrawl-search/SKILL.md | 2 +- skills/firecrawl/SKILL.md | 2 +- src/__tests__/commands/feedback.test.ts | 4 +++- src/__tests__/commands/parse.test.ts | 6 ++---- src/__tests__/utils/feedback-invitation.test.ts | 6 ++++-- src/__tests__/utils/feedback-preference.test.ts | 6 ++++-- src/commands/feedback.ts | 17 ++++++++--------- src/commands/parse.ts | 9 +-------- src/commands/scrape.ts | 7 +------ src/commands/search.ts | 8 ++------ src/index.ts | 6 +++--- src/utils/client.ts | 2 -- src/utils/feedback-invitation.ts | 15 --------------- src/utils/feedback-settings.ts | 6 ------ 15 files changed, 31 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index a2147d1f6c..6f5ae7026a 100644 --- a/README.md +++ b/README.md @@ -495,10 +495,7 @@ firecrawl feedback scrape 0193f6c5-1234-7890-abcd-1234567890ab \ Keep notes and metadata small. Do not send raw scrape or parse outputs as feedback. -Set `FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` to make `firecrawl feedback` skip -endpoint feedback calls silently. Search, Scrape, and Parse also send -`x-firecrawl-no-feedback: 1` so the API does not issue or count invitations -that the CLI suppresses. +Set `FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` or `FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK=1` to skip authenticated endpoint feedback calls. These flags do not suppress keyless invitations or submissions. The API controls keyless invitation frequency and eligibility. Submitting feedback remains optional and is never required for continued keyless access. #### Feedback Options diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index 8d9f285f9d..f7d77278c9 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -52,7 +52,7 @@ Use `firecrawl feedback search ` with `--rating`, `--task`, `--assessment Task, assessment, and observation detail each require 10-2000 characters. Use only evidence already available. Do not ask the user to complete an interview, investigate further, guess missing content, or diagnose causes merely to submit feedback. An empty result set can support a missing-information observation if the response includes an eligible job reference. -One new submission is accepted per keyless identity per UTC day across Search, Scrape, Parse, API, MCP, and CLI. Job references expire after 24 hours. Feedback does not consume or restore operation allowance. Do not submit after every search or retry a daily-limit rejection in a loop. `FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` or `FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK=1` disables this path; respect either flag. +One new submission is accepted per keyless identity per UTC day across Search, Scrape, Parse, API, MCP, and CLI. Job references expire after 24 hours. Feedback does not consume or restore operation allowance. Do not submit after every search or retry a daily-limit rejection in a loop. Client feedback flags do not suppress keyless invitations or submissions. Submitting feedback remains optional. Run `firecrawl feedback --help` for the full evidence contract. Old instructions using `firecrawl search-feedback` require authentication and do not work for keyless jobs. diff --git a/skills/firecrawl/SKILL.md b/skills/firecrawl/SKILL.md index 25dab2dd7b..ccfcb2c186 100644 --- a/skills/firecrawl/SKILL.md +++ b/skills/firecrawl/SKILL.md @@ -128,7 +128,7 @@ firecrawl feedback scrape "$SCRAPE_ID" \ Keep generic feedback small: issue codes, tags, short notes, URLs, page numbers, and small metadata objects — never raw scrape/parse outputs or full page contents. -**Opt out:** `export FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` makes the CLI skip every endpoint feedback call silently. Respect that flag — do not try to work around it. +**Authenticated feedback preference:** `FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` or `FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK=1` skips authenticated endpoint feedback calls. Respect these flags for authenticated jobs. Keyless jobs retain server-issued invitations and optional submissions regardless of these flags. ## Parallelization diff --git a/src/__tests__/commands/feedback.test.ts b/src/__tests__/commands/feedback.test.ts index 119a50c1db..bcc4c09a0a 100644 --- a/src/__tests__/commands/feedback.test.ts +++ b/src/__tests__/commands/feedback.test.ts @@ -36,9 +36,11 @@ describe('executeEndpointFeedback', () => { }); it.each([undefined, 'https://api.firecrawl.dev'])( - 'submits keyless evidence with API URL %s', + 'submits keyless evidence despite authenticated opt-out with API URL %s', async (apiUrl) => { vi.stubEnv('FIRECRAWL_API_KEY', ''); + vi.stubEnv('FIRECRAWL_NO_ENDPOINT_FEEDBACK', '1'); + vi.stubEnv('FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK', '1'); initializeConfig({ apiKey: undefined, apiUrl: 'https://api.firecrawl.dev', diff --git a/src/__tests__/commands/parse.test.ts b/src/__tests__/commands/parse.test.ts index 17c758fc1e..106cb58d16 100644 --- a/src/__tests__/commands/parse.test.ts +++ b/src/__tests__/commands/parse.test.ts @@ -72,14 +72,12 @@ describe('executeParse', () => { expect(init.body.get('file')).toBeInstanceOf(Blob); }); - it('forwards local invitation opt-out with the uploaded file', async () => { + it('does not send an invitation opt-out with the keyless uploaded file', async () => { vi.stubEnv('FIRECRAWL_NO_ENDPOINT_FEEDBACK', 'true'); initializeConfig({ apiUrl: 'https://api.firecrawl.dev' }); const result = await executeParse({ file: filePath }); expect(result.success).toBe(true); - expect(mockFetch.mock.calls[0][1].headers).toEqual({ - 'x-firecrawl-no-feedback': '1', - }); + expect(mockFetch.mock.calls[0][1].headers).toEqual({}); }); it('includes the bearer token when an API key is configured', async () => { diff --git a/src/__tests__/utils/feedback-invitation.test.ts b/src/__tests__/utils/feedback-invitation.test.ts index 71ca334b78..8477d83302 100644 --- a/src/__tests__/utils/feedback-invitation.test.ts +++ b/src/__tests__/utils/feedback-invitation.test.ts @@ -21,7 +21,7 @@ describe('feedback invitation output', () => { 'firecrawl feedback parse job-1' ); }); - it('suppresses invitations when feedback is disabled locally', () => { + it('retains keyless invitations despite authenticated feedback preferences', () => { process.env.FIRECRAWL_NO_ENDPOINT_FEEDBACK = 'true'; const stderr = vi.spyOn(process.stderr, 'write').mockReturnValue(true); reportFeedbackInvitation( @@ -31,7 +31,9 @@ describe('feedback invitation output', () => { }, 'search' ); - expect(stderr).not.toHaveBeenCalled(); + expect(stderr.mock.calls.flat().join('')).toContain( + 'firecrawl feedback search job-1' + ); }); it('does not invent invitations when metadata is absent', () => { const stderr = vi.spyOn(process.stderr, 'write').mockReturnValue(true); diff --git a/src/__tests__/utils/feedback-preference.test.ts b/src/__tests__/utils/feedback-preference.test.ts index 8db2c01b5e..1b0bb1259f 100644 --- a/src/__tests__/utils/feedback-preference.test.ts +++ b/src/__tests__/utils/feedback-preference.test.ts @@ -10,7 +10,7 @@ afterEach(() => { }); it.each(['/v2/search', '/v2/scrape'])( - 'forwards invitation opt-out for %s without changing the operation', + 'does not send an invitation opt-out for keyless %s', async (path) => { vi.stubEnv('FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK', 'true'); const fetch = vi @@ -23,10 +23,12 @@ it.each(['/v2/search', '/v2/scrape'])( expect(fetch.mock.calls[0][1]).toMatchObject({ headers: { 'Content-Type': 'application/json', - 'x-firecrawl-no-feedback': '1', }, body: JSON.stringify({ example: 'fixture' }), }); expect(fetch.mock.calls[0][1].headers.Authorization).toBeUndefined(); + expect( + fetch.mock.calls[0][1].headers['x-firecrawl-no-feedback'] + ).toBeUndefined(); } ); diff --git a/src/commands/feedback.ts b/src/commands/feedback.ts index e62cea50e8..5d0c151824 100644 --- a/src/commands/feedback.ts +++ b/src/commands/feedback.ts @@ -242,18 +242,17 @@ export function parseEndpointFeedbackCliOptions(options: { export async function executeEndpointFeedback( options: EndpointFeedbackOptions ): Promise { - if (isEndpointFeedbackDisabledLocally()) { - return { - success: true, - disabled: true, - disabledSource: 'env', - creditsRefunded: 0, - }; - } - try { const config = getConfig(); const apiKey = options.apiKey || config.apiKey; + if (apiKey && isEndpointFeedbackDisabledLocally()) { + return { + success: true, + disabled: true, + disabledSource: 'env', + creditsRefunded: 0, + }; + } const apiUrl = (options.apiUrl || config.apiUrl || DEFAULT_API_URL).replace( /\/$/, '' diff --git a/src/commands/parse.ts b/src/commands/parse.ts index 61d62bfae0..76fbd296d9 100644 --- a/src/commands/parse.ts +++ b/src/commands/parse.ts @@ -1,8 +1,4 @@ -import { - reportFeedbackInvitation, - filterFeedbackMetadata, -} from '../utils/feedback-invitation'; -import { feedbackPreferenceHeaders } from '../utils/feedback-settings'; +import { reportFeedbackInvitation } from '../utils/feedback-invitation'; /** * Parse command implementation * @@ -191,7 +187,6 @@ export async function executeParse( method: 'POST', headers: { ...(!keyless && apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), - ...feedbackPreferenceHeaders(), }, body: form, }); @@ -201,8 +196,6 @@ export async function executeParse( const payload = (await response.json().catch(() => ({}))) as any; - if (keyless && payload?.data?.metadata) - payload.data.metadata = filterFeedbackMetadata(payload.data.metadata); if (keyless) reportFeedbackInvitation( payload?.data?.metadata ?? payload?.metadata, diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index 09d198eb74..47c8860147 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -1,7 +1,4 @@ -import { - reportFeedbackInvitation, - filterFeedbackMetadata, -} from '../utils/feedback-invitation'; +import { reportFeedbackInvitation } from '../utils/feedback-invitation'; /** * Scrape command implementation */ @@ -155,8 +152,6 @@ export async function executeScrape( ...scrapeParams, }); result = json?.data ?? json; - if (result?.metadata) - result.metadata = filterFeedbackMetadata(result.metadata); reportFeedbackInvitation(result?.metadata, 'scrape'); } else { const app = getClient({ diff --git a/src/commands/search.ts b/src/commands/search.ts index 20fd5f6625..6fe559ac9b 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -1,7 +1,4 @@ -import { - reportFeedbackInvitation, - filterFeedbackMetadata, -} from '../utils/feedback-invitation'; +import { reportFeedbackInvitation } from '../utils/feedback-invitation'; /** * Search command implementation */ @@ -114,6 +111,7 @@ export async function executeSearch( string, any >; + reportFeedbackInvitation(envelope.metadata, 'search'); } else { const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); const httpResponse = await (app as any).http.post( @@ -122,8 +120,6 @@ export async function executeSearch( ); envelope = (httpResponse?.data ?? {}) as Record; } - envelope.metadata = filterFeedbackMetadata(envelope.metadata); - reportFeedbackInvitation(envelope.metadata, 'search'); const payload = (envelope.data ?? {}) as Record; const data: SearchResultData = {}; diff --git a/src/index.ts b/src/index.ts index 5515b9a781..65e5262ea0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -418,7 +418,7 @@ function createScrapeCommand(): Command { .addHelpText( 'after', - '\nOptional feedback: firecrawl feedback --rating --task --assessment --observations-file . Use only evidence already available. Invitations and job references appear in metadata or stderr. Feedback does not consume operation quota.' + '\nOptional keyless feedback: firecrawl feedback --rating --task --assessment --observations-file . Use only evidence already available. Invitations and job references appear in metadata or stderr. Feedback does not consume operation quota.' ) .action(async (positionalArgs, options) => { // Collect URLs from positional args and --url option @@ -871,7 +871,7 @@ Max upload size: 50 MB ) .addHelpText( 'after', - '\nOptional feedback: firecrawl feedback --rating --task --assessment --observations-file . Use only evidence already available. Invitations and job references appear in metadata or stderr. Feedback does not consume operation quota.' + '\nOptional keyless feedback: firecrawl feedback --rating --task --assessment --observations-file . Use only evidence already available. Invitations and job references appear in metadata or stderr. Feedback does not consume operation quota.' ) .action(async (file: string, options) => { let format: string | undefined; @@ -983,7 +983,7 @@ function createSearchCommand(): Command { .option('--json', 'Output as compact JSON', false) .addHelpText( 'after', - '\nOptional feedback: firecrawl feedback --rating --task --assessment --observations-file . Use only evidence already available. Invitations and job references appear in metadata or stderr. Feedback does not consume operation quota.' + '\nOptional keyless feedback: firecrawl feedback --rating --task --assessment --observations-file . Use only evidence already available. Invitations and job references appear in metadata or stderr. Feedback does not consume operation quota.' ) .action(async (query, options) => { // Parse sources diff --git a/src/utils/client.ts b/src/utils/client.ts index f338ff001b..728e6811dd 100644 --- a/src/utils/client.ts +++ b/src/utils/client.ts @@ -1,5 +1,4 @@ import { reportFeedbackInvitation } from './feedback-invitation'; -import { feedbackPreferenceHeaders } from './feedback-settings'; /** * Firecrawl client utility * Provides a singleton client instance initialized with global configuration @@ -40,7 +39,6 @@ export async function keylessRequest( method: 'POST', headers: { 'Content-Type': 'application/json', - ...feedbackPreferenceHeaders(), }, body: JSON.stringify(body), }); diff --git a/src/utils/feedback-invitation.ts b/src/utils/feedback-invitation.ts index c314d24df6..ce97c80bee 100644 --- a/src/utils/feedback-invitation.ts +++ b/src/utils/feedback-invitation.ts @@ -1,22 +1,7 @@ -import { isEndpointFeedbackDisabledLocally } from './feedback-settings'; - -export function filterFeedbackMetadata(metadata: any): any { - if ( - !isEndpointFeedbackDisabledLocally() || - !metadata || - typeof metadata !== 'object' - ) - return metadata; - const { feedback: _feedback, ...rest } = metadata; - return rest; -} - export function reportFeedbackInvitation( metadata: any, endpoint: string ): void { - if (isEndpointFeedbackDisabledLocally()) return; - metadata = filterFeedbackMetadata(metadata); if (typeof metadata?.jobId === 'string') { process.stderr.write(`Feedback job (${endpoint}): ${metadata.jobId}\n`); } diff --git a/src/utils/feedback-settings.ts b/src/utils/feedback-settings.ts index d461e32515..7eca34d9ba 100644 --- a/src/utils/feedback-settings.ts +++ b/src/utils/feedback-settings.ts @@ -10,9 +10,3 @@ export function isEndpointFeedbackDisabledLocally( /^(1|true|yes|on)$/i.test(env[key]?.trim() ?? '') ); } - -export function feedbackPreferenceHeaders(): Record { - return isEndpointFeedbackDisabledLocally() - ? { 'x-firecrawl-no-feedback': '1' } - : {}; -} From 80a8808ad1e81f830c917928ed5a4de082d6c8aa Mon Sep 17 00:00:00 2001 From: Max Loffgren Date: Sat, 12 Sep 2026 17:40:32 -0500 Subject: [PATCH 7/8] feat(cli): support structured keyless feedback categories --- README.md | 11 ++++++- src/__tests__/commands/feedback.test.ts | 29 +++++++++++++++++-- .../utils/feedback-invitation.test.ts | 3 ++ src/commands/feedback.ts | 11 +++++++ src/index.ts | 13 ++++++--- src/utils/feedback-invitation.ts | 2 +- 6 files changed, 61 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6f5ae7026a..4a595d9542 100644 --- a/README.md +++ b/README.md @@ -469,7 +469,7 @@ Paper ids accept `pmid:`, `pmcid:`, `doi:`, and `arxiv:` forms, plus canonical ` Send optional evidence through `/v2/feedback`. Keyless `search`, `scrape`, and `parse` jobs require `--rating`, `--task`, `--assessment`, and 1-20 observations -provided through `--observations` or `--observations-file`. Use the returned job +provided through `--observations` or `--observations-file`. Keyless Parse also requires `--doc-class born_digital|scanned|mixed|unknown` once per submission. Use the returned job reference and evidence already available; no user interview or additional investigation is required. Run `firecrawl feedback --help` for category fields. @@ -495,6 +495,14 @@ firecrawl feedback scrape 0193f6c5-1234-7890-abcd-1234567890ab \ Keep notes and metadata small. Do not send raw scrape or parse outputs as feedback. +Search: useful and irrelevant require a one-based position within the delivered group. source is web, images, or news; required for jobs requesting multiple sources, otherwise defaults to web. The position must exist in that requested group. irrelevant requires reason: aggregator_over_official, off_topic, stale, wrong_content_type, snippet_misleading, or blocked_or_paywalled. vertical is required on missing and optional on useful/irrelevant: web_general, social, business, research, developer, news, government, finance, or other. missing may include topic (up to 200 characters) and knownSources (up to 20 HTTP(S) URLs). + +Scrape: kind correct, wrong_success, incomplete, or incorrect. wrong_success requires reason: blocked_shell, login_required, paywall, empty, wrong_page, stale, or wrong_locale. incomplete requires reason: partial_content, dynamic_content, pagination, main_content_stripped, or format_lost. incorrect requires reason: wrong, hallucinated, or missing_fields. correct has no reason. Optional location is up to 200 characters. No retryOutcome. Hard-failed Scrape jobs receive no feedback invitation. + +Parse: `--doc-class` is required once per submission: born_digital, scanned, mixed, or unknown. Observation kind: correct, text_ocr, table, formula, chart_figure, reading_order, headers_footers, headings_formatting, completeness, images_dropped, or incorrect. text_ocr requires reason: misread_chars, garbled, or missing_text. table requires reason: structure, cells_glued, or digits. completeness requires reason: pages_missing, truncated_at_max_pages, or sections_dropped. incorrect requires reason: wrong, hallucinated, or missing_fields. Other kinds have no reason subtype. Optional page is a one-based positive integer. + +Scrape and Parse: format must be a format type the job requested. It is required for output and source_comparison observations when multiple formats were requested; optional for expectation observations and single-format jobs. All observations retain detail and basis; source_comparison requires comparison: {reference, detail}. + Set `FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` or `FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK=1` to skip authenticated endpoint feedback calls. These flags do not suppress keyless invitations or submissions. The API controls keyless invitation frequency and eligibility. Submitting feedback remains optional and is never required for continued keyless access. #### Feedback Options @@ -503,6 +511,7 @@ Set `FIRECRAWL_NO_ENDPOINT_FEEDBACK=1` or `FIRECRAWL_DISABLE_ENDPOINT_FEEDBACK=1 | -------------------------------- | ---------------------------------------------------- | | `--rating ` | Required: `good`, `partial`, or `bad` | | `--task ` | Task intent, required for keyless feedback | +| `--doc-class ` | Document class, required for keyless Parse | | `--assessment ` | Assessment, required for keyless feedback | | `--observations ` | JSON array of category-specific keyless observations | | `--observations-file ` | File containing the observations JSON array | diff --git a/src/__tests__/commands/feedback.test.ts b/src/__tests__/commands/feedback.test.ts index bcc4c09a0a..08b4690336 100644 --- a/src/__tests__/commands/feedback.test.ts +++ b/src/__tests__/commands/feedback.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { executeEndpointFeedback, + parseEndpointFeedbackCliOptions, handleEndpointFeedbackCommand, parseEndpointFeedbackEndpoint, parseFeedbackListArg, @@ -56,15 +57,18 @@ describe('executeEndpointFeedback', () => { }); const observations = [ { - kind: 'table', + kind: 'incorrect', + reason: 'missing_fields', + format: 'json', basis: 'output', detail: 'The table contains the expected column headings.', - location: 'Page 2', + page: 2, }, ]; const result = await executeEndpointFeedback({ apiUrl, endpoint: 'parse', + docClass: 'born_digital', jobId: '00000000-0000-4000-8000-000000000001', rating: 'good', task: 'Read the table headings', @@ -76,6 +80,7 @@ describe('executeEndpointFeedback', () => { expect(init.headers.Authorization).toBeUndefined(); expect(JSON.parse(init.body)).toMatchObject({ endpoint: 'parse', + docClass: 'born_digital', observations, origin: 'cli', integration: 'cli', @@ -246,3 +251,23 @@ describe('feedback parsing', () => { expect(parsePageNumbersArg('[4,5]')).toEqual([4, 5]); }); }); + +describe('keyless document class option', () => { + it.each(['born_digital', 'scanned', 'mixed', 'unknown'] as const)( + 'preserves %s for submission', + (docClass) => { + expect( + parseEndpointFeedbackCliOptions({ rating: 'partial', docClass }) + .docClass + ).toBe(docClass); + } + ); + it('rejects an unsupported class without changing authenticated defaults', () => { + expect(() => + parseEndpointFeedbackCliOptions({ rating: 'partial', docClass: 'pdf' }) + ).toThrow('--doc-class'); + expect( + parseEndpointFeedbackCliOptions({ rating: 'good' }).docClass + ).toBeUndefined(); + }); +}); diff --git a/src/__tests__/utils/feedback-invitation.test.ts b/src/__tests__/utils/feedback-invitation.test.ts index 8477d83302..553e997b15 100644 --- a/src/__tests__/utils/feedback-invitation.test.ts +++ b/src/__tests__/utils/feedback-invitation.test.ts @@ -20,6 +20,9 @@ describe('feedback invitation output', () => { expect(stderr.mock.calls.flat().join('')).toContain( 'firecrawl feedback parse job-1' ); + expect(stderr.mock.calls.flat().join('')).toContain( + '--doc-class ' + ); }); it('retains keyless invitations despite authenticated feedback preferences', () => { process.env.FIRECRAWL_NO_ENDPOINT_FEEDBACK = 'true'; diff --git a/src/commands/feedback.ts b/src/commands/feedback.ts index 5d0c151824..9daec0a8c3 100644 --- a/src/commands/feedback.ts +++ b/src/commands/feedback.ts @@ -22,6 +22,7 @@ export interface EndpointFeedbackOptions { rating: SearchFeedbackRating; task?: string; assessment?: string; + docClass?: 'born_digital' | 'scanned' | 'mixed' | 'unknown'; observations?: Record[]; issues?: string[]; tags?: string[]; @@ -223,8 +224,17 @@ export function parseEndpointFeedbackCliOptions(options: { rating?: string; observations?: string; observationsFile?: string; + docClass?: string; }) { + if ( + options.docClass !== undefined && + !['born_digital', 'scanned', 'mixed', 'unknown'].includes(options.docClass) + ) + throw new Error( + '--doc-class must be one of: born_digital, scanned, mixed, unknown' + ); return { + docClass: options.docClass as EndpointFeedbackOptions['docClass'], observations: parseObservations( options.observations, options.observationsFile @@ -272,6 +282,7 @@ export async function executeEndpointFeedback( ['note', options.note], ['task', options.task], ['assessment', options.assessment], + ['docClass', options.docClass], ['observations', options.observations], ['valuableSources', options.valuableSources], ['missingContent', options.missingContent], diff --git a/src/index.ts b/src/index.ts index 65e5262ea0..6f0eb3a0bd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1456,6 +1456,10 @@ function createFeedbackCommand(): Command { '--assessment ', 'Meaningful assessment, required for keyless feedback' ) + .option( + '--doc-class ', + 'Document class, required once for keyless Parse: born_digital | scanned | mixed | unknown' + ) .option( '--observations ', 'JSON array of category-specific observations with kind, detail, and basis (output, source_comparison, or expectation)' @@ -1501,10 +1505,10 @@ function createFeedbackCommand(): Command { .addHelpText( 'after', '\nKeyless evidence: task, assessment, and each observation detail must contain 10-2000 characters. Submit 1-20 observations.\n' + - 'Search: kind useful or irrelevant, source web/images/news, and one-based position within that delivered group; or kind missing with topic and optional knownSources URLs.\n' + - 'Scrape: kind correct, missing, incorrect, or failure; optional location and already-observed retryOutcome.\n' + - 'Parse: kind correct, text, table, layout, or completeness; optional location.\n' + - 'All observations require detail and basis: output, source_comparison, or expectation. source_comparison also requires comparison: {reference, detail}.\n' + + 'Search: useful and irrelevant require a one-based position within the delivered group. source is web, images, or news; required for jobs requesting multiple sources, otherwise defaults to web. The position must exist in that requested group. irrelevant requires reason: aggregator_over_official, off_topic, stale, wrong_content_type, snippet_misleading, or blocked_or_paywalled. vertical is required on missing and optional on useful/irrelevant: web_general, social, business, research, developer, news, government, finance, or other. missing may include topic (up to 200 characters) and knownSources (up to 20 HTTP(S) URLs).\n' + + 'Scrape: kind correct, wrong_success, incomplete, or incorrect. wrong_success requires reason: blocked_shell, login_required, paywall, empty, wrong_page, stale, or wrong_locale. incomplete requires reason: partial_content, dynamic_content, pagination, main_content_stripped, or format_lost. incorrect requires reason: wrong, hallucinated, or missing_fields. correct has no reason. Optional location is up to 200 characters. No retryOutcome. Hard-failed Scrape jobs receive no feedback invitation.\n' + + 'Parse: --doc-class is required once per submission: born_digital, scanned, mixed, or unknown. Observation kind: correct, text_ocr, table, formula, chart_figure, reading_order, headers_footers, headings_formatting, completeness, images_dropped, or incorrect. text_ocr requires reason: misread_chars, garbled, or missing_text. table requires reason: structure, cells_glued, or digits. completeness requires reason: pages_missing, truncated_at_max_pages, or sections_dropped. incorrect requires reason: wrong, hallucinated, or missing_fields. Other kinds have no reason subtype. Optional page is a one-based positive integer.\n' + + 'Scrape and Parse: format must be a format type the job requested. It is required for output and source_comparison observations when multiple formats were requested; optional for expectation observations and single-format jobs. All observations retain detail and basis; source_comparison requires comparison: {reference, detail}.\n' + 'Use only evidence already available. One accepted submission per keyless identity per UTC day, shared across Search, Scrape, Parse, and all clients.' ) .action(async (endpointArg: string, jobId: string, options: any) => { @@ -1533,6 +1537,7 @@ function createFeedbackCommand(): Command { note: options.note, task: options.task, assessment: options.assessment, + docClass: parsed.docClass, observations: parsed.observations, valuableSources: parsed.valuableSources, missingContent: parsed.missingContent, diff --git a/src/utils/feedback-invitation.ts b/src/utils/feedback-invitation.ts index ce97c80bee..fd699c1a7b 100644 --- a/src/utils/feedback-invitation.ts +++ b/src/utils/feedback-invitation.ts @@ -7,7 +7,7 @@ export function reportFeedbackInvitation( } if (typeof metadata?.feedback?.message === 'string') { process.stderr.write( - `${metadata.feedback.message}\nUse: firecrawl feedback ${endpoint} ${metadata.feedback.jobId} --rating --task --assessment --observations-file \n` + `${metadata.feedback.message}\nUse: firecrawl feedback ${endpoint} ${metadata.feedback.jobId} --rating --task --assessment --observations-file ${endpoint === 'parse' ? ' --doc-class ' : ''}\n` ); } } From fc5f6795acda2c6dc3b2abaa6c6a58e556e4ccb4 Mon Sep 17 00:00:00 2001 From: Max Loffgren Date: Sat, 12 Sep 2026 17:45:00 -0500 Subject: [PATCH 8/8] docs(cli): clarify search feedback source attribution --- README.md | 2 +- src/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4a595d9542..3b1e5b9bf6 100644 --- a/README.md +++ b/README.md @@ -495,7 +495,7 @@ firecrawl feedback scrape 0193f6c5-1234-7890-abcd-1234567890ab \ Keep notes and metadata small. Do not send raw scrape or parse outputs as feedback. -Search: useful and irrelevant require a one-based position within the delivered group. source is web, images, or news; required for jobs requesting multiple sources, otherwise defaults to web. The position must exist in that requested group. irrelevant requires reason: aggregator_over_official, off_topic, stale, wrong_content_type, snippet_misleading, or blocked_or_paywalled. vertical is required on missing and optional on useful/irrelevant: web_general, social, business, research, developer, news, government, finance, or other. missing may include topic (up to 200 characters) and knownSources (up to 20 HTTP(S) URLs). +Search: useful and irrelevant require a one-based position within the delivered group. source names the response group the position refers to: web, images, or news. It is required only when the job requested multiple sources; otherwise it defaults to web. The position must exist in that requested group. irrelevant requires reason: aggregator_over_official, off_topic, stale, wrong_content_type, snippet_misleading, or blocked_or_paywalled. vertical is required on missing and optional on useful/irrelevant: web_general, social, business, research, developer, news, government, finance, or other. missing may include topic (up to 200 characters) and knownSources (up to 20 HTTP(S) URLs). Do not submit engine attribution; it comes from the stored category tag at that position. Scrape: kind correct, wrong_success, incomplete, or incorrect. wrong_success requires reason: blocked_shell, login_required, paywall, empty, wrong_page, stale, or wrong_locale. incomplete requires reason: partial_content, dynamic_content, pagination, main_content_stripped, or format_lost. incorrect requires reason: wrong, hallucinated, or missing_fields. correct has no reason. Optional location is up to 200 characters. No retryOutcome. Hard-failed Scrape jobs receive no feedback invitation. diff --git a/src/index.ts b/src/index.ts index 6f0eb3a0bd..eed2a9ba3c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1505,7 +1505,7 @@ function createFeedbackCommand(): Command { .addHelpText( 'after', '\nKeyless evidence: task, assessment, and each observation detail must contain 10-2000 characters. Submit 1-20 observations.\n' + - 'Search: useful and irrelevant require a one-based position within the delivered group. source is web, images, or news; required for jobs requesting multiple sources, otherwise defaults to web. The position must exist in that requested group. irrelevant requires reason: aggregator_over_official, off_topic, stale, wrong_content_type, snippet_misleading, or blocked_or_paywalled. vertical is required on missing and optional on useful/irrelevant: web_general, social, business, research, developer, news, government, finance, or other. missing may include topic (up to 200 characters) and knownSources (up to 20 HTTP(S) URLs).\n' + + 'Search: useful and irrelevant require a one-based position within the delivered group. source names the response group the position refers to: web, images, or news. It is required only when the job requested multiple sources; otherwise it defaults to web. The position must exist in that requested group. irrelevant requires reason: aggregator_over_official, off_topic, stale, wrong_content_type, snippet_misleading, or blocked_or_paywalled. vertical is required on missing and optional on useful/irrelevant: web_general, social, business, research, developer, news, government, finance, or other. missing may include topic (up to 200 characters) and knownSources (up to 20 HTTP(S) URLs). Do not submit engine attribution; it comes from the stored category tag at that position.\n' + 'Scrape: kind correct, wrong_success, incomplete, or incorrect. wrong_success requires reason: blocked_shell, login_required, paywall, empty, wrong_page, stale, or wrong_locale. incomplete requires reason: partial_content, dynamic_content, pagination, main_content_stripped, or format_lost. incorrect requires reason: wrong, hallucinated, or missing_fields. correct has no reason. Optional location is up to 200 characters. No retryOutcome. Hard-failed Scrape jobs receive no feedback invitation.\n' + 'Parse: --doc-class is required once per submission: born_digital, scanned, mixed, or unknown. Observation kind: correct, text_ocr, table, formula, chart_figure, reading_order, headers_footers, headings_formatting, completeness, images_dropped, or incorrect. text_ocr requires reason: misread_chars, garbled, or missing_text. table requires reason: structure, cells_glued, or digits. completeness requires reason: pages_missing, truncated_at_max_pages, or sections_dropped. incorrect requires reason: wrong, hallucinated, or missing_fields. Other kinds have no reason subtype. Optional page is a one-based positive integer.\n' + 'Scrape and Parse: format must be a format type the job requested. It is required for output and source_comparison observations when multiple formats were requested; optional for expectation observations and single-format jobs. All observations retain detail and basis; source_comparison requires comparison: {reference, detail}.\n' +