diff --git a/CHANGELOG.md b/CHANGELOG.md index fefba3d..5ee3479 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,9 @@ ## Unreleased -Bug fixes from a review of 0.1.3. No new commands; two behaviour changes are marked. +Bug fixes from a review of 0.1.3, and live progress on a terminal. No new commands; two behaviour changes are marked. +- **Live progress on a terminal.** A spinner with a label and elapsed time covers every wait over about 300 ms (`scrape`, `scrape -`, `batch submit`/`wait`, and any other API call), and `scrape`, `scrape -` and `batch wait` end with a `✓`/`✗` summary line (page, status, credits, duration) on stderr. API errors keep their `ERR::` code and hint in a short block. Everything is on stderr, only when stderr is a terminal, and off with `--json`, `-q`, `NO_COLOR`, `TERM=dumb` or `CI`: stdout and the plain output are unchanged. New dependency: `golang.org/x/term`, for the terminal width. - **Retries are safe.** A connection that drops after the request was sent now exits 12 ("may have run and been billed"), not 10 ("nothing was sent, safe to retry"). A gateway 5xx without an API problem document is retried only for GET/HEAD, and is no longer mistaken for the target's status under `--original-status`. A 301/302/303 on a POST is an error naming the Location instead of a silent GET. `Retry-After` waits are capped at 60 s and announced on stderr. - **Ctrl-C exits 130** with "interrupted", including while `scrape -` waits on stdin. A mistyped subcommand (`spicrawl batch wiat`) is a usage error with a suggestion, not help and exit 0. - **Keys:** a trailing CR/LF or space on the key (flag, env or file) is trimmed. The config file is written via a private temp file, short keys are fully masked, and commands that get the key and base URL from flags work without `$HOME`. diff --git a/DESIGN.md b/DESIGN.md index 669f814..d8cd746 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -10,6 +10,12 @@ never invent a field the API does not accept (the API rejects unknown fields). ## Contract agents rely on 1. **stdout is data only.** Progress, warnings and errors go to stderr. + Live progress (spinner, colour, the `✓`/`✗` summary line, the error block) is + the one thing a person gets that an agent does not, so it is gated hard: only + when stderr is a terminal, and never with `--json`, `-q`, `NO_COLOR`, + `TERM=dumb` or `CI`. Switched off, the output is the plain text it always was. + It never prints an engine or browser name; it says what the request asked for + ("rendering JavaScript"), not which tier the API picked. 2. **JSON when not a terminal.** `--json` forces it. Human mode prints tables or the document itself. 3. **Never prompts without a TTY.** Anything interactive has a flag form. @@ -85,6 +91,7 @@ internal/api HTTP client, Problem, DoWithRetry internal/config key/base resolution, config file internal/exitcode exit codes internal/output Printer (JSON/human/JSONL, stderr info) +internal/ui stderr spinner, summary line, error block, colour (off unless a terminal) internal/cmd one file per command group; each registers in init() ``` diff --git a/README.md b/README.md index 2444382..fb2aac5 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,12 @@ For other clients (Gemini CLI, OpenCode, Factory Droid, Devin and more) use the `spicrawl --help` and `spicrawl --help` list every flag. Global flags: `--api-key`, `--base-url`, `--json`, `-q/--quiet` and `--timeout` (default `3m`). +### Progress on a terminal + +When stderr is a terminal, a wait longer than about 300 ms shows a spinner with what the CLI is doing and how long it has taken (`⠹ Scraping example.com/pricing (rendering JavaScript) · 4.2s`, `⠸ Batch job 3/10 done · 12.1s`). It is cleared before the result prints, and `scrape`, `scrape -` and `batch wait` end with a `✓` or `✗` line on stderr: page, target status, credits charged, duration. API errors keep their `ERR::` code and hint in a short block. + +All of it is on stderr and stdout is byte-for-byte unchanged. It switches itself off, and the CLI prints exactly what it printed before, when stderr is not a terminal (a pipe, a file, an agent or CI log), with `--json` or `-q/--quiet`, with `NO_COLOR` set, with `TERM=dumb`, or when `CI` is set. + ### How to convert a web page to Markdown from the command line ```sh diff --git a/go.mod b/go.mod index f6a187d..b0ec7a9 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/mattn/go-isatty v0.0.20 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.9 + golang.org/x/term v0.6.0 ) require ( diff --git a/go.sum b/go.sum index d589b7c..8edb9ab 100644 --- a/go.sum +++ b/go.sum @@ -11,4 +11,6 @@ github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/api/client.go b/internal/api/client.go index 773c33d..0f2f21e 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -33,6 +33,10 @@ type Client struct { // Notify, when set, receives progress lines (DoWithRetry waiting between // attempts). The caller decides where they go and whether --quiet mutes them. Notify func(format string, a ...any) + // Wait, when set, is called as each round trip starts and the func it + // returns as the round trip ends (answer read, or failed). The commands use + // it to show a spinner while a call is outstanding. + Wait func(method, path string) (done func()) } func New(baseURL, apiKey string, timeout time.Duration) *Client { @@ -188,6 +192,12 @@ func (c *Client) Do(ctx context.Context, req Request) (*Response, error) { hr.Header.Set("Accept", accept) hr.Header.Set("User-Agent", "spicrawl-cli/"+Version) + if c.Wait != nil { + if done := c.Wait(method, req.Path); done != nil { + defer done() + } + } + var sent atomic.Bool hr = hr.WithContext(httptrace.WithClientTrace(hr.Context(), &httptrace.ClientTrace{ WroteHeaders: func() { sent.Store(true) }, diff --git a/internal/cmd/agents.go b/internal/cmd/agents.go index de74132..3f5abe8 100644 --- a/internal/cmd/agents.go +++ b/internal/cmd/agents.go @@ -36,6 +36,7 @@ import ( "github.com/OfficialSpicrawl/cli/internal/config" "github.com/OfficialSpicrawl/cli/internal/output" + "github.com/OfficialSpicrawl/cli/internal/ui" ) const ( @@ -422,11 +423,13 @@ func agentApply(rs []*agentResult) error { } // agentReport prints results (JSON list, or a table) plus secret guidance. +// On a terminal the table gets marks and colour and a count line under it. func agentReport(p *output.Printer, rs []*agentResult, s agentScope) error { if rs == nil { rs = []*agentResult{} } err := p.Result(rs, func(w io.Writer) { + u := outUI(p) rows := make([][]string, 0, len(rs)) for _, r := range rs { note := r.Reason @@ -436,14 +439,75 @@ func agentReport(p *output.Printer, rs []*agentResult, s agentScope) error { } note += "backup: " + agentDisplayPath(r.Backup, s) } - rows = append(rows, []string{r.Client, r.Kind, r.Action, agentDisplayPath(r.File, s), note}) + action := r.Action + if u.On() { + action = agentActionMark[r.Action] + " " + action + } + rows = append(rows, []string{r.Client, r.Kind, action, agentDisplayPath(r.File, s), note}) + } + viewTable(p, u, []string{"CLIENT", "KIND", "ACTION", "FILE", "NOTE"}, rows, func(col int, cell string) string { + switch { + case col == 2 && strings.HasPrefix(cell, "✓"): + return u.Green(cell) + case col == 2 || col == 4: + return u.Dim(cell) + } + return cell + }) + if u.On() { + agentSummary(w, u, rs) } - p.Table([]string{"CLIENT", "KIND", "ACTION", "FILE", "NOTE"}, rows) }) agentSecretGuidance(p, rs, s) return err } +// agentActionMark is the mark in front of an action on a terminal. +var agentActionMark = map[string]string{ + agentActionCreated: "✓", + agentActionUpdated: "✓", + agentActionUnchange: "=", + agentActionSkipped: "–", +} + +// agentSummary is the line under the table: what was written, kept, skipped. +func agentSummary(w io.Writer, u *ui.UI, rs []*agentResult) { + n := map[string]int{} + for _, r := range rs { + n[r.Action]++ + } + var parts []string + if c := n[agentActionCreated] + n[agentActionUpdated]; c > 0 { + parts = append(parts, fmt.Sprintf("%d written", c)) + } + for _, a := range []string{agentActionUnchange, agentActionSkipped} { + if n[a] > 0 { + parts = append(parts, fmt.Sprintf("%d %s", n[a], a)) + } + } + if len(parts) == 0 { + return + } + m := markOK + if n[agentActionCreated]+n[agentActionUpdated]+n[agentActionUnchange] == 0 { + m = markNeutral + } + fmt.Fprintf(w, "\n%s %s\n", viewMark(u, m), strings.Join(parts, " · ")) +} + +// agentChanged is the names of the clients whose file was just written. +func agentChanged(rs []*agentResult) []string { + var names []string + seen := map[string]bool{} + for _, r := range rs { + if (r.Action == agentActionCreated || r.Action == agentActionUpdated) && !seen[r.Client] { + seen[r.Client] = true + names = append(names, agentClientByID(r.Client).Name) + } + } + return names +} + // agentSecretGuidance warns about literal keys, with .gitignore advice for // files inside the project. func agentSecretGuidance(p *output.Printer, rs []*agentResult, s agentScope) { diff --git a/internal/cmd/agents_golden_test.go b/internal/cmd/agents_golden_test.go new file mode 100644 index 0000000..dd30d01 --- /dev/null +++ b/internal/cmd/agents_golden_test.go @@ -0,0 +1,57 @@ +package cmd + +import ( + "net/http" + "testing" +) + +// The agent commands (init, mcp install, skill install) write config files and +// print what they did. Their base URL is an unreachable address so the MCP and +// docs URLs they write are fixed; the skill comes from a fake docs server. +const ( + gAgentBase = "http://127.0.0.1:1" + gAgentSkill = "---\nname: spicrawl-web-data-api\ndescription: golden test skill\n---\n\n# Spicrawl\n\nUse https://api.spicrawl.com for scraping.\n" +) + +func gSkillAPI() func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"GET /skill.md": func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/markdown") + _, _ = w.Write([]byte(gAgentSkill)) + }}) +} + +func init() { + env := map[string]string{agentEnvSkillURL: "{server}/skill.md"} + // agent builds a case that runs in a fresh project directory. + agent := func(name string, human bool, files []string, setup func(t *testing.T, e *goldenEnv), args ...string) goldenCase { + return goldenCase{name: name, human: human, args: args, baseURL: gAgentBase, env: env, server: gSkillAPI(), files: files, setup: setup} + } + allMCP := []string{".mcp.json", ".cursor/mcp.json", ".vscode/mcp.json", ".codex/config.toml"} + existing := func(t *testing.T, e *goldenEnv) { + goldenWrite(t, e.Dir, ".mcp.json", `{"zeta": 1, "mcpServers": {"other": {"command": "x"}}}`) + } + + c := agent("agents/mcp_print_all", false, nil, nil, "mcp", "install", "--client", "all", "--dir", "{dir}", "--print") + c2 := agent("agents/mcp_install_claude", false, []string{".mcp.json"}, nil, "mcp", "install", "--client", "claude", "--dir", "{dir}") + c3 := agent("agents/mcp_install_all", false, allMCP, nil, "mcp", "install", "--client", "all", "--dir", "{dir}") + c4 := agent("agents/mcp_install_merges_existing", false, []string{".mcp.json"}, existing, "mcp", "install", "--client", "claude", "--dir", "{dir}") + c5 := agent("agents/mcp_err_no_client_detected", false, nil, nil, "mcp", "install", "--dir", "{dir}") + c5.stderr = true + c6 := agent("agents/skill_print", false, nil, nil, "skill", "install", "--print", "--dir", "{dir}") + c7 := agent("agents/skill_install_claude", false, []string{".claude/skills/spicrawl/SKILL.md"}, nil, "skill", "install", "--client", "claude", "--dir", "{dir}") + c8 := agent("agents/skill_install_codex_agents_md", false, []string{".agents/skills/spicrawl/SKILL.md", "AGENTS.md"}, nil, "skill", "install", "--client", "codex", "--agents-md", "--dir", "{dir}") + c9 := agent("agents/init_yes", false, []string{".mcp.json", ".claude/skills/spicrawl/SKILL.md", ".cursor/mcp.json"}, nil, "init", "--client", "claude,cursor", "--yes", "--dir", "{dir}") + c10 := agent("agents/init_needs_yes_without_terminal", false, nil, nil, "init", "--client", "claude", "--dir", "{dir}") + c10.stderr = true + c11 := agent("agents/init_err_no_client_detected", false, nil, nil, "init", "--yes", "--dir", "{dir}") + c11.stderr = true + + goldenAdd(c, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, + agent("agents_human/mcp_print_all", true, nil, nil, "mcp", "install", "--client", "all", "--dir", "{dir}", "--print"), + agent("agents_human/mcp_install_all", true, nil, nil, "mcp", "install", "--client", "all", "--dir", "{dir}"), + agent("agents_human/mcp_install_merges_existing", true, nil, existing, "mcp", "install", "--client", "claude", "--dir", "{dir}"), + agent("agents_human/skill_print", true, nil, nil, "skill", "install", "--print", "--dir", "{dir}"), + agent("agents_human/skill_install_claude", true, nil, nil, "skill", "install", "--client", "claude", "--dir", "{dir}"), + agent("agents_human/init_yes", true, nil, nil, "init", "--client", "claude,cursor", "--yes", "--dir", "{dir}"), + ) +} diff --git a/internal/cmd/agents_test.go b/internal/cmd/agents_test.go index 13873e5..e987510 100644 --- a/internal/cmd/agents_test.go +++ b/internal/cmd/agents_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" "testing" + "time" ) const agentTestSkill = "---\nname: spicrawl-web-data-api\ndescription: test skill\n---\n\n# Spicrawl\n" @@ -879,3 +880,145 @@ func TestMCPInstallNonSecretKeepsMode(t *testing.T) { t.Errorf("mode = %v, want 0664 kept", fi.Mode().Perm()) } } + +// ---- terminal output ---- + +func TestInitHumanPlainIsUnchanged(t *testing.T) { + dir, _ := agentTestEnv(t) + agentWrite(t, filepath.Join(dir, "CLAUDE.md"), "x") + out, errOut, code := runCLIHuman(t, agentTestBase, "init", "--yes", "--dir", dir) + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + if !strings.HasPrefix(out, "CLIENT KIND ACTION FILE") || strings.Contains(out, "✓") { + t.Errorf("table changed:\n%s", out) + } + if !strings.HasPrefix(errOut, "clients: Claude Code\nMCP server: "+agentTestMCPURL+"\n") { + t.Errorf("stderr changed:\n%s", errOut) + } + if strings.Contains(errOut, "next") { + t.Errorf("hints without a terminal:\n%s", errOut) + } + noEscapes(t, "stdout", out) + noEscapes(t, "stderr", errOut) +} + +func TestInitLive(t *testing.T) { + liveUI(t) + dir, _ := agentTestEnv(t) + agentWrite(t, filepath.Join(dir, "CLAUDE.md"), "x") + out, errOut, code := runCLIHuman(t, agentTestBase, "init", "--yes", "--dir", dir) + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + plain := stripANSI(out) + for _, want := range []string{"CLIENT", "✓ created", ".mcp.json", ".claude/skills/spicrawl/SKILL.md", "✓ 2 written"} { + if !strings.Contains(plain, want) { + t.Errorf("stdout lacks %q:\n%s", want, plain) + } + } + e := stripANSI(errOut) + for _, want := range []string{"Setting up Spicrawl", "clients Claude Code", "MCP server " + agentTestMCPURL, "next", "restart Claude Code"} { + if !strings.Contains(e, want) { + t.Errorf("stderr lacks %q:\n%s", want, e) + } + } + if strings.Contains(e, "clients: ") { + t.Errorf("plain intro lines next to the styled one:\n%s", e) + } + + // Nothing changed the second time: no "next" list, and the count says so. + out, errOut, _ = runCLIHuman(t, agentTestBase, "init", "--yes", "--dir", dir) + if !strings.Contains(stripANSI(out), "✓ 2 unchanged") || strings.Contains(stripANSI(errOut), "next") { + t.Errorf("second run:\n%s\n%s", stripANSI(out), stripANSI(errOut)) + } + + // --quiet and --json stay plain. + for _, flag := range []string{"-q", "--json"} { + out, errOut, _ = runCLIHuman(t, agentTestBase, flag, "init", "--yes", "--dir", dir) + noEscapes(t, flag+" stdout", out) + noEscapes(t, flag+" stderr", errOut) + } +} + +func TestInitLivePlanAndSkillSpinner(t *testing.T) { + spinNow(t) + dir, _ := agentTestEnv(t) + slow, _ := newRecordingServer(t, func(w http.ResponseWriter, _ *http.Request) { + time.Sleep(40 * time.Millisecond) + _, _ = w.Write([]byte(agentTestSkill)) + }) + t.Setenv(agentEnvSkillURL, slow.URL+"/skill.md") + orig, origIn := initStdinIsTerminal, initStdin + initStdinIsTerminal = func() bool { return true } + t.Cleanup(func() { initStdinIsTerminal, initStdin = orig, origIn }) + initStdin = strings.NewReader("n\n") + + out, errOut, code := runCLIHuman(t, agentTestBase, "init", "--client", "codex", "--dir", dir) + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + if !strings.Contains(errOut, "Downloading the Spicrawl skill") { + t.Errorf("no spinner while the skill downloads: %q", errOut) + } + e := stripANSI(errOut) + if !strings.Contains(e, "plan:\n create codex mcp") || !strings.Contains(e, "apply these changes? [y/N]") { + t.Errorf("plan:\n%s", e) + } + if !strings.Contains(stripANSI(out), "– skipped") || !strings.Contains(stripANSI(out), "not confirmed") { + t.Errorf("report:\n%s", stripANSI(out)) + } +} + +func TestMCPAndSkillInstallLive(t *testing.T) { + liveUI(t) + dir, _ := agentTestEnv(t) + out, errOut, code := runCLIHuman(t, agentTestBase, "mcp", "install", "--client", "claude", "--dir", dir) + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + if !strings.Contains(stripANSI(out), "✓ created") || !strings.Contains(stripANSI(errOut), "restart Claude Code to load the MCP server") { + t.Errorf("mcp install:\n%s\n%s", stripANSI(out), stripANSI(errOut)) + } + out, errOut, code = runCLIHuman(t, agentTestBase, "skill", "install", "--client", "claude", "--dir", dir) + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + if !strings.Contains(stripANSI(out), "✓ created") || !strings.Contains(stripANSI(errOut), "start a new session in Claude Code to use the skill") { + t.Errorf("skill install:\n%s\n%s", stripANSI(out), stripANSI(errOut)) + } + // Idempotent: nothing written, nothing to restart. + _, errOut, _ = runCLIHuman(t, agentTestBase, "skill", "install", "--client", "claude", "--dir", dir) + if strings.Contains(stripANSI(errOut), "next") { + t.Errorf("hint after a no-op:\n%s", errOut) + } +} + +func TestMCPInstallPrintHeadingStyleOnlyOnATerminal(t *testing.T) { + dir, _ := agentTestEnv(t) + plain, _, _ := runCLIHuman(t, agentTestBase, "mcp", "install", "--client", "claude", "--print", "--dir", dir) + if !strings.HasPrefix(plain, "# Claude Code: merge into .mcp.json\n{") { + t.Fatalf("plain:\n%s", plain) + } + noEscapes(t, "plain", plain) + + liveUI(t) + live, _, _ := runCLIHuman(t, agentTestBase, "mcp", "install", "--client", "claude", "--print", "--dir", dir) + if !strings.Contains(live, "\x1b[") || stripANSI(live) != plain { + t.Errorf("styled heading must be the only difference:\n%q\n%q", live, plain) + } +} + +func TestSkillInstallPrintUnchangedWhenLive(t *testing.T) { + dir, _ := agentTestEnv(t) + plain, _, code := runCLIHuman(t, agentTestBase, "skill", "install", "--print", "--dir", dir) + if code != 0 || !strings.Contains(plain, "# Spicrawl") { + t.Fatalf("exit %d %q", code, plain) + } + spinNow(t) + live, _, code := runCLIHuman(t, agentTestBase, "skill", "install", "--print", "--dir", dir) + if code != 0 || live != plain { + t.Errorf("--print must write the skill and nothing else: exit %d\n%q\n%q", code, live, plain) + } + noEscapes(t, "stdout", live) +} diff --git a/internal/cmd/auth.go b/internal/cmd/auth.go index 7fb9392..6e795ae 100644 --- a/internal/cmd/auth.go +++ b/internal/cmd/auth.go @@ -17,6 +17,7 @@ import ( "github.com/OfficialSpicrawl/cli/internal/config" "github.com/OfficialSpicrawl/cli/internal/exitcode" "github.com/OfficialSpicrawl/cli/internal/output" + "github.com/OfficialSpicrawl/cli/internal/ui" ) // authKeysURL is where a person creates an API key. @@ -121,7 +122,10 @@ func authRunLogin(cmd *cobra.Command, _ []string) error { if err != nil { return err } - if err := authCheckKey(rootCmd.Context(), r.BaseURL, key); err != nil { + sp := spin(p, "Checking the key with %s", r.BaseURL) + err = authCheckKey(rootCmd.Context(), r.BaseURL, key) + sp.Stop() + if err != nil { return err } @@ -137,10 +141,27 @@ func authRunLogin(cmd *cobra.Command, _ []string) error { return err } res := map[string]any{"saved": r.Path, "key": config.MaskKey(key), "base_url": r.BaseURL} - return p.Result(res, func(w io.Writer) { + err = p.Result(res, func(w io.Writer) { + if u := outUI(p); u.On() { + fmt.Fprintf(w, "%s Logged in\n", u.Green("✓")) + fmt.Fprint(w, viewRows(u, " ", []ui.Row{ + {Key: "key", Val: config.MaskKey(key)}, + {Key: "base url", Val: r.BaseURL}, + {Key: "saved to", Val: r.Path}, + })) + return + } fmt.Fprintf(w, "Logged in. Key %s saved to %s\n", config.MaskKey(key), r.Path) fmt.Fprintf(w, "Base URL: %s\n", r.BaseURL) }) + if err == nil { + viewNext(p, + "spicrawl status check the API, your key and the workers", + "spicrawl scrape fetch a page", + "spicrawl init set up your AI agent clients", + ) + } + return err } // authPromptKey asks for the key on the terminal. Without golang.org/x/term @@ -150,9 +171,13 @@ func authPromptKey() (string, error) { if w == nil { w = stderr } - fmt.Fprintf(w, "Create an API key at %s\n", authKeysURL) + var u *ui.UI + if authPromptW == nil { + u = errUI(Printer()) + } + fmt.Fprintf(w, "Create an API key at %s\n", u.Cyan(authKeysURL)) fmt.Fprintln(w, "warning: the key will be visible as you type or paste it") - fmt.Fprint(w, "API key: ") + fmt.Fprint(w, u.Bold("API key: ")) line, err := bufio.NewReader(authStdin).ReadString('\n') if err != nil && !errors.Is(err, io.EOF) { return "", fmt.Errorf("read key: %w", err) @@ -195,10 +220,11 @@ func authRunLogout(_ *cobra.Command, _ []string) error { p.Warn("%s is still set in the environment and will keep being used", config.EnvAPIKey) } return p.Result(map[string]any{"path": path, "removed": removed}, func(w io.Writer) { + u := outUI(p) if removed { - fmt.Fprintf(w, "Removed the API key from %s\n", path) + fmt.Fprintf(w, "%sRemoved the API key from %s\n", viewLead(u, markOK), path) } else { - fmt.Fprintf(w, "No API key saved in %s\n", path) + fmt.Fprintf(w, "%sNo API key saved in %s\n", viewLead(u, markNeutral), path) } }) } @@ -222,7 +248,9 @@ func authRunStatus(cmd *cobra.Command, _ []string) error { case r.APIKey == "": fail = ErrNoKey case !authStatusOffline: + sp := spin(p, "Checking your key with %s", r.BaseURL) err := authCheckKey(rootCmd.Context(), r.BaseURL, r.APIKey) + sp.Stop() if prob, ok := api.AsProblem(err); ok && exitcode.ForCode(prob.Code) == exitcode.Auth { res["key_valid"] = false res["check_error"] = prob.Error() @@ -250,6 +278,10 @@ func authRunStatus(cmd *cobra.Command, _ []string) error { case false: valid = "no" } + if u := outUI(p); u.On() { + authPrintLive(w, u, r, key, valid) + return + } p.Table([]string{"FIELD", "VALUE"}, [][]string{ {"api key", key}, {"base url", r.BaseURL + " (from " + r.BaseURLSource + ")"}, @@ -262,3 +294,22 @@ func authRunStatus(cmd *cobra.Command, _ []string) error { } return fail } + +// authPrintLive is the terminal view of "auth status": a verdict line, then +// the facts the plain table lists. +func authPrintLive(w io.Writer, u *ui.UI, r config.Resolved, key, valid string) { + verdict := map[string]string{ + "yes": u.Green("✓") + " " + u.Bold("The API accepts your key"), + "no": u.Red("✗") + " " + u.Bold("The API rejected your key"), + "not checked": viewMark(u, markNeutral) + " " + u.Bold("Key not checked"), + }[valid] + if r.APIKey == "" { + verdict = u.Red("✗") + " " + u.Bold("No API key is configured") + } + fmt.Fprintln(w, verdict) + fmt.Fprint(w, viewRows(u, " ", []ui.Row{ + {Key: "api key", Val: key}, + {Key: "base url", Val: r.BaseURL + " (from " + r.BaseURLSource + ")"}, + {Key: "config", Val: r.Path}, + })) +} diff --git a/internal/cmd/auth_test.go b/internal/cmd/auth_test.go index ff09c2b..b8176d3 100644 --- a/internal/cmd/auth_test.go +++ b/internal/cmd/auth_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/OfficialSpicrawl/cli/internal/config" ) @@ -197,3 +198,132 @@ func TestAuthStatusNoKey(t *testing.T) { t.Fatalf("exit %d %v", code, m) } } + +// authRunHuman is authRun in human mode (no --json) with extra global args. +func authRunHuman(t *testing.T, cfgPath, envKey string, args ...string) (string, string, int) { + t.Helper() + t.Setenv("SPICRAWL_CONFIG", cfgPath) + t.Setenv("SPICRAWL_API_KEY", envKey) + t.Setenv("SPICRAWL_BASE_URL", "") + var out, errb bytes.Buffer + stdout, stderr, humanOutput = &out, &errb, true + defer func() { + stdout, stderr, humanOutput = os.Stdout, os.Stderr, false + resetFlags(rootCmd) + }() + rootCmd.SetArgs(args) + code := Execute() + return out.String(), errb.String(), code +} + +func TestAuthLoginHumanPlainIsUnchanged(t *testing.T) { + base, _ := authOKServer(t) + cfg := filepath.Join(t.TempDir(), "config.json") + out, errb, code := authRunHuman(t, cfg, "", "login", "--api-key", authGoodKey, "--base-url", base) + if code != 0 { + t.Fatalf("exit %d: %s", code, errb) + } + want := "Logged in. Key " + config.MaskKey(authGoodKey) + " saved to " + cfg + "\nBase URL: " + base + "\n" + if out != want || errb != "" { + t.Errorf("stdout %q stderr %q, want %q and nothing", out, errb, want) + } +} + +func TestAuthLoginLive(t *testing.T) { + liveUI(t) + base, _ := authOKServer(t) + cfg := filepath.Join(t.TempDir(), "config.json") + out, errb, code := authRunHuman(t, cfg, "", "login", "--api-key", authGoodKey, "--base-url", base) + if code != 0 { + t.Fatalf("exit %d: %s", code, errb) + } + plain := stripANSI(out) + for _, want := range []string{"✓ Logged in", config.MaskKey(authGoodKey), cfg, base} { + if !strings.Contains(plain, want) { + t.Errorf("stdout lacks %q:\n%s", want, plain) + } + } + if e := stripANSI(errb); !strings.Contains(e, "next") || !strings.Contains(e, "spicrawl status") { + t.Errorf("no next steps on stderr: %q", e) + } + + // --quiet and --json get neither colour nor hints. + for _, flag := range []string{"-q", "--json"} { + out, errb, _ = authRunHuman(t, cfg, "", flag, "login", "--api-key", authGoodKey, "--base-url", base) + noEscapes(t, flag+" stdout", out) + if strings.Contains(errb, "next") { + t.Errorf("%s printed hints: %q", flag, errb) + } + } +} + +func TestAuthLoginSpinnerWhileCheckingTheKey(t *testing.T) { + spinNow(t) + srv, _ := newRecordingServer(t, func(w http.ResponseWriter, r *http.Request) { + time.Sleep(40 * time.Millisecond) + _, _ = w.Write([]byte(`{"data":[],"page":{}}`)) + }) + cfg := filepath.Join(t.TempDir(), "config.json") + out, errb, code := authRunHuman(t, cfg, "", "login", "--api-key", authGoodKey, "--base-url", srv.URL) + if code != 0 { + t.Fatalf("exit %d: %s", code, errb) + } + if !strings.Contains(errb, "Checking the key with "+srv.URL) { + t.Errorf("no spinner label: %q", errb) + } + if strings.Contains(out, "Checking the key") { + t.Errorf("spinner on stdout: %q", out) + } +} + +func TestAuthStatusLive(t *testing.T) { + base, _ := authOKServer(t) + cfg := filepath.Join(t.TempDir(), "config.json") + + out, _, code := authRunHuman(t, cfg, authGoodKey, "auth", "status", "--base-url", base) + if code != 0 || !strings.Contains(out, "FIELD") || !strings.Contains(out, "key works") { + t.Fatalf("plain: exit %d\n%s", code, out) + } + noEscapes(t, "plain stdout", out) + + liveUI(t) + out, _, code = authRunHuman(t, cfg, authGoodKey, "auth", "status", "--base-url", base) + plain := stripANSI(out) + if code != 0 || !strings.Contains(plain, "✓ The API accepts your key") || !strings.Contains(plain, "api key") || strings.Contains(plain, "FIELD") { + t.Errorf("accepted: exit %d\n%s", code, plain) + } + out, _, code = authRunHuman(t, cfg, "spicrawl_live_wrong_key_1234567", "auth", "status", "--base-url", base) + if code != 3 || !strings.Contains(stripANSI(out), "✗ The API rejected your key") { + t.Errorf("rejected: exit %d\n%s", code, stripANSI(out)) + } + out, _, code = authRunHuman(t, cfg, "", "auth", "status", "--base-url", base) + if code != 3 || !strings.Contains(stripANSI(out), "✗ No API key is configured") { + t.Errorf("no key: exit %d\n%s", code, stripANSI(out)) + } + out, _, code = authRunHuman(t, cfg, authGoodKey, "auth", "status", "--offline", "--base-url", base) + if code != 0 || !strings.Contains(stripANSI(out), "Key not checked") { + t.Errorf("offline: exit %d\n%s", code, stripANSI(out)) + } +} + +func TestAuthLogoutHuman(t *testing.T) { + base, _ := authOKServer(t) + cfg := filepath.Join(t.TempDir(), "config.json") + authRun(t, cfg, "", "login", "--api-key", authGoodKey, "--base-url", base) + + out, _, _ := authRunHuman(t, cfg, "", "logout") + if out != "Removed the API key from "+cfg+"\n" { + t.Errorf("plain: %q", out) + } + out, _, _ = authRunHuman(t, cfg, "", "logout") + if out != "No API key saved in "+cfg+"\n" { + t.Errorf("plain, nothing to remove: %q", out) + } + + liveUI(t) + authRun(t, cfg, "", "login", "--api-key", authGoodKey, "--base-url", base) + out, _, _ = authRunHuman(t, cfg, "", "logout") + if stripANSI(out) != "✓ Removed the API key from "+cfg+"\n" { + t.Errorf("live: %q", stripANSI(out)) + } +} diff --git a/internal/cmd/batch.go b/internal/cmd/batch.go index 91f7f05..8e53b87 100644 --- a/internal/cmd/batch.go +++ b/internal/cmd/batch.go @@ -19,6 +19,7 @@ import ( "github.com/OfficialSpicrawl/cli/internal/api" "github.com/OfficialSpicrawl/cli/internal/exitcode" "github.com/OfficialSpicrawl/cli/internal/output" + "github.com/OfficialSpicrawl/cli/internal/ui" ) // batchSubmitScrapeFlags are the scrape flags "batch submit" registers: only @@ -210,7 +211,9 @@ func batchRunSubmit(cmd *cobra.Command, args []string) error { if err != nil { return err } + spin := p.UI().Spin("Submitting %d items", len(urls)+len(items)) resp, err := c.Do(cmd.Context(), api.Request{Method: "POST", Path: "/v1/batch", Body: body}) + spin.Stop() if err != nil { return err } @@ -266,6 +269,7 @@ failing that way gives up with that error.`, } func batchWaitAndPrint(cmd *cobra.Command, c *api.Client, p *output.Printer, id string, poll, timeout time.Duration) error { + start := time.Now() raw, werr := batchWaitJob(cmd.Context(), c, p, id, poll, timeout) if raw != nil { var job batchJob @@ -275,6 +279,9 @@ func batchWaitAndPrint(cmd *cobra.Command, c *api.Client, p *output.Printer, id if err := batchPrintJob(p, raw, job); err != nil { return err } + if werr == nil { + batchSummary(p, job, time.Since(start)) + } if werr == nil && !p.JSON && job.Status != "" { p.Info("\nnext: spicrawl batch results %s --all -o %s.jsonl", job.ID, job.ID) } @@ -289,6 +296,19 @@ func batchWaitAndPrint(cmd *cobra.Command, c *api.Client, p *output.Printer, id return werr } +// batchSummary is the ✓/✗ line after a finished wait (terminal only). +func batchSummary(p *output.Printer, job batchJob, took time.Duration) { + pr := job.Progress + parts := []string{"batch " + job.ID, job.Status, fmt.Sprintf("%d/%d items", pr.Completed, pr.Total)} + if pr.Failed > 0 { + parts = append(parts, fmt.Sprintf("%d failed", pr.Failed)) + } + if pr.CreditsCharged > 0 { + parts = append(parts, fmt.Sprintf("%d credits", pr.CreditsCharged)) + } + p.UI().Done(job.Status == "completed", append(parts, ui.Elapsed(took))...) +} + var batchGetCmd = &cobra.Command{ Use: "get ", Short: "Show a batch job and its progress", diff --git a/internal/cmd/batch_golden_test.go b/internal/cmd/batch_golden_test.go new file mode 100644 index 0000000..3ef11e9 --- /dev/null +++ b/internal/cmd/batch_golden_test.go @@ -0,0 +1,193 @@ +package cmd + +import ( + "fmt" + "net/http" + "sync/atomic" + "testing" +) + +const gBatchID = "01J9Z6V0Q8M4K2T7R3N5B1C9XA" + +// gJob renders a BatchJob the way the API does. +type gJob struct { + Status string + Done, Failed int + Total int + Open bool + Name string + Err string // error_code + Finished, Exp bool +} + +func (j gJob) JSON() string { + ok := j.Done - j.Failed + extra := "" + if j.Err != "" { + extra += fmt.Sprintf(`,"error_code":%q,"error_message":"failure threshold reached"`, j.Err) + } + if j.Status != "queued" { + extra += `,"started_at":"2026-09-22T06:00:05Z"` + } + if j.Finished { + extra += `,"finished_at":"2026-09-22T06:03:41Z"` + } + if j.Exp { + extra += `,"results_expire_at":"2026-09-25T06:00:00Z"` + } + pct := 0.0 + if j.Total > 0 { + pct = float64(j.Done) * 100 / float64(j.Total) + } + return fmt.Sprintf(`{"id":%q,"project_id":"proj_01J9Z","name":%q,"status":%q,"open":%t,`+ + `"status_url":"/v1/batch/%s","results_url":"/v1/batch/%s/results","params":{"js_render":true,"response_format":"markdown"},`+ + `"total_items":%d,"concurrency":10,"priority":100,"max_attempts":3,"estimated_credits":%d,"estimated_credits_micro":%d,`+ + `"progress":{"total":%d,"completed":%d,"succeeded":%d,"failed":%d,"cancelled":0,"skipped":0,"remaining":%d,"percent_complete":%.1f,"credits_charged":%d,"credits_charged_micro":%d,"bytes":%d},`+ + `"submitted_at":"2026-09-22T06:00:00Z"%s}`, + gBatchID, j.Name, j.Status, j.Open, gBatchID, gBatchID, j.Total, j.Total*3, j.Total*3000000, + j.Total, j.Done, ok, j.Failed, j.Total-j.Done, pct, j.Done*3, j.Done*3000000, j.Done*20480, extra) +} + +// gSeq answers each request with the next body (the last one repeats): a job +// that moves through queued, running and completed while the CLI polls it. +func gSeq(bodies ...string) http.HandlerFunc { + var n atomic.Int32 + return func(w http.ResponseWriter, r *http.Request) { + i := int(n.Add(1)) - 1 + gJSON(http.StatusOK, bodies[min(i, len(bodies)-1)])(w, r) + } +} + +const gBatchResults = `{"seq":0,"external_id":"sku-1","url":"https://example.com/a","status":"succeeded","http_status":200,"bytes":2048,"result":{"content":"\"

A

\\n\""}} +{"seq":1,"external_id":"","url":"https://example.com/b","status":"failed","http_status":null,"bytes":0,"error":{"code":"ERR::UPSTREAM::TIMEOUT","message":"the target did not answer"}} +{"seq":2,"external_id":"sku-3","url":"https://example.com/c?x=1&y=2","status":"succeeded","http_status":404,"bytes":512,"result":{"content":"\"not found\""}} +` + +const gBatchURLs = "# nightly targets\nhttps://example.com/a\n\nhttps://example.com/b\n" + +func gBatchSetupURLs(t *testing.T, e *goldenEnv) { goldenWrite(t, e.Dir, "urls.txt", gBatchURLs) } + +func init() { + queued := gJob{Status: "queued", Total: 2, Name: "nightly"} + running := gJob{Status: "running", Done: 1, Total: 2, Name: "nightly"} + completed := gJob{Status: "completed", Done: 3, Failed: 1, Total: 3, Name: "nightly", Finished: true, Exp: true} + failed := gJob{Status: "failed", Done: 2, Failed: 2, Total: 10, Name: "nightly", Finished: true, Err: "ERR::LIMIT::FAILURE_THRESHOLD"} + cancelled := gJob{Status: "cancelled", Done: 1, Total: 5, Name: "nightly", Finished: true} + open := gJob{Status: "running", Done: 4, Total: 9, Name: "crawl", Open: true} + + submit := goldenServe(map[string]http.HandlerFunc{"POST /v1/batch": gJSON(202, queued.JSON())}) + get := func(j gJob) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}": gJSON(200, j.JSON())}) + } + submitWait := func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{ + "POST /v1/batch": gJSON(202, queued.JSON()), + "GET /v1/batch/{id}": gSeq(queued.JSON(), running.JSON(), completed.JSON()), + })() + } + list := func(body string) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"GET /v1/batch": gJSON(200, body)}) + } + two := `{"batches":[` + completed.JSON() + `,` + open.JSON() + `]}` + results := func(next string) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/results": func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/x-ndjson") + if next != "" && r.URL.Query().Get("cursor") == "" { + w.Header().Set("X-Next-Cursor", next) + } + _, _ = fmt.Fprint(w, gBatchResults) + }}) + } + job := func(method string, j gJob) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{method + " /v1/batch/{id}/{action}": gJSON(200, j.JSON())}) + } + wrapped := func(field string, n int) string { + return fmt.Sprintf(`{"job":%s,"%s":%d,"items_dispatched":%d}`, running.JSON(), field, n, n-1) + } + + goldenAdd( + // submit + goldenCase{name: "batch/submit_urls", args: []string{"batch", "submit", "{dir}/urls.txt", "--render", "--name", "nightly"}, setup: gBatchSetupURLs, server: submit}, + goldenCase{name: "batch/submit_items_jsonl", args: []string{"batch", "submit", "{dir}/items.jsonl", "--format", "markdown"}, + setup: func(t *testing.T, e *goldenEnv) { + goldenWrite(t, e.Dir, "items.jsonl", `{"url":"https://example.com/a","external_id":"sku-1"}`+"\n"+`{"url":"https://example.com/b","external_id":"sku-2","js_render":false}`+"\n") + }, server: submit}, + goldenCase{name: "batch/submit_stdin", args: []string{"batch", "submit", "-"}, stdin: gBatchURLs, server: submit}, + goldenCase{name: "batch/submit_wait", args: []string{"batch", "submit", "{dir}/urls.txt", "--wait", "--poll", "1ms"}, setup: gBatchSetupURLs, server: submitWait}, + goldenCase{name: "batch/submit_wait_open_is_usage_error", args: []string{"batch", "submit", "{dir}/urls.txt", "--wait", "--open"}, stderr: true, setup: gBatchSetupURLs, server: submit}, + goldenCase{name: "batch/submit_err_request", args: []string{"batch", "submit", "{dir}/urls.txt", "--format", "pdf"}, stderr: true, setup: gBatchSetupURLs, + server: goldenServe(map[string]http.HandlerFunc{"POST /v1/batch": gProblem(400, "ERR::REQUEST::INVALID_PARAMETER", "response_format pdf is not supported on batch")})}, + goldenCase{name: "batch_human/submit_urls", human: true, args: []string{"batch", "submit", "{dir}/urls.txt", "--name", "nightly"}, setup: gBatchSetupURLs, server: submit}, + goldenCase{name: "batch_human/submit_wait", human: true, args: []string{"batch", "submit", "{dir}/urls.txt", "--wait", "--poll", "1ms"}, setup: gBatchSetupURLs, server: submitWait}, + + // wait + goldenCase{name: "batch/wait_completed", args: []string{"batch", "wait", gBatchID, "--poll", "1ms"}, server: get(completed)}, + goldenCase{name: "batch/wait_sequence", args: []string{"batch", "wait", gBatchID, "--poll", "1ms"}, + server: func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}": gSeq(queued.JSON(), running.JSON(), completed.JSON())})() + }}, + goldenCase{name: "batch/wait_failed_exit4", args: []string{"batch", "wait", gBatchID, "--poll", "1ms"}, stderr: true, server: get(failed)}, + goldenCase{name: "batch/wait_cancelled_exit4", args: []string{"batch", "wait", gBatchID, "--poll", "1ms"}, stderr: true, server: get(cancelled)}, + goldenCase{name: "batch/wait_pending_exit11", args: []string{"batch", "wait", gBatchID, "--poll", "10ms", "--max-wait", "60ms"}, stderr: true, server: get(running)}, + goldenCase{name: "batch_human/wait_completed", human: true, args: []string{"batch", "wait", gBatchID, "--poll", "1ms"}, server: get(completed)}, + goldenCase{name: "batch_human/wait_failed_exit4", human: true, args: []string{"batch", "wait", gBatchID, "--poll", "1ms"}, server: get(failed)}, + + // get + goldenCase{name: "batch/get", args: []string{"batch", "get", gBatchID}, server: get(completed)}, + goldenCase{name: "batch/get_not_found", args: []string{"batch", "get", gBatchID}, stderr: true, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}": gProblem(404, "ERR::REQUEST::NOT_FOUND", "no such batch")})}, + goldenCase{name: "batch_human/get_completed", human: true, args: []string{"batch", "get", gBatchID}, server: get(completed)}, + goldenCase{name: "batch_human/get_open", human: true, args: []string{"batch", "get", gBatchID}, server: get(open)}, + goldenCase{name: "batch_human/get_failed", human: true, args: []string{"batch", "get", gBatchID}, server: get(failed)}, + + // list + goldenCase{name: "batch/list", args: []string{"batch", "list"}, server: list(two)}, + goldenCase{name: "batch/list_empty", args: []string{"batch", "list", "--status", "paused"}, server: list(`{"batches":[]}`)}, + goldenCase{name: "batch/list_next_cursor", args: []string{"batch", "list", "--limit", "2"}, server: list(`{"batches":[` + completed.JSON() + `],"next_cursor":"1790000000000000000.ABC"}`)}, + goldenCase{name: "batch/list_all_pages", args: []string{"batch", "list", "--all"}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch": func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("cursor") == "" { + gJSON(200, `{"batches":[`+completed.JSON()+`],"next_cursor":"1790000000000000000.ABC"}`)(w, r) + return + } + gJSON(200, `{"batches":[`+open.JSON()+`]}`)(w, r) + }})}, + goldenCase{name: "batch_human/list", human: true, args: []string{"batch", "list"}, server: list(two)}, + goldenCase{name: "batch_human/list_empty", human: true, args: []string{"batch", "list"}, server: list(`{"batches":[]}`)}, + goldenCase{name: "batch_human/list_next_cursor", human: true, args: []string{"batch", "list"}, server: list(`{"batches":[` + completed.JSON() + `],"next_cursor":"1790000000000000000.ABC"}`)}, + + // results and content + goldenCase{name: "batch/results", args: []string{"batch", "results", gBatchID}, server: results("")}, + goldenCase{name: "batch/results_more_pages", args: []string{"batch", "results", gBatchID, "--limit", "3"}, server: results("c2")}, + goldenCase{name: "batch/results_all_pages", args: []string{"batch", "results", gBatchID, "--all"}, server: results("c2")}, + goldenCase{name: "batch/results_output_file", args: []string{"batch", "results", gBatchID, "-o", "{dir}/results.jsonl"}, server: results(""), files: []string{"results.jsonl"}}, + goldenCase{name: "batch/results_empty", args: []string{"batch", "results", gBatchID}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/results": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(200) }})}, + goldenCase{name: "batch/results_expired", args: []string{"batch", "results", gBatchID}, stderr: true, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/results": gProblem(410, "ERR::REQUEST::BEYOND_RETENTION", "results expired")})}, + goldenCase{name: "batch_human/results", human: true, args: []string{"batch", "results", gBatchID}, server: results("")}, + goldenCase{name: "batch_human/results_empty", human: true, args: []string{"batch", "results", gBatchID}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/results": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(200) }})}, + goldenCase{name: "batch/content", args: []string{"batch", "content", gBatchID, "0"}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/tasks/{seq}/content": gJSON(200, `"

A

\n"`)})}, + goldenCase{name: "batch/content_output_file", args: []string{"batch", "content", gBatchID, "0", "-o", "{dir}/a.html"}, files: []string{"a.html"}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/tasks/{seq}/content": gJSON(200, `"

A

\n"`)})}, + goldenCase{name: "batch/content_not_finished", args: []string{"batch", "content", gBatchID, "7"}, stderr: true, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/tasks/{seq}/content": gProblem(409, "ERR::REQUEST::NOT_READY", "item 7 has not finished")})}, + goldenCase{name: "batch/content_bad_seq", args: []string{"batch", "content", gBatchID, "x"}, stderr: true}, + goldenCase{name: "batch_human/content", human: true, args: []string{"batch", "content", gBatchID, "0"}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/batch/{id}/tasks/{seq}/content": gJSON(200, `"

A

\n"`)})}, + + // cancel, close, retry, append + goldenCase{name: "batch/cancel_yes", args: []string{"batch", "cancel", gBatchID, "--yes"}, server: job("POST", cancelled)}, + goldenCase{name: "batch/cancel_needs_yes_without_terminal", args: []string{"batch", "cancel", gBatchID}, stderr: true, server: job("POST", cancelled)}, + goldenCase{name: "batch_human/cancel_yes", human: true, args: []string{"batch", "cancel", gBatchID, "--yes"}, server: job("POST", cancelled)}, + goldenCase{name: "batch/close", args: []string{"batch", "close", gBatchID}, server: job("POST", running)}, + goldenCase{name: "batch/retry", args: []string{"batch", "retry", gBatchID}, server: goldenServe(map[string]http.HandlerFunc{"POST /v1/batch/{id}/retry": gJSON(200, wrapped("items_reset", 2))})}, + goldenCase{name: "batch_human/retry", human: true, args: []string{"batch", "retry", gBatchID}, server: goldenServe(map[string]http.HandlerFunc{"POST /v1/batch/{id}/retry": gJSON(200, wrapped("items_reset", 2))})}, + goldenCase{name: "batch/append", args: []string{"batch", "append", gBatchID, "{dir}/urls.txt"}, setup: gBatchSetupURLs, + server: goldenServe(map[string]http.HandlerFunc{"POST /v1/batch/{id}/items": gJSON(200, wrapped("items_added", 2))})}, + goldenCase{name: "batch_human/append", human: true, args: []string{"batch", "append", gBatchID, "{dir}/urls.txt"}, setup: gBatchSetupURLs, + server: goldenServe(map[string]http.HandlerFunc{"POST /v1/batch/{id}/items": gJSON(200, wrapped("items_added", 2))})}, + ) +} diff --git a/internal/cmd/batch_wait.go b/internal/cmd/batch_wait.go index c2f07a2..a32450d 100644 --- a/internal/cmd/batch_wait.go +++ b/internal/cmd/batch_wait.go @@ -12,6 +12,7 @@ import ( "github.com/OfficialSpicrawl/cli/internal/api" "github.com/OfficialSpicrawl/cli/internal/exitcode" "github.com/OfficialSpicrawl/cli/internal/output" + "github.com/OfficialSpicrawl/cli/internal/ui" ) // batchTerminal are the job statuses at which polling stops. @@ -109,12 +110,14 @@ func batchWaitJob(ctx context.Context, c *api.Client, p *output.Printer, id stri } } -// batchProgress renders the wait progress line on stderr in human mode. On a -// TTY the line is rewritten in place with \r; otherwise a new line is written -// only when the counters change, so logs stay readable. JSON mode and --quiet +// batchProgress renders the wait progress on stderr. On a live terminal (see +// ui.Live) it is a spinner, "Batch job 3/10 done · 12.1s". Otherwise, in human +// mode, a TTY line is rewritten in place with \r and a non-TTY gets a new line +// only when the counters change, so logs stay readable. --json and --quiet // print nothing. type batchProgress struct { p *output.Printer + spin *ui.Spinner enabled bool tty bool last string @@ -122,10 +125,28 @@ type batchProgress struct { } func batchNewProgress(p *output.Printer) *batchProgress { - return &batchProgress{p: p, enabled: !p.JSON && !p.Quiet, tty: batchIsTerminal(p.Err)} + return &batchProgress{p: p, spin: p.UI().Spin("Batch job starting"), enabled: !p.JSON && !p.Quiet, tty: batchIsTerminal(p.Err)} +} + +// batchSpinLabel is "Batch job 3/10 done (1 failed)", or just the status +// before the job has any items counted. +func batchSpinLabel(j batchJob) string { + pr := j.Progress + if pr.Total == 0 { + return "Batch job " + j.Status + } + l := fmt.Sprintf("Batch job %d/%d done", pr.Completed, pr.Total) + if pr.Failed > 0 { + l += fmt.Sprintf(" (%d failed)", pr.Failed) + } + return l } func (b *batchProgress) update(j batchJob) { + if b.spin != nil { + b.spin.Label("%s", batchSpinLabel(j)) + return + } if !b.enabled { return } @@ -145,6 +166,10 @@ func (b *batchProgress) update(j batchJob) { } func (b *batchProgress) note(format string, a ...any) { + if b.spin != nil { + ui.Fprintf(b.p.Err, format+"\n", a...) // above the spinner + return + } if !b.enabled { return } @@ -153,6 +178,7 @@ func (b *batchProgress) note(format string, a ...any) { } func (b *batchProgress) done() { + b.spin.Stop() if b.dirty { fmt.Fprintln(b.p.Err) b.dirty = false diff --git a/internal/cmd/config_golden_test.go b/internal/cmd/config_golden_test.go new file mode 100644 index 0000000..0f39c7d --- /dev/null +++ b/internal/cmd/config_golden_test.go @@ -0,0 +1,119 @@ +package cmd + +import ( + "net/http" + "testing" +) + +const gSavedKey = "spicrawl_live_0123456789abcdefghijklmnop" + +func gSetupConfig(t *testing.T, e *goldenEnv) { + goldenWrite(t, e.CfgDir, "config.json", `{"api_key":"`+gSavedKey+`","base_url":"https://api.example.test"}`) +} + +func gSetupKeyOnlyConfig(t *testing.T, e *goldenEnv) { + goldenWrite(t, e.CfgDir, "config.json", `{"api_key":"`+gSavedKey+`"}`) +} + +// gAuthAPI answers the one call login and auth status make: GET /v1/requests?limit=1. +func gAuthAPI(status int) func() http.HandlerFunc { + h := gJSON(200, `{"data":[],`+gLogPageEnd+`}`) + if status == 401 { + h = gProblem(401, "ERR::AUTH::INVALID_KEY", "the API key is not valid") + } + return goldenServe(map[string]http.HandlerFunc{"GET /v1/requests": h}) +} + +const gDocsHome = "# Spicrawl\n\n- [Quickstart](https://docs.spicrawl.com/quickstart.md)\n- [Anti-bot](https://docs.spicrawl.com/guides/anti-bot.md)\n" + +func gDocsAPI() func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{ + "GET /llms.txt": func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte(gDocsHome)) + }, + "GET /guides/anti-bot.md": func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/markdown") + _, _ = w.Write([]byte("# Anti-bot\n\nUse --stealth only when a page needs it.\n")) + }, + "GET /no-newline.md": func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("# No newline")) }, + }) +} + +func init() { + docsEnv := map[string]string{docsEnvURL: "{server}/"} + + goldenAdd( + // config + goldenCase{name: "config/get_all", args: []string{"config", "get"}, setup: gSetupConfig}, + goldenCase{name: "config/get_all_empty", args: []string{"config", "get"}}, + goldenCase{name: "config/get_base_url", args: []string{"config", "get", "base_url"}, setup: gSetupConfig}, + goldenCase{name: "config/get_api_key_masked", args: []string{"config", "get", "api_key"}, setup: gSetupConfig}, + goldenCase{name: "config/get_api_key_show_secret", args: []string{"config", "get", "api_key", "--show-secret"}, setup: gSetupConfig}, + goldenCase{name: "config/get_unset", args: []string{"config", "get", "base_url"}}, + goldenCase{name: "config/set_base_url", args: []string{"config", "set", "base_url", "http://localhost:8080/"}}, + goldenCase{name: "config/set_api_key", args: []string{"config", "set", "api_key", gSavedKey}}, + goldenCase{name: "config/unset_api_key", args: []string{"config", "unset", "api_key"}, setup: gSetupConfig}, + goldenCase{name: "config/path", args: []string{"config", "path"}}, + goldenCase{name: "config/err_unknown_key", args: []string{"config", "get", "nope"}, stderr: true}, + goldenCase{name: "config/err_bad_base_url", args: []string{"config", "set", "base_url", "ftp://x"}, stderr: true}, + goldenCase{name: "config_human/get_all", human: true, args: []string{"config", "get"}, setup: gSetupConfig}, + goldenCase{name: "config_human/get_all_empty", human: true, args: []string{"config", "get"}}, + goldenCase{name: "config_human/get_base_url", human: true, args: []string{"config", "get", "base_url"}, setup: gSetupConfig}, + goldenCase{name: "config_human/get_unset", human: true, args: []string{"config", "get", "base_url"}}, + goldenCase{name: "config_human/set_api_key", human: true, args: []string{"config", "set", "api_key", gSavedKey}}, + goldenCase{name: "config_human/unset_api_key", human: true, args: []string{"config", "unset", "api_key"}, setup: gSetupConfig}, + goldenCase{name: "config_human/path", human: true, args: []string{"config", "path"}}, + + // auth + goldenCase{name: "auth/status_ok", args: []string{"auth", "status"}, server: gAuthAPI(200)}, + goldenCase{name: "auth/status_offline", args: []string{"auth", "status", "--offline"}}, + goldenCase{name: "auth/status_key_from_file", args: []string{"auth", "status", "--offline"}, noKey: true, setup: gSetupKeyOnlyConfig}, + goldenCase{name: "auth/status_rejected_exit3", args: []string{"auth", "status"}, stderr: true, server: gAuthAPI(401)}, + goldenCase{name: "auth/status_no_key_exit3", args: []string{"auth", "status"}, noKey: true, stderr: true}, + goldenCase{name: "auth_human/status_ok", human: true, args: []string{"auth", "status"}, server: gAuthAPI(200)}, + goldenCase{name: "auth_human/status_rejected_exit3", human: true, args: []string{"auth", "status"}, server: gAuthAPI(401)}, + goldenCase{name: "auth_human/status_no_key_exit3", human: true, args: []string{"auth", "status"}, noKey: true}, + goldenCase{name: "auth/login_ok", args: []string{"login", "--api-key", gSavedKey}, server: gAuthAPI(200)}, + goldenCase{name: "auth/login_rejected_exit3", args: []string{"login", "--api-key", gSavedKey}, stderr: true, server: gAuthAPI(401)}, + goldenCase{name: "auth/login_needs_key_without_terminal", args: []string{"login"}, noKey: true, stderr: true, server: gAuthAPI(200)}, + goldenCase{name: "auth_human/login_ok", human: true, args: []string{"login", "--api-key", gSavedKey}, server: gAuthAPI(200)}, + goldenCase{name: "auth/logout_removed", args: []string{"logout"}, setup: gSetupConfig}, + goldenCase{name: "auth/logout_nothing_saved", args: []string{"logout"}}, + goldenCase{name: "auth_human/logout_removed", human: true, args: []string{"logout"}, setup: gSetupConfig}, + goldenCase{name: "auth_human/logout_nothing_saved", human: true, args: []string{"logout"}}, + + // version, exit codes, root errors + goldenCase{name: "meta/version", args: []string{"version"}}, + goldenCase{name: "meta/version_human", human: true, args: []string{"version"}}, + goldenCase{name: "meta/exit_codes", args: []string{"exit-codes"}}, + goldenCase{name: "meta/exit_codes_human", human: true, args: []string{"exit-codes"}}, + goldenCase{name: "meta/err_unknown_command", args: []string{"zzzzzz"}, stderr: true}, + goldenCase{name: "meta/err_unknown_subcommand_suggests", args: []string{"batch", "lst"}, stderr: true}, + goldenCase{name: "meta/err_unknown_flag", args: []string{"scrape", "https://example.com", "--nope"}, stderr: true}, + + // schema: printed from the embedded files (no network) + goldenCase{name: "schema/list", args: []string{"schema"}}, + goldenCase{name: "schema/list_flag", args: []string{"schema", "--list"}}, + goldenCase{name: "schema/session", args: []string{"schema", "session"}, digest: true}, + goldenCase{name: "schema/scrape", args: []string{"schema", "scrape"}, digest: true}, + goldenCase{name: "schema/batch", args: []string{"schema", "batch"}, digest: true}, + goldenCase{name: "schema/batch_item", args: []string{"schema", "batch-item"}, digest: true}, + goldenCase{name: "schema/err_unknown", args: []string{"schema", "nope"}, stderr: true}, + goldenCase{name: "schema_human/list", human: true, args: []string{"schema"}}, + goldenCase{name: "schema_human/session", human: true, args: []string{"schema", "session"}, digest: true}, + goldenCase{name: "schema_human/scrape", human: true, args: []string{"schema", "scrape"}, digest: true}, + goldenCase{name: "schema_human/batch", human: true, args: []string{"schema", "batch"}, digest: true}, + goldenCase{name: "schema_human/batch_item", human: true, args: []string{"schema", "batch-item"}, digest: true}, + + // docs: the Markdown is the data, so it is printed as is whether or not stdout is a terminal + goldenCase{name: "docs/topic_raw_markdown", args: []string{"docs", "guides/anti-bot"}, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs/list_raw_markdown", args: []string{"docs", "--list"}, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs/no_topic_is_the_index", args: []string{"docs"}, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs/adds_missing_newline", args: []string{"docs", "no-newline"}, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs/json_flag_wraps", args: []string{"docs", "guides/anti-bot"}, explicitJSON: true, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs/err_unknown_topic", args: []string{"docs", "missing"}, stderr: true, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs/err_bad_topic", args: []string{"docs", "../etc/passwd"}, stderr: true, env: docsEnv, server: gDocsAPI()}, + goldenCase{name: "docs_human/topic_raw_markdown", human: true, args: []string{"docs", "guides/anti-bot"}, env: docsEnv, server: gDocsAPI()}, + ) +} diff --git a/internal/cmd/configcmd.go b/internal/cmd/configcmd.go index 37deddb..ac81fcf 100644 --- a/internal/cmd/configcmd.go +++ b/internal/cmd/configcmd.go @@ -70,7 +70,7 @@ value (null in JSON).`, } rows = append(rows, []string{k, v}) } - return p.Result(all, func(io.Writer) { p.Table([]string{"KEY", "VALUE"}, rows) }) + return p.Result(all, func(io.Writer) { viewTable(p, outUI(p), []string{"KEY", "VALUE"}, rows, nil) }) }, } @@ -182,11 +182,13 @@ func configCmdWrite(key, val string) error { if shown != "" { jv = shown } - return Printer().Result(map[string]any{"path": path, "key": key, "value": jv}, func(w io.Writer) { + p := Printer() + return p.Result(map[string]any{"path": path, "key": key, "value": jv}, func(w io.Writer) { + lead := viewLead(outUI(p), markOK) if val == "" { - fmt.Fprintf(w, "Removed %s from %s\n", key, path) + fmt.Fprintf(w, "%sRemoved %s from %s\n", lead, key, path) } else { - fmt.Fprintf(w, "Set %s = %s in %s\n", key, shown, path) + fmt.Fprintf(w, "%sSet %s = %s in %s\n", lead, key, shown, path) } }) } diff --git a/internal/cmd/configcmd_test.go b/internal/cmd/configcmd_test.go index 1f6b63e..51dadfc 100644 --- a/internal/cmd/configcmd_test.go +++ b/internal/cmd/configcmd_test.go @@ -2,6 +2,7 @@ package cmd import ( "path/filepath" + "strings" "testing" ) @@ -55,3 +56,41 @@ func TestConfigPath(t *testing.T) { t.Fatalf("exit %d %s", code, out) } } + +func TestConfigHumanPlainAndLive(t *testing.T) { + cfg := filepath.Join(t.TempDir(), "config.json") + out, _, _ := authRunHuman(t, cfg, "", "config", "set", "base_url", "http://localhost:8080/") + if want := "Set base_url = http://localhost:8080 in " + cfg + "\n"; out != want { + t.Errorf("plain set: %q, want %q", out, want) + } + plainTable, _, _ := authRunHuman(t, cfg, "", "config", "get") + noEscapes(t, "plain table", plainTable) + if val, _, _ := authRunHuman(t, cfg, "", "config", "get", "base_url"); val != "http://localhost:8080\n" { + t.Errorf("get must stay a bare value, got %q", val) + } + + liveUI(t) + out, _, _ = authRunHuman(t, cfg, "", "config", "set", "base_url", "http://localhost:9090") + if want := "✓ Set base_url = http://localhost:9090 in " + cfg + "\n"; stripANSI(out) != want { + t.Errorf("live set: %q, want %q", stripANSI(out), want) + } + table, _, _ := authRunHuman(t, cfg, "", "config", "get") + if !strings.Contains(table, "\x1b[") || !strings.Contains(stripANSI(table), "base_url http://localhost:9090") { + t.Errorf("live table: %q", table) + } + if val, _, _ := authRunHuman(t, cfg, "", "config", "get", "base_url"); val != "http://localhost:9090\n" { + t.Errorf("get must stay a bare value on a terminal too, got %q", val) + } + if p, _, _ := authRunHuman(t, cfg, "", "config", "path"); p != cfg+"\n" { + t.Errorf("path must stay bare, got %q", p) + } + out, _, _ = authRunHuman(t, cfg, "", "config", "unset", "base_url") + if want := "✓ Removed base_url from " + cfg + "\n"; stripANSI(out) != want { + t.Errorf("live unset: %q", stripANSI(out)) + } + + // Machine output is untouched. + jout, _, _ := authRun(t, cfg, "", "config", "set", "base_url", "http://localhost:7070") + noEscapes(t, "json", jout) + authDecode(t, jout) +} diff --git a/internal/cmd/docs.go b/internal/cmd/docs.go index 587b394..a60f3ac 100644 --- a/internal/cmd/docs.go +++ b/internal/cmd/docs.go @@ -70,11 +70,17 @@ An unknown topic exits 2.`, } u = base + "/" + topic + ".md" } + p := Printer() + label := "page " + topic + if topic == "llms.txt" { + label = "index" + } + sp := spin(p, "Fetching the docs %s", label) md, err := docsFetch(rootCmd.Context(), u) + sp.Stop() if err != nil { return err } - p := Printer() // Only an explicit --json wraps the page: the Markdown is the data. if flagJSON { return p.Value(map[string]any{"topic": topic, "url": u, "markdown": md}) diff --git a/internal/cmd/docs_test.go b/internal/cmd/docs_test.go index 00dc99d..764be87 100644 --- a/internal/cmd/docs_test.go +++ b/internal/cmd/docs_test.go @@ -4,6 +4,7 @@ import ( "net/http" "strings" "testing" + "time" "github.com/OfficialSpicrawl/cli/internal/config" ) @@ -160,3 +161,38 @@ func TestDocsDerived(t *testing.T) { t.Errorf("docsResolveURL() with the default base URL = %q, %v, want https://docs.spicrawl.com", got, err) } } + +// The Markdown is the data: a terminal sees exactly the bytes a pipe does, +// and only stderr shows that something is being fetched. +func TestDocsOutputIdenticalLiveAndSpinnerOnStderr(t *testing.T) { + srv, _ := newRecordingServer(t, func(w http.ResponseWriter, r *http.Request) { + time.Sleep(40 * time.Millisecond) + _, _ = w.Write([]byte("# Anti-bot\n")) + }) + t.Setenv(docsEnvURL, srv.URL) + t.Setenv(docsEnvHost, "") + plain, perr, code := runCLIHuman(t, "http://127.0.0.1:1", "docs", "guides/anti-bot") + if code != 0 || plain != "# Anti-bot\n" || perr != "" { + t.Fatalf("plain: exit %d stdout %q stderr %q", code, plain, perr) + } + + spinNow(t) + live, lerr, code := runCLIHuman(t, "http://127.0.0.1:1", "docs", "guides/anti-bot") + if code != 0 || live != plain { + t.Errorf("stdout changed: exit %d %q", code, live) + } + if !strings.Contains(lerr, "Fetching the docs page guides/anti-bot") { + t.Errorf("no spinner label: %q", lerr) + } + _, lerr, _ = runCLIHuman(t, "http://127.0.0.1:1", "docs", "--list") + if !strings.Contains(lerr, "Fetching the docs index") { + t.Errorf("no index label: %q", lerr) + } + + // --json wraps the page and stays silent. + out, errb, _ := runCLIHuman(t, "http://127.0.0.1:1", "--json", "docs", "guides/anti-bot") + noEscapes(t, "json stdout", out) + if errb != "" { + t.Errorf("--json stderr %q", errb) + } +} diff --git a/internal/cmd/harness_golden_test.go b/internal/cmd/harness_golden_test.go new file mode 100644 index 0000000..de5858f --- /dev/null +++ b/internal/cmd/harness_golden_test.go @@ -0,0 +1,464 @@ +package cmd + +// Golden tests: they lock what the CLI writes to stdout (byte for byte) and the +// exit code it returns, for every command and its main modes, so that terminal +// work (spinners, colour, TUI) cannot change what scripts and AI agents parse. +// DESIGN.md "Contract agents rely on" is the rule they enforce: stdout is +// data only, JSON when stdout is not a terminal, stable exit codes, and the +// problem document on stderr in JSON mode. +// +// Every case runs three times against its own fake API server, with the +// environment a non-interactive caller can have: nothing set, NO_COLOR=1 and +// CI=true. All three must produce the same stdout, the same exit code and a +// stderr (and stdout) with no ANSI escape, so a feature that colours or +// animates must stay off when the output is not a terminal. +// +// Files (testdata/golden/.): +// +// .stdout stdout, normalized (see goldenNormalize) +// .stdout.sha256 instead of .stdout for outputs too big to review (digest case flag) +// .exit the exit code +// .stderr stderr, only for cases where stderr is itself machine output +// (JSON mode: the problem document); other stderr is prose +// .file. a file the command was asked to write (-o, --screenshot-dir) +// +// Update them after an intended change, then review the diff: +// +// go test ./internal/cmd -run TestGolden -update (or GOLDEN_UPDATE=1 go test ./...) +// +// (-update is a flag of this package's test binary only, so use GOLDEN_UPDATE=1 +// when running ./... .) -update also deletes golden files no case produces. +// +// The default mode of a case is what a script sees: stdout not a terminal, so +// JSON unless the command says otherwise (docs, raw documents). A case with +// human set runs the code path a person at a terminal gets (humanOutput), which +// is the only way to reach it without a pty. + +import ( + "bytes" + "crypto/sha256" + "flag" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "testing" + + "github.com/OfficialSpicrawl/cli/internal/api" +) + +var goldenUpdate = flag.Bool("update", false, "rewrite the files under testdata/golden (or set GOLDEN_UPDATE=1)") + +func goldenWantUpdate() bool { return *goldenUpdate || os.Getenv("GOLDEN_UPDATE") == "1" } + +// goldenKey is the fake API key every case runs with (unless noKey). +const goldenKey = "spicrawl_test_0123456789abcdefghij" + +// goldenCase is one invocation of the CLI. +type goldenCase struct { + name string // "group/case": the golden file stem; unique + args []string // after the global flags; {server}, {dir} and {home} are replaced + human bool // run the human (terminal) code path instead of the script one + + explicitJSON bool // pass --json (the default for a script is the same) + noKey bool // no API key in the environment + baseURL string // --base-url, default: the fake server + server func() http.HandlerFunc // the fake API; built fresh for every run + env map[string]string // extra environment; tokens as in args + stdin string // what os.Stdin holds (never a terminal) + setup func(t *testing.T, e *goldenEnv) + + stderr bool // golden stderr too: it is machine-readable in this case + digest bool // golden a sha256 of stdout instead of the bytes (big outputs) + files []string // paths under {dir} the command writes; their bytes are golden too +} + +// goldenEnv is the sandbox of one run. +type goldenEnv struct { + Dir string // working/project directory + Home string // $HOME + CfgDir string // holds config.json ($SPICRAWL_CONFIG) + Server string // fake API URL, "" when the case has none +} + +var goldenCases []goldenCase + +// goldenAdd registers cases; each *_golden_test.go calls it from init(). +func goldenAdd(cs ...goldenCase) { + seen := map[string]bool{} + for _, c := range goldenCases { + seen[c.name] = true + } + for _, c := range cs { + if seen[c.name] { + panic("duplicate golden case " + c.name) + } + seen[c.name] = true + goldenCases = append(goldenCases, c) + } +} + +// goldenVariants are the environments every case must be insensitive to. +var goldenVariants = []struct { + name string + env map[string]string +}{ + {"plain", nil}, + {"NO_COLOR", map[string]string{"NO_COLOR": "1"}}, + {"CI", map[string]string{"CI": "true"}}, +} + +func TestGolden(t *testing.T) { + for _, c := range goldenCases { + t.Run(strings.ReplaceAll(c.name, "/", "."), func(t *testing.T) { + for _, v := range goldenVariants { + t.Run(v.name, func(t *testing.T) { + goldenRun(t, c, goldenWantUpdate() && v.name == "plain", v.env) + }) + } + }) + } +} + +func goldenRun(t *testing.T, c goldenCase, update bool, variantEnv map[string]string) { + t.Helper() + r, e := goldenExec(t, c, variantEnv) + norm := func(s string) string { return goldenNormalize(s, e) } + + // Not a terminal, so never colour or animation, whatever NO_COLOR/CI say. + for name, s := range map[string]string{"stdout": r.stdout, "stderr": r.stderr} { + if strings.Contains(s, "\x1b") { + t.Errorf("%s has an ANSI escape on a non-terminal: %q", name, s) + } + } + if strings.Contains(r.stderr, "\r") { + t.Errorf("stderr has a carriage return (an in-place spinner or progress line) on a non-terminal: %q", r.stderr) + } + if r.stray != "" { + t.Errorf("the command wrote to os.Stdout directly instead of the stdout writer: %q", r.stray) + } + + out := norm(r.stdout) + if c.digest { + sum := sha256.Sum256([]byte(out)) + goldenCompare(t, update, c.name, ".stdout.sha256", []byte(fmt.Sprintf("sha256 %x\nbytes %d\n", sum, len(out)))) + } else { + goldenCompare(t, update, c.name, ".stdout", []byte(out)) + } + goldenCompare(t, update, c.name, ".exit", []byte(fmt.Sprintf("%d\n", r.code))) + if c.stderr { + goldenCompare(t, update, c.name, ".stderr", []byte(norm(r.stderr))) + } + for _, rel := range c.files { + b, err := os.ReadFile(filepath.Join(e.Dir, rel)) + if err != nil { + t.Errorf("expected the command to write %s: %v", rel, err) + continue + } + goldenCompare(t, update, c.name, ".file."+strings.ReplaceAll(rel, "/", "__"), []byte(norm(string(b)))) + } +} + +type goldenResult struct { + stdout, stderr, stray string + code int +} + +// goldenExec runs c in a fresh sandbox with the variant's extra environment. +func goldenExec(t *testing.T, c goldenCase, variantEnv map[string]string) (goldenResult, *goldenEnv) { + t.Helper() + e := &goldenEnv{Dir: t.TempDir(), Home: t.TempDir(), CfgDir: t.TempDir()} + + // A clean, deterministic environment, whatever the developer's shell has. + for _, k := range []string{ + "NO_COLOR", "CI", "FORCE_COLOR", "CLICOLOR", "CLICOLOR_FORCE", "SPICRAWL_API_KEY", "SPICRAWL_BASE_URL", + agentEnvMCPURL, agentEnvSkillURL, docsEnvURL, docsEnvHost, + } { + goldenUnsetenv(t, k) + } + t.Setenv("HOME", e.Home) + t.Setenv("XDG_CONFIG_HOME", "") + t.Setenv("CODEX_HOME", "") + t.Setenv("TERM", "xterm-256color") // a colour-capable TERM: only the non-terminal stdout may keep colour off + t.Setenv("COLORTERM", "truecolor") + t.Setenv("SPICRAWL_CONFIG", filepath.Join(e.CfgDir, "config.json")) + if !c.noKey { + t.Setenv("SPICRAWL_API_KEY", goldenKey) + } + + if c.server != nil { + srv := httptest.NewServer(c.server()) + t.Cleanup(srv.Close) + e.Server = srv.URL + } + tokens := strings.NewReplacer("{server}", e.Server, "{dir}", e.Dir, "{home}", e.Home) + for k, v := range c.env { + t.Setenv(k, tokens.Replace(v)) + } + for k, v := range variantEnv { + t.Setenv(k, v) + } + if c.setup != nil { + c.setup(t, e) + } + + base := c.baseURL + if base == "" { + base = e.Server + } + args := []string{"--base-url", base} + if c.explicitJSON { + args = append(args, "--json") + } + for _, a := range c.args { + args = append(args, tokens.Replace(a)) + } + + // Never a terminal: not even when `go test` itself runs in one. + tmp := func(name string) *os.File { + f, err := os.CreateTemp(t.TempDir(), name) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { f.Close() }) + return f + } + in, so, se := tmp("stdin"), tmp("stdout"), tmp("stderr") + _, _ = in.WriteString(c.stdin) + _, _ = in.Seek(0, 0) + + var ( + ob, eb bytes.Buffer + code int + ) + func() { + prevIn, prevOut, prevErr := os.Stdin, os.Stdout, os.Stderr + prevW, prevEW, prevHuman := stdout, stderr, humanOutput + os.Stdin, os.Stdout, os.Stderr = in, so, se + stdout, stderr, humanOutput = &ob, &eb, c.human + defer func() { + os.Stdin, os.Stdout, os.Stderr = prevIn, prevOut, prevErr + stdout, stderr, humanOutput = prevW, prevEW, prevHuman + resetFlags(rootCmd) + }() + rootCmd.SetArgs(args) + code = Execute() + }() + stray, _ := os.ReadFile(so.Name()) + return goldenResult{stdout: ob.String(), stderr: eb.String(), stray: string(stray), code: code}, e +} + +// goldenUnsetenv removes k for the test (t.Setenv alone can only set it). +func goldenUnsetenv(t *testing.T, k string) { + t.Helper() + t.Setenv(k, "") + _ = os.Unsetenv(k) +} + +// goldenCompare checks got against testdata/golden/, or rewrites +// the file when update is set. +func goldenCompare(t *testing.T, update bool, name, ext string, got []byte) { + t.Helper() + path := goldenPath(name, ext) + if update { + if old, err := os.ReadFile(path); err == nil && bytes.Equal(old, got) { + return + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, got, 0o644); err != nil { + t.Fatal(err) + } + return + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("no golden file %s (run: go test ./internal/cmd -run TestGolden -update)", path) + } + if !bytes.Equal(want, got) { + t.Errorf("%s differs from the golden file (intended? run: go test ./internal/cmd -run TestGolden -update)\n%s", path, goldenDiff(want, got)) + } +} + +func goldenPath(name, ext string) string { + return filepath.Join("testdata", "golden", filepath.FromSlash(name)+ext) +} + +// goldenDiff shows the first line where want and got differ. +func goldenDiff(want, got []byte) string { + w, g := strings.Split(string(want), "\n"), strings.Split(string(got), "\n") + for i := 0; i < len(w) || i < len(g); i++ { + var wl, gl string + if i < len(w) { + wl = w[i] + } + if i < len(g) { + gl = g[i] + } + if wl != gl || i >= len(w) || i >= len(g) { + return fmt.Sprintf("first difference at line %d:\n want %q\n got %q\n(want %d bytes, got %d bytes)", i+1, wl, gl, len(want), len(got)) + } + } + return "(same lines, different bytes)" +} + +// TestGoldenNoOrphans fails when testdata/golden holds a file no registered +// case produces (a renamed or deleted case); -update removes them. +func TestGoldenNoOrphans(t *testing.T) { + want := map[string]bool{} + for _, c := range goldenCases { + stdoutExt := ".stdout" + if c.digest { + stdoutExt = ".stdout.sha256" + } + exts := []string{stdoutExt, ".exit"} + if c.stderr { + exts = append(exts, ".stderr") + } + for _, f := range c.files { + exts = append(exts, ".file."+strings.ReplaceAll(f, "/", "__")) + } + for _, ext := range exts { + want[filepath.ToSlash(goldenPath(c.name, ext))] = true + } + } + var orphans []string + _ = filepath.WalkDir(filepath.Join("testdata", "golden"), func(p string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() && d.Name() != ".gitattributes" && !want[filepath.ToSlash(p)] { + orphans = append(orphans, p) + } + return nil + }) + sort.Strings(orphans) + for _, p := range orphans { + if goldenWantUpdate() { + _ = os.Remove(p) + continue + } + t.Errorf("orphan golden file %s: no case produces it (run: go test ./internal/cmd -run TestGolden -update)", p) + } +} + +// ---- normalization ---------------------------------------------------------- + +var ( + goldenTimestampRE = regexp.MustCompile(`\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})`) + goldenReqIDJSONRE = regexp.MustCompile(`("request_id":\s*")[^"]+(")`) + goldenReqIDTextRE = regexp.MustCompile(`(request id: )\S+( \(spicrawl logs get )\S+(\))`) +) + +// goldenNormalize turns the parts of an output that differ between runs and +// machines into fixed placeholders, so a golden file changes only when the +// CLI's output does. It is applied to stdout, stderr and written files: +// +// - the fake server's address (random port) -> SERVER (127.0.0.1:41233 -> SERVER) +// - the sandbox directories (random temp paths) -> , , +// - build identity: CLI version, Go version, OS and architecture +// (api.Version is stamped by -ldflags; the others are the host's) +// -> , , , +// - RFC 3339 timestamps (the fixtures' and any the CLI might stamp) -> +// - non-empty request ids ("request_id" fields, "request id:" lines) -> +// +// Empty values are left alone, so a field that goes missing or empty still +// shows. The CLI prints no measured durations today (the durations in its output +// are flags echoed back, e.g. "after 30m0s"), so there is nothing to mask; if +// one is ever added, add its pattern here rather than letting goldens flake. +func goldenNormalize(s string, e *goldenEnv) string { + if e != nil { + for _, d := range []struct{ path, name string }{{e.Dir, ""}, {e.Home, ""}, {e.CfgDir, ""}} { + s = strings.ReplaceAll(s, d.path, d.name) + if real, err := filepath.EvalSymlinks(d.path); err == nil && real != d.path { // macOS: /var -> /private/var + s = strings.ReplaceAll(s, real, d.name) + } + } + if e.Server != "" { + s = strings.ReplaceAll(s, strings.TrimPrefix(e.Server, "http://"), "SERVER") + } + } + for _, r := range [][2]string{ + {`"version": "` + api.Version + `"`, `"version": ""`}, + {"spicrawl " + api.Version + " (", "spicrawl ("}, + {runtime.Version(), ""}, + {`"os": "` + runtime.GOOS + `"`, `"os": ""`}, + {`"arch": "` + runtime.GOARCH + `"`, `"arch": ""`}, + {runtime.GOOS + "/" + runtime.GOARCH + ")", "/)"}, + } { + s = strings.ReplaceAll(s, r[0], r[1]) + } + s = goldenTimestampRE.ReplaceAllString(s, "") + s = goldenReqIDJSONRE.ReplaceAllString(s, "${1}${2}") + s = goldenReqIDTextRE.ReplaceAllString(s, "${1}${2}${3}") + return s +} + +func TestGoldenNormalize(t *testing.T) { + e := &goldenEnv{Dir: "/tmp/x/001", Home: "/tmp/x/002", CfgDir: "/tmp/x/003", Server: "http://127.0.0.1:41233"} + in := "ws://127.0.0.1:41233/v1 /tmp/x/001/a /tmp/x/003/config.json 2026-09-22T06:00:00.5Z 2026-09-22T06:00:00+02:00 " + + `"request_id": "01J9ZQ5N" "request_id": "" request id: 01ABC (spicrawl logs get 01ABC) ` + + `"version": "` + api.Version + `" ` + runtime.Version() + " 2026-09-22" + want := "ws://SERVER/v1 /a /config.json " + + `"request_id": "" "request_id": "" request id: (spicrawl logs get ) ` + + `"version": "" 2026-09-22` + if got := goldenNormalize(in, e); got != want { + t.Errorf("got\n%s\nwant\n%s", got, want) + } +} + +// ---- fake API building blocks ------------------------------------------------- + +// goldenServe builds a fake API from "METHOD /path" patterns (net/http ServeMux +// syntax, so "GET /v1/batch/{id}" works). Anything else is a 404 problem. +func goldenServe(routes map[string]http.HandlerFunc) func() http.HandlerFunc { + return func() http.HandlerFunc { + mux := http.NewServeMux() + for pattern, h := range routes { + mux.HandleFunc(pattern, h) + } + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + writeProblem(w, http.StatusNotFound, "ERR::REQUEST::NOT_FOUND", false) + }) + return mux.ServeHTTP + } +} + +// gJSON answers with a JSON body. +func gJSON(status int, body string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, body) + } +} + +// gProblem answers with an RFC 7807 problem document (a full one: detail, +// request id and a hint, so the human error renderer has every line to show). +func gProblem(status int, code, detail string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/problem+json") + w.Header().Set("X-Request-Id", "01J9ZQ9ERR0000000000000000") + w.WriteHeader(status) + fmt.Fprintf(w, `{"type":"https://docs.spicrawl.com/errors","title":%q,"status":%d,"code":%q,"detail":%q,"retryable":false,"doc_url":"https://docs.spicrawl.com/errors","request_id":"01J9ZQ9ERR0000000000000000","target_status":null,"diagnostics":{"hint":"check the request and try again"}}`, + http.StatusText(status), status, code, detail) + } +} + +// goldenWrite creates path (relative to dir) with content, making parent +// directories, for a case's setup. +func goldenWrite(t *testing.T, dir, rel, content string) string { + t.Helper() + p := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return p +} diff --git a/internal/cmd/initcmd.go b/internal/cmd/initcmd.go index b7bc085..3bbd45a 100644 --- a/internal/cmd/initcmd.go +++ b/internal/cmd/initcmd.go @@ -11,6 +11,7 @@ import ( "github.com/OfficialSpicrawl/cli/internal/config" "github.com/OfficialSpicrawl/cli/internal/output" + "github.com/OfficialSpicrawl/cli/internal/ui" ) var initFlags struct { @@ -89,8 +90,13 @@ func runInit(cmd *cobra.Command, _ []string) error { for i, c := range clients { names[i] = c.Name } - p.Info("clients: %s", strings.Join(names, ", ")) - p.Info("MCP server: %s", ep.MCP) + if !viewIntro(p, "Setting up Spicrawl", + ui.Row{Key: "clients", Val: strings.Join(names, ", ")}, + ui.Row{Key: "MCP server", Val: ep.MCP}, + ) { + p.Info("clients: %s", strings.Join(names, ", ")) + p.Info("MCP server: %s", ep.MCP) + } key, _ := agentAPIKey() opt := mcpOptions{URL: ep.MCP, UseEnv: initFlags.useEnv, Key: key, Force: initFlags.force} @@ -124,7 +130,7 @@ func runInit(cmd *cobra.Command, _ []string) error { } } if pending > 0 && !initFlags.yes { - initPrintPlan(p.Err, rs, s) + initPrintPlan(p.Err, errUI(p), rs, s) if !initStdinIsTerminal() { return Usagef("init changes files and stdin is not a terminal: re-run with --yes to apply the plan above") } @@ -153,11 +159,17 @@ func runInit(cmd *cobra.Command, _ []string) error { if key == "" { p.Warn("no API key is configured: run \"spicrawl login\" or set %s", config.EnvAPIKey) } + if names := agentChanged(rs); len(names) > 0 { + viewNext(p, + "restart "+strings.Join(names, ", ")+" to load the MCP server and the skill", + "then ask your agent to scrape a page, or try: spicrawl scrape https://example.com", + ) + } return nil } -func initPrintPlan(w io.Writer, rs []*agentResult, s agentScope) { - fmt.Fprintln(w, "plan:") +func initPrintPlan(w io.Writer, u *ui.UI, rs []*agentResult, s agentScope) { + fmt.Fprintln(w, u.Bold("plan:")) for _, r := range rs { verb := map[string]string{ agentActionCreated: "create", @@ -165,9 +177,15 @@ func initPrintPlan(w io.Writer, rs []*agentResult, s agentScope) { agentActionUnchange: "keep ", agentActionSkipped: "skip ", }[r.Action] + switch r.Action { + case agentActionCreated, agentActionUpdated: + verb = u.Green(verb) + default: + verb = u.Dim(verb) + } line := fmt.Sprintf(" %s %-7s %-6s %s", verb, r.Client, r.Kind, agentDisplayPath(r.File, s)) if r.Reason != "" { - line += " (" + r.Reason + ")" + line += " " + u.Dim("("+r.Reason+")") } fmt.Fprintln(w, line) } diff --git a/internal/cmd/live_test.go b/internal/cmd/live_test.go new file mode 100644 index 0000000..b2f9cc7 --- /dev/null +++ b/internal/cmd/live_test.go @@ -0,0 +1,315 @@ +package cmd + +import ( + "bytes" + "fmt" + "net/http" + "os" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/OfficialSpicrawl/cli/internal/exitcode" + "github.com/OfficialSpicrawl/cli/internal/ui" +) + +// lockedBuf is a stderr the spinner goroutine and the command both write to. +type lockedBuf struct { + mu sync.Mutex + b bytes.Buffer +} + +func (l *lockedBuf) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.b.Write(p) +} + +func (l *lockedBuf) String() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.b.String() +} + +// runHuman runs the CLI in human mode (stdout is not a terminal, but the +// command behaves as if it were). With live, stderr is a stand-in terminal: +// the spinner, colour and summary lines are drawn on it, as they would be on a +// real one. Without it stderr is a plain pipe, which is what scripts and +// agents get. +func runHuman(t *testing.T, baseURL string, live bool, args ...string) (stdoutS, stderrS string, code int) { + t.Helper() + t.Setenv("SPICRAWL_CONFIG", t.TempDir()+"/config.json") + t.Setenv("SPICRAWL_API_KEY", "spicrawl_test_key") + d, i := ui.Delay, ui.Interval + ui.Delay, ui.Interval = 20*time.Millisecond, 5*time.Millisecond + var out bytes.Buffer + var errb lockedBuf + stdout, stderr, humanOutput = &out, &errb, true + if live { + liveTerm = ui.Force(&errb) + } + defer func() { + stdout, stderr, humanOutput, liveTerm = os.Stdout, os.Stderr, false, nil + ui.Delay, ui.Interval = d, i + resetFlags(rootCmd) + }() + rootCmd.SetArgs(append([]string{"--base-url", baseURL}, args...)) + code = Execute() + return out.String(), errb.String(), code +} + +// slow answers after d, so the spinner has time to appear. +func slow(d time.Duration, h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + time.Sleep(d) + h(w, r) + } +} + +var ansi = regexp.MustCompile(`\x1b\[[0-9;?]*[A-Za-z]`) + +// internalNames must never reach a person's screen from the CLI's own text. +var internalNames = regexp.MustCompile(`(?i)moli|chromium|chrome|camoufox|obscura|browserless|headful|headless|playwright|puppeteer`) + +// A handler that leaks an internal engine name in a header the CLI reads. +func scrapeLeakyHandler(w http.ResponseWriter, r *http.Request) { + scrapeMarkdownHandler(w, r) + w.Header().Set("X-Engine", "moli-chromium") +} + +func TestLiveScrapeShowsSpinnerAndSummaryButNoNewStdout(t *testing.T) { + srv, _ := newRecordingServer(t, slow(150*time.Millisecond, scrapeLeakyHandler)) + args := []string{"scrape", "https://example.com/pricing", "--render", "--format", "markdown"} + + plainOut, plainErr, code := runHuman(t, srv.URL, false, args...) + if code != 0 { + t.Fatalf("plain: exit %d: %s", code, plainErr) + } + liveOut, liveErr, code := runHuman(t, srv.URL, true, args...) + if code != 0 { + t.Fatalf("live: exit %d: %s", code, liveErr) + } + + if liveOut != plainOut || liveOut != "# Hello\n" { + t.Errorf("stdout changed: live %q, plain %q", liveOut, plainOut) + } + if strings.Contains(plainErr, "\x1b") || strings.Contains(plainErr, "✓") { + t.Errorf("plain stderr has live output: %q", plainErr) + } + if plainErr != "warning: FORMAT_COERCED: one\n" { + t.Errorf("plain stderr changed: %q", plainErr) + } + + for _, want := range []string{ + "\x1b[?25l", // cursor hidden while it spins + "Scraping example.com/pricing (rendering JavaScript)", + "\x1b[?25h", // and shown again + "✓", "example.com/pricing", "200", "1 credit", + "warning: FORMAT_COERCED: one\n", + } { + if !strings.Contains(liveErr, want) { + t.Errorf("live stderr lacks %q: %q", want, liveErr) + } + } + if internalNames.MatchString(ansi.ReplaceAllString(liveErr, "")) { + t.Errorf("live stderr names an internal engine: %q", liveErr) + } + // The spinner is gone before the summary: the cursor comes back first. + if strings.LastIndex(liveErr, "\x1b[?25h") > strings.Index(liveErr, "✓") { + t.Errorf("the spinner was still up when the summary printed: %q", liveErr) + } + // Without the escape codes the summary line is "✓ url status · credits · time". + line := regexp.MustCompile(`✓ example.com/pricing 200 · 1 credit · \d+\.\ds`) + if !line.MatchString(ansi.ReplaceAllString(liveErr, "")) { + t.Errorf("summary line not as designed: %q", ansi.ReplaceAllString(liveErr, "")) + } +} + +func TestLiveIsOffForJSONAndQuiet(t *testing.T) { + srv, _ := newRecordingServer(t, slow(100*time.Millisecond, scrapeMarkdownHandler)) + _, wantErr, _ := runHuman(t, srv.URL, false, "scrape", "https://example.com", "--json") + for _, flag := range []string{"--json", "--quiet"} { + out, errOut, code := runHuman(t, srv.URL, true, "scrape", "https://example.com", flag) + if code != 0 { + t.Fatalf("%s: exit %d: %s", flag, code, errOut) + } + if strings.Contains(errOut, "\x1b") || strings.Contains(errOut, "✓") { + t.Errorf("%s: live output on stderr: %q", flag, errOut) + } + if flag == "--json" { + if errOut != wantErr { + t.Errorf("--json stderr changed: %q vs %q", errOut, wantErr) + } + if !strings.HasPrefix(out, "{") { + t.Errorf("--json stdout is not JSON: %q", out) + } + } + } +} + +func TestFastCallNeverFlashesASpinner(t *testing.T) { + srv, _ := newRecordingServer(t, scrapeMarkdownHandler) + d := ui.Delay + defer func() { ui.Delay = d }() + // runHuman lowers the delay to 20 ms; an instant answer is still faster. + _, errOut, code := runHuman(t, srv.URL, true, "scrape", "https://example.com") + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + if strings.Contains(errOut, "\x1b[?25l") || strings.Contains(errOut, "Scraping") { + t.Errorf("a fast call drew a spinner: %q", errOut) + } + if !strings.Contains(errOut, "✓") { + t.Errorf("no summary line: %q", errOut) + } +} + +func TestLiveErrorKeepsCodeAndHint(t *testing.T) { + srv, _ := newRecordingServer(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `{"title":"blocked","status":403,"code":"ERR::TARGET::BLOCKED","detail":"the site refused the request","retryable":true,"request_id":"01REQ","target_status":403,"diagnostics":{"hint":"try --render"}}`) + }) + args := []string{"scrape", "https://example.com/x"} + _, plainErr, pc := runHuman(t, srv.URL, false, args...) + _, liveErr, lc := runHuman(t, srv.URL, true, args...) + if pc != lc || pc == 0 { + t.Fatalf("exit codes differ: plain %d, live %d", pc, lc) + } + for _, want := range []string{"error: ERR::TARGET::BLOCKED: the site refused the request\n", "hint: try --render\n", "request id: 01REQ"} { + if !strings.Contains(plainErr, want) { + t.Errorf("plain stderr lost %q: %q", want, plainErr) + } + } + if strings.Contains(plainErr, "\x1b") { + t.Errorf("plain error has escape codes: %q", plainErr) + } + flat := ansi.ReplaceAllString(liveErr, "") + for _, want := range []string{"✗ ERR::TARGET::BLOCKED: the site refused the request", "try --render", "01REQ", "https://example.com/x", "after"} { + if !strings.Contains(flat, want) { + t.Errorf("live error lacks %q: %q", want, flat) + } + } +} + +func TestLiveBatchWaitShowsProgress(t *testing.T) { + gets := 0 + var mu sync.Mutex + srv, _ := newRecordingServer(t, func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + gets++ + n := gets + mu.Unlock() + time.Sleep(40 * time.Millisecond) + switch { + case n < 3: + fmt.Fprint(w, batchJobJSON("J9", "running", n, 4)) + default: + fmt.Fprint(w, batchJobJSON("J9", "completed", 4, 4)) + } + }) + args := []string{"batch", "wait", "J9", "--poll", "10ms"} + plainOut, plainErr, _ := runHuman(t, srv.URL, false, args...) + mu.Lock() + gets = 0 + mu.Unlock() + liveOut, liveErr, code := runHuman(t, srv.URL, true, args...) + if code != 0 { + t.Fatalf("exit %d: %s", code, liveErr) + } + if liveOut != plainOut { + t.Errorf("stdout changed:\nlive %q\nplain %q", liveOut, plainOut) + } + flat := ansi.ReplaceAllString(liveErr, "") + for _, want := range []string{"Batch job 1/4 done", "✓ batch J9 completed · 4/4 items"} { + if !strings.Contains(flat, want) { + t.Errorf("live stderr lacks %q: %q", want, flat) + } + } + // The plain path keeps its per-change progress lines (stderr is not a terminal). + if !strings.Contains(plainErr, "batch J9: running 1/4") || strings.Contains(plainErr, "\x1b") { + t.Errorf("plain progress changed: %q", plainErr) + } +} + +func TestLiveScrapeManyKeepsStdoutLines(t *testing.T) { + srv, _ := newRecordingServer(t, slow(60*time.Millisecond, scrapeMarkdownHandler)) + run := func(live bool) (string, string) { + scrapeWithStdin(t, "https://example.com/a\n# skip\nhttps://example.com/b\n") + out, errOut, code := runHuman(t, srv.URL, live, "scrape", "-", "--concurrency", "1") + if code != 0 { + t.Fatalf("live=%v exit %d: %s", live, code, errOut) + } + return out, errOut + } + plainOut, plainErr := run(false) + liveOut, liveErr := run(true) + if strings.Count(liveOut, "\n") != 2 || strings.Contains(liveOut, "\x1b") { + t.Errorf("live stdout: %q", liveOut) + } + // Lines come in completion order, which with one worker is input order; the + // request ids and timings are identical, so the bytes match. + if liveOut != plainOut { + t.Errorf("stdout changed:\nlive %q\nplain %q", liveOut, plainOut) + } + if !strings.Contains(plainErr, "2 ok, 0 failed") { + t.Errorf("plain summary changed: %q", plainErr) + } + flat := ansi.ReplaceAllString(liveErr, "") + for _, want := range []string{"Scraping 1/2 done", "✓ 2 URLs 2 ok"} { + if !strings.Contains(flat, want) { + t.Errorf("live stderr lacks %q: %q", want, flat) + } + } +} + +func TestLiveBareCallGetsALabel(t *testing.T) { + srv, _ := newRecordingServer(t, slow(100*time.Millisecond, func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprint(w, `{"data":[],"page":{"has_more":false,"retention_hours":24}}`) + })) + _, errOut, code := runHuman(t, srv.URL, true, "logs") + if code != 0 { + t.Fatalf("exit %d: %s", code, errOut) + } + if !strings.Contains(errOut, "Fetching the request log") { + t.Errorf("no spinner label for a slow call: %q", errOut) + } +} + +// Ctrl-C while the spinner is up: the cursor comes back, the line is cleared, +// and the run ends with exit 130 and "interrupted". +func TestLiveInterruptRestoresTheCursor(t *testing.T) { + if os.PathSeparator == '\\' { + t.Skip("cannot signal self on Windows") + } + inflight, release := make(chan struct{}), make(chan struct{}) + srv, _ := newRecordingServer(t, func(_ http.ResponseWriter, _ *http.Request) { + select { + case inflight <- struct{}{}: + default: + } + <-release + }) + defer close(release) + go func() { + <-inflight + time.Sleep(80 * time.Millisecond) // long enough for the spinner to be drawn + p, _ := os.FindProcess(os.Getpid()) + _ = p.Signal(os.Interrupt) + }() + _, errOut, code := runHuman(t, srv.URL, true, "scrape", "https://example.com") + if code != exitcode.Interrupted { + t.Fatalf("exit %d, want %d: %q", code, exitcode.Interrupted, errOut) + } + if !strings.Contains(errOut, "\x1b[?25l") { + t.Errorf("the spinner never drew: %q", errOut) + } + if strings.LastIndex(errOut, "\x1b[?25h") < strings.LastIndex(errOut, "\x1b[?25l") { + t.Errorf("cursor left hidden: %q", errOut) + } + if !strings.Contains(ansi.ReplaceAllString(errOut, ""), "✗ interrupted") { + t.Errorf("no interrupted line: %q", errOut) + } +} diff --git a/internal/cmd/logs.go b/internal/cmd/logs.go index 1df7d2c..cbc9350 100644 --- a/internal/cmd/logs.go +++ b/internal/cmd/logs.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/cobra" "github.com/OfficialSpicrawl/cli/internal/api" + "github.com/OfficialSpicrawl/cli/internal/ui" ) // logsEntry is the subset of RequestLogEntry the human views read. JSON mode @@ -134,8 +135,11 @@ Status is one of: success, failed, timeout, blocked, rejected, cancelled. return err } p := Printer() + sp := spin(p, "Fetching the request log") + defer sp.Stop() var all []json.RawMessage var last logsPage + read := 0 for { resp, err := c.Do(cmd.Context(), api.Request{Path: "/v1/requests", Query: q}) if err != nil { @@ -147,13 +151,17 @@ Status is one of: success, failed, timeout, blocked, rejected, cancelled. } if logsJSONL { for _, r := range page.Data { - if err := p.Line(r); err != nil { - return err + var perr error + sp.Around(func() { perr = p.Line(r) }) // stdout shares the terminal with the spinner + if perr != nil { + return perr } } } else { all = append(all, page.Data...) } + read += len(page.Data) + sp.Label("Fetching the request log · %d so far", read) last = page if !logsAll || !page.Page.HasMore || page.Page.NextBefore == "" || page.Page.NextBeforeID == "" { break @@ -161,6 +169,7 @@ Status is one of: success, failed, timeout, blocked, rejected, cancelled. q.Set("before", page.Page.NextBefore) q.Set("before_id", page.Page.NextBeforeID) } + sp.Stop() if logsJSONL { return nil } @@ -190,7 +199,17 @@ Status is one of: success, failed, timeout, blocked, rejected, cancelled. if logsAllProjects { header = append(header, "PROJECT") } - p.Table(append(header, "STATUS", "HTTP", "ENGINE", "CREDITS", "TOTAL_MS", "URL"), rows) + u := outUI(p) + statusCol := len(header) // TIME (and PROJECT) come first + viewTable(p, u, append(header, "STATUS", "HTTP", "ENGINE", "CREDITS", "TOTAL_MS", "URL"), rows, func(col int, cell string) string { + switch col { + case 0: + return u.Dim(cell) + case statusCol: + return logsStatusCell(u, cell) + } + return cell + }) if last.Page.HasMore && !logsAll { p.Info("more records exist; pass --all to fetch every page") } @@ -214,11 +233,13 @@ shown as — was not measured, which is not the same as 0.`, if err != nil { return err } + p := Printer() + sp := spin(p, "Fetching request %s", args[0]) resp, err := c.Do(cmd.Context(), api.Request{Path: "/v1/requests/" + url.PathEscape(args[0])}) + sp.Stop() if err != nil { return err } - p := Printer() if p.JSON { return p.RawJSON(resp.Body) } @@ -226,15 +247,37 @@ shown as — was not measured, which is not the same as 0.`, if err := resp.Decode(&e); err != nil { return err } - logsPrintEntry(p.Out, &e) + logsWriteEntry(p.Out, outUI(p), &e) return nil }, } -func logsPrintEntry(w io.Writer, e *logsEntry) { +func logsPrintEntry(w io.Writer, e *logsEntry) { logsWriteEntry(w, nil, e) } + +// logsStatusCell colours a request status: success green, trouble red. +func logsStatusCell(u *ui.UI, s string) string { + return viewStatus(u, s, []string{"success"}, []string{"failed", "timeout", "blocked", "rejected"}) +} + +// logsWriteEntry writes one record as key/value lines. With u on, a verdict +// line leads, keys are dimmed and the section titles are bold. +func logsWriteEntry(w io.Writer, u *ui.UI, e *logsEntry) { + if u.On() { + m := markBad + switch e.Status { + case "success": + m = markOK + case "cancelled": + m = markNeutral + } + fmt.Fprintf(w, "%s %s %s\n\n", viewMark(u, m), u.Bold(logsDash(e.Status)), e.URL) + } kv := func(k, v string) { if v != "" { - fmt.Fprintf(w, "%-18s %s\n", k+":", v) + if k == "status" { + v = logsStatusCell(u, v) + } + fmt.Fprintf(w, "%s %s\n", viewKey(u, 18, k+":"), v) } } kv("id", e.ID) @@ -254,7 +297,7 @@ func logsPrintEntry(w io.Writer, e *logsEntry) { kv("bytes", fmt.Sprintf("in %d, out %d, via proxy %d", e.BytesIn, e.BytesOut, e.ProxyBytes)) } - fmt.Fprintln(w, "\nspans (ms):") + fmt.Fprintln(w, "\n"+u.Bold("spans (ms):")) s := e.Spans for _, sp := range []struct { name string @@ -263,14 +306,14 @@ func logsPrintEntry(w io.Writer, e *logsEntry) { {"admission", s.Admission}, {"dispatch", s.Dispatch}, {"worker", s.Worker}, {"queue", s.Queue}, {"engine", s.Engine}, {"upstream", s.Upstream}, {"extract", s.Extract}, {"total", s.Total}, } { - fmt.Fprintf(w, " %-16s %s\n", sp.name, logsSpan(sp.v)) + fmt.Fprintf(w, " %s %s\n", viewKey(u, 16, sp.name), logsSpan(sp.v)) } pr := e.Proxy - fmt.Fprintln(w, "\nproxy:") + fmt.Fprintln(w, "\n"+u.Bold("proxy:")) pkv := func(k, v string) { if v != "" { - fmt.Fprintf(w, " %-16s %s\n", k+":", v) + fmt.Fprintf(w, " %s %s\n", viewKey(u, 16, k+":"), v) } } pkv("source", logsDash(pr.Source)) @@ -288,7 +331,7 @@ func logsPrintEntry(w io.Writer, e *logsEntry) { } if e.Blocked != nil { - fmt.Fprintln(w, "\nblocked:") + fmt.Fprintln(w, "\n"+u.Bold("blocked:")) pkv("vendor", logsDash(e.Blocked.Vendor)) pkv("rule", e.Blocked.Rule) pkv("signal", e.Blocked.Signal) diff --git a/internal/cmd/logs_test.go b/internal/cmd/logs_test.go index 1fa0e79..36e4e89 100644 --- a/internal/cmd/logs_test.go +++ b/internal/cmd/logs_test.go @@ -3,8 +3,10 @@ package cmd import ( "encoding/json" "net/http" + "net/http/httptest" "strings" "testing" + "time" ) const logsTestEntry = `{"id":"R1","created_at":"2026-09-22T10:00:00.5Z","engine":"fetch","status":"blocked","http_status":403,"url":"https://example.com","credits_micro":0,"proxy":{"source":"pool","country":"de"},"spans":{"admission_ms":1,"dispatch_ms":null,"worker_ms":null,"queue_ms":0,"engine_ms":null,"upstream_ms":120,"extract_ms":null,"total_ms":130},"blocked":{"vendor":"cloudflare","rule":"r","signal":"cf-mitigated"}}` @@ -135,3 +137,91 @@ func TestLogsAllProjectsHumanShowsTheProject(t *testing.T) { t.Errorf("project table has a project column:\n%s", out) } } + +func logsOneEntryServer(t *testing.T, delay time.Duration) *httptest.Server { + srv, _ := newRecordingServer(t, func(w http.ResponseWriter, r *http.Request) { + time.Sleep(delay) + if strings.HasPrefix(r.URL.Path, "/v1/requests/") { + _, _ = w.Write([]byte(logsTestEntry)) + return + } + _, _ = w.Write([]byte(`{"data":[` + logsTestEntry + `],"page":{"has_more":false,"retention_hours":24,"retained_since":"x"}}`)) + }) + return srv +} + +// On a terminal the list is the same table with the status coloured; stripped +// of colour it is byte for byte the plain one. +func TestLogsListLiveIsThePlainTableInColour(t *testing.T) { + srv := logsOneEntryServer(t, 0) + plain, _, code := runCLIHuman(t, srv.URL, "logs") + if code != 0 || !strings.Contains(plain, "STATUS") { + t.Fatalf("exit %d\n%s", code, plain) + } + noEscapes(t, "plain", plain) + + liveUI(t) + live, _, _ := runCLIHuman(t, srv.URL, "logs") + if !strings.Contains(live, "\x1b[31mblocked") { + t.Errorf("blocked is not red: %q", live) + } + if stripANSI(live) != plain { + t.Errorf("columns moved:\n%q\n%q", stripANSI(live), plain) + } +} + +func TestLogsGetLive(t *testing.T) { + srv := logsOneEntryServer(t, 0) + plain, _, code := runCLIHuman(t, srv.URL, "logs", "get", "R1") + if code != 0 { + t.Fatalf("exit %d", code) + } + noEscapes(t, "plain", plain) + + liveUI(t) + live, _, _ := runCLIHuman(t, srv.URL, "logs", "get", "R1") + want := "✗ blocked https://example.com\n\n" + plain + if stripANSI(live) != want { + t.Errorf("live view differs from verdict + plain view:\n%q\n%q", stripANSI(live), want) + } + if !strings.Contains(live, "\x1b[1mspans (ms):") { + t.Errorf("section titles are not bold: %q", live) + } + + // JSON keeps the API's record, unstyled. + out, errb, _ := runCLI(t, srv.URL, "logs", "get", "R1") + noEscapes(t, "json stdout", out) + noEscapes(t, "json stderr", errb) +} + +func TestLogsSpinnerAndJSONLStayClean(t *testing.T) { + spinNow(t) + srv := logsOneEntryServer(t, 40*time.Millisecond) + + _, errb, code := runCLIHuman(t, srv.URL, "logs") + if code != 0 || !strings.Contains(errb, "Fetching the request log") { + t.Errorf("exit %d, stderr %q", code, errb) + } + _, errb, _ = runCLIHuman(t, srv.URL, "logs", "get", "R1") + if !strings.Contains(errb, "Fetching request R1") { + t.Errorf("stderr %q", errb) + } + + out, errb, code := runCLIHuman(t, srv.URL, "logs", "--jsonl") + if code != 0 { + t.Fatalf("exit %d: %s", code, errb) + } + noEscapes(t, "jsonl stdout", out) + if strings.Count(out, "\n") != 1 || json.Unmarshal([]byte(out), new(map[string]any)) != nil { + t.Errorf("jsonl stream damaged: %q", out) + } + if !strings.Contains(errb, "Fetching the request log") { + t.Errorf("no spinner on stderr while streaming: %q", errb) + } + + // --json and -q never spin. + for _, flag := range []string{"--json", "-q"} { + _, errb, _ = runCLIHuman(t, srv.URL, flag, "logs") + noEscapes(t, flag+" stderr", errb) + } +} diff --git a/internal/cmd/mcpcmd.go b/internal/cmd/mcpcmd.go index 06c49d4..a4ae0d0 100644 --- a/internal/cmd/mcpcmd.go +++ b/internal/cmd/mcpcmd.go @@ -132,6 +132,9 @@ func runMCPInstall(cmd *cobra.Command, _ []string) error { return ErrNoKey } } + if names := agentChanged(rs); len(names) > 0 { + viewNext(p, "restart "+strings.Join(names, ", ")+" to load the MCP server") + } return nil } @@ -1126,7 +1129,8 @@ func mcpPrint(jsonMode bool, clients []*agentClient, s agentScope, opt mcpOption if i > 0 { fmt.Fprintln(w) } - fmt.Fprintf(w, "# %s: merge into %s\n%s", agentClientByID(sn.Client).Name, agentDisplayPath(sn.File, s), sn.Snippet) + head := fmt.Sprintf("# %s: merge into %s", agentClientByID(sn.Client).Name, agentDisplayPath(sn.File, s)) + fmt.Fprintf(w, "%s\n%s", outUI(p).Bold(head), sn.Snippet) } }) } diff --git a/internal/cmd/ops_golden_test.go b/internal/cmd/ops_golden_test.go new file mode 100644 index 0000000..bf3c944 --- /dev/null +++ b/internal/cmd/ops_golden_test.go @@ -0,0 +1,174 @@ +package cmd + +import ( + "net/http" +) + +// ---- logs ---- + +const ( + gLogOK = `{"id":"01M0FK1EP2E7FBMWAR7W10SMXW","created_at":"2026-09-22T10:00:00Z","project_id":"proj_01J9Z","status":"success","http_status":200,"engine":"fetch",` + + `"credits_micro":1500000,"url":"https://example.com/a","attempts":2,"bytes_in":2048,"bytes_out":512,"proxy_bytes":4096,` + + `"spans":{"admission_ms":1,"dispatch_ms":2,"worker_ms":3,"queue_ms":0,"engine_ms":80,"upstream_ms":120,"extract_ms":null,"total_ms":210},` + + `"proxy":{"source":"pool","reason":"default","provider":"dc-1","requested_tier":"datacenter","requested_country":"de","country":"de","endpoint_id":"ep-1","endpoints":["ep-1","ep-2"],"sticky":true}}` + + gLogBlocked = `{"id":"01M0FK1EP2E7FBMWAR7W10SMXX","created_at":"2026-09-22T10:00:01Z","project_id":"proj_01J9Z","status":"blocked","http_status":403,"engine":"",` + + `"credits_micro":0,"url":"https://example.com/b?x=1&y=2","error_code":"ERR::UPSTREAM::BLOCKED","error_detail":"bot challenge",` + + `"spans":{"admission_ms":1,"dispatch_ms":null,"worker_ms":null,"queue_ms":0,"engine_ms":null,"upstream_ms":120,"extract_ms":null,"total_ms":130},` + + `"proxy":{"source":"pool"},"blocked":{"vendor":"cloudflare","rule":"r-17","signal":"cf-mitigated"}}` + + gLogPageEnd = `"page":{"has_more":false,"retention_hours":24,"retained_since":"2026-09-21T10:00:00Z"}` + gLogPageMore = `"page":{"has_more":true,"next_before":"2026-09-22T10:00:00.5Z","next_before_id":"01M0FK1EP2E7FBMWAR7W10SMXW","retention_hours":24,"retained_since":"2026-09-21T10:00:00Z"}` +) + +// ---- status ---- + +func gStatusAPI(ready bool, keyStatus int, workers string) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{ + "GET /readyz": func(w http.ResponseWriter, r *http.Request) { + if ready { + gJSON(200, `{"status":"ready","checks":{"postgres":"ok","redis":"ok"},"cdp":"enabled"}`)(w, r) + return + } + gJSON(503, `{"status":"not_ready","checks":{"postgres":"ok","redis":"unreachable"},"cdp":"disabled"}`)(w, r) + }, + "GET /v1/requests": func(w http.ResponseWriter, r *http.Request) { + switch keyStatus { + case 401: + gProblem(401, "ERR::AUTH::INVALID_KEY", "the API key is not valid")(w, r) + case 403: + gProblem(403, "ERR::AUTH::INSUFFICIENT_SCOPE", "the key lacks the read scope")(w, r) + default: + gJSON(200, `{"data":[],`+gLogPageEnd+`}`)(w, r) + } + }, + "GET /v1/workers": gJSON(200, workers), + }) +} + +// gStatusAPIWorkersDown is a healthy API whose fleet endpoint fails. +func gStatusAPIWorkersDown() func() http.HandlerFunc { + ok := gStatusAPI(true, 200, "")() + return func() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/workers" { + gProblem(503, "ERR::INTERNAL::UNAVAILABLE", "fleet registry is down")(w, r) + return + } + ok(w, r) + } + } +} + +const gWorkers = `{"kinds":[{"kind":"browser","present":3,"ready":2,"workers":[]},{"kind":"egress","present":1,"ready":1,"workers":[]},{"kind":"batch","present":2,"ready":0,"workers":[]}]}` + +// ---- usage ---- + +const ( + gUsageDay = `{"period":{"start":"2026-08-01","end":"2026-08-04","days":3,"bounds":"[start, end)"},"group_by":"day","groups":[` + + `{"key":"2026-08-01","label":"","metrics":{"requests":10,"credits":12500000,"engine_ms":4200}},` + + `{"key":"2026-08-02","label":"","metrics":{"requests":15,"credits":18500000,"engine_ms":4800}}],` + + `"totals":{"requests":25,"credits":31000000,"engine_ms":9000},"unattributed":{"requests":0,"credits":0,"engine_ms":0},` + + `"usage_as_of":"2026-09-22T06:00:00Z","stale_seconds":120,"warnings":["rollup is partial for 2026-08-03"]}` + + gUsageEngine = `{"period":{"start":"2026-08-01","end":"2026-09-01","days":31,"bounds":"[start, end)"},"group_by":"engine","groups":[` + + `{"key":"fetch","label":"Fetch","metrics":{"requests":40,"credits":40000000}},{"key":"chromium","label":"","metrics":{"requests":5,"credits":25000000}}],` + + `"totals":{"requests":47,"credits":66000000},"unattributed":{"requests":2,"credits":1000000},"usage_as_of":null,"stale_seconds":7200}` + + gUsageKey = `{"period":{"start":"2026-08-01","end":"2026-09-01","days":31,"bounds":"[start, end)"},"group_by":"key","keys":[` + + `{"key_id":"K1","name":"ci-key","prefix":"spicrawl_live_ab12","project_id":"proj_01J9Z","revoked":false,"days":[],"totals":{"requests":7,"client_cli":3}},` + + `{"key_id":"K2","name":"old","prefix":"spicrawl_live_cd34","project_id":"proj_01J9Z","revoked":true,"days":[],"totals":{"requests":1}}]}` + + gUsageSummary = `{"period":{"start":"2026-09-01","end":"2026-10-01","days":30,"bounds":"[start, end)"},"period_elapsed_days":22,` + + `"plan":{"code":"starter","name":"Starter","subscription_status":"active","period_source":"subscription"},` + + `"credits":{"included_micro":50000000000,"used_micro":12345670000,"remaining_micro":37654330000,"overage_micro":0,"used_basis_points":2469},` + + `"allowance":{"limit_micro":50000000000,"used_micro":12400000000,"remaining_micro":37600000000,"unlimited":false,"resets_at":"2026-10-01T00:00:00Z"},` + + `"metrics":{"requests":1234,"credits":12345670000,"extractions":55},"warnings":["allowance counter is 40s behind"]}` + + gUsageRecon = `{"day":"2026-09-21","interpretation":"The request log holds no rows for the day; drift is not evidence of overbilling.",` + + `"drift":[{"metric":"requests","billed":10,"derived":0}]}` +) + +func init() { + get := func(path, body string) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"GET " + path: gJSON(200, body)}) + } + logs := func(body string) func() http.HandlerFunc { return get("/v1/requests", body) } + both := `{"data":[` + gLogOK + `,` + gLogBlocked + `],` + gLogPageEnd + `}` + logsPaged := goldenServe(map[string]http.HandlerFunc{"GET /v1/requests": func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("before") == "" { + gJSON(200, `{"data":[`+gLogOK+`],`+gLogPageMore+`}`)(w, r) + return + } + gJSON(200, `{"data":[`+gLogBlocked+`],`+gLogPageEnd+`}`)(w, r) + }}) + logGet := func(body string) func() http.HandlerFunc { return get("/v1/requests/{id}", body) } + + goldenAdd( + goldenCase{name: "logs/list", args: []string{"logs"}, server: logs(both)}, + goldenCase{name: "logs/errors", args: []string{"logs", "--errors", "--limit", "20"}, server: logs(`{"data":[` + gLogBlocked + `],` + gLogPageEnd + `}`)}, + goldenCase{name: "logs/jsonl", args: []string{"logs", "--jsonl"}, server: logs(both)}, + goldenCase{name: "logs/all_pages", args: []string{"logs", "--all"}, server: logsPaged}, + goldenCase{name: "logs/all_pages_jsonl", args: []string{"logs", "--all", "--jsonl"}, server: logsPaged}, + goldenCase{name: "logs/has_more", args: []string{"logs"}, server: logs(`{"data":[` + gLogOK + `],` + gLogPageMore + `}`)}, + goldenCase{name: "logs/empty", args: []string{"logs"}, server: logs(`{"data":[],` + gLogPageEnd + `}`)}, + goldenCase{name: "logs/get", args: []string{"logs", "get", "01M0FK1EP2E7FBMWAR7W10SMXW"}, server: logGet(gLogOK)}, + goldenCase{name: "logs/get_not_found", args: []string{"logs", "get", "nope"}, stderr: true, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/requests/{id}": gProblem(404, "ERR::REQUEST::NOT_FOUND", "no such request")})}, + goldenCase{name: "logs/err_usage_status", args: []string{"logs", "--status", "weird"}, stderr: true}, + goldenCase{name: "logs/err_usage_errors_and_status", args: []string{"logs", "--errors", "--status", "failed"}, stderr: true}, + goldenCase{name: "logs_human/list", human: true, args: []string{"logs"}, server: logs(both)}, + goldenCase{name: "logs_human/all_projects", human: true, args: []string{"logs", "--all-projects", "--errors"}, server: logs(both)}, + goldenCase{name: "logs_human/has_more", human: true, args: []string{"logs"}, server: logs(`{"data":[` + gLogOK + `],` + gLogPageMore + `}`)}, + goldenCase{name: "logs_human/empty", human: true, args: []string{"logs"}, server: logs(`{"data":[],` + gLogPageEnd + `}`)}, + goldenCase{name: "logs_human/get_full", human: true, args: []string{"logs", "get", "x"}, server: logGet(gLogOK)}, + goldenCase{name: "logs_human/get_blocked", human: true, args: []string{"logs", "get", "x"}, server: logGet(gLogBlocked)}, + + // status: the report is on stdout in every outcome, exit 0 / 3 / 10 + goldenCase{name: "status/ok", args: []string{"status"}, server: gStatusAPI(true, 200, gWorkers)}, + goldenCase{name: "status/not_ready", args: []string{"status"}, server: gStatusAPI(false, 200, gWorkers)}, + goldenCase{name: "status/key_invalid_exit3", args: []string{"status"}, stderr: true, server: gStatusAPI(true, 401, gWorkers)}, + goldenCase{name: "status/key_without_read_scope", args: []string{"status"}, server: gStatusAPI(true, 403, gWorkers)}, + goldenCase{name: "status/no_key_exit3", args: []string{"status"}, noKey: true, stderr: true, server: gStatusAPI(true, 200, gWorkers)}, + goldenCase{name: "status/workers_error", args: []string{"status"}, server: gStatusAPIWorkersDown()}, + goldenCase{name: "status/unreachable_exit10", args: []string{"status"}, baseURL: "http://127.0.0.1:1"}, + goldenCase{name: "status_human/ok", human: true, args: []string{"status"}, server: gStatusAPI(true, 200, gWorkers)}, + goldenCase{name: "status_human/not_ready", human: true, args: []string{"status"}, server: gStatusAPI(false, 200, gWorkers)}, + goldenCase{name: "status_human/key_invalid_exit3", human: true, args: []string{"status"}, server: gStatusAPI(true, 401, gWorkers)}, + goldenCase{name: "status_human/no_key_exit3", human: true, args: []string{"status"}, noKey: true, server: gStatusAPI(true, 200, gWorkers)}, + goldenCase{name: "status_human/unreachable_exit10", human: true, args: []string{"status"}, baseURL: "http://127.0.0.1:1"}, + + // usage + goldenCase{name: "usage/by_day", args: []string{"usage", "--from", "2026-08-01", "--to", "2026-08-04"}, server: get("/v1/usage", gUsageDay)}, + goldenCase{name: "usage/by_key", args: []string{"usage", "--group-by", "key"}, server: get("/v1/usage", gUsageKey)}, + goldenCase{name: "usage/summary", args: []string{"usage", "summary"}, server: get("/v1/usage/summary", gUsageSummary)}, + goldenCase{name: "usage/reconciliation", args: []string{"usage", "reconciliation", "--day", "2026-09-21"}, server: get("/v1/usage/reconciliation", gUsageRecon)}, + goldenCase{name: "usage/err_scope", args: []string{"usage"}, stderr: true, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/usage": gProblem(403, "ERR::AUTH::INSUFFICIENT_SCOPE", "the key lacks the read scope")})}, + goldenCase{name: "usage/err_usage_to_before_from", args: []string{"usage", "--from", "2026-09-01", "--to", "2026-09-01"}, stderr: true}, + goldenCase{name: "usage_human/by_day", human: true, args: []string{"usage"}, server: get("/v1/usage", gUsageDay)}, + goldenCase{name: "usage_human/by_engine_unattributed", human: true, args: []string{"usage", "--group-by", "engine"}, server: get("/v1/usage", gUsageEngine)}, + goldenCase{name: "usage_human/by_key", human: true, args: []string{"usage", "--group-by", "key"}, server: get("/v1/usage", gUsageKey)}, + goldenCase{name: "usage_human/summary", human: true, args: []string{"usage", "summary"}, server: get("/v1/usage/summary", gUsageSummary)}, + goldenCase{name: "usage_human/reconciliation", human: true, args: []string{"usage", "reconciliation"}, server: get("/v1/usage/reconciliation", gUsageRecon)}, + goldenCase{name: "usage_human/err_scope", human: true, args: []string{"usage"}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/usage": gProblem(403, "ERR::AUTH::INSUFFICIENT_SCOPE", "the key lacks the read scope")})}, + + // browser url + goldenCase{name: "browser/url", args: []string{"browser", "url", "--country", "de"}, server: gBrowser(200)}, + goldenCase{name: "browser/url_quiet_prints_bare_url", args: []string{"-q", "browser", "url"}, server: gBrowser(200)}, + goldenCase{name: "browser/url_quiet_json", args: []string{"-q", "--json", "browser", "url"}, server: gBrowser(200)}, + goldenCase{name: "browser/not_enabled_exit8", args: []string{"browser", "url"}, stderr: true, server: gBrowser(501)}, + goldenCase{name: "browser/err_usage_ttl", args: []string{"browser", "url", "--ttl", "5"}, stderr: true}, + goldenCase{name: "browser/err_no_key", args: []string{"browser", "url"}, noKey: true, stderr: true, server: gBrowser(200)}, + goldenCase{name: "browser_human/url", human: true, args: []string{"browser", "url"}, server: gBrowser(200)}, + ) +} + +func gBrowser(status int) func() http.HandlerFunc { + h := gJSON(200, `{"path":"/v1/browser?token=wbt_testtoken&engine=chromium&proxy_country=de","expires_at":"2026-09-22T06:01:00Z"}`) + if status == 501 { + h = gProblem(501, "ERR::ENGINE::UNAVAILABLE", "cloud browser is not enabled") + } + return goldenServe(map[string]http.HandlerFunc{"POST /v1/browser/token": h}) +} diff --git a/internal/cmd/root.go b/internal/cmd/root.go index bf3e0bc..b6bbedc 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "net/http" "os" "os/signal" "strings" @@ -20,6 +21,7 @@ import ( "github.com/OfficialSpicrawl/cli/internal/config" "github.com/OfficialSpicrawl/cli/internal/exitcode" "github.com/OfficialSpicrawl/cli/internal/output" + "github.com/OfficialSpicrawl/cli/internal/ui" ) // Global flags. @@ -71,6 +73,9 @@ var ( stdout io.Writer = os.Stdout stderr io.Writer = os.Stderr humanOutput bool + // liveTerm, when set by a test, stands in for the stderr terminal: spinner, + // colour and summary lines are drawn on it unless --json or --quiet. + liveTerm *ui.UI ) // Printer returns the output printer for the current invocation. @@ -80,6 +85,9 @@ func Printer() *output.Printer { p.JSON = false } p.Out, p.Err = stdout, stderr + if liveTerm != nil && !p.Plain { + p.Term = liveTerm + } return p } @@ -93,10 +101,42 @@ func Client() (*api.Client, error) { return nil, ErrNoKey } c := api.New(r.BaseURL, r.APIKey, flagTimeout) - c.Notify = Printer().Info // retry waits; muted by --quiet + p := Printer() + c.Notify = p.Info // retry waits; muted by --quiet + if u := p.UI(); u.On() { + // Every call shows a spinner once it has been outstanding for a moment. + // A command that already runs a spinner (scrape, batch wait) keeps its + // own label: only one spinner runs at a time. + c.Wait = func(method, path string) func() { + s := u.Spin("%s", waitLabel(method, path)) + return func() { s.Stop() } + } + } return c, nil } +// waitLabel says what a bare API call is doing, for its spinner. +func waitLabel(method, path string) string { + switch { + case strings.HasPrefix(path, "/v1/requests"): + return "Fetching the request log" + case strings.HasPrefix(path, "/v1/usage"): + return "Fetching usage" + case strings.HasPrefix(path, "/v1/sessions"): + if method == http.MethodPost { + return "Creating a session" + } + return "Fetching sessions" + case strings.HasPrefix(path, "/v1/batch"): + return "Contacting the batch API" + case strings.HasPrefix(path, "/v1/browser"): + return "Requesting a browser URL" + case strings.HasPrefix(path, "/v1/workers"): + return "Checking the workers" + } + return "Waiting for the Spicrawl API" +} + // ErrNoKey is returned when no API key is configured anywhere. var ErrNoKey = errors.New(`no API key: run "spicrawl login", set SPICRAWL_API_KEY, or pass --api-key`) @@ -128,11 +168,16 @@ func Execute() int { rootCmd.InitDefaultCompletionCmd() // cobra adds it lazily; groups are fixed up below requireKnownSubcommand(rootCmd) err := rootCmd.ExecuteContext(ctx) + ui.StopAll() // a spinner a command left running (Ctrl-C) must not outlive it if err == nil { return exitcode.OK } if ctx.Err() != nil { // Ctrl-C: whatever error the command made of it, say so briefly - Printer().Info("interrupted") + if p := Printer(); p.UI().On() { + p.UI().Done(false, "interrupted") + } else { + p.Info("interrupted") + } return exitcode.Interrupted } return report(err) @@ -226,6 +271,8 @@ func report(err error) int { if p.JSON { p.Out = p.Err _ = p.Value(prob) + } else if u := p.UI(); u.On() { + u.Fail(prob.Error(), append(problemRows(prob), failRows...)...) } else { fmt.Fprintf(stderr, "error: %s\n", prob.Error()) if h := prob.Hint(); h != "" { @@ -248,12 +295,36 @@ func report(err error) int { if p.JSON { p.Out = p.Err _ = p.Value(map[string]any{"error": err.Error(), "exit_code": code}) + } else if u := p.UI(); u.On() { + u.Fail(err.Error(), failRows...) } else { fmt.Fprintf(stderr, "error: %s\n", err) } return code } +// failRows are extra facts a command wants under its error block on a +// terminal (the URL it was on, how long it ran). One command runs per process. +var failRows []ui.Row + +func failContext(rows ...ui.Row) { failRows = append(failRows, rows...) } + +// problemRows are the facts under an API error's headline on a terminal: the +// same ones the plain output lists, the ERR:: code stays in the headline. +func problemRows(prob *api.Problem) []ui.Row { + rows := []ui.Row{{Key: "hint", Val: prob.Hint()}} + if prob.TargetStatus != nil { + rows = append(rows, ui.Row{Key: "target", Val: fmt.Sprintf("returned %d", *prob.TargetStatus)}) + } + if prob.Retryable { + rows = append(rows, ui.Row{Key: "retryable", Val: "yes"}) + } + if prob.RequestID != "" { + rows = append(rows, ui.Row{Key: "request", Val: fmt.Sprintf("%s (spicrawl logs get %s)", prob.RequestID, prob.RequestID)}) + } + return rows +} + // isCobraUsage recognises cobra's own flag/arg errors, which are plain errors. func isCobraUsage(err error) bool { msg := err.Error() diff --git a/internal/cmd/scrape.go b/internal/cmd/scrape.go index 930bcaf..8ba68d5 100644 --- a/internal/cmd/scrape.go +++ b/internal/cmd/scrape.go @@ -9,11 +9,13 @@ import ( "io" "mime" "net/http" + "net/url" "os" "path/filepath" "strconv" "strings" "sync" + "time" "unicode/utf8" "github.com/spf13/cobra" @@ -21,6 +23,7 @@ import ( "github.com/OfficialSpicrawl/cli/internal/api" "github.com/OfficialSpicrawl/cli/internal/exitcode" "github.com/OfficialSpicrawl/cli/internal/output" + "github.com/OfficialSpicrawl/cli/internal/ui" ) var ( @@ -174,8 +177,14 @@ func scrapeRun(cmd *cobra.Command, args []string) error { return err } body["url"] = args[0] + term := p.UI() + spin := term.Spin("%s", scrapeLabel(args[0], body)) + defer spin.Stop() + start := time.Now() resp, err := scrapeDo(cmd.Context(), client, body) + spin.Stop() if err != nil { + failContext(ui.Row{Key: "url", Val: args[0]}, ui.Row{Key: "after", Val: ui.Elapsed(time.Since(start))}) return err } res, err := scrapeResultOf(args[0], resp) @@ -183,6 +192,10 @@ func scrapeRun(cmd *cobra.Command, args []string) error { return err } targetErr := res.checkTarget(body) + // The last thing on stderr, after the document: ✓/✗ url status credits time. + defer func() { + term.Done(targetErr == nil, scrapeSummary(args[0], res, body, time.Since(start))...) + }() // An error page must not replace a good file from an earlier run: -o (and // the screenshots next to it) is written only for a target that is ok. write := outFile != "" @@ -648,6 +661,10 @@ func scrapeRunMany(ctx context.Context, p *output.Printer, base map[string]any) // requests nobody can read are not worth billing. runCtx, stop := context.WithCancel(ctx) defer stop() + term := p.UI() + spin := term.Spin("Scraping %d URLs", len(urls)) + defer spin.Stop() + start := time.Now() var ( mu sync.Mutex ok int @@ -666,8 +683,15 @@ func scrapeRunMany(ctx context.Context, p *output.Printer, base map[string]any) } else { ok++ } + if failed > 0 { + spin.Label("Scraping %d/%d done (%d failed)", ok+failed, len(urls), failed) + } else { + spin.Label("Scraping %d/%d done", ok+failed, len(urls)) + } if writeErr == nil { - if writeErr = p.Line(line); writeErr != nil { + // The spinner shares the terminal with stdout: step it aside. + spin.Around(func() { writeErr = p.Line(line) }) + if writeErr != nil { stop() } } @@ -725,8 +749,18 @@ feed: } close(jobs) wg.Wait() + spin.Stop() - if !p.JSON { + if term.On() { + parts := []string{fmt.Sprintf("%d URLs", len(urls)), fmt.Sprintf("%d ok", ok)} + if failed > 0 { + parts = append(parts, fmt.Sprintf("%d failed", failed)) + } + if notRun := len(urls) - ok - failed; notRun > 0 { + parts = append(parts, fmt.Sprintf("%d not run", notRun)) + } + term.Done(failed == 0 && ok == len(urls), append(parts, ui.Elapsed(time.Since(start)))...) + } else if !p.JSON { if notRun := len(urls) - ok - failed; notRun > 0 { p.Info("%d ok, %d failed, %d not run", ok, failed, notRun) } else { @@ -755,3 +789,61 @@ func scrapeErrorObject(err error) (any, int) { } return map[string]any{"detail": err.Error(), "exit_code": code}, code } + +// scrapeLabel is the spinner text for one scrape: the page, and what the user +// asked the API to do that makes it slow. Only what the request says is shown, +// never the tier the API picks, and never an engine name. +func scrapeLabel(target string, body map[string]any) string { + label := "Scraping " + scrapeShortURL(target) + var does []string + switch { + case body["stealth"] == true, scrapeBrowserEngine(body["engine"]), body["actions"] != nil: + does = append(does, "using a full browser") + case body["js_render"] == true: + does = append(does, "rendering JavaScript") + } + if body["ai_extract"] != nil { + does = append(does, "extracting with AI") + } + if len(does) > 0 { + label += " (" + strings.Join(does, ", ") + ")" + } + return label +} + +// scrapeBrowserEngine reports whether a pinned engine is a browser. +func scrapeBrowserEngine(v any) bool { + e, _ := v.(string) + return e != "" && e != "fetch" +} + +// scrapeShortURL is host and path without the scheme, query or fragment. +func scrapeShortURL(raw string) string { + u, err := url.Parse(raw) + if err != nil || u.Host == "" { + return raw + } + return u.Host + strings.TrimRight(u.Path, "/") +} + +// scrapeSummary is the facts of the ✓/✗ line: the page, the target's status, +// the credits charged and how long the call took. +func scrapeSummary(target string, r *scrapeResult, body map[string]any, took time.Duration) []string { + parts := []string{scrapeShortURL(target)} + if st := r.targetStatus(body); st != 0 { + parts = append(parts, strconv.Itoa(st)) + } + h := r.resp.Header + credits := strings.TrimSpace(h.Get("X-Credits-Charged")) + if credits == "" && r.obj["credits"] != nil { + credits = fmt.Sprint(r.obj["credits"]) + } + switch credits { + case "": + case "1": + parts = append(parts, "1 credit") + default: + parts = append(parts, credits+" credits") + } + return append(parts, ui.Elapsed(took)) +} diff --git a/internal/cmd/scrape_golden_test.go b/internal/cmd/scrape_golden_test.go new file mode 100644 index 0000000..1a78c87 --- /dev/null +++ b/internal/cmd/scrape_golden_test.go @@ -0,0 +1,136 @@ +package cmd + +import ( + "encoding/json" + "io" + "net/http" + "strconv" + "strings" +) + +// gScrapeRaw answers POST /v1/scrape with the raw document and the metadata +// headers the CLI wraps into its JSON object. +func gScrapeRaw(contentType, body string, target int, warnings ...string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + h := w.Header() + h.Set("Content-Type", contentType) + h.Set("X-Request-Id", "01J9ZQ5N8S4T9WY3L6P7Q1C2DE") + h.Set("X-Credits-Charged", "2") + h.Set("X-Request-Cost", "2") + h.Set("X-Engine", "fetch") + h.Set("X-Target-Status", strconv.Itoa(target)) + h.Set("X-Proxy-Source", "pool") + h.Set("X-Final-Url", "https://example.com/final") + h.Set("Cache-State", "miss") + for _, wn := range warnings { + h.Add("X-Warning", wn) + } + _, _ = io.WriteString(w, body) + } +} + +// gScrapeEnv answers with the API's JSON envelope (response_format=json, or +// any of extract, ai, autoparse, links, network capture, screenshot). +func gScrapeEnv(extra string) http.HandlerFunc { + body := `{"url":"https://example.com/product/1","final_url":"https://example.com/product/1","status":200,` + + `"content":"

Anvil

$19.99 & free shipping

","headers":{"content-type":"text/html"},` + + `"truncated":false,"credits":3,"engine":"fetch","proxy_source":"pool","warnings":[]` + extra + `}` + return gJSON(http.StatusOK, body) +} + +func gScrape(h http.HandlerFunc) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{"POST /v1/scrape": h}) +} + +const gHTMLDoc = "\nExample Domain\n

Example Domain

Prices from $19.99 & up.

\n" + +func init() { + const u = "https://example.com/product/1" + md := gScrapeRaw("text/markdown; charset=utf-8", "# Example Domain\n\nPrices from $19.99 & up.\n", 200, "FORMAT_COERCED: markdown requested, html served") + pdf := gScrapeRaw("application/pdf", "%PDF-1.4\n1 0 obj\n<<>>\nendobj\n%%EOF\n", 200) + shot := "UE5HREFUQQ==" // base64 "PNGDATA" + shots := gScrapeEnv(`,"screenshots":[{"label":"page","format":"png","width":1280,"height":720,"data":"` + shot + `"},{"label":"page","format":"png","width":1280,"height":720,"data":"` + shot + `"}]`) + + goldenAdd( + // ---- what a script gets: stdout is not a terminal, so one JSON object ---- + goldenCase{name: "scrape/html", args: []string{"scrape", u, "--format", "html"}, server: gScrape(gScrapeRaw("text/html; charset=utf-8", gHTMLDoc, 200))}, + goldenCase{name: "scrape/markdown", args: []string{"scrape", u, "--format", "markdown"}, server: gScrape(md)}, + goldenCase{name: "scrape/markdown_json_flag", args: []string{"scrape", u, "--format", "markdown"}, explicitJSON: true, server: gScrape(md)}, + goldenCase{name: "scrape/markdown_jsonl", args: []string{"scrape", u, "--format", "markdown", "--jsonl"}, server: gScrape(md)}, + goldenCase{name: "scrape/text", args: []string{"scrape", u, "--format", "text"}, server: gScrape(gScrapeRaw("text/plain; charset=utf-8", "Example Domain\nPrices from $19.99 & up.", 200))}, + goldenCase{name: "scrape/json_envelope", args: []string{"scrape", u, "--format", "json"}, server: gScrape(gScrapeEnv(`,"links":null`))}, + goldenCase{name: "scrape/json_envelope_jsonl", args: []string{"scrape", u, "--format", "json", "--jsonl"}, server: gScrape(gScrapeEnv(``))}, + goldenCase{name: "scrape/pdf_base64", args: []string{"scrape", u, "--format", "pdf"}, server: gScrape(pdf)}, + goldenCase{name: "scrape/extract", args: []string{"scrape", u, "--extract", `{"title":"h1","price":".price"}`}, server: gScrape(gScrapeEnv(`,"data":{"title":"Anvil","price":"$19.99 & free shipping"}`))}, + goldenCase{name: "scrape/ai", args: []string{"scrape", u, "--ai", "the product name and price"}, server: gScrape(gScrapeEnv(`,"data":{"name":"Anvil","price":19.99}`))}, + goldenCase{name: "scrape/autoparse", args: []string{"scrape", u, "--autoparse"}, server: gScrape(gScrapeEnv(`,"data":{"json_ld":[{"@type":"Product","name":"Anvil"}],"open_graph":{"og:title":"Anvil"}}`))}, + goldenCase{name: "scrape/links", args: []string{"scrape", u, "--links"}, server: gScrape(gScrapeEnv(`,"links":["https://example.com/a","https://example.com/b?x=1&y=2"]`))}, + goldenCase{name: "scrape/network_capture", args: []string{"scrape", u, "--render", "--network-capture", `{"url_patterns":["*/api/*"]}`}, + server: gScrape(gScrapeEnv(`,"network":[{"url":"https://example.com/api/price","status":200,"resource_type":"xhr","content_type":"application/json","body":"{\"price\":19.99}"}]`))}, + goldenCase{name: "scrape/screenshot_dir", args: []string{"scrape", u, "--render", "--screenshot", "--screenshot-dir", "{dir}/shots"}, server: gScrape(shots), + files: []string{"shots/page.png", "shots/page-2.png"}}, + goldenCase{name: "scrape/output_file", args: []string{"scrape", u, "--format", "markdown", "-o", "{dir}/page.md"}, server: gScrape(md), files: []string{"page.md"}}, + goldenCase{name: "scrape/output_stdout_dash", args: []string{"scrape", u, "--format", "markdown", "-o", "-"}, server: gScrape(md)}, + + // target answered non-2xx: the document is still printed, exit 6, target_error + goldenCase{name: "scrape/target_503", args: []string{"scrape", u}, stderr: true, server: gScrape(gScrapeRaw("text/html", "

Service Unavailable

\n", 503))}, + goldenCase{name: "scrape/target_404_allowed", args: []string{"scrape", u, "--allowed-status", "404"}, server: gScrape(gScrapeRaw("text/html", "

Not Found

\n", 404))}, + goldenCase{name: "scrape/target_503_output_file_not_written", args: []string{"scrape", u, "-o", "{dir}/page.html"}, stderr: true, server: gScrape(gScrapeRaw("text/html", "

Down

\n", 503))}, + goldenCase{name: "scrape/target_403_envelope", args: []string{"scrape", u, "--extract", `{"t":"h1"}`}, stderr: true, + server: gScrape(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Target-Status", "403") + gScrapeEnv(`,"data":null`)(w, r) + })}, + + // many URLs on stdin: one compact JSON line each (completion order; --concurrency 1 makes it input order) + goldenCase{name: "scrape/stdin_many", args: []string{"scrape", "-", "--format", "markdown", "--concurrency", "1"}, stderr: true, + stdin: "# targets\nhttps://example.com/ok\n\nhttps://example.com/down\nhttps://example.com/denied\n", + server: gScrape(gScrapeStdinMany)}, + goldenCase{name: "scrape/stdin_many_all_ok", args: []string{"scrape", "-", "--format", "markdown", "--concurrency", "1"}, stdin: "https://example.com/ok\nhttps://example.com/ok2\n", server: gScrape(gScrapeStdinMany)}, + goldenCase{name: "scrape/stdin_many_empty", args: []string{"scrape", "-"}, stderr: true, stdin: "\n# nothing\n", server: gScrape(gScrapeStdinMany)}, + + // failures: the problem document goes to stderr, stdout stays empty + goldenCase{name: "scrape/err_auth", args: []string{"scrape", u}, stderr: true, server: gScrape(gProblem(401, "ERR::AUTH::INVALID_KEY", "the API key is not valid"))}, + goldenCase{name: "scrape/err_limit", args: []string{"scrape", u}, stderr: true, server: gScrape(gProblem(429, "ERR::LIMIT::RATE", "slow down"))}, + goldenCase{name: "scrape/err_request", args: []string{"scrape", u, "--render", "--engine", "nope"}, stderr: true, server: gScrape(gProblem(400, "ERR::REQUEST::INVALID_PARAMETER", "engine: unknown value"))}, + goldenCase{name: "scrape/err_upstream", args: []string{"scrape", u}, stderr: true, server: gScrape(gProblem(502, "ERR::UPSTREAM::BLOCKED", "the target served a bot challenge"))}, + goldenCase{name: "scrape/err_engine", args: []string{"scrape", u, "--ai", "x"}, stderr: true, server: gScrape(gProblem(501, "ERR::ENGINE::UNAVAILABLE", "ai_extract is not available"))}, + goldenCase{name: "scrape/err_usage_ai_exclusive", args: []string{"scrape", u, "--ai", "x", "--ai-schema", `{"type":"object"}`}, stderr: true, server: gScrape(md)}, + goldenCase{name: "scrape/err_usage_bad_extract", args: []string{"scrape", u, "--extract", "{nope"}, server: gScrape(md)}, + goldenCase{name: "scrape/err_usage_no_url", args: []string{"scrape"}, stderr: true, server: gScrape(md)}, + goldenCase{name: "scrape/err_no_key", args: []string{"scrape", u}, noKey: true, stderr: true, server: gScrape(md)}, + goldenCase{name: "scrape/err_unreachable", args: []string{"scrape", u}, baseURL: "http://127.0.0.1:1"}, // stderr names the OS's dial error + + // ---- what a person gets (human mode): the document itself ---- + goldenCase{name: "scrape_human/html", human: true, args: []string{"scrape", u, "--format", "html"}, server: gScrape(gScrapeRaw("text/html", gHTMLDoc, 200))}, + goldenCase{name: "scrape_human/markdown", human: true, args: []string{"scrape", u, "--format", "markdown"}, server: gScrape(md)}, + goldenCase{name: "scrape_human/markdown_meta", human: true, args: []string{"scrape", u, "--format", "markdown", "--meta"}, server: gScrape(md)}, + goldenCase{name: "scrape_human/text_no_trailing_newline", human: true, args: []string{"scrape", u, "--format", "text"}, server: gScrape(gScrapeRaw("text/plain", "Example Domain", 200))}, + goldenCase{name: "scrape_human/extract_data", human: true, args: []string{"scrape", u, "--extract", `{"title":"h1"}`}, server: gScrape(gScrapeEnv(`,"data":{"title":"Anvil","price":"$19.99 & free shipping"}`))}, + goldenCase{name: "scrape_human/links", human: true, args: []string{"scrape", u, "--links", "--format", "markdown"}, server: gScrape(gScrapeEnv(`,"links":["https://example.com/a","https://example.com/b?x=1&y=2"]`))}, + goldenCase{name: "scrape_human/network", human: true, args: []string{"scrape", u, "--render", "--network-capture", `{}`}, + server: gScrape(gScrapeEnv(`,"network":[{"url":"https://example.com/api/price","status":200,"resource_type":"xhr","content_type":"application/json"},{"url":"https://example.com/x.png","resource_type":"image"}]`))}, + goldenCase{name: "scrape_human/target_503", human: true, args: []string{"scrape", u}, server: gScrape(gScrapeRaw("text/html", "

Service Unavailable

\n", 503))}, + goldenCase{name: "scrape_human/output_file", human: true, args: []string{"scrape", u, "--format", "markdown", "-o", "{dir}/page.md"}, server: gScrape(md), files: []string{"page.md"}}, + goldenCase{name: "scrape_human/pdf_needs_file", human: true, args: []string{"scrape", u, "--format", "pdf"}, server: gScrape(pdf)}, + goldenCase{name: "scrape_human/pdf_output_file", human: true, args: []string{"scrape", u, "--format", "pdf", "-o", "{dir}/page.pdf"}, server: gScrape(pdf), files: []string{"page.pdf"}}, + goldenCase{name: "scrape_human/err_auth", human: true, args: []string{"scrape", u}, server: gScrape(gProblem(401, "ERR::AUTH::INVALID_KEY", "the API key is not valid"))}, + ) +} + +// gScrapeStdinMany answers per URL: /ok* is a page, /down a target 503, /denied an API problem. +func gScrapeStdinMany(w http.ResponseWriter, r *http.Request) { + var req struct { + URL string `json:"url"` + } + b, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(b, &req) + switch { + case strings.HasSuffix(req.URL, "/down"): + gScrapeRaw("text/markdown", "# Down\n", 503)(w, r) + case strings.HasSuffix(req.URL, "/denied"): + gProblem(403, "ERR::UPSTREAM::BLOCKED", "blocked by the target")(w, r) + default: + gScrapeRaw("text/markdown", "# "+req.URL+"\n", 200)(w, r) + } +} diff --git a/internal/cmd/sessions.go b/internal/cmd/sessions.go index 2cbe109..b70c894 100644 --- a/internal/cmd/sessions.go +++ b/internal/cmd/sessions.go @@ -15,6 +15,7 @@ import ( "github.com/OfficialSpicrawl/cli/internal/api" "github.com/OfficialSpicrawl/cli/internal/output" + "github.com/OfficialSpicrawl/cli/internal/ui" ) // sessionsRecord is the subset of the Session schema the human views read. @@ -148,7 +149,9 @@ whose session_context member is sent.`, if err != nil { return err } + sp := spin(Printer(), "Creating a session") resp, err := c.Do(cmd.Context(), api.Request{Method: http.MethodPost, Path: "/v1/sessions", Body: body}) + sp.Stop() if err != nil { return err } @@ -189,6 +192,9 @@ until the API says there are no more.`, if err != nil { return err } + p := Printer() + sp := spin(p, "Fetching sessions") + defer sp.Stop() var all []json.RawMessage var next string for { @@ -204,6 +210,7 @@ until the API says there are no more.`, return err } all = append(all, page.Sessions...) + sp.Label("Fetching sessions · %d so far", len(all)) next = page.NextCursor if !sessionsListAll || next == "" { break @@ -217,7 +224,7 @@ until the API says there are no more.`, if next != "" { out["next_cursor"] = next } - p := Printer() + sp.Stop() return p.Result(out, func(w io.Writer) { if len(all) == 0 { p.Info("no sessions") @@ -233,7 +240,13 @@ until the API says there are no more.`, } rows = append(rows, []string{s.ID, s.Engine, s.Status, exit, strconv.Itoa(s.UsageCount), s.ExpiresAt}) } - p.Table([]string{"ID", "ENGINE", "STATUS", "EXIT", "USES", "EXPIRES"}, rows) + u := outUI(p) + viewTable(p, u, []string{"ID", "ENGINE", "STATUS", "EXIT", "USES", "EXPIRES"}, rows, func(col int, cell string) string { + if col == 2 { + return sessionsStatusCell(u, cell) + } + return cell + }) if next != "" { p.Info("more sessions exist; pass --all to fetch every page") } @@ -251,7 +264,9 @@ var sessionsGetCmd = &cobra.Command{ if err != nil { return err } + sp := spin(Printer(), "Fetching session %s", args[0]) resp, err := c.Do(cmd.Context(), api.Request{Path: "/v1/sessions/" + url.PathEscape(args[0])}) + sp.Stop() if err != nil { return err } @@ -272,11 +287,13 @@ session to carry a login over.`, if err != nil { return err } + p := Printer() + sp := spin(p, "Fetching the context of session %s", args[0]) resp, err := c.Do(cmd.Context(), api.Request{Path: "/v1/sessions/" + url.PathEscape(args[0]) + "/context"}) + sp.Stop() if err != nil { return err } - p := Printer() if !p.JSON { p.Warn("this output contains live cookies; treat it like a password") } @@ -302,11 +319,13 @@ lease (that render fails).`, if sessionsForce { q = url.Values{"force": {"true"}} } + sp := spin(Printer(), "Ending session %s", args[0]) resp, err := c.Do(cmd.Context(), api.Request{ Method: http.MethodPost, Path: "/v1/sessions/" + url.PathEscape(args[0]) + "/release", Query: q, }) + sp.Stop() if err != nil { return err } @@ -342,16 +361,19 @@ terminal; without a terminal you must pass --yes.`, if sessionsForce { q = url.Values{"force": {"true"}} } - if _, err := c.Do(cmd.Context(), api.Request{ + p := Printer() + sp := spin(p, "Deleting session %s", id) + _, err = c.Do(cmd.Context(), api.Request{ Method: http.MethodDelete, Path: "/v1/sessions/" + url.PathEscape(id), Query: q, - }); err != nil { + }) + sp.Stop() + if err != nil { return err } - p := Printer() return p.Result(map[string]any{"id": id, "deleted": true}, func(w io.Writer) { - fmt.Fprintf(w, "deleted session %s\n", id) + fmt.Fprintf(w, "%sdeleted session %s\n", viewLead(outUI(p), markOK), id) }) }, } @@ -403,9 +425,16 @@ func sessionsPrintOne(body []byte, created bool) error { return p.RawJSON(body) } w := p.Out + u := outUI(p) + if created && u.On() { + fmt.Fprintf(w, "%s Session created\n\n", u.Green("✓")) + } kv := func(k, v string) { if v != "" { - fmt.Fprintf(w, "%-16s %s\n", k+":", v) + if k == "status" { + v = sessionsStatusCell(u, v) + } + fmt.Fprintf(w, "%s %s\n", viewKey(u, 16, k+":"), v) } } kv("id", s.ID) @@ -441,6 +470,11 @@ func sessionsPrintOne(body []byte, created bool) error { return nil } +// sessionsStatusCell colours a session status: active green, the rest dim. +func sessionsStatusCell(u *ui.UI, s string) string { + return viewStatus(u, s, []string{"active"}, nil) +} + func init() { cf := sessionsCreateCmd.Flags() cf.StringVar(&sessionsEngine, "engine", "", "engine to pin: fetch|obscura|chromium|camoufox") diff --git a/internal/cmd/sessions_golden_test.go b/internal/cmd/sessions_golden_test.go new file mode 100644 index 0000000..6decc8b --- /dev/null +++ b/internal/cmd/sessions_golden_test.go @@ -0,0 +1,69 @@ +package cmd + +import ( + "net/http" + "testing" +) + +const ( + gSessionID = "01J9ZQ4M7R3T8VX2K5N6P0B1CD" + + gSession = `{"id":"` + gSessionID + `","project_id":"proj_01J9Z","engine":"chromium","status":"active","sticky_key":"login-1",` + + `"proxy":{"tier":"residential","country":"de","region_pool":"eu"},"created_at":"2026-09-22T00:00:00Z","last_used_at":"2026-09-22T00:30:00Z",` + + `"expires_at":"2026-09-22T02:00:00Z","hard_expires_at":"2026-09-23T00:00:00Z","usage_count":7,"context":null}` + + gSessionNew = `{"id":"` + gSessionID + `","project_id":"proj_01J9Z","engine":"obscura","status":"active","sticky_key":null,` + + `"proxy":{"tier":"datacenter"},"created_at":"2026-09-22T00:00:00Z","last_used_at":null,"expires_at":"2026-09-22T00:30:00Z",` + + `"usage_count":0,"warnings":["engine obscura cannot persist IndexedDB"]}` + + gSessionReleased = `{"id":"` + gSessionID + `","project_id":"proj_01J9Z","engine":"chromium","status":"released","sticky_key":"login-1",` + + `"proxy":{"tier":"residential","country":"de"},"created_at":"2026-09-22T00:00:00Z","last_used_at":"2026-09-22T00:30:00Z",` + + `"expires_at":"2026-09-22T02:00:00Z","released_at":"2026-09-22T00:45:00Z","retired":"released by the caller","usage_count":7}` + + gSessionContext = `{"session_id":"` + gSessionID + `","session_context":{"cookies":[{"name":"sid","value":"abc123","domain":"example.com","path":"/"}],"local_storage":{"theme":"dark"}}}` +) + +func init() { + one := func(method, path, body string, status int) func() http.HandlerFunc { + return goldenServe(map[string]http.HandlerFunc{method + " " + path: gJSON(status, body)}) + } + list := func(body string) func() http.HandlerFunc { return one("GET", "/v1/sessions", body, 200) } + page := `{"sessions":[` + gSession + `,` + gSessionReleased + `]}` + more := `{"sessions":[` + gSession + `],"next_cursor":"1790000000000000000.ABC"}` + del := goldenServe(map[string]http.HandlerFunc{"DELETE /v1/sessions/{id}": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }}) + + goldenAdd( + goldenCase{name: "sessions/create", args: []string{"sessions", "create", "--engine", "obscura"}, server: one("POST", "/v1/sessions", gSessionNew, 201)}, + goldenCase{name: "sessions/create_session_context_file", args: []string{"sessions", "create", "--session-context", "{dir}/ctx.json"}, server: one("POST", "/v1/sessions", gSession, 201), + setup: func(t *testing.T, e *goldenEnv) { goldenWrite(t, e.Dir, "ctx.json", gSessionContext) }}, + goldenCase{name: "sessions/create_usage_error", args: []string{"sessions", "create", "--rotate-ip", "--sticky-key", "k1"}, stderr: true}, + goldenCase{name: "sessions/list", args: []string{"sessions", "list"}, server: list(page)}, + goldenCase{name: "sessions/list_empty", args: []string{"sessions", "list", "--status", "expired"}, server: list(`{"sessions":[]}`)}, + goldenCase{name: "sessions/list_next_cursor", args: []string{"sessions", "list", "--limit", "1"}, server: list(more)}, + goldenCase{name: "sessions/list_all_pages", args: []string{"sessions", "list", "--all"}, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/sessions": func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("cursor") == "" { + gJSON(200, more)(w, r) + return + } + gJSON(200, `{"sessions":[`+gSessionReleased+`]}`)(w, r) + }})}, + goldenCase{name: "sessions/get", args: []string{"sessions", "get", gSessionID}, server: one("GET", "/v1/sessions/{id}", gSession, 200)}, + goldenCase{name: "sessions/get_not_found", args: []string{"sessions", "get", gSessionID}, stderr: true, + server: goldenServe(map[string]http.HandlerFunc{"GET /v1/sessions/{id}": gProblem(404, "ERR::SESSION::NOT_FOUND", "no such session")})}, + goldenCase{name: "sessions/context", args: []string{"sessions", "context", gSessionID}, server: one("GET", "/v1/sessions/{id}/context", gSessionContext, 200)}, + goldenCase{name: "sessions/release", args: []string{"sessions", "release", gSessionID}, server: one("POST", "/v1/sessions/{id}/release", gSessionReleased, 200)}, + goldenCase{name: "sessions/delete_yes", args: []string{"sessions", "delete", gSessionID, "--yes"}, server: del}, + goldenCase{name: "sessions/delete_needs_yes_without_terminal", args: []string{"sessions", "delete", gSessionID}, stderr: true, server: del}, + + goldenCase{name: "sessions_human/create", human: true, args: []string{"sessions", "create"}, server: one("POST", "/v1/sessions", gSessionNew, 201)}, + goldenCase{name: "sessions_human/list", human: true, args: []string{"sessions", "list"}, server: list(page)}, + goldenCase{name: "sessions_human/list_empty", human: true, args: []string{"sessions", "list"}, server: list(`{"sessions":[]}`)}, + goldenCase{name: "sessions_human/list_next_cursor", human: true, args: []string{"sessions", "list"}, server: list(more)}, + goldenCase{name: "sessions_human/get", human: true, args: []string{"sessions", "get", gSessionID}, server: one("GET", "/v1/sessions/{id}", gSession, 200)}, + goldenCase{name: "sessions_human/get_released", human: true, args: []string{"sessions", "get", gSessionID}, server: one("GET", "/v1/sessions/{id}", gSessionReleased, 200)}, + goldenCase{name: "sessions_human/context", human: true, args: []string{"sessions", "context", gSessionID}, server: one("GET", "/v1/sessions/{id}/context", gSessionContext, 200)}, + goldenCase{name: "sessions_human/release", human: true, args: []string{"sessions", "release", gSessionID}, server: one("POST", "/v1/sessions/{id}/release", gSessionReleased, 200)}, + goldenCase{name: "sessions_human/delete_yes", human: true, args: []string{"sessions", "delete", gSessionID, "--yes"}, server: del}, + ) +} diff --git a/internal/cmd/sessions_test.go b/internal/cmd/sessions_test.go index 4a4f716..cc05d6c 100644 --- a/internal/cmd/sessions_test.go +++ b/internal/cmd/sessions_test.go @@ -3,10 +3,12 @@ package cmd import ( "encoding/json" "net/http" + "net/http/httptest" "os" "reflect" "strings" "testing" + "time" ) const sessionsTestBody = `{"id":"01J9ZQ4M7R3T8VX2K5N6P0B1CD","project_id":"p","engine":"chromium","status":"active","sticky_key":"s1","proxy":{"tier":"residential","country":"de"},"created_at":"2026-09-22T00:00:00Z","last_used_at":null,"expires_at":"2026-09-22T02:00:00Z","usage_count":0,"context":null}` @@ -195,3 +197,92 @@ func TestSessionsCreateSessionContextBadFileIsUsageError(t *testing.T) { } } } + +func sessionsServer(t *testing.T, delay time.Duration) *httptest.Server { + srv, _ := newRecordingServer(t, func(w http.ResponseWriter, r *http.Request) { + time.Sleep(delay) + switch { + case r.URL.Path == "/v1/sessions" && r.Method == http.MethodGet: + _, _ = w.Write([]byte(`{"sessions":[` + sessionsTestBody + `]}`)) + case r.Method == http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodPost && r.URL.Path == "/v1/sessions": + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(sessionsTestBody)) + default: + _, _ = w.Write([]byte(sessionsTestBody)) + } + }) + return srv +} + +// Each human view is the plain one with colour and, for create, a headline. +func TestSessionsLiveViews(t *testing.T) { + srv := sessionsServer(t, 0) + plain := map[string]string{} + cmds := map[string][]string{ + "list": {"sessions", "list"}, + "get": {"sessions", "get", "S1"}, + "create": {"sessions", "create"}, + "release": {"sessions", "release", "S1"}, + "delete": {"sessions", "delete", "S1", "--yes"}, + } + for name, args := range cmds { + out, errb, code := runCLIHuman(t, srv.URL, args...) + if code != 0 { + t.Fatalf("%s: exit %d: %s", name, code, errb) + } + noEscapes(t, name+" plain stdout", out) + plain[name] = out + } + if plain["delete"] != "deleted session S1\n" || !strings.HasPrefix(plain["get"], "id:") { + t.Fatalf("plain views changed: %q %q", plain["delete"], plain["get"]) + } + + liveUI(t) + for name, args := range cmds { + out, _, _ := runCLIHuman(t, srv.URL, args...) + want := plain[name] + switch name { + case "create": + want = "✓ Session created\n\n" + want + case "delete": + want = "✓ " + want + } + if stripANSI(out) != want { + t.Errorf("%s: live view differs:\n%q\nwant\n%q", name, stripANSI(out), want) + } + if !strings.Contains(out, "\x1b[") { + t.Errorf("%s: no colour: %q", name, out) + } + } + list, _, _ := runCLIHuman(t, srv.URL, "sessions", "list") + if !strings.Contains(list, "\x1b[32mactive") { + t.Errorf("active is not green: %q", list) + } +} + +func TestSessionsSpinnerNamesTheWait(t *testing.T) { + spinNow(t) + srv := sessionsServer(t, 40*time.Millisecond) + for label, args := range map[string][]string{ + "Creating a session": {"sessions", "create"}, + "Fetching sessions": {"sessions", "list"}, + "Fetching session S1": {"sessions", "get", "S1"}, + "Fetching the context of session S1": {"sessions", "context", "S1"}, + "Ending session S1": {"sessions", "release", "S1"}, + "Deleting session S1": {"sessions", "delete", "S1", "--yes"}, + } { + out, errb, code := runCLIHuman(t, srv.URL, args...) + if code != 0 || !strings.Contains(errb, label) { + t.Errorf("%v: exit %d, stderr %q, want %q", args, code, errb, label) + } + if strings.Contains(out, label) { + t.Errorf("%v: spinner on stdout", args) + } + } + // --json: no spinner and the API's body. + out, errb, _ := runCLIHuman(t, srv.URL, "--json", "sessions", "get", "S1") + noEscapes(t, "json stdout", out) + noEscapes(t, "json stderr", errb) +} diff --git a/internal/cmd/skill.go b/internal/cmd/skill.go index 098df5b..32ae7a5 100644 --- a/internal/cmd/skill.go +++ b/internal/cmd/skill.go @@ -121,14 +121,22 @@ func runSkillInstall(cmd *cobra.Command, _ []string) error { if err := agentApply(rs); err != nil { return err } - return agentReport(p, rs, s) + if err := agentReport(p, rs, s); err != nil { + return err + } + if names := agentChanged(rs); len(names) > 0 { + viewNext(p, "start a new session in "+strings.Join(names, ", ")+" to use the skill") + } + return nil } // skillFetch downloads the skill from ep.Skill, falling back to the embedded // copy, and points its URLs at this deployment. The source is the URL, or // "embedded". func skillFetch(ctx context.Context, p *output.Printer, ep agentEndpoints) ([]byte, string) { + sp := spin(p, "Downloading the Spicrawl skill") b, err := skillDownload(ctx, ep.Skill) + sp.Stop() if err != nil { p.Warn("could not download the skill from %s (%v); using the copy built into this CLI", ep.Skill, err) return skillRewrite(skillNormalize(skillEmbedded), ep), "embedded" diff --git a/internal/cmd/status.go b/internal/cmd/status.go index edec13e..81b3065 100644 --- a/internal/cmd/status.go +++ b/internal/cmd/status.go @@ -17,6 +17,7 @@ import ( "github.com/OfficialSpicrawl/cli/internal/config" "github.com/OfficialSpicrawl/cli/internal/exitcode" "github.com/OfficialSpicrawl/cli/internal/output" + "github.com/OfficialSpicrawl/cli/internal/ui" ) type statusWorkers struct { @@ -67,10 +68,13 @@ fails during the key check. A reachable API that reports not ready still exits 0 c := api.New(r.BaseURL, r.APIKey, flagTimeout) rep := statusReport{BaseURL: r.BaseURL, KeySource: r.APIKeySource} + p := Printer() + sp := spin(p, "Checking the API") + defer sp.Stop() var readyErr error rep.APIReachable, rep.Ready, rep.Checks, rep.CDP, readyErr = statusReadyz(ctx, c) - p := Printer() if !rep.APIReachable { + sp.Stop() if err := statusPrint(p, &rep); err != nil { return err } @@ -83,6 +87,7 @@ fails during the key check. A reachable API that reports not ready still exits 0 rep.KeyValid = &f rep.KeyError = "no API key configured" } else { + sp.Label("Checking your key") _, err := c.Do(ctx, api.Request{Path: "/v1/requests", Query: url.Values{"limit": {"1"}}}) t := true switch prob, isProb := api.AsProblem(err); { @@ -102,6 +107,7 @@ fails during the key check. A reachable API that reports not ready still exits 0 } if rep.KeyValid != nil && *rep.KeyValid { + sp.Label("Reading worker status") rep.Workers, err = statusFleet(ctx, c) if err != nil { rep.WorkersError = err.Error() @@ -109,6 +115,7 @@ fails during the key check. A reachable API that reports not ready still exits 0 } } + sp.Stop() if err := statusPrint(p, &rep); err != nil { return err } @@ -175,59 +182,79 @@ func statusFleet(ctx context.Context, c *api.Client) (map[string]statusWorkers, return out, nil } +func statusYN(b bool) string { + if b { + return "yes" + } + return "no" +} + +// statusReadyText is the ready value: yes, or no with the checks that failed. +func statusReadyText(rep *statusReport) string { + ready := statusYN(rep.Ready) + if !rep.Ready && len(rep.Checks) > 0 { + var bad []string + for k, v := range rep.Checks { + if v != "ok" { + bad = append(bad, k+"="+v) + } + } + sort.Strings(bad) + if len(bad) > 0 { + ready += " (" + strings.Join(bad, ", ") + ")" + } + } + return ready +} + +func statusCDPText(rep *statusReport) string { + cdp := rep.CDP + switch cdp { + case "": + cdp = "unknown (this API does not report it)" + case "disabled": + cdp = "disabled (cloud browser /v1/browser is not available on this deployment)" + case "unknown": + cdp = "unknown (the browser worker could not be asked)" + } + return cdp +} + +func statusKeyText(rep *statusReport) string { + key := "unknown" + if rep.KeyValid != nil { + key = map[bool]string{true: "valid", false: "invalid"}[*rep.KeyValid] + } + if rep.KeySource != "" { + key += " (from " + rep.KeySource + ")" + } + if rep.KeyError != "" { + key += ": " + rep.KeyError + } + return key +} + +var statusWorkerKinds = []string{"browser", "egress", "extract", "batch"} + func statusPrint(p *output.Printer, rep *statusReport) error { return p.Result(rep, func(w io.Writer) { - yn := func(b bool) string { - if b { - return "yes" - } - return "no" + if u := outUI(p); u.On() { + statusPrintLive(w, u, rep) + return } fmt.Fprintf(w, "%-12s %s\n", "base url:", rep.BaseURL) - fmt.Fprintf(w, "%-12s %s\n", "reachable:", yn(rep.APIReachable)) + fmt.Fprintf(w, "%-12s %s\n", "reachable:", statusYN(rep.APIReachable)) if !rep.APIReachable { return } - ready := yn(rep.Ready) - if !rep.Ready && len(rep.Checks) > 0 { - var bad []string - for k, v := range rep.Checks { - if v != "ok" { - bad = append(bad, k+"="+v) - } - } - sort.Strings(bad) - if len(bad) > 0 { - ready += " (" + strings.Join(bad, ", ") + ")" - } - } - fmt.Fprintf(w, "%-12s %s\n", "ready:", ready) - cdp := rep.CDP - switch cdp { - case "": - cdp = "unknown (this API does not report it)" - case "disabled": - cdp = "disabled (cloud browser /v1/browser is not available on this deployment)" - case "unknown": - cdp = "unknown (the browser worker could not be asked)" - } - fmt.Fprintf(w, "%-12s %s\n", "cdp:", cdp) - key := "unknown" - if rep.KeyValid != nil { - key = map[bool]string{true: "valid", false: "invalid"}[*rep.KeyValid] - } - if rep.KeySource != "" { - key += " (from " + rep.KeySource + ")" - } - if rep.KeyError != "" { - key += ": " + rep.KeyError - } - fmt.Fprintf(w, "%-12s %s\n", "key:", key) + fmt.Fprintf(w, "%-12s %s\n", "ready:", statusReadyText(rep)) + fmt.Fprintf(w, "%-12s %s\n", "cdp:", statusCDPText(rep)) + fmt.Fprintf(w, "%-12s %s\n", "key:", statusKeyText(rep)) if rep.WorkersError != "" { fmt.Fprintf(w, "%-12s %s\n", "workers:", rep.WorkersError) } else if len(rep.Workers) > 0 { fmt.Fprintln(w, "workers:") - for _, k := range []string{"browser", "egress", "extract", "batch"} { + for _, k := range statusWorkerKinds { if v, ok := rep.Workers[k]; ok { fmt.Fprintf(w, " %-10s %d ready of %d\n", k, v.Ready, v.Present) } @@ -236,6 +263,74 @@ func statusPrint(p *output.Printer, rep *statusReport) error { }) } +// statusPrintLive is the checklist a person sees on a terminal: one mark per +// check and a closing line that counts the failures. +func statusPrintLive(w io.Writer, u *ui.UI, rep *statusReport) { + failed := 0 + check := func(m int, label, val string) { + if m == markBad { + failed++ + } + fmt.Fprintf(w, " %s %s %s\n", viewMark(u, m), viewKey(u, 9, label), val) + } + pick := func(ok bool) int { + if ok { + return markOK + } + return markBad + } + + fmt.Fprintf(w, "%s %s\n\n", u.Bold("Spicrawl API"), rep.BaseURL) + check(pick(rep.APIReachable), "reachable", statusYN(rep.APIReachable)) + if !rep.APIReachable { + fmt.Fprintf(w, "\n%s %s\n", u.Red("✗"), "the API could not be reached") + return + } + check(pick(rep.Ready), "ready", statusReadyText(rep)) + cdp := markNeutral + if rep.CDP == "enabled" { + cdp = markOK + } + check(cdp, "cdp", statusCDPText(rep)) + key := markNeutral + if rep.KeyValid != nil { + key = pick(*rep.KeyValid) + } + check(key, "key", statusKeyText(rep)) + + switch { + case rep.WorkersError != "": + check(markNeutral, "workers", rep.WorkersError) + case len(rep.Workers) > 0: + fmt.Fprintf(w, "\n%s\n", u.Bold("workers")) + for _, k := range statusWorkerKinds { + v, ok := rep.Workers[k] + if !ok { + continue + } + m := markWarn + switch { + case v.Present == 0: + m = markNeutral // none of this kind runs here + case v.Ready == 0: + m = markBad + case v.Ready >= v.Present: + m = markOK + } + if m == markBad { + failed++ + } + fmt.Fprintf(w, " %s %s %d ready of %d\n", viewMark(u, m), viewKey(u, 9, k), v.Ready, v.Present) + } + } + + if failed == 0 { + fmt.Fprintf(w, "\n%s all checks passed\n", u.Green("✓")) + } else { + fmt.Fprintf(w, "\n%s %d %s failed\n", u.Red("✗"), failed, map[bool]string{true: "check", false: "checks"}[failed == 1]) + } +} + func init() { rootCmd.AddCommand(statusCmd) } diff --git a/internal/cmd/status_test.go b/internal/cmd/status_test.go index 01c9886..ed8ff9f 100644 --- a/internal/cmd/status_test.go +++ b/internal/cmd/status_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" ) func statusServer(t *testing.T, keyOK bool) (*httptest.Server, *[]recordedRequest) { @@ -100,3 +101,144 @@ func TestStatusReportsCDP(t *testing.T) { t.Errorf("human output lacks the cdp line:\n%s", human) } } + +// The checklist is for a person at a terminal. Piped, or with the live output +// off, the old key/value lines come out byte for byte. +func TestStatusHumanPlainIsUnchanged(t *testing.T) { + srv, _ := statusServer(t, true) + out, errb, code := runCLIHuman(t, srv.URL, "status") + if code != 0 { + t.Fatalf("exit %d: %s", code, errb) + } + want := "base url: " + srv.URL + "\n" + + "reachable: yes\n" + + "ready: no (redis=unreachable)\n" + + "cdp: disabled (cloud browser /v1/browser is not available on this deployment)\n" + + "key: valid (from env)\n" + + "workers:\n" + + " browser 1 ready of 2\n" + + " egress 0 ready of 0\n" + if out != want { + t.Errorf("stdout changed:\n%q\nwant\n%q", out, want) + } + noEscapes(t, "stdout", out) + noEscapes(t, "stderr", errb) +} + +func TestStatusLiveChecklist(t *testing.T) { + liveUI(t) + srv, _ := statusServer(t, true) + out, errb, code := runCLIHuman(t, srv.URL, "status") + if code != 0 { + t.Fatalf("exit %d: %s", code, errb) + } + if !strings.Contains(out, "\x1b[") { + t.Errorf("no colour on a terminal: %q", out) + } + plain := stripANSI(out) + for _, want := range []string{ + "Spicrawl API " + srv.URL, + "✓ reachable yes", + "✗ ready no (redis=unreachable)", + "– cdp disabled", + "✓ key valid (from env)", + "workers", + "! browser 1 ready of 2", + "– egress 0 ready of 0", + "✗ 1 check failed", + } { + if !strings.Contains(plain, want) { + t.Errorf("missing %q in\n%s", want, plain) + } + } +} + +func TestStatusLiveAllGood(t *testing.T) { + liveUI(t) + srv, _ := newRecordingServer(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/readyz": + _, _ = w.Write([]byte(`{"status":"ready","checks":{"postgres":"ok"},"cdp":"enabled"}`)) + case "/v1/workers": + _, _ = w.Write([]byte(`{"kinds":[{"kind":"browser","present":2,"ready":2}]}`)) + default: + _, _ = w.Write([]byte(`{"data":[],"page":{}}`)) + } + }) + out, _, code := runCLIHuman(t, srv.URL, "status") + if code != 0 { + t.Fatalf("exit %d", code) + } + plain := stripANSI(out) + if !strings.Contains(plain, "✓ cdp enabled") || !strings.Contains(plain, "✓ browser 2 ready of 2") || !strings.Contains(plain, "✓ all checks passed") { + t.Errorf("\n%s", plain) + } +} + +func TestStatusLiveUnreachableAndBadKey(t *testing.T) { + liveUI(t) + srv := httptest.NewServer(http.NotFoundHandler()) + base := srv.URL + srv.Close() + out, _, code := runCLIHuman(t, base, "status") + if code != 10 || !strings.Contains(stripANSI(out), "✗ reachable no") || !strings.Contains(stripANSI(out), "could not be reached") { + t.Errorf("exit %d\n%s", code, stripANSI(out)) + } + + bad, _ := statusServer(t, false) + out, _, code = runCLIHuman(t, bad.URL, "status") + if code != 3 || !strings.Contains(stripANSI(out), "✗ key") { + t.Errorf("exit %d\n%s", code, stripANSI(out)) + } +} + +// JSON, --json and --quiet leave both streams free of escapes even when the +// streams would be terminals. +func TestStatusLiveNeverTouchesMachineOutput(t *testing.T) { + spinNow(t) + srv, _ := statusServer(t, true) + out, errb, code := runCLI(t, srv.URL, "status") + if code != 0 { + t.Fatalf("exit %d: %s", code, errb) + } + var rep statusReport + if err := json.Unmarshal([]byte(out), &rep); err != nil { + t.Fatalf("stdout is not JSON: %v\n%s", err, out) + } + noEscapes(t, "stdout", out) + noEscapes(t, "stderr", errb) + + out, errb, _ = runCLIHuman(t, srv.URL, "-q", "status") + noEscapes(t, "quiet stdout", out) + noEscapes(t, "quiet stderr", errb) +} + +func TestStatusSpinnerNamesTheWait(t *testing.T) { + spinNow(t) + srv, _ := newRecordingServer(t, func(w http.ResponseWriter, r *http.Request) { + time.Sleep(30 * time.Millisecond) + switch r.URL.Path { + case "/readyz": + _, _ = w.Write([]byte(`{"status":"ready","checks":{},"cdp":"enabled"}`)) + case "/v1/workers": + _, _ = w.Write([]byte(`{"kinds":[]}`)) + default: + _, _ = w.Write([]byte(`{"data":[],"page":{}}`)) + } + }) + out, errb, code := runCLIHuman(t, srv.URL, "status") + if code != 0 { + t.Fatalf("exit %d: %s", code, errb) + } + for _, want := range []string{"Checking the API", "Checking your key", "Reading worker status"} { + if !strings.Contains(errb, want) { + t.Errorf("stderr lacks %q: %q", want, errb) + } + } + if strings.Contains(out, "Checking") { + t.Errorf("spinner text on stdout: %q", out) + } + if !strings.Contains(errb, "\x1b[?25h") { + t.Errorf("cursor not restored: %q", errb) + } +} diff --git a/internal/cmd/testdata/golden/.gitattributes b/internal/cmd/testdata/golden/.gitattributes new file mode 100644 index 0000000..fa1385d --- /dev/null +++ b/internal/cmd/testdata/golden/.gitattributes @@ -0,0 +1 @@ +* -text diff --git a/internal/cmd/testdata/golden/agents/init_err_no_client_detected.exit b/internal/cmd/testdata/golden/agents/init_err_no_client_detected.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_err_no_client_detected.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/agents/init_err_no_client_detected.stderr b/internal/cmd/testdata/golden/agents/init_err_no_client_detected.stderr new file mode 100644 index 0000000..28738e9 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_err_no_client_detected.stderr @@ -0,0 +1,4 @@ +{ + "error": "no agent client detected in or your home directory; pass --client claude|cursor|vscode|codex or all", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/agents/init_err_no_client_detected.stdout b/internal/cmd/testdata/golden/agents/init_err_no_client_detected.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.exit b/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.stderr b/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.stderr new file mode 100644 index 0000000..2f9e2aa --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.stderr @@ -0,0 +1,9 @@ +clients: Claude Code +MCP server: http://127.0.0.1:1/mcp +plan: + create claude mcp .mcp.json + create claude skill .claude/skills/spicrawl/SKILL.md +{ + "error": "init changes files and stdin is not a terminal: re-run with --yes to apply the plan above", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.stdout b/internal/cmd/testdata/golden/agents/init_needs_yes_without_terminal.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/agents/init_yes.exit b/internal/cmd/testdata/golden/agents/init_yes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_yes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/init_yes.file..claude__skills__spicrawl__SKILL.md b/internal/cmd/testdata/golden/agents/init_yes.file..claude__skills__spicrawl__SKILL.md new file mode 100644 index 0000000..9a9132e --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_yes.file..claude__skills__spicrawl__SKILL.md @@ -0,0 +1,8 @@ +--- +name: spicrawl +description: golden test skill +--- + +# Spicrawl + +Use http://127.0.0.1:1 for scraping. diff --git a/internal/cmd/testdata/golden/agents/init_yes.file..cursor__mcp.json b/internal/cmd/testdata/golden/agents/init_yes.file..cursor__mcp.json new file mode 100644 index 0000000..4277328 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_yes.file..cursor__mcp.json @@ -0,0 +1,10 @@ +{ + "mcpServers": { + "spicrawl": { + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${env:SPICRAWL_API_KEY}" + } + } + } +} diff --git a/internal/cmd/testdata/golden/agents/init_yes.file..mcp.json b/internal/cmd/testdata/golden/agents/init_yes.file..mcp.json new file mode 100644 index 0000000..698f8f0 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_yes.file..mcp.json @@ -0,0 +1,11 @@ +{ + "mcpServers": { + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${SPICRAWL_API_KEY}" + } + } + } +} diff --git a/internal/cmd/testdata/golden/agents/init_yes.stdout b/internal/cmd/testdata/golden/agents/init_yes.stdout new file mode 100644 index 0000000..82dcdd7 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/init_yes.stdout @@ -0,0 +1,27 @@ +[ + { + "client": "claude", + "kind": "mcp", + "file": "/.mcp.json", + "action": "created" + }, + { + "client": "cursor", + "kind": "mcp", + "file": "/.cursor/mcp.json", + "action": "created" + }, + { + "client": "claude", + "kind": "skill", + "file": "/.claude/skills/spicrawl/SKILL.md", + "action": "created" + }, + { + "client": "cursor", + "kind": "skill", + "file": "/.claude/skills/spicrawl/SKILL.md", + "action": "skipped", + "reason": "already reads this skill" + } +] diff --git a/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.exit b/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.stderr b/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.stderr new file mode 100644 index 0000000..28738e9 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.stderr @@ -0,0 +1,4 @@ +{ + "error": "no agent client detected in or your home directory; pass --client claude|cursor|vscode|codex or all", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.stdout b/internal/cmd/testdata/golden/agents/mcp_err_no_client_detected.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/agents/mcp_install_all.exit b/internal/cmd/testdata/golden/agents/mcp_install_all.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_all.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/mcp_install_all.file..codex__config.toml b/internal/cmd/testdata/golden/agents/mcp_install_all.file..codex__config.toml new file mode 100644 index 0000000..feae81b --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_all.file..codex__config.toml @@ -0,0 +1,3 @@ +[mcp_servers.spicrawl] +url = "http://127.0.0.1:1/mcp" +bearer_token_env_var = "SPICRAWL_API_KEY" diff --git a/internal/cmd/testdata/golden/agents/mcp_install_all.file..cursor__mcp.json b/internal/cmd/testdata/golden/agents/mcp_install_all.file..cursor__mcp.json new file mode 100644 index 0000000..4277328 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_all.file..cursor__mcp.json @@ -0,0 +1,10 @@ +{ + "mcpServers": { + "spicrawl": { + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${env:SPICRAWL_API_KEY}" + } + } + } +} diff --git a/internal/cmd/testdata/golden/agents/mcp_install_all.file..mcp.json b/internal/cmd/testdata/golden/agents/mcp_install_all.file..mcp.json new file mode 100644 index 0000000..698f8f0 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_all.file..mcp.json @@ -0,0 +1,11 @@ +{ + "mcpServers": { + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${SPICRAWL_API_KEY}" + } + } + } +} diff --git a/internal/cmd/testdata/golden/agents/mcp_install_all.file..vscode__mcp.json b/internal/cmd/testdata/golden/agents/mcp_install_all.file..vscode__mcp.json new file mode 100644 index 0000000..8c81ec7 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_all.file..vscode__mcp.json @@ -0,0 +1,19 @@ +{ + "servers": { + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${input:spicrawl-api-key}" + } + } + }, + "inputs": [ + { + "type": "promptString", + "id": "spicrawl-api-key", + "description": "Spicrawl API key", + "password": true + } + ] +} diff --git a/internal/cmd/testdata/golden/agents/mcp_install_all.stdout b/internal/cmd/testdata/golden/agents/mcp_install_all.stdout new file mode 100644 index 0000000..3ff9d2e --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_all.stdout @@ -0,0 +1,26 @@ +[ + { + "client": "claude", + "kind": "mcp", + "file": "/.mcp.json", + "action": "created" + }, + { + "client": "cursor", + "kind": "mcp", + "file": "/.cursor/mcp.json", + "action": "created" + }, + { + "client": "vscode", + "kind": "mcp", + "file": "/.vscode/mcp.json", + "action": "created" + }, + { + "client": "codex", + "kind": "mcp", + "file": "/.codex/config.toml", + "action": "created" + } +] diff --git a/internal/cmd/testdata/golden/agents/mcp_install_claude.exit b/internal/cmd/testdata/golden/agents/mcp_install_claude.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_claude.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/mcp_install_claude.file..mcp.json b/internal/cmd/testdata/golden/agents/mcp_install_claude.file..mcp.json new file mode 100644 index 0000000..698f8f0 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_claude.file..mcp.json @@ -0,0 +1,11 @@ +{ + "mcpServers": { + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${SPICRAWL_API_KEY}" + } + } + } +} diff --git a/internal/cmd/testdata/golden/agents/mcp_install_claude.stdout b/internal/cmd/testdata/golden/agents/mcp_install_claude.stdout new file mode 100644 index 0000000..d72903e --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_claude.stdout @@ -0,0 +1,8 @@ +[ + { + "client": "claude", + "kind": "mcp", + "file": "/.mcp.json", + "action": "created" + } +] diff --git a/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.exit b/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.file..mcp.json b/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.file..mcp.json new file mode 100644 index 0000000..4d19b70 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.file..mcp.json @@ -0,0 +1,15 @@ +{ + "zeta": 1, + "mcpServers": { + "other": { + "command": "x" + }, + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${SPICRAWL_API_KEY}" + } + } + } +} diff --git a/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.stdout b/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.stdout new file mode 100644 index 0000000..edcf64a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_install_merges_existing.stdout @@ -0,0 +1,9 @@ +[ + { + "client": "claude", + "kind": "mcp", + "file": "/.mcp.json", + "action": "updated", + "backup": "/.mcp.json.bak" + } +] diff --git a/internal/cmd/testdata/golden/agents/mcp_print_all.exit b/internal/cmd/testdata/golden/agents/mcp_print_all.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_print_all.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/mcp_print_all.stdout b/internal/cmd/testdata/golden/agents/mcp_print_all.stdout new file mode 100644 index 0000000..72f63cc --- /dev/null +++ b/internal/cmd/testdata/golden/agents/mcp_print_all.stdout @@ -0,0 +1,26 @@ +[ + { + "client": "claude", + "file": "/.mcp.json", + "format": "json", + "snippet": "{\n \"mcpServers\": {\n \"spicrawl\": {\n \"type\": \"http\",\n \"url\": \"http://127.0.0.1:1/mcp\",\n \"headers\": {\n \"Authorization\": \"Bearer ${SPICRAWL_API_KEY}\"\n }\n }\n }\n}\n" + }, + { + "client": "cursor", + "file": "/.cursor/mcp.json", + "format": "json", + "snippet": "{\n \"mcpServers\": {\n \"spicrawl\": {\n \"url\": \"http://127.0.0.1:1/mcp\",\n \"headers\": {\n \"Authorization\": \"Bearer ${env:SPICRAWL_API_KEY}\"\n }\n }\n }\n}\n" + }, + { + "client": "vscode", + "file": "/.vscode/mcp.json", + "format": "json", + "snippet": "{\n \"servers\": {\n \"spicrawl\": {\n \"type\": \"http\",\n \"url\": \"http://127.0.0.1:1/mcp\",\n \"headers\": {\n \"Authorization\": \"Bearer ${input:spicrawl-api-key}\"\n }\n }\n },\n \"inputs\": [\n {\n \"type\": \"promptString\",\n \"id\": \"spicrawl-api-key\",\n \"description\": \"Spicrawl API key\",\n \"password\": true\n }\n ]\n}\n" + }, + { + "client": "codex", + "file": "/.codex/config.toml", + "format": "toml", + "snippet": "[mcp_servers.spicrawl]\nurl = \"http://127.0.0.1:1/mcp\"\nbearer_token_env_var = \"SPICRAWL_API_KEY\"\n" + } +] diff --git a/internal/cmd/testdata/golden/agents/skill_install_claude.exit b/internal/cmd/testdata/golden/agents/skill_install_claude.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_claude.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/skill_install_claude.file..claude__skills__spicrawl__SKILL.md b/internal/cmd/testdata/golden/agents/skill_install_claude.file..claude__skills__spicrawl__SKILL.md new file mode 100644 index 0000000..9a9132e --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_claude.file..claude__skills__spicrawl__SKILL.md @@ -0,0 +1,8 @@ +--- +name: spicrawl +description: golden test skill +--- + +# Spicrawl + +Use http://127.0.0.1:1 for scraping. diff --git a/internal/cmd/testdata/golden/agents/skill_install_claude.stdout b/internal/cmd/testdata/golden/agents/skill_install_claude.stdout new file mode 100644 index 0000000..1f9c4d9 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_claude.stdout @@ -0,0 +1,8 @@ +[ + { + "client": "claude", + "kind": "skill", + "file": "/.claude/skills/spicrawl/SKILL.md", + "action": "created" + } +] diff --git a/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.exit b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.file..agents__skills__spicrawl__SKILL.md b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.file..agents__skills__spicrawl__SKILL.md new file mode 100644 index 0000000..9a9132e --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.file..agents__skills__spicrawl__SKILL.md @@ -0,0 +1,8 @@ +--- +name: spicrawl +description: golden test skill +--- + +# Spicrawl + +Use http://127.0.0.1:1 for scraping. diff --git a/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.file.AGENTS.md b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.file.AGENTS.md new file mode 100644 index 0000000..54caf39 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.file.AGENTS.md @@ -0,0 +1,10 @@ + +## Web data (Spicrawl) + +To fetch, render or extract data from web pages, use Spicrawl instead of ad-hoc +HTTP requests: the `spicrawl` MCP server (tools `spicrawl_*`) or the CLI +(`spicrawl scrape --format markdown`, `spicrawl batch submit urls.txt`). +The API key is read from `$SPICRAWL_API_KEY`; never print or commit it. +Start cheap (plain fetch), then add `--render`, `--premium-proxy`, `--stealth` +only when the page needs it. Full guide: the `spicrawl` skill, or http://SERVER/skill.md. + diff --git a/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.stdout b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.stdout new file mode 100644 index 0000000..2c047d7 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_install_codex_agents_md.stdout @@ -0,0 +1,14 @@ +[ + { + "client": "codex", + "kind": "skill", + "file": "/.agents/skills/spicrawl/SKILL.md", + "action": "created" + }, + { + "client": "codex", + "kind": "agents_md", + "file": "/AGENTS.md", + "action": "created" + } +] diff --git a/internal/cmd/testdata/golden/agents/skill_print.exit b/internal/cmd/testdata/golden/agents/skill_print.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_print.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents/skill_print.stdout b/internal/cmd/testdata/golden/agents/skill_print.stdout new file mode 100644 index 0000000..ccd5736 --- /dev/null +++ b/internal/cmd/testdata/golden/agents/skill_print.stdout @@ -0,0 +1,4 @@ +{ + "content": "---\nname: spicrawl\ndescription: golden test skill\n---\n\n# Spicrawl\n\nUse http://127.0.0.1:1 for scraping.\n", + "source": "http://SERVER/skill.md" +} diff --git a/internal/cmd/testdata/golden/agents_human/init_yes.exit b/internal/cmd/testdata/golden/agents_human/init_yes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/init_yes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents_human/init_yes.stdout b/internal/cmd/testdata/golden/agents_human/init_yes.stdout new file mode 100644 index 0000000..3073b3e --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/init_yes.stdout @@ -0,0 +1,5 @@ +CLIENT KIND ACTION FILE NOTE +claude mcp created .mcp.json +cursor mcp created .cursor/mcp.json +claude skill created .claude/skills/spicrawl/SKILL.md +cursor skill skipped .claude/skills/spicrawl/SKILL.md already reads this skill diff --git a/internal/cmd/testdata/golden/agents_human/mcp_install_all.exit b/internal/cmd/testdata/golden/agents_human/mcp_install_all.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/mcp_install_all.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents_human/mcp_install_all.stdout b/internal/cmd/testdata/golden/agents_human/mcp_install_all.stdout new file mode 100644 index 0000000..3042314 --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/mcp_install_all.stdout @@ -0,0 +1,5 @@ +CLIENT KIND ACTION FILE NOTE +claude mcp created .mcp.json +cursor mcp created .cursor/mcp.json +vscode mcp created .vscode/mcp.json +codex mcp created .codex/config.toml diff --git a/internal/cmd/testdata/golden/agents_human/mcp_install_merges_existing.exit b/internal/cmd/testdata/golden/agents_human/mcp_install_merges_existing.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/mcp_install_merges_existing.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents_human/mcp_install_merges_existing.stdout b/internal/cmd/testdata/golden/agents_human/mcp_install_merges_existing.stdout new file mode 100644 index 0000000..fccf189 --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/mcp_install_merges_existing.stdout @@ -0,0 +1,2 @@ +CLIENT KIND ACTION FILE NOTE +claude mcp updated .mcp.json backup: .mcp.json.bak diff --git a/internal/cmd/testdata/golden/agents_human/mcp_print_all.exit b/internal/cmd/testdata/golden/agents_human/mcp_print_all.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/mcp_print_all.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents_human/mcp_print_all.stdout b/internal/cmd/testdata/golden/agents_human/mcp_print_all.stdout new file mode 100644 index 0000000..beb2ecf --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/mcp_print_all.stdout @@ -0,0 +1,50 @@ +# Claude Code: merge into .mcp.json +{ + "mcpServers": { + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${SPICRAWL_API_KEY}" + } + } + } +} + +# Cursor: merge into .cursor/mcp.json +{ + "mcpServers": { + "spicrawl": { + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${env:SPICRAWL_API_KEY}" + } + } + } +} + +# VS Code (Copilot): merge into .vscode/mcp.json +{ + "servers": { + "spicrawl": { + "type": "http", + "url": "http://127.0.0.1:1/mcp", + "headers": { + "Authorization": "Bearer ${input:spicrawl-api-key}" + } + } + }, + "inputs": [ + { + "type": "promptString", + "id": "spicrawl-api-key", + "description": "Spicrawl API key", + "password": true + } + ] +} + +# Codex: merge into .codex/config.toml +[mcp_servers.spicrawl] +url = "http://127.0.0.1:1/mcp" +bearer_token_env_var = "SPICRAWL_API_KEY" diff --git a/internal/cmd/testdata/golden/agents_human/skill_install_claude.exit b/internal/cmd/testdata/golden/agents_human/skill_install_claude.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/skill_install_claude.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents_human/skill_install_claude.stdout b/internal/cmd/testdata/golden/agents_human/skill_install_claude.stdout new file mode 100644 index 0000000..4fe0518 --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/skill_install_claude.stdout @@ -0,0 +1,2 @@ +CLIENT KIND ACTION FILE NOTE +claude skill created .claude/skills/spicrawl/SKILL.md diff --git a/internal/cmd/testdata/golden/agents_human/skill_print.exit b/internal/cmd/testdata/golden/agents_human/skill_print.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/skill_print.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/agents_human/skill_print.stdout b/internal/cmd/testdata/golden/agents_human/skill_print.stdout new file mode 100644 index 0000000..9a9132e --- /dev/null +++ b/internal/cmd/testdata/golden/agents_human/skill_print.stdout @@ -0,0 +1,8 @@ +--- +name: spicrawl +description: golden test skill +--- + +# Spicrawl + +Use http://127.0.0.1:1 for scraping. diff --git a/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.exit b/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.stderr b/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.stderr new file mode 100644 index 0000000..2472206 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.stderr @@ -0,0 +1,4 @@ +{ + "error": "no API key given and stdin is not a terminal: pass --api-key or set SPICRAWL_API_KEY (create a key at https://app.spicrawl.com/dashboard/keys)", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.stdout b/internal/cmd/testdata/golden/auth/login_needs_key_without_terminal.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/auth/login_ok.exit b/internal/cmd/testdata/golden/auth/login_ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth/login_ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth/login_ok.stdout b/internal/cmd/testdata/golden/auth/login_ok.stdout new file mode 100644 index 0000000..b58b7b9 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/login_ok.stdout @@ -0,0 +1,5 @@ +{ + "base_url": "http://SERVER", + "key": "spicrawl_live_0123…mnop", + "saved": "/config.json" +} diff --git a/internal/cmd/testdata/golden/auth/login_rejected_exit3.exit b/internal/cmd/testdata/golden/auth/login_rejected_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/auth/login_rejected_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/auth/login_rejected_exit3.stderr b/internal/cmd/testdata/golden/auth/login_rejected_exit3.stderr new file mode 100644 index 0000000..cbdc3b7 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/login_rejected_exit3.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Unauthorized", + "status": 401, + "code": "ERR::AUTH::INVALID_KEY", + "detail": "the API key is not valid", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/auth/login_rejected_exit3.stdout b/internal/cmd/testdata/golden/auth/login_rejected_exit3.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/auth/logout_nothing_saved.exit b/internal/cmd/testdata/golden/auth/logout_nothing_saved.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth/logout_nothing_saved.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth/logout_nothing_saved.stdout b/internal/cmd/testdata/golden/auth/logout_nothing_saved.stdout new file mode 100644 index 0000000..a00ad10 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/logout_nothing_saved.stdout @@ -0,0 +1,4 @@ +{ + "path": "/config.json", + "removed": false +} diff --git a/internal/cmd/testdata/golden/auth/logout_removed.exit b/internal/cmd/testdata/golden/auth/logout_removed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth/logout_removed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth/logout_removed.stdout b/internal/cmd/testdata/golden/auth/logout_removed.stdout new file mode 100644 index 0000000..8f7f696 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/logout_removed.stdout @@ -0,0 +1,4 @@ +{ + "path": "/config.json", + "removed": true +} diff --git a/internal/cmd/testdata/golden/auth/status_key_from_file.exit b/internal/cmd/testdata/golden/auth/status_key_from_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_key_from_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth/status_key_from_file.stdout b/internal/cmd/testdata/golden/auth/status_key_from_file.stdout new file mode 100644 index 0000000..4ae02a9 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_key_from_file.stdout @@ -0,0 +1,8 @@ +{ + "api_key": "spicrawl_live_0123…mnop", + "api_key_source": "file", + "base_url": "https://api.spicrawl.com", + "base_url_source": "default", + "config_path": "/config.json", + "key_valid": null +} diff --git a/internal/cmd/testdata/golden/auth/status_no_key_exit3.exit b/internal/cmd/testdata/golden/auth/status_no_key_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_no_key_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/auth/status_no_key_exit3.stderr b/internal/cmd/testdata/golden/auth/status_no_key_exit3.stderr new file mode 100644 index 0000000..6b478b2 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_no_key_exit3.stderr @@ -0,0 +1,4 @@ +{ + "error": "no API key: run \"spicrawl login\", set SPICRAWL_API_KEY, or pass --api-key", + "exit_code": 3 +} diff --git a/internal/cmd/testdata/golden/auth/status_no_key_exit3.stdout b/internal/cmd/testdata/golden/auth/status_no_key_exit3.stdout new file mode 100644 index 0000000..be37f14 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_no_key_exit3.stdout @@ -0,0 +1,8 @@ +{ + "api_key": null, + "api_key_source": null, + "base_url": "https://api.spicrawl.com", + "base_url_source": "default", + "config_path": "/config.json", + "key_valid": null +} diff --git a/internal/cmd/testdata/golden/auth/status_offline.exit b/internal/cmd/testdata/golden/auth/status_offline.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_offline.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth/status_offline.stdout b/internal/cmd/testdata/golden/auth/status_offline.stdout new file mode 100644 index 0000000..232df19 --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_offline.stdout @@ -0,0 +1,8 @@ +{ + "api_key": "spicrawl_test_0123…ghij", + "api_key_source": "env", + "base_url": "https://api.spicrawl.com", + "base_url_source": "default", + "config_path": "/config.json", + "key_valid": null +} diff --git a/internal/cmd/testdata/golden/auth/status_ok.exit b/internal/cmd/testdata/golden/auth/status_ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth/status_ok.stdout b/internal/cmd/testdata/golden/auth/status_ok.stdout new file mode 100644 index 0000000..3f3611c --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_ok.stdout @@ -0,0 +1,8 @@ +{ + "api_key": "spicrawl_test_0123…ghij", + "api_key_source": "env", + "base_url": "http://SERVER", + "base_url_source": "flag", + "config_path": "/config.json", + "key_valid": true +} diff --git a/internal/cmd/testdata/golden/auth/status_rejected_exit3.exit b/internal/cmd/testdata/golden/auth/status_rejected_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_rejected_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/auth/status_rejected_exit3.stderr b/internal/cmd/testdata/golden/auth/status_rejected_exit3.stderr new file mode 100644 index 0000000..47478ac --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_rejected_exit3.stderr @@ -0,0 +1,4 @@ +{ + "error": "the API rejected the key: ERR::AUTH::INVALID_KEY: the API key is not valid", + "exit_code": 3 +} diff --git a/internal/cmd/testdata/golden/auth/status_rejected_exit3.stdout b/internal/cmd/testdata/golden/auth/status_rejected_exit3.stdout new file mode 100644 index 0000000..a9c0aad --- /dev/null +++ b/internal/cmd/testdata/golden/auth/status_rejected_exit3.stdout @@ -0,0 +1,9 @@ +{ + "api_key": "spicrawl_test_0123…ghij", + "api_key_source": "env", + "base_url": "http://SERVER", + "base_url_source": "flag", + "check_error": "ERR::AUTH::INVALID_KEY: the API key is not valid", + "config_path": "/config.json", + "key_valid": false +} diff --git a/internal/cmd/testdata/golden/auth_human/login_ok.exit b/internal/cmd/testdata/golden/auth_human/login_ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/login_ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth_human/login_ok.stdout b/internal/cmd/testdata/golden/auth_human/login_ok.stdout new file mode 100644 index 0000000..69d5c68 --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/login_ok.stdout @@ -0,0 +1,2 @@ +Logged in. Key spicrawl_live_0123…mnop saved to /config.json +Base URL: http://SERVER diff --git a/internal/cmd/testdata/golden/auth_human/logout_nothing_saved.exit b/internal/cmd/testdata/golden/auth_human/logout_nothing_saved.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/logout_nothing_saved.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth_human/logout_nothing_saved.stdout b/internal/cmd/testdata/golden/auth_human/logout_nothing_saved.stdout new file mode 100644 index 0000000..fd5b5ae --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/logout_nothing_saved.stdout @@ -0,0 +1 @@ +No API key saved in /config.json diff --git a/internal/cmd/testdata/golden/auth_human/logout_removed.exit b/internal/cmd/testdata/golden/auth_human/logout_removed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/logout_removed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth_human/logout_removed.stdout b/internal/cmd/testdata/golden/auth_human/logout_removed.stdout new file mode 100644 index 0000000..9fe5b85 --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/logout_removed.stdout @@ -0,0 +1 @@ +Removed the API key from /config.json diff --git a/internal/cmd/testdata/golden/auth_human/status_no_key_exit3.exit b/internal/cmd/testdata/golden/auth_human/status_no_key_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/status_no_key_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/auth_human/status_no_key_exit3.stdout b/internal/cmd/testdata/golden/auth_human/status_no_key_exit3.stdout new file mode 100644 index 0000000..8281987 --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/status_no_key_exit3.stdout @@ -0,0 +1,5 @@ +FIELD VALUE +api key (none) +base url https://api.spicrawl.com (from default) +config /config.json +key works not checked diff --git a/internal/cmd/testdata/golden/auth_human/status_ok.exit b/internal/cmd/testdata/golden/auth_human/status_ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/status_ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/auth_human/status_ok.stdout b/internal/cmd/testdata/golden/auth_human/status_ok.stdout new file mode 100644 index 0000000..ff80741 --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/status_ok.stdout @@ -0,0 +1,5 @@ +FIELD VALUE +api key spicrawl_test_0123…ghij (from env) +base url http://SERVER (from flag) +config /config.json +key works yes diff --git a/internal/cmd/testdata/golden/auth_human/status_rejected_exit3.exit b/internal/cmd/testdata/golden/auth_human/status_rejected_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/status_rejected_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/auth_human/status_rejected_exit3.stdout b/internal/cmd/testdata/golden/auth_human/status_rejected_exit3.stdout new file mode 100644 index 0000000..7fc2897 --- /dev/null +++ b/internal/cmd/testdata/golden/auth_human/status_rejected_exit3.stdout @@ -0,0 +1,5 @@ +FIELD VALUE +api key spicrawl_test_0123…ghij (from env) +base url http://SERVER (from flag) +config /config.json +key works no diff --git a/internal/cmd/testdata/golden/batch/append.exit b/internal/cmd/testdata/golden/batch/append.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/append.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/append.stdout b/internal/cmd/testdata/golden/batch/append.stdout new file mode 100644 index 0000000..d89a0bf --- /dev/null +++ b/internal/cmd/testdata/golden/batch/append.stdout @@ -0,0 +1,38 @@ +{ + "job": { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "running", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 1, + "succeeded": 1, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 1, + "percent_complete": 50.0, + "credits_charged": 3, + "credits_charged_micro": 3000000, + "bytes": 20480 + }, + "submitted_at": "", + "started_at": "" + }, + "items_added": 2, + "items_dispatched": 1 +} diff --git a/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.exit b/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.stderr b/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.stderr new file mode 100644 index 0000000..578268c --- /dev/null +++ b/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.stderr @@ -0,0 +1,4 @@ +{ + "error": "refusing to cancel batch 01J9Z6V0Q8M4K2T7R3N5B1C9XA without confirmation: cancelling cannot be undone; pass --yes", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.stdout b/internal/cmd/testdata/golden/batch/cancel_needs_yes_without_terminal.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/cancel_yes.exit b/internal/cmd/testdata/golden/batch/cancel_yes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/cancel_yes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/cancel_yes.stdout b/internal/cmd/testdata/golden/batch/cancel_yes.stdout new file mode 100644 index 0000000..55bef45 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/cancel_yes.stdout @@ -0,0 +1,35 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "cancelled", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 5, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 15, + "estimated_credits_micro": 15000000, + "progress": { + "total": 5, + "completed": 1, + "succeeded": 1, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 4, + "percent_complete": 20.0, + "credits_charged": 3, + "credits_charged_micro": 3000000, + "bytes": 20480 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/close.exit b/internal/cmd/testdata/golden/batch/close.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/close.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/close.stdout b/internal/cmd/testdata/golden/batch/close.stdout new file mode 100644 index 0000000..211ab47 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/close.stdout @@ -0,0 +1,34 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "running", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 1, + "succeeded": 1, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 1, + "percent_complete": 50.0, + "credits_charged": 3, + "credits_charged_micro": 3000000, + "bytes": 20480 + }, + "submitted_at": "", + "started_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/content.exit b/internal/cmd/testdata/golden/batch/content.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/content.stdout b/internal/cmd/testdata/golden/batch/content.stdout new file mode 100644 index 0000000..619f212 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content.stdout @@ -0,0 +1 @@ +"

A

\n" diff --git a/internal/cmd/testdata/golden/batch/content_bad_seq.exit b/internal/cmd/testdata/golden/batch/content_bad_seq.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content_bad_seq.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/batch/content_bad_seq.stderr b/internal/cmd/testdata/golden/batch/content_bad_seq.stderr new file mode 100644 index 0000000..7a43a17 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content_bad_seq.stderr @@ -0,0 +1,4 @@ +{ + "error": "seq \"x\": want the item's 0-based position (\"seq\" on a result line)", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/batch/content_bad_seq.stdout b/internal/cmd/testdata/golden/batch/content_bad_seq.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/content_not_finished.exit b/internal/cmd/testdata/golden/batch/content_not_finished.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content_not_finished.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch/content_not_finished.stderr b/internal/cmd/testdata/golden/batch/content_not_finished.stderr new file mode 100644 index 0000000..2cb7ffa --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content_not_finished.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Conflict", + "status": 409, + "code": "ERR::REQUEST::NOT_READY", + "detail": "item 7 has not finished", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/batch/content_not_finished.stdout b/internal/cmd/testdata/golden/batch/content_not_finished.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/content_output_file.exit b/internal/cmd/testdata/golden/batch/content_output_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content_output_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/content_output_file.file.a.html b/internal/cmd/testdata/golden/batch/content_output_file.file.a.html new file mode 100644 index 0000000..cd15ba6 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/content_output_file.file.a.html @@ -0,0 +1 @@ +

A

diff --git a/internal/cmd/testdata/golden/batch/content_output_file.stdout b/internal/cmd/testdata/golden/batch/content_output_file.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/get.exit b/internal/cmd/testdata/golden/batch/get.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/get.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/get.stdout b/internal/cmd/testdata/golden/batch/get.stdout new file mode 100644 index 0000000..f30f585 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/get.stdout @@ -0,0 +1,36 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/get_not_found.exit b/internal/cmd/testdata/golden/batch/get_not_found.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/get_not_found.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch/get_not_found.stderr b/internal/cmd/testdata/golden/batch/get_not_found.stderr new file mode 100644 index 0000000..e52b1ba --- /dev/null +++ b/internal/cmd/testdata/golden/batch/get_not_found.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Not Found", + "status": 404, + "code": "ERR::REQUEST::NOT_FOUND", + "detail": "no such batch", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/batch/get_not_found.stdout b/internal/cmd/testdata/golden/batch/get_not_found.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/list.exit b/internal/cmd/testdata/golden/batch/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/list.stdout b/internal/cmd/testdata/golden/batch/list.stdout new file mode 100644 index 0000000..d93642d --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list.stdout @@ -0,0 +1,74 @@ +{ + "batches": [ + { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" + }, + { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "crawl", + "status": "running", + "open": true, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 9, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 27, + "estimated_credits_micro": 27000000, + "progress": { + "total": 9, + "completed": 4, + "succeeded": 4, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 5, + "percent_complete": 44.4, + "credits_charged": 12, + "credits_charged_micro": 12000000, + "bytes": 81920 + }, + "submitted_at": "", + "started_at": "" + } + ] +} diff --git a/internal/cmd/testdata/golden/batch/list_all_pages.exit b/internal/cmd/testdata/golden/batch/list_all_pages.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list_all_pages.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/list_all_pages.stdout b/internal/cmd/testdata/golden/batch/list_all_pages.stdout new file mode 100644 index 0000000..d93642d --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list_all_pages.stdout @@ -0,0 +1,74 @@ +{ + "batches": [ + { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" + }, + { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "crawl", + "status": "running", + "open": true, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 9, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 27, + "estimated_credits_micro": 27000000, + "progress": { + "total": 9, + "completed": 4, + "succeeded": 4, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 5, + "percent_complete": 44.4, + "credits_charged": 12, + "credits_charged_micro": 12000000, + "bytes": 81920 + }, + "submitted_at": "", + "started_at": "" + } + ] +} diff --git a/internal/cmd/testdata/golden/batch/list_empty.exit b/internal/cmd/testdata/golden/batch/list_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/list_empty.stdout b/internal/cmd/testdata/golden/batch/list_empty.stdout new file mode 100644 index 0000000..fef4d1b --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list_empty.stdout @@ -0,0 +1,3 @@ +{ + "batches": [] +} diff --git a/internal/cmd/testdata/golden/batch/list_next_cursor.exit b/internal/cmd/testdata/golden/batch/list_next_cursor.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list_next_cursor.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/list_next_cursor.stdout b/internal/cmd/testdata/golden/batch/list_next_cursor.stdout new file mode 100644 index 0000000..2518c02 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/list_next_cursor.stdout @@ -0,0 +1,41 @@ +{ + "batches": [ + { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" + } + ], + "next_cursor": "1790000000000000000.ABC" +} diff --git a/internal/cmd/testdata/golden/batch/results.exit b/internal/cmd/testdata/golden/batch/results.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/results.stdout b/internal/cmd/testdata/golden/batch/results.stdout new file mode 100644 index 0000000..a3ce132 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results.stdout @@ -0,0 +1,3 @@ +{"seq":0,"external_id":"sku-1","url":"https://example.com/a","status":"succeeded","http_status":200,"bytes":2048,"result":{"content":"\"

A

\\n\""}} +{"seq":1,"external_id":"","url":"https://example.com/b","status":"failed","http_status":null,"bytes":0,"error":{"code":"ERR::UPSTREAM::TIMEOUT","message":"the target did not answer"}} +{"seq":2,"external_id":"sku-3","url":"https://example.com/c?x=1&y=2","status":"succeeded","http_status":404,"bytes":512,"result":{"content":"\"not found\""}} diff --git a/internal/cmd/testdata/golden/batch/results_all_pages.exit b/internal/cmd/testdata/golden/batch/results_all_pages.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_all_pages.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/results_all_pages.stdout b/internal/cmd/testdata/golden/batch/results_all_pages.stdout new file mode 100644 index 0000000..bcb5037 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_all_pages.stdout @@ -0,0 +1,6 @@ +{"seq":0,"external_id":"sku-1","url":"https://example.com/a","status":"succeeded","http_status":200,"bytes":2048,"result":{"content":"\"

A

\\n\""}} +{"seq":1,"external_id":"","url":"https://example.com/b","status":"failed","http_status":null,"bytes":0,"error":{"code":"ERR::UPSTREAM::TIMEOUT","message":"the target did not answer"}} +{"seq":2,"external_id":"sku-3","url":"https://example.com/c?x=1&y=2","status":"succeeded","http_status":404,"bytes":512,"result":{"content":"\"not found\""}} +{"seq":0,"external_id":"sku-1","url":"https://example.com/a","status":"succeeded","http_status":200,"bytes":2048,"result":{"content":"\"

A

\\n\""}} +{"seq":1,"external_id":"","url":"https://example.com/b","status":"failed","http_status":null,"bytes":0,"error":{"code":"ERR::UPSTREAM::TIMEOUT","message":"the target did not answer"}} +{"seq":2,"external_id":"sku-3","url":"https://example.com/c?x=1&y=2","status":"succeeded","http_status":404,"bytes":512,"result":{"content":"\"not found\""}} diff --git a/internal/cmd/testdata/golden/batch/results_empty.exit b/internal/cmd/testdata/golden/batch/results_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/results_empty.stdout b/internal/cmd/testdata/golden/batch/results_empty.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/results_expired.exit b/internal/cmd/testdata/golden/batch/results_expired.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_expired.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch/results_expired.stderr b/internal/cmd/testdata/golden/batch/results_expired.stderr new file mode 100644 index 0000000..b68510a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_expired.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Gone", + "status": 410, + "code": "ERR::REQUEST::BEYOND_RETENTION", + "detail": "results expired", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/batch/results_expired.stdout b/internal/cmd/testdata/golden/batch/results_expired.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/results_more_pages.exit b/internal/cmd/testdata/golden/batch/results_more_pages.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_more_pages.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/results_more_pages.stdout b/internal/cmd/testdata/golden/batch/results_more_pages.stdout new file mode 100644 index 0000000..a3ce132 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_more_pages.stdout @@ -0,0 +1,3 @@ +{"seq":0,"external_id":"sku-1","url":"https://example.com/a","status":"succeeded","http_status":200,"bytes":2048,"result":{"content":"\"

A

\\n\""}} +{"seq":1,"external_id":"","url":"https://example.com/b","status":"failed","http_status":null,"bytes":0,"error":{"code":"ERR::UPSTREAM::TIMEOUT","message":"the target did not answer"}} +{"seq":2,"external_id":"sku-3","url":"https://example.com/c?x=1&y=2","status":"succeeded","http_status":404,"bytes":512,"result":{"content":"\"not found\""}} diff --git a/internal/cmd/testdata/golden/batch/results_output_file.exit b/internal/cmd/testdata/golden/batch/results_output_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_output_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/results_output_file.file.results.jsonl b/internal/cmd/testdata/golden/batch/results_output_file.file.results.jsonl new file mode 100644 index 0000000..a3ce132 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/results_output_file.file.results.jsonl @@ -0,0 +1,3 @@ +{"seq":0,"external_id":"sku-1","url":"https://example.com/a","status":"succeeded","http_status":200,"bytes":2048,"result":{"content":"\"

A

\\n\""}} +{"seq":1,"external_id":"","url":"https://example.com/b","status":"failed","http_status":null,"bytes":0,"error":{"code":"ERR::UPSTREAM::TIMEOUT","message":"the target did not answer"}} +{"seq":2,"external_id":"sku-3","url":"https://example.com/c?x=1&y=2","status":"succeeded","http_status":404,"bytes":512,"result":{"content":"\"not found\""}} diff --git a/internal/cmd/testdata/golden/batch/results_output_file.stdout b/internal/cmd/testdata/golden/batch/results_output_file.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/retry.exit b/internal/cmd/testdata/golden/batch/retry.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/retry.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/retry.stdout b/internal/cmd/testdata/golden/batch/retry.stdout new file mode 100644 index 0000000..351570f --- /dev/null +++ b/internal/cmd/testdata/golden/batch/retry.stdout @@ -0,0 +1,38 @@ +{ + "job": { + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "running", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 1, + "succeeded": 1, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 1, + "percent_complete": 50.0, + "credits_charged": 3, + "credits_charged_micro": 3000000, + "bytes": 20480 + }, + "submitted_at": "", + "started_at": "" + }, + "items_reset": 2, + "items_dispatched": 1 +} diff --git a/internal/cmd/testdata/golden/batch/submit_err_request.exit b/internal/cmd/testdata/golden/batch/submit_err_request.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_err_request.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch/submit_err_request.stderr b/internal/cmd/testdata/golden/batch/submit_err_request.stderr new file mode 100644 index 0000000..69386d4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_err_request.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Bad Request", + "status": 400, + "code": "ERR::REQUEST::INVALID_PARAMETER", + "detail": "response_format pdf is not supported on batch", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/batch/submit_err_request.stdout b/internal/cmd/testdata/golden/batch/submit_err_request.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/submit_items_jsonl.exit b/internal/cmd/testdata/golden/batch/submit_items_jsonl.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_items_jsonl.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/submit_items_jsonl.stdout b/internal/cmd/testdata/golden/batch/submit_items_jsonl.stdout new file mode 100644 index 0000000..2f58ee9 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_items_jsonl.stdout @@ -0,0 +1,33 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "queued", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 0, + "succeeded": 0, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 2, + "percent_complete": 0.0, + "credits_charged": 0, + "credits_charged_micro": 0, + "bytes": 0 + }, + "submitted_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/submit_stdin.exit b/internal/cmd/testdata/golden/batch/submit_stdin.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_stdin.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/submit_stdin.stdout b/internal/cmd/testdata/golden/batch/submit_stdin.stdout new file mode 100644 index 0000000..2f58ee9 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_stdin.stdout @@ -0,0 +1,33 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "queued", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 0, + "succeeded": 0, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 2, + "percent_complete": 0.0, + "credits_charged": 0, + "credits_charged_micro": 0, + "bytes": 0 + }, + "submitted_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/submit_urls.exit b/internal/cmd/testdata/golden/batch/submit_urls.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_urls.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/submit_urls.stdout b/internal/cmd/testdata/golden/batch/submit_urls.stdout new file mode 100644 index 0000000..2f58ee9 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_urls.stdout @@ -0,0 +1,33 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "queued", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 0, + "succeeded": 0, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 2, + "percent_complete": 0.0, + "credits_charged": 0, + "credits_charged_micro": 0, + "bytes": 0 + }, + "submitted_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/submit_wait.exit b/internal/cmd/testdata/golden/batch/submit_wait.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_wait.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/submit_wait.stdout b/internal/cmd/testdata/golden/batch/submit_wait.stdout new file mode 100644 index 0000000..f30f585 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_wait.stdout @@ -0,0 +1,36 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.exit b/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.stderr b/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.stderr new file mode 100644 index 0000000..8b9f9fa --- /dev/null +++ b/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.stderr @@ -0,0 +1,4 @@ +{ + "error": "--wait with --open would never finish: an open job completes only after \"spicrawl batch close\"", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.stdout b/internal/cmd/testdata/golden/batch/submit_wait_open_is_usage_error.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.exit b/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.stderr b/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.stderr new file mode 100644 index 0000000..bd49396 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.stderr @@ -0,0 +1,4 @@ +{ + "error": "batch 01J9Z6V0Q8M4K2T7R3N5B1C9XA cancelled (1 of 5 items finished)", + "exit_code": 4 +} diff --git a/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.stdout b/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.stdout new file mode 100644 index 0000000..55bef45 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_cancelled_exit4.stdout @@ -0,0 +1,35 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "cancelled", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 5, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 15, + "estimated_credits_micro": 15000000, + "progress": { + "total": 5, + "completed": 1, + "succeeded": 1, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 4, + "percent_complete": 20.0, + "credits_charged": 3, + "credits_charged_micro": 3000000, + "bytes": 20480 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/wait_completed.exit b/internal/cmd/testdata/golden/batch/wait_completed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_completed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/wait_completed.stdout b/internal/cmd/testdata/golden/batch/wait_completed.stdout new file mode 100644 index 0000000..f30f585 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_completed.stdout @@ -0,0 +1,36 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/wait_failed_exit4.exit b/internal/cmd/testdata/golden/batch/wait_failed_exit4.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_failed_exit4.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch/wait_failed_exit4.stderr b/internal/cmd/testdata/golden/batch/wait_failed_exit4.stderr new file mode 100644 index 0000000..93c36d9 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_failed_exit4.stderr @@ -0,0 +1,4 @@ +{ + "error": "batch 01J9Z6V0Q8M4K2T7R3N5B1C9XA failed (2 of 10 items finished): ERR::LIMIT::FAILURE_THRESHOLD failure threshold reached", + "exit_code": 4 +} diff --git a/internal/cmd/testdata/golden/batch/wait_failed_exit4.stdout b/internal/cmd/testdata/golden/batch/wait_failed_exit4.stdout new file mode 100644 index 0000000..4f404fb --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_failed_exit4.stdout @@ -0,0 +1,37 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "failed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 10, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 30, + "estimated_credits_micro": 30000000, + "progress": { + "total": 10, + "completed": 2, + "succeeded": 0, + "failed": 2, + "cancelled": 0, + "skipped": 0, + "remaining": 8, + "percent_complete": 20.0, + "credits_charged": 6, + "credits_charged_micro": 6000000, + "bytes": 40960 + }, + "submitted_at": "", + "error_code": "ERR::LIMIT::FAILURE_THRESHOLD", + "error_message": "failure threshold reached", + "started_at": "", + "finished_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/wait_pending_exit11.exit b/internal/cmd/testdata/golden/batch/wait_pending_exit11.exit new file mode 100644 index 0000000..b4de394 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_pending_exit11.exit @@ -0,0 +1 @@ +11 diff --git a/internal/cmd/testdata/golden/batch/wait_pending_exit11.stderr b/internal/cmd/testdata/golden/batch/wait_pending_exit11.stderr new file mode 100644 index 0000000..e3b9954 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_pending_exit11.stderr @@ -0,0 +1,4 @@ +{ + "error": "batch 01J9Z6V0Q8M4K2T7R3N5B1C9XA is still running after 60ms; resume with: spicrawl batch wait 01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "exit_code": 11 +} diff --git a/internal/cmd/testdata/golden/batch/wait_pending_exit11.stdout b/internal/cmd/testdata/golden/batch/wait_pending_exit11.stdout new file mode 100644 index 0000000..211ab47 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_pending_exit11.stdout @@ -0,0 +1,34 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "running", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 2, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 6, + "estimated_credits_micro": 6000000, + "progress": { + "total": 2, + "completed": 1, + "succeeded": 1, + "failed": 0, + "cancelled": 0, + "skipped": 0, + "remaining": 1, + "percent_complete": 50.0, + "credits_charged": 3, + "credits_charged_micro": 3000000, + "bytes": 20480 + }, + "submitted_at": "", + "started_at": "" +} diff --git a/internal/cmd/testdata/golden/batch/wait_sequence.exit b/internal/cmd/testdata/golden/batch/wait_sequence.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_sequence.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch/wait_sequence.stdout b/internal/cmd/testdata/golden/batch/wait_sequence.stdout new file mode 100644 index 0000000..f30f585 --- /dev/null +++ b/internal/cmd/testdata/golden/batch/wait_sequence.stdout @@ -0,0 +1,36 @@ +{ + "id": "01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "project_id": "proj_01J9Z", + "name": "nightly", + "status": "completed", + "open": false, + "status_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA", + "results_url": "/v1/batch/01J9Z6V0Q8M4K2T7R3N5B1C9XA/results", + "params": { + "js_render": true, + "response_format": "markdown" + }, + "total_items": 3, + "concurrency": 10, + "priority": 100, + "max_attempts": 3, + "estimated_credits": 9, + "estimated_credits_micro": 9000000, + "progress": { + "total": 3, + "completed": 3, + "succeeded": 2, + "failed": 1, + "cancelled": 0, + "skipped": 0, + "remaining": 0, + "percent_complete": 100.0, + "credits_charged": 9, + "credits_charged_micro": 9000000, + "bytes": 61440 + }, + "submitted_at": "", + "started_at": "", + "finished_at": "", + "results_expire_at": "" +} diff --git a/internal/cmd/testdata/golden/batch_human/append.exit b/internal/cmd/testdata/golden/batch_human/append.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/append.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/append.stdout b/internal/cmd/testdata/golden/batch_human/append.stdout new file mode 100644 index 0000000..5f03531 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/append.stdout @@ -0,0 +1,10 @@ +added 2 items (1 dispatched now, the rest by the recovery sweep; do not re-append them) + +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status running +progress 1/2 (50.0%) succeeded 1 failed 0 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 3 charged, 6 reserved +submitted +started diff --git a/internal/cmd/testdata/golden/batch_human/cancel_yes.exit b/internal/cmd/testdata/golden/batch_human/cancel_yes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/cancel_yes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/cancel_yes.stdout b/internal/cmd/testdata/golden/batch_human/cancel_yes.stdout new file mode 100644 index 0000000..8d0f250 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/cancel_yes.stdout @@ -0,0 +1,9 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status cancelled +progress 1/5 (20.0%) succeeded 1 failed 0 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 3 charged, 15 reserved +submitted +started +finished diff --git a/internal/cmd/testdata/golden/batch_human/content.exit b/internal/cmd/testdata/golden/batch_human/content.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/content.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/content.stdout b/internal/cmd/testdata/golden/batch_human/content.stdout new file mode 100644 index 0000000..cd15ba6 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/content.stdout @@ -0,0 +1 @@ +

A

diff --git a/internal/cmd/testdata/golden/batch_human/get_completed.exit b/internal/cmd/testdata/golden/batch_human/get_completed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/get_completed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/get_completed.stdout b/internal/cmd/testdata/golden/batch_human/get_completed.stdout new file mode 100644 index 0000000..2f5584a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/get_completed.stdout @@ -0,0 +1,10 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status completed +progress 3/3 (100.0%) succeeded 2 failed 1 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 9 charged, 9 reserved +submitted +started +finished +results readable until diff --git a/internal/cmd/testdata/golden/batch_human/get_failed.exit b/internal/cmd/testdata/golden/batch_human/get_failed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/get_failed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/get_failed.stdout b/internal/cmd/testdata/golden/batch_human/get_failed.stdout new file mode 100644 index 0000000..a36a5fb --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/get_failed.stdout @@ -0,0 +1,10 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status failed +progress 2/10 (20.0%) succeeded 0 failed 2 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 6 charged, 30 reserved +submitted +started +finished +error ERR::LIMIT::FAILURE_THRESHOLD failure threshold reached diff --git a/internal/cmd/testdata/golden/batch_human/get_open.exit b/internal/cmd/testdata/golden/batch_human/get_open.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/get_open.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/get_open.stdout b/internal/cmd/testdata/golden/batch_human/get_open.stdout new file mode 100644 index 0000000..59e4918 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/get_open.stdout @@ -0,0 +1,8 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name crawl +status running (open: accepting items until closed) +progress 4/9 (44.4%) succeeded 4 failed 0 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 12 charged, 27 reserved +submitted +started diff --git a/internal/cmd/testdata/golden/batch_human/list.exit b/internal/cmd/testdata/golden/batch_human/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/list.stdout b/internal/cmd/testdata/golden/batch_human/list.stdout new file mode 100644 index 0000000..8bcaf3b --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/list.stdout @@ -0,0 +1,3 @@ +ID STATUS NAME DONE SUBMITTED +01J9Z6V0Q8M4K2T7R3N5B1C9XA completed nightly 3/3 +01J9Z6V0Q8M4K2T7R3N5B1C9XA running crawl 4/9 diff --git a/internal/cmd/testdata/golden/batch_human/list_empty.exit b/internal/cmd/testdata/golden/batch_human/list_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/list_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/list_empty.stdout b/internal/cmd/testdata/golden/batch_human/list_empty.stdout new file mode 100644 index 0000000..ddc789b --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/list_empty.stdout @@ -0,0 +1 @@ +no batch jobs diff --git a/internal/cmd/testdata/golden/batch_human/list_next_cursor.exit b/internal/cmd/testdata/golden/batch_human/list_next_cursor.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/list_next_cursor.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/list_next_cursor.stdout b/internal/cmd/testdata/golden/batch_human/list_next_cursor.stdout new file mode 100644 index 0000000..d31059a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/list_next_cursor.stdout @@ -0,0 +1,2 @@ +ID STATUS NAME DONE SUBMITTED +01J9Z6V0Q8M4K2T7R3N5B1C9XA completed nightly 3/3 diff --git a/internal/cmd/testdata/golden/batch_human/results.exit b/internal/cmd/testdata/golden/batch_human/results.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/results.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/results.stdout b/internal/cmd/testdata/golden/batch_human/results.stdout new file mode 100644 index 0000000..279d254 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/results.stdout @@ -0,0 +1,4 @@ +SEQ STATUS HTTP URL BYTES +0 succeeded 200 https://example.com/a [sku-1] 2048 +1 failed ERR::UPSTREAM::TIMEOUT - https://example.com/b 0 +2 succeeded 404 https://example.com/c?x=1&y=2 [sku-3] 512 diff --git a/internal/cmd/testdata/golden/batch_human/results_empty.exit b/internal/cmd/testdata/golden/batch_human/results_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/results_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/results_empty.stdout b/internal/cmd/testdata/golden/batch_human/results_empty.stdout new file mode 100644 index 0000000..22cb73b --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/results_empty.stdout @@ -0,0 +1 @@ +no finished items yet diff --git a/internal/cmd/testdata/golden/batch_human/retry.exit b/internal/cmd/testdata/golden/batch_human/retry.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/retry.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/retry.stdout b/internal/cmd/testdata/golden/batch_human/retry.stdout new file mode 100644 index 0000000..5e8da5c --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/retry.stdout @@ -0,0 +1,10 @@ +reset 2 failed items (1 dispatched now, the rest by the recovery sweep) + +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status running +progress 1/2 (50.0%) succeeded 1 failed 0 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 3 charged, 6 reserved +submitted +started diff --git a/internal/cmd/testdata/golden/batch_human/submit_urls.exit b/internal/cmd/testdata/golden/batch_human/submit_urls.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/submit_urls.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/submit_urls.stdout b/internal/cmd/testdata/golden/batch_human/submit_urls.stdout new file mode 100644 index 0000000..babaef9 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/submit_urls.stdout @@ -0,0 +1,7 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status queued +progress 0/2 (0.0%) succeeded 0 failed 0 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 0 charged, 6 reserved +submitted diff --git a/internal/cmd/testdata/golden/batch_human/submit_wait.exit b/internal/cmd/testdata/golden/batch_human/submit_wait.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/submit_wait.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/submit_wait.stdout b/internal/cmd/testdata/golden/batch_human/submit_wait.stdout new file mode 100644 index 0000000..2f5584a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/submit_wait.stdout @@ -0,0 +1,10 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status completed +progress 3/3 (100.0%) succeeded 2 failed 1 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 9 charged, 9 reserved +submitted +started +finished +results readable until diff --git a/internal/cmd/testdata/golden/batch_human/wait_completed.exit b/internal/cmd/testdata/golden/batch_human/wait_completed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/wait_completed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/batch_human/wait_completed.stdout b/internal/cmd/testdata/golden/batch_human/wait_completed.stdout new file mode 100644 index 0000000..2f5584a --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/wait_completed.stdout @@ -0,0 +1,10 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status completed +progress 3/3 (100.0%) succeeded 2 failed 1 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 9 charged, 9 reserved +submitted +started +finished +results readable until diff --git a/internal/cmd/testdata/golden/batch_human/wait_failed_exit4.exit b/internal/cmd/testdata/golden/batch_human/wait_failed_exit4.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/wait_failed_exit4.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/batch_human/wait_failed_exit4.stdout b/internal/cmd/testdata/golden/batch_human/wait_failed_exit4.stdout new file mode 100644 index 0000000..a36a5fb --- /dev/null +++ b/internal/cmd/testdata/golden/batch_human/wait_failed_exit4.stdout @@ -0,0 +1,10 @@ +id 01J9Z6V0Q8M4K2T7R3N5B1C9XA +name nightly +status failed +progress 2/10 (20.0%) succeeded 0 failed 2 cancelled 0 skipped 0 +run concurrency 10 priority 100 max attempts 3 +credits 6 charged, 30 reserved +submitted +started +finished +error ERR::LIMIT::FAILURE_THRESHOLD failure threshold reached diff --git a/internal/cmd/testdata/golden/browser/err_no_key.exit b/internal/cmd/testdata/golden/browser/err_no_key.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/browser/err_no_key.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/browser/err_no_key.stderr b/internal/cmd/testdata/golden/browser/err_no_key.stderr new file mode 100644 index 0000000..6b478b2 --- /dev/null +++ b/internal/cmd/testdata/golden/browser/err_no_key.stderr @@ -0,0 +1,4 @@ +{ + "error": "no API key: run \"spicrawl login\", set SPICRAWL_API_KEY, or pass --api-key", + "exit_code": 3 +} diff --git a/internal/cmd/testdata/golden/browser/err_no_key.stdout b/internal/cmd/testdata/golden/browser/err_no_key.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/browser/err_usage_ttl.exit b/internal/cmd/testdata/golden/browser/err_usage_ttl.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/browser/err_usage_ttl.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/browser/err_usage_ttl.stderr b/internal/cmd/testdata/golden/browser/err_usage_ttl.stderr new file mode 100644 index 0000000..79518e5 --- /dev/null +++ b/internal/cmd/testdata/golden/browser/err_usage_ttl.stderr @@ -0,0 +1,4 @@ +{ + "error": "--ttl must be between 60 and 900 seconds", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/browser/err_usage_ttl.stdout b/internal/cmd/testdata/golden/browser/err_usage_ttl.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/browser/not_enabled_exit8.exit b/internal/cmd/testdata/golden/browser/not_enabled_exit8.exit new file mode 100644 index 0000000..45a4fb7 --- /dev/null +++ b/internal/cmd/testdata/golden/browser/not_enabled_exit8.exit @@ -0,0 +1 @@ +8 diff --git a/internal/cmd/testdata/golden/browser/not_enabled_exit8.stderr b/internal/cmd/testdata/golden/browser/not_enabled_exit8.stderr new file mode 100644 index 0000000..ec405ec --- /dev/null +++ b/internal/cmd/testdata/golden/browser/not_enabled_exit8.stderr @@ -0,0 +1,4 @@ +{ + "error": "cloud browser (CDP) is not enabled on this server; \"spicrawl status\" shows its cdp field", + "exit_code": 8 +} diff --git a/internal/cmd/testdata/golden/browser/not_enabled_exit8.stdout b/internal/cmd/testdata/golden/browser/not_enabled_exit8.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/browser/url.exit b/internal/cmd/testdata/golden/browser/url.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/browser/url.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/browser/url.stdout b/internal/cmd/testdata/golden/browser/url.stdout new file mode 100644 index 0000000..6b234ac --- /dev/null +++ b/internal/cmd/testdata/golden/browser/url.stdout @@ -0,0 +1,5 @@ +{ + "expires_at": "", + "single_use": true, + "url": "ws://SERVER/v1/browser?token=wbt_testtoken&engine=chromium&proxy_country=de" +} diff --git a/internal/cmd/testdata/golden/browser/url_quiet_json.exit b/internal/cmd/testdata/golden/browser/url_quiet_json.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/browser/url_quiet_json.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/browser/url_quiet_json.stdout b/internal/cmd/testdata/golden/browser/url_quiet_json.stdout new file mode 100644 index 0000000..6b234ac --- /dev/null +++ b/internal/cmd/testdata/golden/browser/url_quiet_json.stdout @@ -0,0 +1,5 @@ +{ + "expires_at": "", + "single_use": true, + "url": "ws://SERVER/v1/browser?token=wbt_testtoken&engine=chromium&proxy_country=de" +} diff --git a/internal/cmd/testdata/golden/browser/url_quiet_prints_bare_url.exit b/internal/cmd/testdata/golden/browser/url_quiet_prints_bare_url.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/browser/url_quiet_prints_bare_url.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/browser/url_quiet_prints_bare_url.stdout b/internal/cmd/testdata/golden/browser/url_quiet_prints_bare_url.stdout new file mode 100644 index 0000000..c1c624d --- /dev/null +++ b/internal/cmd/testdata/golden/browser/url_quiet_prints_bare_url.stdout @@ -0,0 +1 @@ +ws://SERVER/v1/browser?token=wbt_testtoken&engine=chromium&proxy_country=de diff --git a/internal/cmd/testdata/golden/browser_human/url.exit b/internal/cmd/testdata/golden/browser_human/url.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/browser_human/url.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/browser_human/url.stdout b/internal/cmd/testdata/golden/browser_human/url.stdout new file mode 100644 index 0000000..c1c624d --- /dev/null +++ b/internal/cmd/testdata/golden/browser_human/url.stdout @@ -0,0 +1 @@ +ws://SERVER/v1/browser?token=wbt_testtoken&engine=chromium&proxy_country=de diff --git a/internal/cmd/testdata/golden/config/err_bad_base_url.exit b/internal/cmd/testdata/golden/config/err_bad_base_url.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/config/err_bad_base_url.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/config/err_bad_base_url.stderr b/internal/cmd/testdata/golden/config/err_bad_base_url.stderr new file mode 100644 index 0000000..bfc7ab2 --- /dev/null +++ b/internal/cmd/testdata/golden/config/err_bad_base_url.stderr @@ -0,0 +1,4 @@ +{ + "error": "base_url must be an http(s) URL, got \"ftp://x\"", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/config/err_bad_base_url.stdout b/internal/cmd/testdata/golden/config/err_bad_base_url.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/config/err_unknown_key.exit b/internal/cmd/testdata/golden/config/err_unknown_key.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/config/err_unknown_key.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/config/err_unknown_key.stderr b/internal/cmd/testdata/golden/config/err_unknown_key.stderr new file mode 100644 index 0000000..4778ffc --- /dev/null +++ b/internal/cmd/testdata/golden/config/err_unknown_key.stderr @@ -0,0 +1,4 @@ +{ + "error": "unknown config key \"nope\" (keys: api_key, base_url)", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/config/err_unknown_key.stdout b/internal/cmd/testdata/golden/config/err_unknown_key.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/config/get_all.exit b/internal/cmd/testdata/golden/config/get_all.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_all.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/get_all.stdout b/internal/cmd/testdata/golden/config/get_all.stdout new file mode 100644 index 0000000..66518d7 --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_all.stdout @@ -0,0 +1,4 @@ +{ + "api_key": "spicrawl_live_0123…mnop", + "base_url": "https://api.example.test" +} diff --git a/internal/cmd/testdata/golden/config/get_all_empty.exit b/internal/cmd/testdata/golden/config/get_all_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_all_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/get_all_empty.stdout b/internal/cmd/testdata/golden/config/get_all_empty.stdout new file mode 100644 index 0000000..04e5250 --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_all_empty.stdout @@ -0,0 +1,4 @@ +{ + "api_key": null, + "base_url": null +} diff --git a/internal/cmd/testdata/golden/config/get_api_key_masked.exit b/internal/cmd/testdata/golden/config/get_api_key_masked.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_api_key_masked.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/get_api_key_masked.stdout b/internal/cmd/testdata/golden/config/get_api_key_masked.stdout new file mode 100644 index 0000000..2acb265 --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_api_key_masked.stdout @@ -0,0 +1,4 @@ +{ + "key": "api_key", + "value": "spicrawl_live_0123…mnop" +} diff --git a/internal/cmd/testdata/golden/config/get_api_key_show_secret.exit b/internal/cmd/testdata/golden/config/get_api_key_show_secret.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_api_key_show_secret.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/get_api_key_show_secret.stdout b/internal/cmd/testdata/golden/config/get_api_key_show_secret.stdout new file mode 100644 index 0000000..98a7f3a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_api_key_show_secret.stdout @@ -0,0 +1,4 @@ +{ + "key": "api_key", + "value": "spicrawl_live_0123456789abcdefghijklmnop" +} diff --git a/internal/cmd/testdata/golden/config/get_base_url.exit b/internal/cmd/testdata/golden/config/get_base_url.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_base_url.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/get_base_url.stdout b/internal/cmd/testdata/golden/config/get_base_url.stdout new file mode 100644 index 0000000..b70c081 --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_base_url.stdout @@ -0,0 +1,4 @@ +{ + "key": "base_url", + "value": "https://api.example.test" +} diff --git a/internal/cmd/testdata/golden/config/get_unset.exit b/internal/cmd/testdata/golden/config/get_unset.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_unset.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/get_unset.stdout b/internal/cmd/testdata/golden/config/get_unset.stdout new file mode 100644 index 0000000..78c4aa4 --- /dev/null +++ b/internal/cmd/testdata/golden/config/get_unset.stdout @@ -0,0 +1,4 @@ +{ + "key": "base_url", + "value": null +} diff --git a/internal/cmd/testdata/golden/config/path.exit b/internal/cmd/testdata/golden/config/path.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/path.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/path.stdout b/internal/cmd/testdata/golden/config/path.stdout new file mode 100644 index 0000000..026f21c --- /dev/null +++ b/internal/cmd/testdata/golden/config/path.stdout @@ -0,0 +1,3 @@ +{ + "path": "/config.json" +} diff --git a/internal/cmd/testdata/golden/config/set_api_key.exit b/internal/cmd/testdata/golden/config/set_api_key.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/set_api_key.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/set_api_key.stdout b/internal/cmd/testdata/golden/config/set_api_key.stdout new file mode 100644 index 0000000..7bb7b4c --- /dev/null +++ b/internal/cmd/testdata/golden/config/set_api_key.stdout @@ -0,0 +1,5 @@ +{ + "key": "api_key", + "path": "/config.json", + "value": "spicrawl_live_0123…mnop" +} diff --git a/internal/cmd/testdata/golden/config/set_base_url.exit b/internal/cmd/testdata/golden/config/set_base_url.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/set_base_url.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/set_base_url.stdout b/internal/cmd/testdata/golden/config/set_base_url.stdout new file mode 100644 index 0000000..a9c825a --- /dev/null +++ b/internal/cmd/testdata/golden/config/set_base_url.stdout @@ -0,0 +1,5 @@ +{ + "key": "base_url", + "path": "/config.json", + "value": "http://localhost:8080" +} diff --git a/internal/cmd/testdata/golden/config/unset_api_key.exit b/internal/cmd/testdata/golden/config/unset_api_key.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config/unset_api_key.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config/unset_api_key.stdout b/internal/cmd/testdata/golden/config/unset_api_key.stdout new file mode 100644 index 0000000..9c2295e --- /dev/null +++ b/internal/cmd/testdata/golden/config/unset_api_key.stdout @@ -0,0 +1,5 @@ +{ + "key": "api_key", + "path": "/config.json", + "value": null +} diff --git a/internal/cmd/testdata/golden/config_human/get_all.exit b/internal/cmd/testdata/golden/config_human/get_all.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_all.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/get_all.stdout b/internal/cmd/testdata/golden/config_human/get_all.stdout new file mode 100644 index 0000000..25d4d5a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_all.stdout @@ -0,0 +1,3 @@ +KEY VALUE +api_key spicrawl_live_0123…mnop +base_url https://api.example.test diff --git a/internal/cmd/testdata/golden/config_human/get_all_empty.exit b/internal/cmd/testdata/golden/config_human/get_all_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_all_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/get_all_empty.stdout b/internal/cmd/testdata/golden/config_human/get_all_empty.stdout new file mode 100644 index 0000000..cf47445 --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_all_empty.stdout @@ -0,0 +1,3 @@ +KEY VALUE +api_key +base_url diff --git a/internal/cmd/testdata/golden/config_human/get_base_url.exit b/internal/cmd/testdata/golden/config_human/get_base_url.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_base_url.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/get_base_url.stdout b/internal/cmd/testdata/golden/config_human/get_base_url.stdout new file mode 100644 index 0000000..d07c895 --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_base_url.stdout @@ -0,0 +1 @@ +https://api.example.test diff --git a/internal/cmd/testdata/golden/config_human/get_unset.exit b/internal/cmd/testdata/golden/config_human/get_unset.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_unset.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/get_unset.stdout b/internal/cmd/testdata/golden/config_human/get_unset.stdout new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/get_unset.stdout @@ -0,0 +1 @@ + diff --git a/internal/cmd/testdata/golden/config_human/path.exit b/internal/cmd/testdata/golden/config_human/path.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/path.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/path.stdout b/internal/cmd/testdata/golden/config_human/path.stdout new file mode 100644 index 0000000..ac2f8ba --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/path.stdout @@ -0,0 +1 @@ +/config.json diff --git a/internal/cmd/testdata/golden/config_human/set_api_key.exit b/internal/cmd/testdata/golden/config_human/set_api_key.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/set_api_key.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/set_api_key.stdout b/internal/cmd/testdata/golden/config_human/set_api_key.stdout new file mode 100644 index 0000000..7862456 --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/set_api_key.stdout @@ -0,0 +1 @@ +Set api_key = spicrawl_live_0123…mnop in /config.json diff --git a/internal/cmd/testdata/golden/config_human/unset_api_key.exit b/internal/cmd/testdata/golden/config_human/unset_api_key.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/unset_api_key.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/config_human/unset_api_key.stdout b/internal/cmd/testdata/golden/config_human/unset_api_key.stdout new file mode 100644 index 0000000..e3caede --- /dev/null +++ b/internal/cmd/testdata/golden/config_human/unset_api_key.stdout @@ -0,0 +1 @@ +Removed api_key from /config.json diff --git a/internal/cmd/testdata/golden/docs/adds_missing_newline.exit b/internal/cmd/testdata/golden/docs/adds_missing_newline.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/docs/adds_missing_newline.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/docs/adds_missing_newline.stdout b/internal/cmd/testdata/golden/docs/adds_missing_newline.stdout new file mode 100644 index 0000000..2876d08 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/adds_missing_newline.stdout @@ -0,0 +1 @@ +# No newline diff --git a/internal/cmd/testdata/golden/docs/err_bad_topic.exit b/internal/cmd/testdata/golden/docs/err_bad_topic.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/err_bad_topic.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/docs/err_bad_topic.stderr b/internal/cmd/testdata/golden/docs/err_bad_topic.stderr new file mode 100644 index 0000000..5ddc308 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/err_bad_topic.stderr @@ -0,0 +1,4 @@ +{ + "error": "invalid docs topic \"../etc/passwd\": use a page path like guides/anti-bot (see spicrawl docs --list)", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/docs/err_bad_topic.stdout b/internal/cmd/testdata/golden/docs/err_bad_topic.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/docs/err_unknown_topic.exit b/internal/cmd/testdata/golden/docs/err_unknown_topic.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/err_unknown_topic.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/docs/err_unknown_topic.stderr b/internal/cmd/testdata/golden/docs/err_unknown_topic.stderr new file mode 100644 index 0000000..09c70ed --- /dev/null +++ b/internal/cmd/testdata/golden/docs/err_unknown_topic.stderr @@ -0,0 +1,4 @@ +{ + "error": "no docs page at http://SERVER/missing.md (see spicrawl docs --list)", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/docs/err_unknown_topic.stdout b/internal/cmd/testdata/golden/docs/err_unknown_topic.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/docs/json_flag_wraps.exit b/internal/cmd/testdata/golden/docs/json_flag_wraps.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/docs/json_flag_wraps.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/docs/json_flag_wraps.stdout b/internal/cmd/testdata/golden/docs/json_flag_wraps.stdout new file mode 100644 index 0000000..254440e --- /dev/null +++ b/internal/cmd/testdata/golden/docs/json_flag_wraps.stdout @@ -0,0 +1,5 @@ +{ + "markdown": "# Anti-bot\n\nUse --stealth only when a page needs it.\n", + "topic": "guides/anti-bot", + "url": "http://SERVER/guides/anti-bot.md" +} diff --git a/internal/cmd/testdata/golden/docs/list_raw_markdown.exit b/internal/cmd/testdata/golden/docs/list_raw_markdown.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/docs/list_raw_markdown.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/docs/list_raw_markdown.stdout b/internal/cmd/testdata/golden/docs/list_raw_markdown.stdout new file mode 100644 index 0000000..8229e13 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/list_raw_markdown.stdout @@ -0,0 +1,4 @@ +# Spicrawl + +- [Quickstart](https://docs.spicrawl.com/quickstart.md) +- [Anti-bot](https://docs.spicrawl.com/guides/anti-bot.md) diff --git a/internal/cmd/testdata/golden/docs/no_topic_is_the_index.exit b/internal/cmd/testdata/golden/docs/no_topic_is_the_index.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/docs/no_topic_is_the_index.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/docs/no_topic_is_the_index.stdout b/internal/cmd/testdata/golden/docs/no_topic_is_the_index.stdout new file mode 100644 index 0000000..8229e13 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/no_topic_is_the_index.stdout @@ -0,0 +1,4 @@ +# Spicrawl + +- [Quickstart](https://docs.spicrawl.com/quickstart.md) +- [Anti-bot](https://docs.spicrawl.com/guides/anti-bot.md) diff --git a/internal/cmd/testdata/golden/docs/topic_raw_markdown.exit b/internal/cmd/testdata/golden/docs/topic_raw_markdown.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/docs/topic_raw_markdown.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/docs/topic_raw_markdown.stdout b/internal/cmd/testdata/golden/docs/topic_raw_markdown.stdout new file mode 100644 index 0000000..298b286 --- /dev/null +++ b/internal/cmd/testdata/golden/docs/topic_raw_markdown.stdout @@ -0,0 +1,3 @@ +# Anti-bot + +Use --stealth only when a page needs it. diff --git a/internal/cmd/testdata/golden/docs_human/topic_raw_markdown.exit b/internal/cmd/testdata/golden/docs_human/topic_raw_markdown.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/docs_human/topic_raw_markdown.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/docs_human/topic_raw_markdown.stdout b/internal/cmd/testdata/golden/docs_human/topic_raw_markdown.stdout new file mode 100644 index 0000000..298b286 --- /dev/null +++ b/internal/cmd/testdata/golden/docs_human/topic_raw_markdown.stdout @@ -0,0 +1,3 @@ +# Anti-bot + +Use --stealth only when a page needs it. diff --git a/internal/cmd/testdata/golden/logs/all_pages.exit b/internal/cmd/testdata/golden/logs/all_pages.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/all_pages.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/all_pages.stdout b/internal/cmd/testdata/golden/logs/all_pages.stdout new file mode 100644 index 0000000..066b2c2 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/all_pages.stdout @@ -0,0 +1,77 @@ +{ + "data": [ + { + "id": "01M0FK1EP2E7FBMWAR7W10SMXW", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "success", + "http_status": 200, + "engine": "fetch", + "credits_micro": 1500000, + "url": "https://example.com/a", + "attempts": 2, + "bytes_in": 2048, + "bytes_out": 512, + "proxy_bytes": 4096, + "spans": { + "admission_ms": 1, + "dispatch_ms": 2, + "worker_ms": 3, + "queue_ms": 0, + "engine_ms": 80, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 210 + }, + "proxy": { + "source": "pool", + "reason": "default", + "provider": "dc-1", + "requested_tier": "datacenter", + "requested_country": "de", + "country": "de", + "endpoint_id": "ep-1", + "endpoints": [ + "ep-1", + "ep-2" + ], + "sticky": true + } + }, + { + "id": "01M0FK1EP2E7FBMWAR7W10SMXX", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "blocked", + "http_status": 403, + "engine": "", + "credits_micro": 0, + "url": "https://example.com/b?x=1&y=2", + "error_code": "ERR::UPSTREAM::BLOCKED", + "error_detail": "bot challenge", + "spans": { + "admission_ms": 1, + "dispatch_ms": null, + "worker_ms": null, + "queue_ms": 0, + "engine_ms": null, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 130 + }, + "proxy": { + "source": "pool" + }, + "blocked": { + "vendor": "cloudflare", + "rule": "r-17", + "signal": "cf-mitigated" + } + } + ], + "page": { + "has_more": false, + "retained_since": "", + "retention_hours": 24 + } +} diff --git a/internal/cmd/testdata/golden/logs/all_pages_jsonl.exit b/internal/cmd/testdata/golden/logs/all_pages_jsonl.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/all_pages_jsonl.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/all_pages_jsonl.stdout b/internal/cmd/testdata/golden/logs/all_pages_jsonl.stdout new file mode 100644 index 0000000..22f025f --- /dev/null +++ b/internal/cmd/testdata/golden/logs/all_pages_jsonl.stdout @@ -0,0 +1,2 @@ +{"id":"01M0FK1EP2E7FBMWAR7W10SMXW","created_at":"","project_id":"proj_01J9Z","status":"success","http_status":200,"engine":"fetch","credits_micro":1500000,"url":"https://example.com/a","attempts":2,"bytes_in":2048,"bytes_out":512,"proxy_bytes":4096,"spans":{"admission_ms":1,"dispatch_ms":2,"worker_ms":3,"queue_ms":0,"engine_ms":80,"upstream_ms":120,"extract_ms":null,"total_ms":210},"proxy":{"source":"pool","reason":"default","provider":"dc-1","requested_tier":"datacenter","requested_country":"de","country":"de","endpoint_id":"ep-1","endpoints":["ep-1","ep-2"],"sticky":true}} +{"id":"01M0FK1EP2E7FBMWAR7W10SMXX","created_at":"","project_id":"proj_01J9Z","status":"blocked","http_status":403,"engine":"","credits_micro":0,"url":"https://example.com/b?x=1&y=2","error_code":"ERR::UPSTREAM::BLOCKED","error_detail":"bot challenge","spans":{"admission_ms":1,"dispatch_ms":null,"worker_ms":null,"queue_ms":0,"engine_ms":null,"upstream_ms":120,"extract_ms":null,"total_ms":130},"proxy":{"source":"pool"},"blocked":{"vendor":"cloudflare","rule":"r-17","signal":"cf-mitigated"}} diff --git a/internal/cmd/testdata/golden/logs/empty.exit b/internal/cmd/testdata/golden/logs/empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/empty.stdout b/internal/cmd/testdata/golden/logs/empty.stdout new file mode 100644 index 0000000..1ac3e6d --- /dev/null +++ b/internal/cmd/testdata/golden/logs/empty.stdout @@ -0,0 +1,8 @@ +{ + "data": [], + "page": { + "has_more": false, + "retained_since": "", + "retention_hours": 24 + } +} diff --git a/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.exit b/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.stderr b/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.stderr new file mode 100644 index 0000000..02a65d0 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.stderr @@ -0,0 +1,4 @@ +{ + "error": "--errors and --status cannot be combined", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.stdout b/internal/cmd/testdata/golden/logs/err_usage_errors_and_status.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/logs/err_usage_status.exit b/internal/cmd/testdata/golden/logs/err_usage_status.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/err_usage_status.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/logs/err_usage_status.stderr b/internal/cmd/testdata/golden/logs/err_usage_status.stderr new file mode 100644 index 0000000..e4d9101 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/err_usage_status.stderr @@ -0,0 +1,4 @@ +{ + "error": "--status must be one of success, failed, timeout, blocked, rejected, cancelled, not \"weird\"", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/logs/err_usage_status.stdout b/internal/cmd/testdata/golden/logs/err_usage_status.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/logs/errors.exit b/internal/cmd/testdata/golden/logs/errors.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/errors.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/errors.stdout b/internal/cmd/testdata/golden/logs/errors.stdout new file mode 100644 index 0000000..3cfb105 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/errors.stdout @@ -0,0 +1,39 @@ +{ + "data": [ + { + "id": "01M0FK1EP2E7FBMWAR7W10SMXX", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "blocked", + "http_status": 403, + "engine": "", + "credits_micro": 0, + "url": "https://example.com/b?x=1&y=2", + "error_code": "ERR::UPSTREAM::BLOCKED", + "error_detail": "bot challenge", + "spans": { + "admission_ms": 1, + "dispatch_ms": null, + "worker_ms": null, + "queue_ms": 0, + "engine_ms": null, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 130 + }, + "proxy": { + "source": "pool" + }, + "blocked": { + "vendor": "cloudflare", + "rule": "r-17", + "signal": "cf-mitigated" + } + } + ], + "page": { + "has_more": false, + "retained_since": "", + "retention_hours": 24 + } +} diff --git a/internal/cmd/testdata/golden/logs/get.exit b/internal/cmd/testdata/golden/logs/get.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/get.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/get.stdout b/internal/cmd/testdata/golden/logs/get.stdout new file mode 100644 index 0000000..82c6926 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/get.stdout @@ -0,0 +1,38 @@ +{ + "id": "01M0FK1EP2E7FBMWAR7W10SMXW", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "success", + "http_status": 200, + "engine": "fetch", + "credits_micro": 1500000, + "url": "https://example.com/a", + "attempts": 2, + "bytes_in": 2048, + "bytes_out": 512, + "proxy_bytes": 4096, + "spans": { + "admission_ms": 1, + "dispatch_ms": 2, + "worker_ms": 3, + "queue_ms": 0, + "engine_ms": 80, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 210 + }, + "proxy": { + "source": "pool", + "reason": "default", + "provider": "dc-1", + "requested_tier": "datacenter", + "requested_country": "de", + "country": "de", + "endpoint_id": "ep-1", + "endpoints": [ + "ep-1", + "ep-2" + ], + "sticky": true + } +} diff --git a/internal/cmd/testdata/golden/logs/get_not_found.exit b/internal/cmd/testdata/golden/logs/get_not_found.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/get_not_found.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/logs/get_not_found.stderr b/internal/cmd/testdata/golden/logs/get_not_found.stderr new file mode 100644 index 0000000..856bebc --- /dev/null +++ b/internal/cmd/testdata/golden/logs/get_not_found.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Not Found", + "status": 404, + "code": "ERR::REQUEST::NOT_FOUND", + "detail": "no such request", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/logs/get_not_found.stdout b/internal/cmd/testdata/golden/logs/get_not_found.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/logs/has_more.exit b/internal/cmd/testdata/golden/logs/has_more.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/has_more.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/has_more.stdout b/internal/cmd/testdata/golden/logs/has_more.stdout new file mode 100644 index 0000000..63c2235 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/has_more.stdout @@ -0,0 +1,49 @@ +{ + "data": [ + { + "id": "01M0FK1EP2E7FBMWAR7W10SMXW", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "success", + "http_status": 200, + "engine": "fetch", + "credits_micro": 1500000, + "url": "https://example.com/a", + "attempts": 2, + "bytes_in": 2048, + "bytes_out": 512, + "proxy_bytes": 4096, + "spans": { + "admission_ms": 1, + "dispatch_ms": 2, + "worker_ms": 3, + "queue_ms": 0, + "engine_ms": 80, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 210 + }, + "proxy": { + "source": "pool", + "reason": "default", + "provider": "dc-1", + "requested_tier": "datacenter", + "requested_country": "de", + "country": "de", + "endpoint_id": "ep-1", + "endpoints": [ + "ep-1", + "ep-2" + ], + "sticky": true + } + } + ], + "page": { + "has_more": true, + "next_before": "", + "next_before_id": "01M0FK1EP2E7FBMWAR7W10SMXW", + "retained_since": "", + "retention_hours": 24 + } +} diff --git a/internal/cmd/testdata/golden/logs/jsonl.exit b/internal/cmd/testdata/golden/logs/jsonl.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/jsonl.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/jsonl.stdout b/internal/cmd/testdata/golden/logs/jsonl.stdout new file mode 100644 index 0000000..22f025f --- /dev/null +++ b/internal/cmd/testdata/golden/logs/jsonl.stdout @@ -0,0 +1,2 @@ +{"id":"01M0FK1EP2E7FBMWAR7W10SMXW","created_at":"","project_id":"proj_01J9Z","status":"success","http_status":200,"engine":"fetch","credits_micro":1500000,"url":"https://example.com/a","attempts":2,"bytes_in":2048,"bytes_out":512,"proxy_bytes":4096,"spans":{"admission_ms":1,"dispatch_ms":2,"worker_ms":3,"queue_ms":0,"engine_ms":80,"upstream_ms":120,"extract_ms":null,"total_ms":210},"proxy":{"source":"pool","reason":"default","provider":"dc-1","requested_tier":"datacenter","requested_country":"de","country":"de","endpoint_id":"ep-1","endpoints":["ep-1","ep-2"],"sticky":true}} +{"id":"01M0FK1EP2E7FBMWAR7W10SMXX","created_at":"","project_id":"proj_01J9Z","status":"blocked","http_status":403,"engine":"","credits_micro":0,"url":"https://example.com/b?x=1&y=2","error_code":"ERR::UPSTREAM::BLOCKED","error_detail":"bot challenge","spans":{"admission_ms":1,"dispatch_ms":null,"worker_ms":null,"queue_ms":0,"engine_ms":null,"upstream_ms":120,"extract_ms":null,"total_ms":130},"proxy":{"source":"pool"},"blocked":{"vendor":"cloudflare","rule":"r-17","signal":"cf-mitigated"}} diff --git a/internal/cmd/testdata/golden/logs/list.exit b/internal/cmd/testdata/golden/logs/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs/list.stdout b/internal/cmd/testdata/golden/logs/list.stdout new file mode 100644 index 0000000..066b2c2 --- /dev/null +++ b/internal/cmd/testdata/golden/logs/list.stdout @@ -0,0 +1,77 @@ +{ + "data": [ + { + "id": "01M0FK1EP2E7FBMWAR7W10SMXW", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "success", + "http_status": 200, + "engine": "fetch", + "credits_micro": 1500000, + "url": "https://example.com/a", + "attempts": 2, + "bytes_in": 2048, + "bytes_out": 512, + "proxy_bytes": 4096, + "spans": { + "admission_ms": 1, + "dispatch_ms": 2, + "worker_ms": 3, + "queue_ms": 0, + "engine_ms": 80, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 210 + }, + "proxy": { + "source": "pool", + "reason": "default", + "provider": "dc-1", + "requested_tier": "datacenter", + "requested_country": "de", + "country": "de", + "endpoint_id": "ep-1", + "endpoints": [ + "ep-1", + "ep-2" + ], + "sticky": true + } + }, + { + "id": "01M0FK1EP2E7FBMWAR7W10SMXX", + "created_at": "", + "project_id": "proj_01J9Z", + "status": "blocked", + "http_status": 403, + "engine": "", + "credits_micro": 0, + "url": "https://example.com/b?x=1&y=2", + "error_code": "ERR::UPSTREAM::BLOCKED", + "error_detail": "bot challenge", + "spans": { + "admission_ms": 1, + "dispatch_ms": null, + "worker_ms": null, + "queue_ms": 0, + "engine_ms": null, + "upstream_ms": 120, + "extract_ms": null, + "total_ms": 130 + }, + "proxy": { + "source": "pool" + }, + "blocked": { + "vendor": "cloudflare", + "rule": "r-17", + "signal": "cf-mitigated" + } + } + ], + "page": { + "has_more": false, + "retained_since": "", + "retention_hours": 24 + } +} diff --git a/internal/cmd/testdata/golden/logs_human/all_projects.exit b/internal/cmd/testdata/golden/logs_human/all_projects.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/all_projects.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs_human/all_projects.stdout b/internal/cmd/testdata/golden/logs_human/all_projects.stdout new file mode 100644 index 0000000..16b9d52 --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/all_projects.stdout @@ -0,0 +1,3 @@ +TIME PROJECT STATUS HTTP ENGINE CREDITS TOTAL_MS URL + proj_01J9Z success 200 fetch 1.5 210 https://example.com/a + proj_01J9Z blocked 403 — 0 130 https://example.com/b?x=1&y=2 diff --git a/internal/cmd/testdata/golden/logs_human/empty.exit b/internal/cmd/testdata/golden/logs_human/empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs_human/empty.stdout b/internal/cmd/testdata/golden/logs_human/empty.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/logs_human/get_blocked.exit b/internal/cmd/testdata/golden/logs_human/get_blocked.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/get_blocked.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs_human/get_blocked.stdout b/internal/cmd/testdata/golden/logs_human/get_blocked.stdout new file mode 100644 index 0000000..8024009 --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/get_blocked.stdout @@ -0,0 +1,28 @@ +id: 01M0FK1EP2E7FBMWAR7W10SMXX +time: +project: proj_01J9Z +status: blocked +url: https://example.com/b?x=1&y=2 +http status: 403 +engine: — +credits: 0 +error: ERR::UPSTREAM::BLOCKED +error detail: bot challenge + +spans (ms): + admission 1 + dispatch — + worker — + queue 0 + engine — + upstream 120 + extract — + total 130 + +proxy: + source: pool + +blocked: + vendor: cloudflare + rule: r-17 + signal: cf-mitigated diff --git a/internal/cmd/testdata/golden/logs_human/get_full.exit b/internal/cmd/testdata/golden/logs_human/get_full.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/get_full.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs_human/get_full.stdout b/internal/cmd/testdata/golden/logs_human/get_full.stdout new file mode 100644 index 0000000..1456d34 --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/get_full.stdout @@ -0,0 +1,30 @@ +id: 01M0FK1EP2E7FBMWAR7W10SMXW +time: +project: proj_01J9Z +status: success +url: https://example.com/a +http status: 200 +engine: fetch +credits: 1.5 +attempts: 2 +bytes: in 2048, out 512, via proxy 4096 + +spans (ms): + admission 1 + dispatch 2 + worker 3 + queue 0 + engine 80 + upstream 120 + extract — + total 210 + +proxy: + source: pool + reason: default + provider: dc-1 + requested: datacenter de + served country: de + endpoint: ep-1 + endpoints: ep-1 -> ep-2 + sticky: yes diff --git a/internal/cmd/testdata/golden/logs_human/has_more.exit b/internal/cmd/testdata/golden/logs_human/has_more.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/has_more.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs_human/has_more.stdout b/internal/cmd/testdata/golden/logs_human/has_more.stdout new file mode 100644 index 0000000..e7be0f5 --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/has_more.stdout @@ -0,0 +1,2 @@ +TIME STATUS HTTP ENGINE CREDITS TOTAL_MS URL + success 200 fetch 1.5 210 https://example.com/a diff --git a/internal/cmd/testdata/golden/logs_human/list.exit b/internal/cmd/testdata/golden/logs_human/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/logs_human/list.stdout b/internal/cmd/testdata/golden/logs_human/list.stdout new file mode 100644 index 0000000..4572b4b --- /dev/null +++ b/internal/cmd/testdata/golden/logs_human/list.stdout @@ -0,0 +1,3 @@ +TIME STATUS HTTP ENGINE CREDITS TOTAL_MS URL + success 200 fetch 1.5 210 https://example.com/a + blocked 403 — 0 130 https://example.com/b?x=1&y=2 diff --git a/internal/cmd/testdata/golden/meta/err_unknown_command.exit b/internal/cmd/testdata/golden/meta/err_unknown_command.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/err_unknown_command.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/meta/err_unknown_command.stderr b/internal/cmd/testdata/golden/meta/err_unknown_command.stderr new file mode 100644 index 0000000..52a5ca0 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/err_unknown_command.stderr @@ -0,0 +1,4 @@ +{ + "error": "unknown command \"zzzzzz\" for \"spicrawl\"", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/meta/err_unknown_command.stdout b/internal/cmd/testdata/golden/meta/err_unknown_command.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/meta/err_unknown_flag.exit b/internal/cmd/testdata/golden/meta/err_unknown_flag.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/err_unknown_flag.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/meta/err_unknown_flag.stderr b/internal/cmd/testdata/golden/meta/err_unknown_flag.stderr new file mode 100644 index 0000000..016d3db --- /dev/null +++ b/internal/cmd/testdata/golden/meta/err_unknown_flag.stderr @@ -0,0 +1,4 @@ +{ + "error": "unknown flag: --nope", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/meta/err_unknown_flag.stdout b/internal/cmd/testdata/golden/meta/err_unknown_flag.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.exit b/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.stderr b/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.stderr new file mode 100644 index 0000000..51e7fb5 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.stderr @@ -0,0 +1,4 @@ +{ + "error": "unknown command \"lst\" for \"spicrawl batch\"; did you mean get or list?", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.stdout b/internal/cmd/testdata/golden/meta/err_unknown_subcommand_suggests.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/meta/exit_codes.exit b/internal/cmd/testdata/golden/meta/exit_codes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/meta/exit_codes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/meta/exit_codes.stdout b/internal/cmd/testdata/golden/meta/exit_codes.stdout new file mode 100644 index 0000000..68909d7 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/exit_codes.stdout @@ -0,0 +1,72 @@ +[ + { + "code": 0, + "name": "ok", + "meaning": "success" + }, + { + "code": 1, + "name": "internal", + "meaning": "CLI bug, or a transient ERR::INTERNAL::* (retryable)" + }, + { + "code": 2, + "name": "usage", + "meaning": "bad flags or arguments, or an unreadable/malformed config file; nothing was sent" + }, + { + "code": 3, + "name": "auth", + "meaning": "ERR::AUTH::*, or no API key configured" + }, + { + "code": 4, + "name": "request", + "meaning": "ERR::REQUEST::*, ERR::SECURITY::*, ERR::EXTRACT::INVALID_RULES: fix the request" + }, + { + "code": 5, + "name": "limit", + "meaning": "ERR::LIMIT::*: rate, concurrency, quota or max_cost" + }, + { + "code": 6, + "name": "upstream", + "meaning": "ERR::UPSTREAM::*: the target site failed or served a bot challenge" + }, + { + "code": 7, + "name": "proxy", + "meaning": "ERR::PROXY::*" + }, + { + "code": 8, + "name": "engine", + "meaning": "ERR::ENGINE::*, ERR::EXTRACT::FAILED, or ERR::INTERNAL::UNAVAILABLE with retryable=false: capability not available on this deployment" + }, + { + "code": 9, + "name": "session", + "meaning": "ERR::SESSION::*" + }, + { + "code": 10, + "name": "network", + "meaning": "the Spicrawl API could not be reached (DNS, refused, TLS); nothing was sent" + }, + { + "code": 11, + "name": "pending", + "meaning": "a --wait gave up before the job finished" + }, + { + "code": 12, + "name": "timeout", + "meaning": "timed out waiting for the API, or the connection dropped after the request was sent; the request may have run and been billed: check `spicrawl logs` before retrying" + }, + { + "code": 130, + "name": "interrupted", + "meaning": "stopped with Ctrl-C" + } +] diff --git a/internal/cmd/testdata/golden/meta/exit_codes_human.exit b/internal/cmd/testdata/golden/meta/exit_codes_human.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/meta/exit_codes_human.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/meta/exit_codes_human.stdout b/internal/cmd/testdata/golden/meta/exit_codes_human.stdout new file mode 100644 index 0000000..9f53803 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/exit_codes_human.stdout @@ -0,0 +1,18 @@ +Exit codes (stable; scripts and agents may branch on them): + + 0 ok success + 1 internal CLI bug, or a transient ERR::INTERNAL::* (retryable) + 2 usage bad flags or arguments, or an unreadable/malformed config file; nothing was sent + 3 auth ERR::AUTH::*, or no API key configured + 4 request ERR::REQUEST::*, ERR::SECURITY::*, ERR::EXTRACT::INVALID_RULES: fix the request + 5 limit ERR::LIMIT::*: rate, concurrency, quota or max_cost + 6 upstream ERR::UPSTREAM::*: the target site failed or served a bot challenge + 7 proxy ERR::PROXY::* + 8 engine ERR::ENGINE::*, ERR::EXTRACT::FAILED, or ERR::INTERNAL::UNAVAILABLE with retryable=false: capability not available on this deployment + 9 session ERR::SESSION::* + 10 network the Spicrawl API could not be reached (DNS, refused, TLS); nothing was sent + 11 pending a --wait gave up before the job finished + 12 timeout timed out waiting for the API, or the connection dropped after the request was sent; the request may have run and been billed: check `spicrawl logs` before retrying + 130 interrupted stopped with Ctrl-C + +In JSON mode a failed command writes the API's problem document to stderr. diff --git a/internal/cmd/testdata/golden/meta/version.exit b/internal/cmd/testdata/golden/meta/version.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/meta/version.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/meta/version.stdout b/internal/cmd/testdata/golden/meta/version.stdout new file mode 100644 index 0000000..ba21f57 --- /dev/null +++ b/internal/cmd/testdata/golden/meta/version.stdout @@ -0,0 +1,6 @@ +{ + "arch": "", + "go": "", + "os": "", + "version": "" +} diff --git a/internal/cmd/testdata/golden/meta/version_human.exit b/internal/cmd/testdata/golden/meta/version_human.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/meta/version_human.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/meta/version_human.stdout b/internal/cmd/testdata/golden/meta/version_human.stdout new file mode 100644 index 0000000..1e73a9b --- /dev/null +++ b/internal/cmd/testdata/golden/meta/version_human.stdout @@ -0,0 +1 @@ +spicrawl (, /) diff --git a/internal/cmd/testdata/golden/schema/batch.exit b/internal/cmd/testdata/golden/schema/batch.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema/batch.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema/batch.stdout.sha256 b/internal/cmd/testdata/golden/schema/batch.stdout.sha256 new file mode 100644 index 0000000..e3a3e98 --- /dev/null +++ b/internal/cmd/testdata/golden/schema/batch.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 266998850ee3992e4bd07755021c5f11ec6077c02834d03c83da88898731eb2a +bytes 102977 diff --git a/internal/cmd/testdata/golden/schema/batch_item.exit b/internal/cmd/testdata/golden/schema/batch_item.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema/batch_item.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema/batch_item.stdout.sha256 b/internal/cmd/testdata/golden/schema/batch_item.stdout.sha256 new file mode 100644 index 0000000..ae9bbaf --- /dev/null +++ b/internal/cmd/testdata/golden/schema/batch_item.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 9039dfe91d5728ae8bf229ddd0560d651b27fbd3c73fb64a14915f5f36a09f9a +bytes 48426 diff --git a/internal/cmd/testdata/golden/schema/err_unknown.exit b/internal/cmd/testdata/golden/schema/err_unknown.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/schema/err_unknown.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/schema/err_unknown.stderr b/internal/cmd/testdata/golden/schema/err_unknown.stderr new file mode 100644 index 0000000..c6f1050 --- /dev/null +++ b/internal/cmd/testdata/golden/schema/err_unknown.stderr @@ -0,0 +1,4 @@ +{ + "error": "unknown schema \"nope\" (available: batch, batch-item, scrape, session)", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/schema/err_unknown.stdout b/internal/cmd/testdata/golden/schema/err_unknown.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/schema/list.exit b/internal/cmd/testdata/golden/schema/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema/list.stdout b/internal/cmd/testdata/golden/schema/list.stdout new file mode 100644 index 0000000..34aaae7 --- /dev/null +++ b/internal/cmd/testdata/golden/schema/list.stdout @@ -0,0 +1,18 @@ +[ + { + "description": "POST /v1/batch and POST /v1/batch/{id}/items body", + "name": "batch" + }, + { + "description": "one element of a batch body's items[] (one line of a JSONL batch file)", + "name": "batch-item" + }, + { + "description": "POST /v1/scrape body (spicrawl scrape --body)", + "name": "scrape" + }, + { + "description": "POST /v1/sessions body", + "name": "session" + } +] diff --git a/internal/cmd/testdata/golden/schema/list_flag.exit b/internal/cmd/testdata/golden/schema/list_flag.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema/list_flag.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema/list_flag.stdout b/internal/cmd/testdata/golden/schema/list_flag.stdout new file mode 100644 index 0000000..34aaae7 --- /dev/null +++ b/internal/cmd/testdata/golden/schema/list_flag.stdout @@ -0,0 +1,18 @@ +[ + { + "description": "POST /v1/batch and POST /v1/batch/{id}/items body", + "name": "batch" + }, + { + "description": "one element of a batch body's items[] (one line of a JSONL batch file)", + "name": "batch-item" + }, + { + "description": "POST /v1/scrape body (spicrawl scrape --body)", + "name": "scrape" + }, + { + "description": "POST /v1/sessions body", + "name": "session" + } +] diff --git a/internal/cmd/testdata/golden/schema/scrape.exit b/internal/cmd/testdata/golden/schema/scrape.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema/scrape.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema/scrape.stdout.sha256 b/internal/cmd/testdata/golden/schema/scrape.stdout.sha256 new file mode 100644 index 0000000..5b7dc24 --- /dev/null +++ b/internal/cmd/testdata/golden/schema/scrape.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 676f4b8f19ba42a3665e03dc792447ae0d0f7994212b5aa6a38eeaaaf6491de8 +bytes 47911 diff --git a/internal/cmd/testdata/golden/schema/session.exit b/internal/cmd/testdata/golden/schema/session.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema/session.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema/session.stdout.sha256 b/internal/cmd/testdata/golden/schema/session.stdout.sha256 new file mode 100644 index 0000000..c53e2ef --- /dev/null +++ b/internal/cmd/testdata/golden/schema/session.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 ec7f95d65b603dca6468abb8aec21ca1acc14905ab47d6843c2633d9b6c7e16e +bytes 8827 diff --git a/internal/cmd/testdata/golden/schema_human/batch.exit b/internal/cmd/testdata/golden/schema_human/batch.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/batch.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema_human/batch.stdout.sha256 b/internal/cmd/testdata/golden/schema_human/batch.stdout.sha256 new file mode 100644 index 0000000..e3a3e98 --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/batch.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 266998850ee3992e4bd07755021c5f11ec6077c02834d03c83da88898731eb2a +bytes 102977 diff --git a/internal/cmd/testdata/golden/schema_human/batch_item.exit b/internal/cmd/testdata/golden/schema_human/batch_item.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/batch_item.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema_human/batch_item.stdout.sha256 b/internal/cmd/testdata/golden/schema_human/batch_item.stdout.sha256 new file mode 100644 index 0000000..ae9bbaf --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/batch_item.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 9039dfe91d5728ae8bf229ddd0560d651b27fbd3c73fb64a14915f5f36a09f9a +bytes 48426 diff --git a/internal/cmd/testdata/golden/schema_human/list.exit b/internal/cmd/testdata/golden/schema_human/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema_human/list.stdout b/internal/cmd/testdata/golden/schema_human/list.stdout new file mode 100644 index 0000000..c50f241 --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/list.stdout @@ -0,0 +1,5 @@ +NAME DESCRIBES +batch POST /v1/batch and POST /v1/batch/{id}/items body +batch-item one element of a batch body's items[] (one line of a JSONL batch file) +scrape POST /v1/scrape body (spicrawl scrape --body) +session POST /v1/sessions body diff --git a/internal/cmd/testdata/golden/schema_human/scrape.exit b/internal/cmd/testdata/golden/schema_human/scrape.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/scrape.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema_human/scrape.stdout.sha256 b/internal/cmd/testdata/golden/schema_human/scrape.stdout.sha256 new file mode 100644 index 0000000..5b7dc24 --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/scrape.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 676f4b8f19ba42a3665e03dc792447ae0d0f7994212b5aa6a38eeaaaf6491de8 +bytes 47911 diff --git a/internal/cmd/testdata/golden/schema_human/session.exit b/internal/cmd/testdata/golden/schema_human/session.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/session.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/schema_human/session.stdout.sha256 b/internal/cmd/testdata/golden/schema_human/session.stdout.sha256 new file mode 100644 index 0000000..c53e2ef --- /dev/null +++ b/internal/cmd/testdata/golden/schema_human/session.stdout.sha256 @@ -0,0 +1,2 @@ +sha256 ec7f95d65b603dca6468abb8aec21ca1acc14905ab47d6843c2633d9b6c7e16e +bytes 8827 diff --git a/internal/cmd/testdata/golden/scrape/ai.exit b/internal/cmd/testdata/golden/scrape/ai.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/ai.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/ai.stdout b/internal/cmd/testdata/golden/scrape/ai.stdout new file mode 100644 index 0000000..0e82125 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/ai.stdout @@ -0,0 +1,18 @@ +{ + "url": "https://example.com/product/1", + "final_url": "https://example.com/product/1", + "status": 200, + "content": "

Anvil

$19.99 & free shipping

", + "headers": { + "content-type": "text/html" + }, + "truncated": false, + "credits": 3, + "engine": "fetch", + "proxy_source": "pool", + "warnings": [], + "data": { + "name": "Anvil", + "price": 19.99 + } +} diff --git a/internal/cmd/testdata/golden/scrape/autoparse.exit b/internal/cmd/testdata/golden/scrape/autoparse.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/autoparse.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/autoparse.stdout b/internal/cmd/testdata/golden/scrape/autoparse.stdout new file mode 100644 index 0000000..ae8c0ab --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/autoparse.stdout @@ -0,0 +1,25 @@ +{ + "url": "https://example.com/product/1", + "final_url": "https://example.com/product/1", + "status": 200, + "content": "

Anvil

$19.99 & free shipping

", + "headers": { + "content-type": "text/html" + }, + "truncated": false, + "credits": 3, + "engine": "fetch", + "proxy_source": "pool", + "warnings": [], + "data": { + "json_ld": [ + { + "@type": "Product", + "name": "Anvil" + } + ], + "open_graph": { + "og:title": "Anvil" + } + } +} diff --git a/internal/cmd/testdata/golden/scrape/err_auth.exit b/internal/cmd/testdata/golden/scrape/err_auth.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_auth.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/scrape/err_auth.stderr b/internal/cmd/testdata/golden/scrape/err_auth.stderr new file mode 100644 index 0000000..cbdc3b7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_auth.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Unauthorized", + "status": 401, + "code": "ERR::AUTH::INVALID_KEY", + "detail": "the API key is not valid", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/scrape/err_auth.stdout b/internal/cmd/testdata/golden/scrape/err_auth.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_engine.exit b/internal/cmd/testdata/golden/scrape/err_engine.exit new file mode 100644 index 0000000..45a4fb7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_engine.exit @@ -0,0 +1 @@ +8 diff --git a/internal/cmd/testdata/golden/scrape/err_engine.stderr b/internal/cmd/testdata/golden/scrape/err_engine.stderr new file mode 100644 index 0000000..fbf1f2f --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_engine.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Not Implemented", + "status": 501, + "code": "ERR::ENGINE::UNAVAILABLE", + "detail": "ai_extract is not available", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/scrape/err_engine.stdout b/internal/cmd/testdata/golden/scrape/err_engine.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_limit.exit b/internal/cmd/testdata/golden/scrape/err_limit.exit new file mode 100644 index 0000000..7ed6ff8 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_limit.exit @@ -0,0 +1 @@ +5 diff --git a/internal/cmd/testdata/golden/scrape/err_limit.stderr b/internal/cmd/testdata/golden/scrape/err_limit.stderr new file mode 100644 index 0000000..eb1a50a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_limit.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Too Many Requests", + "status": 429, + "code": "ERR::LIMIT::RATE", + "detail": "slow down", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/scrape/err_limit.stdout b/internal/cmd/testdata/golden/scrape/err_limit.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_no_key.exit b/internal/cmd/testdata/golden/scrape/err_no_key.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_no_key.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/scrape/err_no_key.stderr b/internal/cmd/testdata/golden/scrape/err_no_key.stderr new file mode 100644 index 0000000..6b478b2 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_no_key.stderr @@ -0,0 +1,4 @@ +{ + "error": "no API key: run \"spicrawl login\", set SPICRAWL_API_KEY, or pass --api-key", + "exit_code": 3 +} diff --git a/internal/cmd/testdata/golden/scrape/err_no_key.stdout b/internal/cmd/testdata/golden/scrape/err_no_key.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_request.exit b/internal/cmd/testdata/golden/scrape/err_request.exit new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_request.exit @@ -0,0 +1 @@ +4 diff --git a/internal/cmd/testdata/golden/scrape/err_request.stderr b/internal/cmd/testdata/golden/scrape/err_request.stderr new file mode 100644 index 0000000..2174ecc --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_request.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Bad Request", + "status": 400, + "code": "ERR::REQUEST::INVALID_PARAMETER", + "detail": "engine: unknown value", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/scrape/err_request.stdout b/internal/cmd/testdata/golden/scrape/err_request.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_unreachable.exit b/internal/cmd/testdata/golden/scrape/err_unreachable.exit new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_unreachable.exit @@ -0,0 +1 @@ +10 diff --git a/internal/cmd/testdata/golden/scrape/err_unreachable.stdout b/internal/cmd/testdata/golden/scrape/err_unreachable.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_upstream.exit b/internal/cmd/testdata/golden/scrape/err_upstream.exit new file mode 100644 index 0000000..1e8b314 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_upstream.exit @@ -0,0 +1 @@ +6 diff --git a/internal/cmd/testdata/golden/scrape/err_upstream.stderr b/internal/cmd/testdata/golden/scrape/err_upstream.stderr new file mode 100644 index 0000000..657d4ce --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_upstream.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Bad Gateway", + "status": 502, + "code": "ERR::UPSTREAM::BLOCKED", + "detail": "the target served a bot challenge", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/scrape/err_upstream.stdout b/internal/cmd/testdata/golden/scrape/err_upstream.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.exit b/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.stderr b/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.stderr new file mode 100644 index 0000000..0b659a1 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.stderr @@ -0,0 +1,4 @@ +{ + "error": "--ai and --ai-schema are mutually exclusive: the API takes a prompt or a schema, not both", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.stdout b/internal/cmd/testdata/golden/scrape/err_usage_ai_exclusive.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_usage_bad_extract.exit b/internal/cmd/testdata/golden/scrape/err_usage_bad_extract.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_usage_bad_extract.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/scrape/err_usage_bad_extract.stdout b/internal/cmd/testdata/golden/scrape/err_usage_bad_extract.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/err_usage_no_url.exit b/internal/cmd/testdata/golden/scrape/err_usage_no_url.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_usage_no_url.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/scrape/err_usage_no_url.stderr b/internal/cmd/testdata/golden/scrape/err_usage_no_url.stderr new file mode 100644 index 0000000..9a97161 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/err_usage_no_url.stderr @@ -0,0 +1,4 @@ +{ + "error": "accepts 1 arg(s), received 0", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/scrape/err_usage_no_url.stdout b/internal/cmd/testdata/golden/scrape/err_usage_no_url.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/extract.exit b/internal/cmd/testdata/golden/scrape/extract.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/extract.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/extract.stdout b/internal/cmd/testdata/golden/scrape/extract.stdout new file mode 100644 index 0000000..559a5e8 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/extract.stdout @@ -0,0 +1,18 @@ +{ + "url": "https://example.com/product/1", + "final_url": "https://example.com/product/1", + "status": 200, + "content": "

Anvil

$19.99 & free shipping

", + "headers": { + "content-type": "text/html" + }, + "truncated": false, + "credits": 3, + "engine": "fetch", + "proxy_source": "pool", + "warnings": [], + "data": { + "title": "Anvil", + "price": "$19.99 & free shipping" + } +} diff --git a/internal/cmd/testdata/golden/scrape/html.exit b/internal/cmd/testdata/golden/scrape/html.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/html.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/html.stdout b/internal/cmd/testdata/golden/scrape/html.stdout new file mode 100644 index 0000000..d3ad295 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/html.stdout @@ -0,0 +1,15 @@ +{ + "cache_state": "miss", + "content": "\nExample Domain\n

Example Domain

Prices from $19.99 & up.

\n", + "content_type": "text/html; charset=utf-8", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/json_envelope.exit b/internal/cmd/testdata/golden/scrape/json_envelope.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/json_envelope.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/json_envelope.stdout b/internal/cmd/testdata/golden/scrape/json_envelope.stdout new file mode 100644 index 0000000..81d4384 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/json_envelope.stdout @@ -0,0 +1,15 @@ +{ + "url": "https://example.com/product/1", + "final_url": "https://example.com/product/1", + "status": 200, + "content": "

Anvil

$19.99 & free shipping

", + "headers": { + "content-type": "text/html" + }, + "truncated": false, + "credits": 3, + "engine": "fetch", + "proxy_source": "pool", + "warnings": [], + "links": null +} diff --git a/internal/cmd/testdata/golden/scrape/json_envelope_jsonl.exit b/internal/cmd/testdata/golden/scrape/json_envelope_jsonl.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/json_envelope_jsonl.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/json_envelope_jsonl.stdout b/internal/cmd/testdata/golden/scrape/json_envelope_jsonl.stdout new file mode 100644 index 0000000..dd48a48 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/json_envelope_jsonl.stdout @@ -0,0 +1 @@ +{"url":"https://example.com/product/1","final_url":"https://example.com/product/1","status":200,"content":"

Anvil

$19.99 & free shipping

","headers":{"content-type":"text/html"},"truncated":false,"credits":3,"engine":"fetch","proxy_source":"pool","warnings":[]} diff --git a/internal/cmd/testdata/golden/scrape/links.exit b/internal/cmd/testdata/golden/scrape/links.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/links.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/links.stdout b/internal/cmd/testdata/golden/scrape/links.stdout new file mode 100644 index 0000000..af3f6c0 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/links.stdout @@ -0,0 +1,18 @@ +{ + "url": "https://example.com/product/1", + "final_url": "https://example.com/product/1", + "status": 200, + "content": "

Anvil

$19.99 & free shipping

", + "headers": { + "content-type": "text/html" + }, + "truncated": false, + "credits": 3, + "engine": "fetch", + "proxy_source": "pool", + "warnings": [], + "links": [ + "https://example.com/a", + "https://example.com/b?x=1&y=2" + ] +} diff --git a/internal/cmd/testdata/golden/scrape/markdown.exit b/internal/cmd/testdata/golden/scrape/markdown.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/markdown.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/markdown.stdout b/internal/cmd/testdata/golden/scrape/markdown.stdout new file mode 100644 index 0000000..982714c --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/markdown.stdout @@ -0,0 +1,17 @@ +{ + "cache_state": "miss", + "content": "# Example Domain\n\nPrices from $19.99 & up.\n", + "content_type": "text/markdown; charset=utf-8", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [ + "FORMAT_COERCED: markdown requested, html served" + ] +} diff --git a/internal/cmd/testdata/golden/scrape/markdown_json_flag.exit b/internal/cmd/testdata/golden/scrape/markdown_json_flag.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/markdown_json_flag.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/markdown_json_flag.stdout b/internal/cmd/testdata/golden/scrape/markdown_json_flag.stdout new file mode 100644 index 0000000..982714c --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/markdown_json_flag.stdout @@ -0,0 +1,17 @@ +{ + "cache_state": "miss", + "content": "# Example Domain\n\nPrices from $19.99 & up.\n", + "content_type": "text/markdown; charset=utf-8", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [ + "FORMAT_COERCED: markdown requested, html served" + ] +} diff --git a/internal/cmd/testdata/golden/scrape/markdown_jsonl.exit b/internal/cmd/testdata/golden/scrape/markdown_jsonl.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/markdown_jsonl.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/markdown_jsonl.stdout b/internal/cmd/testdata/golden/scrape/markdown_jsonl.stdout new file mode 100644 index 0000000..e7ae612 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/markdown_jsonl.stdout @@ -0,0 +1 @@ +{"cache_state":"miss","content":"# Example Domain\n\nPrices from $19.99 & up.\n","content_type":"text/markdown; charset=utf-8","credits":2,"credits_charged":2,"engine":"fetch","final_url":"https://example.com/final","proxy_source":"pool","request_cost":2,"request_id":"","status":200,"url":"https://example.com/product/1","warnings":["FORMAT_COERCED: markdown requested, html served"]} diff --git a/internal/cmd/testdata/golden/scrape/network_capture.exit b/internal/cmd/testdata/golden/scrape/network_capture.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/network_capture.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/network_capture.stdout b/internal/cmd/testdata/golden/scrape/network_capture.stdout new file mode 100644 index 0000000..76c989a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/network_capture.stdout @@ -0,0 +1,23 @@ +{ + "url": "https://example.com/product/1", + "final_url": "https://example.com/product/1", + "status": 200, + "content": "

Anvil

$19.99 & free shipping

", + "headers": { + "content-type": "text/html" + }, + "truncated": false, + "credits": 3, + "engine": "fetch", + "proxy_source": "pool", + "warnings": [], + "network": [ + { + "url": "https://example.com/api/price", + "status": 200, + "resource_type": "xhr", + "content_type": "application/json", + "body": "{\"price\":19.99}" + } + ] +} diff --git a/internal/cmd/testdata/golden/scrape/output_file.exit b/internal/cmd/testdata/golden/scrape/output_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/output_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/output_file.file.page.md b/internal/cmd/testdata/golden/scrape/output_file.file.page.md new file mode 100644 index 0000000..f3485b7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/output_file.file.page.md @@ -0,0 +1,3 @@ +# Example Domain + +Prices from $19.99 & up. diff --git a/internal/cmd/testdata/golden/scrape/output_file.stdout b/internal/cmd/testdata/golden/scrape/output_file.stdout new file mode 100644 index 0000000..2f8fbcb --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/output_file.stdout @@ -0,0 +1,17 @@ +{ + "cache_state": "miss", + "content_type": "text/markdown; charset=utf-8", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "output": "/page.md", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [ + "FORMAT_COERCED: markdown requested, html served" + ] +} diff --git a/internal/cmd/testdata/golden/scrape/output_stdout_dash.exit b/internal/cmd/testdata/golden/scrape/output_stdout_dash.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/output_stdout_dash.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/output_stdout_dash.stdout b/internal/cmd/testdata/golden/scrape/output_stdout_dash.stdout new file mode 100644 index 0000000..982714c --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/output_stdout_dash.stdout @@ -0,0 +1,17 @@ +{ + "cache_state": "miss", + "content": "# Example Domain\n\nPrices from $19.99 & up.\n", + "content_type": "text/markdown; charset=utf-8", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [ + "FORMAT_COERCED: markdown requested, html served" + ] +} diff --git a/internal/cmd/testdata/golden/scrape/pdf_base64.exit b/internal/cmd/testdata/golden/scrape/pdf_base64.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/pdf_base64.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/pdf_base64.stdout b/internal/cmd/testdata/golden/scrape/pdf_base64.stdout new file mode 100644 index 0000000..c513fb9 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/pdf_base64.stdout @@ -0,0 +1,15 @@ +{ + "cache_state": "miss", + "content_base64": "JVBERi0xLjQKMSAwIG9iago8PD4+CmVuZG9iagolJUVPRgo=", + "content_type": "application/pdf", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/screenshot_dir.exit b/internal/cmd/testdata/golden/scrape/screenshot_dir.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/screenshot_dir.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/screenshot_dir.file.shots__page-2.png b/internal/cmd/testdata/golden/scrape/screenshot_dir.file.shots__page-2.png new file mode 100644 index 0000000..0cadfa2 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/screenshot_dir.file.shots__page-2.png @@ -0,0 +1 @@ +PNGDATA \ No newline at end of file diff --git a/internal/cmd/testdata/golden/scrape/screenshot_dir.file.shots__page.png b/internal/cmd/testdata/golden/scrape/screenshot_dir.file.shots__page.png new file mode 100644 index 0000000..0cadfa2 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/screenshot_dir.file.shots__page.png @@ -0,0 +1 @@ +PNGDATA \ No newline at end of file diff --git a/internal/cmd/testdata/golden/scrape/screenshot_dir.stdout b/internal/cmd/testdata/golden/scrape/screenshot_dir.stdout new file mode 100644 index 0000000..678fb28 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/screenshot_dir.stdout @@ -0,0 +1,30 @@ +{ + "content": "

Anvil

$19.99 & free shipping

", + "credits": 3, + "engine": "fetch", + "final_url": "https://example.com/product/1", + "headers": { + "content-type": "text/html" + }, + "proxy_source": "pool", + "screenshots": [ + { + "file": "/shots/page.png", + "format": "png", + "height": 720, + "label": "page", + "width": 1280 + }, + { + "file": "/shots/page-2.png", + "format": "png", + "height": 720, + "label": "page", + "width": 1280 + } + ], + "status": 200, + "truncated": false, + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/stdin_many.exit b/internal/cmd/testdata/golden/scrape/stdin_many.exit new file mode 100644 index 0000000..1e8b314 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many.exit @@ -0,0 +1 @@ +6 diff --git a/internal/cmd/testdata/golden/scrape/stdin_many.stderr b/internal/cmd/testdata/golden/scrape/stdin_many.stderr new file mode 100644 index 0000000..01f0dcb --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many.stderr @@ -0,0 +1,4 @@ +{ + "error": "2 of 3 URLs failed", + "exit_code": 6 +} diff --git a/internal/cmd/testdata/golden/scrape/stdin_many.stdout b/internal/cmd/testdata/golden/scrape/stdin_many.stdout new file mode 100644 index 0000000..57b2c98 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many.stdout @@ -0,0 +1,3 @@ +{"cache_state":"miss","content":"# https://example.com/ok\n","content_type":"text/markdown","credits":2,"credits_charged":2,"engine":"fetch","final_url":"https://example.com/final","index":1,"proxy_source":"pool","request_cost":2,"request_id":"","status":200,"url":"https://example.com/ok","warnings":[]} +{"cache_state":"miss","content":"# Down\n","content_type":"text/markdown","credits":2,"credits_charged":2,"engine":"fetch","final_url":"https://example.com/final","index":2,"proxy_source":"pool","request_cost":2,"request_id":"","status":503,"target_error":"target returned 503","url":"https://example.com/down","warnings":[]} +{"error":{"type":"https://docs.spicrawl.com/errors","title":"Forbidden","status":403,"code":"ERR::UPSTREAM::BLOCKED","detail":"blocked by the target","retryable":false,"doc_url":"https://docs.spicrawl.com/errors","request_id":"","target_status":null,"diagnostics":{"hint":"check the request and try again"}},"index":3,"url":"https://example.com/denied"} diff --git a/internal/cmd/testdata/golden/scrape/stdin_many_all_ok.exit b/internal/cmd/testdata/golden/scrape/stdin_many_all_ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many_all_ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/stdin_many_all_ok.stdout b/internal/cmd/testdata/golden/scrape/stdin_many_all_ok.stdout new file mode 100644 index 0000000..11a25ce --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many_all_ok.stdout @@ -0,0 +1,2 @@ +{"cache_state":"miss","content":"# https://example.com/ok\n","content_type":"text/markdown","credits":2,"credits_charged":2,"engine":"fetch","final_url":"https://example.com/final","index":1,"proxy_source":"pool","request_cost":2,"request_id":"","status":200,"url":"https://example.com/ok","warnings":[]} +{"cache_state":"miss","content":"# https://example.com/ok2\n","content_type":"text/markdown","credits":2,"credits_charged":2,"engine":"fetch","final_url":"https://example.com/final","index":2,"proxy_source":"pool","request_cost":2,"request_id":"","status":200,"url":"https://example.com/ok2","warnings":[]} diff --git a/internal/cmd/testdata/golden/scrape/stdin_many_empty.exit b/internal/cmd/testdata/golden/scrape/stdin_many_empty.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many_empty.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/scrape/stdin_many_empty.stderr b/internal/cmd/testdata/golden/scrape/stdin_many_empty.stderr new file mode 100644 index 0000000..e09eb43 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/stdin_many_empty.stderr @@ -0,0 +1,4 @@ +{ + "error": "no URLs on stdin", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/scrape/stdin_many_empty.stdout b/internal/cmd/testdata/golden/scrape/stdin_many_empty.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape/target_403_envelope.exit b/internal/cmd/testdata/golden/scrape/target_403_envelope.exit new file mode 100644 index 0000000..1e8b314 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_403_envelope.exit @@ -0,0 +1 @@ +6 diff --git a/internal/cmd/testdata/golden/scrape/target_403_envelope.stderr b/internal/cmd/testdata/golden/scrape/target_403_envelope.stderr new file mode 100644 index 0000000..0ba524a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_403_envelope.stderr @@ -0,0 +1,4 @@ +{ + "error": "target returned 403 (pass --allowed-status 403 to accept it)", + "exit_code": 6 +} diff --git a/internal/cmd/testdata/golden/scrape/target_403_envelope.stdout b/internal/cmd/testdata/golden/scrape/target_403_envelope.stdout new file mode 100644 index 0000000..59e30fa --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_403_envelope.stdout @@ -0,0 +1,16 @@ +{ + "content": "

Anvil

$19.99 & free shipping

", + "credits": 3, + "data": null, + "engine": "fetch", + "final_url": "https://example.com/product/1", + "headers": { + "content-type": "text/html" + }, + "proxy_source": "pool", + "status": 200, + "target_error": "target returned 403", + "truncated": false, + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/target_404_allowed.exit b/internal/cmd/testdata/golden/scrape/target_404_allowed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_404_allowed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/target_404_allowed.stdout b/internal/cmd/testdata/golden/scrape/target_404_allowed.stdout new file mode 100644 index 0000000..9f62169 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_404_allowed.stdout @@ -0,0 +1,15 @@ +{ + "cache_state": "miss", + "content": "

Not Found

\n", + "content_type": "text/html", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 404, + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/target_503.exit b/internal/cmd/testdata/golden/scrape/target_503.exit new file mode 100644 index 0000000..1e8b314 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_503.exit @@ -0,0 +1 @@ +6 diff --git a/internal/cmd/testdata/golden/scrape/target_503.stderr b/internal/cmd/testdata/golden/scrape/target_503.stderr new file mode 100644 index 0000000..af601fb --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_503.stderr @@ -0,0 +1,4 @@ +{ + "error": "target returned 503 (pass --allowed-status 503 to accept it)", + "exit_code": 6 +} diff --git a/internal/cmd/testdata/golden/scrape/target_503.stdout b/internal/cmd/testdata/golden/scrape/target_503.stdout new file mode 100644 index 0000000..161f5b7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_503.stdout @@ -0,0 +1,16 @@ +{ + "cache_state": "miss", + "content": "

Service Unavailable

\n", + "content_type": "text/html", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 503, + "target_error": "target returned 503", + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.exit b/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.exit new file mode 100644 index 0000000..1e8b314 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.exit @@ -0,0 +1 @@ +6 diff --git a/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.stderr b/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.stderr new file mode 100644 index 0000000..85c31dc --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.stderr @@ -0,0 +1,5 @@ +warning: not writing /page.html: the target failed (--allowed-status accepts it) +{ + "error": "target returned 503 (pass --allowed-status 503 to accept it)", + "exit_code": 6 +} diff --git a/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.stdout b/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.stdout new file mode 100644 index 0000000..3fee6c4 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/target_503_output_file_not_written.stdout @@ -0,0 +1,16 @@ +{ + "cache_state": "miss", + "content": "

Down

\n", + "content_type": "text/html", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 503, + "target_error": "target returned 503", + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape/text.exit b/internal/cmd/testdata/golden/scrape/text.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/text.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape/text.stdout b/internal/cmd/testdata/golden/scrape/text.stdout new file mode 100644 index 0000000..071e7e5 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape/text.stdout @@ -0,0 +1,15 @@ +{ + "cache_state": "miss", + "content": "Example Domain\nPrices from $19.99 & up.", + "content_type": "text/plain; charset=utf-8", + "credits": 2, + "credits_charged": 2, + "engine": "fetch", + "final_url": "https://example.com/final", + "proxy_source": "pool", + "request_cost": 2, + "request_id": "", + "status": 200, + "url": "https://example.com/product/1", + "warnings": [] +} diff --git a/internal/cmd/testdata/golden/scrape_human/err_auth.exit b/internal/cmd/testdata/golden/scrape_human/err_auth.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/err_auth.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/scrape_human/err_auth.stdout b/internal/cmd/testdata/golden/scrape_human/err_auth.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape_human/extract_data.exit b/internal/cmd/testdata/golden/scrape_human/extract_data.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/extract_data.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/extract_data.stdout b/internal/cmd/testdata/golden/scrape_human/extract_data.stdout new file mode 100644 index 0000000..a2ead9e --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/extract_data.stdout @@ -0,0 +1,4 @@ +{ + "price": "$19.99 & free shipping", + "title": "Anvil" +} diff --git a/internal/cmd/testdata/golden/scrape_human/html.exit b/internal/cmd/testdata/golden/scrape_human/html.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/html.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/html.stdout b/internal/cmd/testdata/golden/scrape_human/html.stdout new file mode 100644 index 0000000..5af3329 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/html.stdout @@ -0,0 +1,3 @@ + +Example Domain +

Example Domain

Prices from $19.99 & up.

diff --git a/internal/cmd/testdata/golden/scrape_human/links.exit b/internal/cmd/testdata/golden/scrape_human/links.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/links.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/links.stdout b/internal/cmd/testdata/golden/scrape_human/links.stdout new file mode 100644 index 0000000..e1e2411 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/links.stdout @@ -0,0 +1,4 @@ +

Anvil

$19.99 & free shipping

+--- links (2) --- +https://example.com/a +https://example.com/b?x=1&y=2 diff --git a/internal/cmd/testdata/golden/scrape_human/markdown.exit b/internal/cmd/testdata/golden/scrape_human/markdown.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/markdown.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/markdown.stdout b/internal/cmd/testdata/golden/scrape_human/markdown.stdout new file mode 100644 index 0000000..f3485b7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/markdown.stdout @@ -0,0 +1,3 @@ +# Example Domain + +Prices from $19.99 & up. diff --git a/internal/cmd/testdata/golden/scrape_human/markdown_meta.exit b/internal/cmd/testdata/golden/scrape_human/markdown_meta.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/markdown_meta.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/markdown_meta.stdout b/internal/cmd/testdata/golden/scrape_human/markdown_meta.stdout new file mode 100644 index 0000000..f3485b7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/markdown_meta.stdout @@ -0,0 +1,3 @@ +# Example Domain + +Prices from $19.99 & up. diff --git a/internal/cmd/testdata/golden/scrape_human/network.exit b/internal/cmd/testdata/golden/scrape_human/network.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/network.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/network.stdout b/internal/cmd/testdata/golden/scrape_human/network.stdout new file mode 100644 index 0000000..b21ace6 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/network.stdout @@ -0,0 +1,4 @@ +

Anvil

$19.99 & free shipping

+--- network (2) --- +200 xhr application/json https://example.com/api/price +- image - https://example.com/x.png diff --git a/internal/cmd/testdata/golden/scrape_human/output_file.exit b/internal/cmd/testdata/golden/scrape_human/output_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/output_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/output_file.file.page.md b/internal/cmd/testdata/golden/scrape_human/output_file.file.page.md new file mode 100644 index 0000000..f3485b7 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/output_file.file.page.md @@ -0,0 +1,3 @@ +# Example Domain + +Prices from $19.99 & up. diff --git a/internal/cmd/testdata/golden/scrape_human/output_file.stdout b/internal/cmd/testdata/golden/scrape_human/output_file.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape_human/pdf_needs_file.exit b/internal/cmd/testdata/golden/scrape_human/pdf_needs_file.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/pdf_needs_file.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/scrape_human/pdf_needs_file.stdout b/internal/cmd/testdata/golden/scrape_human/pdf_needs_file.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape_human/pdf_output_file.exit b/internal/cmd/testdata/golden/scrape_human/pdf_output_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/pdf_output_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/pdf_output_file.file.page.pdf b/internal/cmd/testdata/golden/scrape_human/pdf_output_file.file.page.pdf new file mode 100644 index 0000000..e8cf043 Binary files /dev/null and b/internal/cmd/testdata/golden/scrape_human/pdf_output_file.file.page.pdf differ diff --git a/internal/cmd/testdata/golden/scrape_human/pdf_output_file.stdout b/internal/cmd/testdata/golden/scrape_human/pdf_output_file.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/scrape_human/target_503.exit b/internal/cmd/testdata/golden/scrape_human/target_503.exit new file mode 100644 index 0000000..1e8b314 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/target_503.exit @@ -0,0 +1 @@ +6 diff --git a/internal/cmd/testdata/golden/scrape_human/target_503.stdout b/internal/cmd/testdata/golden/scrape_human/target_503.stdout new file mode 100644 index 0000000..312cd2a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/target_503.stdout @@ -0,0 +1 @@ +

Service Unavailable

diff --git a/internal/cmd/testdata/golden/scrape_human/text_no_trailing_newline.exit b/internal/cmd/testdata/golden/scrape_human/text_no_trailing_newline.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/text_no_trailing_newline.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/scrape_human/text_no_trailing_newline.stdout b/internal/cmd/testdata/golden/scrape_human/text_no_trailing_newline.stdout new file mode 100644 index 0000000..85fedb4 --- /dev/null +++ b/internal/cmd/testdata/golden/scrape_human/text_no_trailing_newline.stdout @@ -0,0 +1 @@ +Example Domain diff --git a/internal/cmd/testdata/golden/sessions/context.exit b/internal/cmd/testdata/golden/sessions/context.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/context.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/context.stdout b/internal/cmd/testdata/golden/sessions/context.stdout new file mode 100644 index 0000000..a0f43bc --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/context.stdout @@ -0,0 +1,16 @@ +{ + "session_id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "session_context": { + "cookies": [ + { + "name": "sid", + "value": "abc123", + "domain": "example.com", + "path": "/" + } + ], + "local_storage": { + "theme": "dark" + } + } +} diff --git a/internal/cmd/testdata/golden/sessions/create.exit b/internal/cmd/testdata/golden/sessions/create.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/create.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/create.stdout b/internal/cmd/testdata/golden/sessions/create.stdout new file mode 100644 index 0000000..562dd8b --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/create.stdout @@ -0,0 +1,17 @@ +{ + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "obscura", + "status": "active", + "sticky_key": null, + "proxy": { + "tier": "datacenter" + }, + "created_at": "", + "last_used_at": null, + "expires_at": "", + "usage_count": 0, + "warnings": [ + "engine obscura cannot persist IndexedDB" + ] +} diff --git a/internal/cmd/testdata/golden/sessions/create_session_context_file.exit b/internal/cmd/testdata/golden/sessions/create_session_context_file.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/create_session_context_file.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/create_session_context_file.stdout b/internal/cmd/testdata/golden/sessions/create_session_context_file.stdout new file mode 100644 index 0000000..c5b29c0 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/create_session_context_file.stdout @@ -0,0 +1,18 @@ +{ + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "active", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de", + "region_pool": "eu" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "hard_expires_at": "", + "usage_count": 7, + "context": null +} diff --git a/internal/cmd/testdata/golden/sessions/create_usage_error.exit b/internal/cmd/testdata/golden/sessions/create_usage_error.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/create_usage_error.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/sessions/create_usage_error.stderr b/internal/cmd/testdata/golden/sessions/create_usage_error.stderr new file mode 100644 index 0000000..8135a56 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/create_usage_error.stderr @@ -0,0 +1,4 @@ +{ + "error": "--rotate-ip and --sticky-key cannot be combined: one lets the exit change, the other pins it", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/sessions/create_usage_error.stdout b/internal/cmd/testdata/golden/sessions/create_usage_error.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.exit b/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.stderr b/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.stderr new file mode 100644 index 0000000..c2f38d6 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.stderr @@ -0,0 +1,4 @@ +{ + "error": "refusing to delete session 01J9ZQ4M7R3T8VX2K5N6P0B1CD without confirmation: pass --yes", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.stdout b/internal/cmd/testdata/golden/sessions/delete_needs_yes_without_terminal.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/sessions/delete_yes.exit b/internal/cmd/testdata/golden/sessions/delete_yes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/delete_yes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/delete_yes.stdout b/internal/cmd/testdata/golden/sessions/delete_yes.stdout new file mode 100644 index 0000000..b8dc394 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/delete_yes.stdout @@ -0,0 +1,4 @@ +{ + "deleted": true, + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD" +} diff --git a/internal/cmd/testdata/golden/sessions/get.exit b/internal/cmd/testdata/golden/sessions/get.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/get.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/get.stdout b/internal/cmd/testdata/golden/sessions/get.stdout new file mode 100644 index 0000000..c5b29c0 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/get.stdout @@ -0,0 +1,18 @@ +{ + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "active", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de", + "region_pool": "eu" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "hard_expires_at": "", + "usage_count": 7, + "context": null +} diff --git a/internal/cmd/testdata/golden/sessions/get_not_found.exit b/internal/cmd/testdata/golden/sessions/get_not_found.exit new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/get_not_found.exit @@ -0,0 +1 @@ +9 diff --git a/internal/cmd/testdata/golden/sessions/get_not_found.stderr b/internal/cmd/testdata/golden/sessions/get_not_found.stderr new file mode 100644 index 0000000..d1b1da5 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/get_not_found.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Not Found", + "status": 404, + "code": "ERR::SESSION::NOT_FOUND", + "detail": "no such session", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/sessions/get_not_found.stdout b/internal/cmd/testdata/golden/sessions/get_not_found.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/sessions/list.exit b/internal/cmd/testdata/golden/sessions/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/list.stdout b/internal/cmd/testdata/golden/sessions/list.stdout new file mode 100644 index 0000000..931ded3 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list.stdout @@ -0,0 +1,39 @@ +{ + "sessions": [ + { + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "active", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de", + "region_pool": "eu" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "hard_expires_at": "", + "usage_count": 7, + "context": null + }, + { + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "released", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "released_at": "", + "retired": "released by the caller", + "usage_count": 7 + } + ] +} diff --git a/internal/cmd/testdata/golden/sessions/list_all_pages.exit b/internal/cmd/testdata/golden/sessions/list_all_pages.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list_all_pages.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/list_all_pages.stdout b/internal/cmd/testdata/golden/sessions/list_all_pages.stdout new file mode 100644 index 0000000..931ded3 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list_all_pages.stdout @@ -0,0 +1,39 @@ +{ + "sessions": [ + { + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "active", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de", + "region_pool": "eu" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "hard_expires_at": "", + "usage_count": 7, + "context": null + }, + { + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "released", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "released_at": "", + "retired": "released by the caller", + "usage_count": 7 + } + ] +} diff --git a/internal/cmd/testdata/golden/sessions/list_empty.exit b/internal/cmd/testdata/golden/sessions/list_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/list_empty.stdout b/internal/cmd/testdata/golden/sessions/list_empty.stdout new file mode 100644 index 0000000..f56218a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list_empty.stdout @@ -0,0 +1,3 @@ +{ + "sessions": [] +} diff --git a/internal/cmd/testdata/golden/sessions/list_next_cursor.exit b/internal/cmd/testdata/golden/sessions/list_next_cursor.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list_next_cursor.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/list_next_cursor.stdout b/internal/cmd/testdata/golden/sessions/list_next_cursor.stdout new file mode 100644 index 0000000..e1a1240 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/list_next_cursor.stdout @@ -0,0 +1,23 @@ +{ + "next_cursor": "1790000000000000000.ABC", + "sessions": [ + { + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "active", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de", + "region_pool": "eu" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "hard_expires_at": "", + "usage_count": 7, + "context": null + } + ] +} diff --git a/internal/cmd/testdata/golden/sessions/release.exit b/internal/cmd/testdata/golden/sessions/release.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/release.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions/release.stdout b/internal/cmd/testdata/golden/sessions/release.stdout new file mode 100644 index 0000000..aaea824 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions/release.stdout @@ -0,0 +1,17 @@ +{ + "id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "project_id": "proj_01J9Z", + "engine": "chromium", + "status": "released", + "sticky_key": "login-1", + "proxy": { + "tier": "residential", + "country": "de" + }, + "created_at": "", + "last_used_at": "", + "expires_at": "", + "released_at": "", + "retired": "released by the caller", + "usage_count": 7 +} diff --git a/internal/cmd/testdata/golden/sessions_human/context.exit b/internal/cmd/testdata/golden/sessions_human/context.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/context.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/context.stdout b/internal/cmd/testdata/golden/sessions_human/context.stdout new file mode 100644 index 0000000..a0f43bc --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/context.stdout @@ -0,0 +1,16 @@ +{ + "session_id": "01J9ZQ4M7R3T8VX2K5N6P0B1CD", + "session_context": { + "cookies": [ + { + "name": "sid", + "value": "abc123", + "domain": "example.com", + "path": "/" + } + ], + "local_storage": { + "theme": "dark" + } + } +} diff --git a/internal/cmd/testdata/golden/sessions_human/create.exit b/internal/cmd/testdata/golden/sessions_human/create.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/create.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/create.stdout b/internal/cmd/testdata/golden/sessions_human/create.stdout new file mode 100644 index 0000000..69f419b --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/create.stdout @@ -0,0 +1,9 @@ +id: 01J9ZQ4M7R3T8VX2K5N6P0B1CD +status: active +engine: obscura +exit: datacenter +sticky key: (rotating) +uses: 0 +created: +last used: never +expires: diff --git a/internal/cmd/testdata/golden/sessions_human/delete_yes.exit b/internal/cmd/testdata/golden/sessions_human/delete_yes.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/delete_yes.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/delete_yes.stdout b/internal/cmd/testdata/golden/sessions_human/delete_yes.stdout new file mode 100644 index 0000000..a452206 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/delete_yes.stdout @@ -0,0 +1 @@ +deleted session 01J9ZQ4M7R3T8VX2K5N6P0B1CD diff --git a/internal/cmd/testdata/golden/sessions_human/get.exit b/internal/cmd/testdata/golden/sessions_human/get.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/get.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/get.stdout b/internal/cmd/testdata/golden/sessions_human/get.stdout new file mode 100644 index 0000000..20dce43 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/get.stdout @@ -0,0 +1,10 @@ +id: 01J9ZQ4M7R3T8VX2K5N6P0B1CD +status: active +engine: chromium +exit: residential (de) pool eu +sticky key: login-1 +uses: 7 +created: +last used: +expires: +hard expiry: diff --git a/internal/cmd/testdata/golden/sessions_human/get_released.exit b/internal/cmd/testdata/golden/sessions_human/get_released.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/get_released.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/get_released.stdout b/internal/cmd/testdata/golden/sessions_human/get_released.stdout new file mode 100644 index 0000000..a3d6efc --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/get_released.stdout @@ -0,0 +1,11 @@ +id: 01J9ZQ4M7R3T8VX2K5N6P0B1CD +status: released +engine: chromium +exit: residential (de) +sticky key: login-1 +uses: 7 +created: +last used: +expires: +released: +retired: released by the caller diff --git a/internal/cmd/testdata/golden/sessions_human/list.exit b/internal/cmd/testdata/golden/sessions_human/list.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/list.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/list.stdout b/internal/cmd/testdata/golden/sessions_human/list.stdout new file mode 100644 index 0000000..aa37fc0 --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/list.stdout @@ -0,0 +1,3 @@ +ID ENGINE STATUS EXIT USES EXPIRES +01J9ZQ4M7R3T8VX2K5N6P0B1CD chromium active residential/de 7 +01J9ZQ4M7R3T8VX2K5N6P0B1CD chromium released residential/de 7 diff --git a/internal/cmd/testdata/golden/sessions_human/list_empty.exit b/internal/cmd/testdata/golden/sessions_human/list_empty.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/list_empty.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/list_empty.stdout b/internal/cmd/testdata/golden/sessions_human/list_empty.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/sessions_human/list_next_cursor.exit b/internal/cmd/testdata/golden/sessions_human/list_next_cursor.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/list_next_cursor.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/list_next_cursor.stdout b/internal/cmd/testdata/golden/sessions_human/list_next_cursor.stdout new file mode 100644 index 0000000..05429ad --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/list_next_cursor.stdout @@ -0,0 +1,2 @@ +ID ENGINE STATUS EXIT USES EXPIRES +01J9ZQ4M7R3T8VX2K5N6P0B1CD chromium active residential/de 7 diff --git a/internal/cmd/testdata/golden/sessions_human/release.exit b/internal/cmd/testdata/golden/sessions_human/release.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/release.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/sessions_human/release.stdout b/internal/cmd/testdata/golden/sessions_human/release.stdout new file mode 100644 index 0000000..a3d6efc --- /dev/null +++ b/internal/cmd/testdata/golden/sessions_human/release.stdout @@ -0,0 +1,11 @@ +id: 01J9ZQ4M7R3T8VX2K5N6P0B1CD +status: released +engine: chromium +exit: residential (de) +sticky key: login-1 +uses: 7 +created: +last used: +expires: +released: +retired: released by the caller diff --git a/internal/cmd/testdata/golden/status/key_invalid_exit3.exit b/internal/cmd/testdata/golden/status/key_invalid_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/status/key_invalid_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/status/key_invalid_exit3.stderr b/internal/cmd/testdata/golden/status/key_invalid_exit3.stderr new file mode 100644 index 0000000..47478ac --- /dev/null +++ b/internal/cmd/testdata/golden/status/key_invalid_exit3.stderr @@ -0,0 +1,4 @@ +{ + "error": "the API rejected the key: ERR::AUTH::INVALID_KEY: the API key is not valid", + "exit_code": 3 +} diff --git a/internal/cmd/testdata/golden/status/key_invalid_exit3.stdout b/internal/cmd/testdata/golden/status/key_invalid_exit3.stdout new file mode 100644 index 0000000..d846600 --- /dev/null +++ b/internal/cmd/testdata/golden/status/key_invalid_exit3.stdout @@ -0,0 +1,14 @@ +{ + "api_reachable": true, + "ready": true, + "checks": { + "postgres": "ok", + "redis": "ok" + }, + "cdp": "enabled", + "key_valid": false, + "key_error": "ERR::AUTH::INVALID_KEY: the API key is not valid", + "key_source": "env", + "base_url": "http://SERVER", + "workers": null +} diff --git a/internal/cmd/testdata/golden/status/key_without_read_scope.exit b/internal/cmd/testdata/golden/status/key_without_read_scope.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/status/key_without_read_scope.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/status/key_without_read_scope.stdout b/internal/cmd/testdata/golden/status/key_without_read_scope.stdout new file mode 100644 index 0000000..dca8245 --- /dev/null +++ b/internal/cmd/testdata/golden/status/key_without_read_scope.stdout @@ -0,0 +1,27 @@ +{ + "api_reachable": true, + "ready": true, + "checks": { + "postgres": "ok", + "redis": "ok" + }, + "cdp": "enabled", + "key_valid": true, + "key_error": "key is valid but lacks the read scope", + "key_source": "env", + "base_url": "http://SERVER", + "workers": { + "batch": { + "present": 2, + "ready": 0 + }, + "browser": { + "present": 3, + "ready": 2 + }, + "egress": { + "present": 1, + "ready": 1 + } + } +} diff --git a/internal/cmd/testdata/golden/status/no_key_exit3.exit b/internal/cmd/testdata/golden/status/no_key_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/status/no_key_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/status/no_key_exit3.stderr b/internal/cmd/testdata/golden/status/no_key_exit3.stderr new file mode 100644 index 0000000..6b478b2 --- /dev/null +++ b/internal/cmd/testdata/golden/status/no_key_exit3.stderr @@ -0,0 +1,4 @@ +{ + "error": "no API key: run \"spicrawl login\", set SPICRAWL_API_KEY, or pass --api-key", + "exit_code": 3 +} diff --git a/internal/cmd/testdata/golden/status/no_key_exit3.stdout b/internal/cmd/testdata/golden/status/no_key_exit3.stdout new file mode 100644 index 0000000..5c64b55 --- /dev/null +++ b/internal/cmd/testdata/golden/status/no_key_exit3.stdout @@ -0,0 +1,14 @@ +{ + "api_reachable": true, + "ready": true, + "checks": { + "postgres": "ok", + "redis": "ok" + }, + "cdp": "enabled", + "key_valid": false, + "key_error": "no API key configured", + "key_source": "", + "base_url": "http://SERVER", + "workers": null +} diff --git a/internal/cmd/testdata/golden/status/not_ready.exit b/internal/cmd/testdata/golden/status/not_ready.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/status/not_ready.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/status/not_ready.stdout b/internal/cmd/testdata/golden/status/not_ready.stdout new file mode 100644 index 0000000..9610bd8 --- /dev/null +++ b/internal/cmd/testdata/golden/status/not_ready.stdout @@ -0,0 +1,26 @@ +{ + "api_reachable": true, + "ready": false, + "checks": { + "postgres": "ok", + "redis": "unreachable" + }, + "cdp": "disabled", + "key_valid": true, + "key_source": "env", + "base_url": "http://SERVER", + "workers": { + "batch": { + "present": 2, + "ready": 0 + }, + "browser": { + "present": 3, + "ready": 2 + }, + "egress": { + "present": 1, + "ready": 1 + } + } +} diff --git a/internal/cmd/testdata/golden/status/ok.exit b/internal/cmd/testdata/golden/status/ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/status/ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/status/ok.stdout b/internal/cmd/testdata/golden/status/ok.stdout new file mode 100644 index 0000000..92d7385 --- /dev/null +++ b/internal/cmd/testdata/golden/status/ok.stdout @@ -0,0 +1,26 @@ +{ + "api_reachable": true, + "ready": true, + "checks": { + "postgres": "ok", + "redis": "ok" + }, + "cdp": "enabled", + "key_valid": true, + "key_source": "env", + "base_url": "http://SERVER", + "workers": { + "batch": { + "present": 2, + "ready": 0 + }, + "browser": { + "present": 3, + "ready": 2 + }, + "egress": { + "present": 1, + "ready": 1 + } + } +} diff --git a/internal/cmd/testdata/golden/status/unreachable_exit10.exit b/internal/cmd/testdata/golden/status/unreachable_exit10.exit new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/internal/cmd/testdata/golden/status/unreachable_exit10.exit @@ -0,0 +1 @@ +10 diff --git a/internal/cmd/testdata/golden/status/unreachable_exit10.stdout b/internal/cmd/testdata/golden/status/unreachable_exit10.stdout new file mode 100644 index 0000000..63919ae --- /dev/null +++ b/internal/cmd/testdata/golden/status/unreachable_exit10.stdout @@ -0,0 +1,9 @@ +{ + "api_reachable": false, + "ready": false, + "cdp": "", + "key_valid": null, + "key_source": "env", + "base_url": "http://127.0.0.1:1", + "workers": null +} diff --git a/internal/cmd/testdata/golden/status/workers_error.exit b/internal/cmd/testdata/golden/status/workers_error.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/status/workers_error.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/status/workers_error.stdout b/internal/cmd/testdata/golden/status/workers_error.stdout new file mode 100644 index 0000000..5266310 --- /dev/null +++ b/internal/cmd/testdata/golden/status/workers_error.stdout @@ -0,0 +1,14 @@ +{ + "api_reachable": true, + "ready": true, + "checks": { + "postgres": "ok", + "redis": "ok" + }, + "cdp": "enabled", + "key_valid": true, + "key_source": "env", + "base_url": "http://SERVER", + "workers": null, + "workers_error": "ERR::INTERNAL::UNAVAILABLE: fleet registry is down" +} diff --git a/internal/cmd/testdata/golden/status_human/key_invalid_exit3.exit b/internal/cmd/testdata/golden/status_human/key_invalid_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/key_invalid_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/status_human/key_invalid_exit3.stdout b/internal/cmd/testdata/golden/status_human/key_invalid_exit3.stdout new file mode 100644 index 0000000..1d516c5 --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/key_invalid_exit3.stdout @@ -0,0 +1,5 @@ +base url: http://SERVER +reachable: yes +ready: yes +cdp: enabled +key: invalid (from env): ERR::AUTH::INVALID_KEY: the API key is not valid diff --git a/internal/cmd/testdata/golden/status_human/no_key_exit3.exit b/internal/cmd/testdata/golden/status_human/no_key_exit3.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/no_key_exit3.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/status_human/no_key_exit3.stdout b/internal/cmd/testdata/golden/status_human/no_key_exit3.stdout new file mode 100644 index 0000000..33c2360 --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/no_key_exit3.stdout @@ -0,0 +1,5 @@ +base url: http://SERVER +reachable: yes +ready: yes +cdp: enabled +key: invalid: no API key configured diff --git a/internal/cmd/testdata/golden/status_human/not_ready.exit b/internal/cmd/testdata/golden/status_human/not_ready.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/not_ready.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/status_human/not_ready.stdout b/internal/cmd/testdata/golden/status_human/not_ready.stdout new file mode 100644 index 0000000..c283eaf --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/not_ready.stdout @@ -0,0 +1,9 @@ +base url: http://SERVER +reachable: yes +ready: no (redis=unreachable) +cdp: disabled (cloud browser /v1/browser is not available on this deployment) +key: valid (from env) +workers: + browser 2 ready of 3 + egress 1 ready of 1 + batch 0 ready of 2 diff --git a/internal/cmd/testdata/golden/status_human/ok.exit b/internal/cmd/testdata/golden/status_human/ok.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/ok.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/status_human/ok.stdout b/internal/cmd/testdata/golden/status_human/ok.stdout new file mode 100644 index 0000000..6604db2 --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/ok.stdout @@ -0,0 +1,9 @@ +base url: http://SERVER +reachable: yes +ready: yes +cdp: enabled +key: valid (from env) +workers: + browser 2 ready of 3 + egress 1 ready of 1 + batch 0 ready of 2 diff --git a/internal/cmd/testdata/golden/status_human/unreachable_exit10.exit b/internal/cmd/testdata/golden/status_human/unreachable_exit10.exit new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/unreachable_exit10.exit @@ -0,0 +1 @@ +10 diff --git a/internal/cmd/testdata/golden/status_human/unreachable_exit10.stdout b/internal/cmd/testdata/golden/status_human/unreachable_exit10.stdout new file mode 100644 index 0000000..df2dcbf --- /dev/null +++ b/internal/cmd/testdata/golden/status_human/unreachable_exit10.stdout @@ -0,0 +1,2 @@ +base url: http://127.0.0.1:1 +reachable: no diff --git a/internal/cmd/testdata/golden/usage/by_day.exit b/internal/cmd/testdata/golden/usage/by_day.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage/by_day.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage/by_day.stdout b/internal/cmd/testdata/golden/usage/by_day.stdout new file mode 100644 index 0000000..837f85f --- /dev/null +++ b/internal/cmd/testdata/golden/usage/by_day.stdout @@ -0,0 +1,44 @@ +{ + "period": { + "start": "2026-08-01", + "end": "2026-08-04", + "days": 3, + "bounds": "[start, end)" + }, + "group_by": "day", + "groups": [ + { + "key": "2026-08-01", + "label": "", + "metrics": { + "requests": 10, + "credits": 12500000, + "engine_ms": 4200 + } + }, + { + "key": "2026-08-02", + "label": "", + "metrics": { + "requests": 15, + "credits": 18500000, + "engine_ms": 4800 + } + } + ], + "totals": { + "requests": 25, + "credits": 31000000, + "engine_ms": 9000 + }, + "unattributed": { + "requests": 0, + "credits": 0, + "engine_ms": 0 + }, + "usage_as_of": "", + "stale_seconds": 120, + "warnings": [ + "rollup is partial for 2026-08-03" + ] +} diff --git a/internal/cmd/testdata/golden/usage/by_key.exit b/internal/cmd/testdata/golden/usage/by_key.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage/by_key.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage/by_key.stdout b/internal/cmd/testdata/golden/usage/by_key.stdout new file mode 100644 index 0000000..925df59 --- /dev/null +++ b/internal/cmd/testdata/golden/usage/by_key.stdout @@ -0,0 +1,34 @@ +{ + "period": { + "start": "2026-08-01", + "end": "2026-09-01", + "days": 31, + "bounds": "[start, end)" + }, + "group_by": "key", + "keys": [ + { + "key_id": "K1", + "name": "ci-key", + "prefix": "spicrawl_live_ab12", + "project_id": "proj_01J9Z", + "revoked": false, + "days": [], + "totals": { + "requests": 7, + "client_cli": 3 + } + }, + { + "key_id": "K2", + "name": "old", + "prefix": "spicrawl_live_cd34", + "project_id": "proj_01J9Z", + "revoked": true, + "days": [], + "totals": { + "requests": 1 + } + } + ] +} diff --git a/internal/cmd/testdata/golden/usage/err_scope.exit b/internal/cmd/testdata/golden/usage/err_scope.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/usage/err_scope.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/usage/err_scope.stderr b/internal/cmd/testdata/golden/usage/err_scope.stderr new file mode 100644 index 0000000..28eb06b --- /dev/null +++ b/internal/cmd/testdata/golden/usage/err_scope.stderr @@ -0,0 +1,14 @@ +{ + "type": "https://docs.spicrawl.com/errors", + "title": "Forbidden", + "status": 403, + "code": "ERR::AUTH::INSUFFICIENT_SCOPE", + "detail": "the key lacks the read scope", + "retryable": false, + "doc_url": "https://docs.spicrawl.com/errors", + "request_id": "", + "target_status": null, + "diagnostics": { + "hint": "check the request and try again" + } +} diff --git a/internal/cmd/testdata/golden/usage/err_scope.stdout b/internal/cmd/testdata/golden/usage/err_scope.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/usage/err_usage_to_before_from.exit b/internal/cmd/testdata/golden/usage/err_usage_to_before_from.exit new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/internal/cmd/testdata/golden/usage/err_usage_to_before_from.exit @@ -0,0 +1 @@ +2 diff --git a/internal/cmd/testdata/golden/usage/err_usage_to_before_from.stderr b/internal/cmd/testdata/golden/usage/err_usage_to_before_from.stderr new file mode 100644 index 0000000..a0cc871 --- /dev/null +++ b/internal/cmd/testdata/golden/usage/err_usage_to_before_from.stderr @@ -0,0 +1,4 @@ +{ + "error": "--to is exclusive and must be after --from", + "exit_code": 2 +} diff --git a/internal/cmd/testdata/golden/usage/err_usage_to_before_from.stdout b/internal/cmd/testdata/golden/usage/err_usage_to_before_from.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/usage/reconciliation.exit b/internal/cmd/testdata/golden/usage/reconciliation.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage/reconciliation.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage/reconciliation.stdout b/internal/cmd/testdata/golden/usage/reconciliation.stdout new file mode 100644 index 0000000..aea1bd6 --- /dev/null +++ b/internal/cmd/testdata/golden/usage/reconciliation.stdout @@ -0,0 +1,11 @@ +{ + "day": "2026-09-21", + "interpretation": "The request log holds no rows for the day; drift is not evidence of overbilling.", + "drift": [ + { + "metric": "requests", + "billed": 10, + "derived": 0 + } + ] +} diff --git a/internal/cmd/testdata/golden/usage/summary.exit b/internal/cmd/testdata/golden/usage/summary.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage/summary.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage/summary.stdout b/internal/cmd/testdata/golden/usage/summary.stdout new file mode 100644 index 0000000..2999943 --- /dev/null +++ b/internal/cmd/testdata/golden/usage/summary.stdout @@ -0,0 +1,37 @@ +{ + "period": { + "start": "2026-09-01", + "end": "2026-10-01", + "days": 30, + "bounds": "[start, end)" + }, + "period_elapsed_days": 22, + "plan": { + "code": "starter", + "name": "Starter", + "subscription_status": "active", + "period_source": "subscription" + }, + "credits": { + "included_micro": 50000000000, + "used_micro": 12345670000, + "remaining_micro": 37654330000, + "overage_micro": 0, + "used_basis_points": 2469 + }, + "allowance": { + "limit_micro": 50000000000, + "used_micro": 12400000000, + "remaining_micro": 37600000000, + "unlimited": false, + "resets_at": "" + }, + "metrics": { + "requests": 1234, + "credits": 12345670000, + "extractions": 55 + }, + "warnings": [ + "allowance counter is 40s behind" + ] +} diff --git a/internal/cmd/testdata/golden/usage_human/by_day.exit b/internal/cmd/testdata/golden/usage_human/by_day.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/by_day.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage_human/by_day.stdout b/internal/cmd/testdata/golden/usage_human/by_day.stdout new file mode 100644 index 0000000..3cb469e --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/by_day.stdout @@ -0,0 +1,6 @@ +period [2026-08-01, 2026-08-04) (3 days), by day + +DAY REQUESTS CREDITS ENGINE_MS +2026-08-01 10 12.5 4200 +2026-08-02 15 18.5 4800 +TOTAL 25 31 9000 diff --git a/internal/cmd/testdata/golden/usage_human/by_engine_unattributed.exit b/internal/cmd/testdata/golden/usage_human/by_engine_unattributed.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/by_engine_unattributed.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage_human/by_engine_unattributed.stdout b/internal/cmd/testdata/golden/usage_human/by_engine_unattributed.stdout new file mode 100644 index 0000000..c18b6a0 --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/by_engine_unattributed.stdout @@ -0,0 +1,7 @@ +period [2026-08-01, 2026-09-01) (31 days), by engine + +ENGINE REQUESTS CREDITS +Fetch 40 40 +chromium 5 25 +(unattributed) 2 1 +TOTAL 47 66 diff --git a/internal/cmd/testdata/golden/usage_human/by_key.exit b/internal/cmd/testdata/golden/usage_human/by_key.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/by_key.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage_human/by_key.stdout b/internal/cmd/testdata/golden/usage_human/by_key.stdout new file mode 100644 index 0000000..34fd23d --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/by_key.stdout @@ -0,0 +1,5 @@ +period [2026-08-01, 2026-09-01) (31 days), by key + +KEY NAME PREFIX REQUESTS CLIENT_CLI +K1 ci-key spicrawl_live_ab12 7 3 +K2 old spicrawl_live_cd34 1 0 diff --git a/internal/cmd/testdata/golden/usage_human/err_scope.exit b/internal/cmd/testdata/golden/usage_human/err_scope.exit new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/err_scope.exit @@ -0,0 +1 @@ +3 diff --git a/internal/cmd/testdata/golden/usage_human/err_scope.stdout b/internal/cmd/testdata/golden/usage_human/err_scope.stdout new file mode 100644 index 0000000..e69de29 diff --git a/internal/cmd/testdata/golden/usage_human/reconciliation.exit b/internal/cmd/testdata/golden/usage_human/reconciliation.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/reconciliation.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage_human/reconciliation.stdout b/internal/cmd/testdata/golden/usage_human/reconciliation.stdout new file mode 100644 index 0000000..aea1bd6 --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/reconciliation.stdout @@ -0,0 +1,11 @@ +{ + "day": "2026-09-21", + "interpretation": "The request log holds no rows for the day; drift is not evidence of overbilling.", + "drift": [ + { + "metric": "requests", + "billed": 10, + "derived": 0 + } + ] +} diff --git a/internal/cmd/testdata/golden/usage_human/summary.exit b/internal/cmd/testdata/golden/usage_human/summary.exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/summary.exit @@ -0,0 +1 @@ +0 diff --git a/internal/cmd/testdata/golden/usage_human/summary.stdout b/internal/cmd/testdata/golden/usage_human/summary.stdout new file mode 100644 index 0000000..96740c8 --- /dev/null +++ b/internal/cmd/testdata/golden/usage_human/summary.stdout @@ -0,0 +1,9 @@ +plan: Starter (active) +period: [2026-09-01, 2026-10-01), day 22 of 30 (subscription) +credits: 12345.67 of 50000 used (24.7%), 37654.33 remaining +allowance: 12400 of 50000 used (incl. credits held for in-flight work), 37600 remaining, resets + +METRIC VALUE +requests 1234 +credits 12345.67 +extractions 55 diff --git a/internal/cmd/view.go b/internal/cmd/view.go new file mode 100644 index 0000000..b8348a3 --- /dev/null +++ b/internal/cmd/view.go @@ -0,0 +1,190 @@ +package cmd + +// Shared pieces of the human-facing output of the setup and read-only +// commands (init, auth, config, status, skill, mcp, docs, logs, sessions). +// +// The rule they all follow: the plain output is what it always was, byte for +// byte. Colour, marks, headings and spinners exist only for a person at a +// terminal, so each helper takes a *ui.UI and does nothing (or prints the old +// text) when it is nil or off. A nil *ui.UI and a nil *ui.Spinner are valid. + +import ( + "fmt" + "io" + "strings" + "unicode/utf8" + + "github.com/OfficialSpicrawl/cli/internal/output" + "github.com/OfficialSpicrawl/cli/internal/ui" +) + +// viewForce draws the live output whatever the streams are. Tests set it. +var viewForce bool + +// errUI is the live output on stderr: spinner, hints, headings. Off when +// stderr is not a terminal, with --json or --quiet, and under NO_COLOR, +// TERM=dumb or CI (see ui.Live). +func errUI(p *output.Printer) *ui.UI { + if viewForce && !p.Plain { + return ui.Force(p.Err) + } + return p.UI() +} + +// outUI is the styling of a human result on stdout. Off unless stdout is a +// terminal showing the human view (never JSON, never piped), and under the +// same switches as errUI. +func outUI(p *output.Printer) *ui.UI { + switch { + case p.JSON || p.Plain: + return nil + case viewForce: + return ui.Force(p.Out) + } + return ui.New(p.Out) +} + +// spin starts a spinner on stderr for a wait. It appears only after +// ui.Delay, so a fast call shows nothing. Stop it before printing anything. +func spin(p *output.Printer, format string, a ...any) *ui.Spinner { + return errUI(p).Spin(format, a...) +} + +// viewIntro prints a heading and key/value rows to stderr and reports true, +// or reports false when the caller should print its plain lines instead. +func viewIntro(p *output.Printer, title string, rows ...ui.Row) bool { + u := errUI(p) + if !u.On() { + return false + } + var b strings.Builder + b.WriteString(u.Bold(title) + "\n") + b.WriteString(viewRows(u, " ", rows)) + u.Printf("%s", b.String()) + return true +} + +// viewNext prints a short "next" list to stderr for a person who just +// finished a setup step. Nothing in JSON mode, with --quiet or off a terminal. +func viewNext(p *output.Printer, steps ...string) { + u := errUI(p) + if p.JSON || !u.On() || len(steps) == 0 { + return + } + var b strings.Builder + b.WriteString("\n" + u.Bold("next") + "\n") + for _, s := range steps { + b.WriteString(" " + u.Cyan("›") + " " + s + "\n") + } + u.Printf("%s", b.String()) +} + +// viewRows renders key/value rows with dimmed, aligned keys. Rows with an +// empty value are skipped. +func viewRows(u *ui.UI, indent string, rows []ui.Row) string { + pad := 0 + for _, r := range rows { + if r.Val != "" && utf8.RuneCountInString(r.Key) > pad { + pad = utf8.RuneCountInString(r.Key) + } + } + var b strings.Builder + for _, r := range rows { + if r.Val != "" { + b.WriteString(indent + viewKey(u, pad, r.Key) + " " + r.Val + "\n") + } + } + return b.String() +} + +// viewKey is a key padded to width, dimmed when u is on. With u off it is +// exactly fmt's %-*s. +func viewKey(u *ui.UI, width int, key string) string { + return u.Dim(fmt.Sprintf("%-*s", width, key)) +} + +// Marks for a check list. Neutral is for "not applicable" or "unknown". +const ( + markOK = iota + markBad + markWarn + markNeutral +) + +func viewMark(u *ui.UI, m int) string { + switch m { + case markOK: + return u.Green("✓") + case markBad: + return u.Red("✗") + case markWarn: + return u.Bold("!") + } + return u.Dim("–") +} + +// viewLead is a mark and a space to put before a line, or "" while u is off, +// so a plain line keeps exactly the text it had. +func viewLead(u *ui.UI, m int) string { + if !u.On() { + return "" + } + return viewMark(u, m) + " " +} + +// viewTable is p.Table, and exactly that while u is off. When it is on the +// header is bold and style, if given, colours each body cell. Padding is +// added around the plain text, so colour never skews the columns. +func viewTable(p *output.Printer, u *ui.UI, headers []string, rows [][]string, style func(col int, cell string) string) { + if !u.On() { + p.Table(headers, rows) + return + } + widths := make([]int, len(headers)) + for i, h := range headers { + widths[i] = utf8.RuneCountInString(h) + } + for _, r := range rows { + for i, c := range r { + if i < len(widths) { + widths[i] = max(widths[i], utf8.RuneCountInString(c)) + } + } + } + line := func(cells []string, paint func(col int, cell string) string) { + var b strings.Builder + for i, c := range cells { + shown := paint(i, c) + b.WriteString(shown) + if i < len(cells)-1 && i < len(widths) { + b.WriteString(strings.Repeat(" ", max(widths[i]-utf8.RuneCountInString(c), 0)+2)) + } + } + b.WriteString("\n") + io.WriteString(p.Out, b.String()) + } + line(headers, func(_ int, c string) string { return u.Bold(c) }) + for _, r := range rows { + line(r, func(col int, c string) string { + if style == nil { + return c + } + return style(col, c) + }) + } +} + +// viewStatus colours a status word: good green, bad red, anything else dim. +func viewStatus(u *ui.UI, s string, good, bad []string) string { + for _, g := range good { + if s == g { + return u.Green(s) + } + } + for _, b := range bad { + if s == b { + return u.Red(s) + } + } + return u.Dim(s) +} diff --git a/internal/cmd/view_test.go b/internal/cmd/view_test.go new file mode 100644 index 0000000..9abaeb9 --- /dev/null +++ b/internal/cmd/view_test.go @@ -0,0 +1,110 @@ +package cmd + +import ( + "bytes" + "regexp" + "strings" + "testing" + "time" + + "github.com/OfficialSpicrawl/cli/internal/output" + "github.com/OfficialSpicrawl/cli/internal/ui" +) + +var viewANSI = regexp.MustCompile("\x1b\\[[0-9;?]*[A-Za-z]") + +// stripANSI removes colour and cursor codes. +func stripANSI(s string) string { return viewANSI.ReplaceAllString(s, "") } + +// liveUI makes the commands draw their terminal output into the test's +// buffers, as if both streams were terminals. The spinner keeps its delay. +func liveUI(t *testing.T) { + t.Helper() + viewForce = true + t.Cleanup(func() { viewForce = false }) +} + +// spinNow also lets the spinner appear at once, so a short slow call shows it. +func spinNow(t *testing.T) { + t.Helper() + liveUI(t) + d, i := ui.Delay, ui.Interval + ui.Delay, ui.Interval = 0, time.Millisecond + t.Cleanup(func() { ui.Delay, ui.Interval = d, i }) +} + +// noEscapes fails when s carries any terminal control code. +func noEscapes(t *testing.T, what, s string) { + t.Helper() + if strings.Contains(s, "\x1b") { + t.Errorf("%s has terminal escapes: %q", what, s) + } +} + +func TestViewGatingFollowsTheStream(t *testing.T) { + var out, errb bytes.Buffer + p := &output.Printer{Out: &out, Err: &errb} + if outUI(p).On() || errUI(p).On() { + t.Error("a buffer is not a terminal: the live output must be off") + } + + liveUI(t) + if !outUI(p).On() || !errUI(p).On() { + t.Error("forced: both should be on") + } + p.JSON = true + if outUI(p).On() { + t.Error("JSON mode must never be styled on stdout") + } + if !errUI(p).On() { + t.Error("JSON picked only because stdout is piped must keep stderr's spinner") + } + p.Plain = true // --json or --quiet + if outUI(p).On() || errUI(p).On() { + t.Error("--json and --quiet must keep both streams plain") + } +} + +func TestViewHelpersAreInertWhenOff(t *testing.T) { + var out, errb bytes.Buffer + p := &output.Printer{Out: &out, Err: &errb} + if sp := spin(p, "waiting"); sp != nil { + t.Error("spin returned a spinner while off") + } + if viewIntro(p, "title", ui.Row{Key: "k", Val: "v"}) { + t.Error("viewIntro drew while off") + } + viewNext(p, "do a thing") + if errb.Len() != 0 { + t.Errorf("wrote %q while off", errb.String()) + } + var u *ui.UI + if viewKey(u, 6, "id:") != "id: " || viewLead(u, markOK) != "" { + t.Error("an off UI must leave keys padded and marks absent") + } +} + +func TestViewTableOffIsExactlyTheTabwriterTable(t *testing.T) { + headers := []string{"A", "BB", "C"} + rows := [][]string{{"one", "x", "—"}, {"two-two", "yy", "z"}} + var plain, off bytes.Buffer + (&output.Printer{Out: &plain}).Table(headers, rows) + p := &output.Printer{Out: &off} + viewTable(p, nil, headers, rows, nil) + if off.String() != plain.String() { + t.Errorf("off table differs:\n%q\n%q", off.String(), plain.String()) + } + + // On: coloured, and the columns line up exactly as the plain table's do. + var live bytes.Buffer + p = &output.Printer{Out: &live} + viewTable(p, ui.Force(&live), headers, rows, func(col int, cell string) string { + return ui.Force(&live).Red(cell) + }) + if !strings.Contains(live.String(), "\x1b[") { + t.Fatal("no colour in the live table") + } + if got := stripANSI(live.String()); got != plain.String() { + t.Errorf("live table misaligned:\n%q\n%q", got, plain.String()) + } +} diff --git a/internal/output/output.go b/internal/output/output.go index 64ab378..7ea9240 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -15,6 +15,8 @@ import ( "text/tabwriter" "github.com/mattn/go-isatty" + + "github.com/OfficialSpicrawl/cli/internal/ui" ) type Printer struct { @@ -22,12 +24,33 @@ type Printer struct { Quiet bool Out io.Writer Err io.Writer + // Plain keeps stderr plain (no spinner, colour or summary lines): --json + // and --quiet set it. The live output is also off whenever Err is not a + // terminal, or NO_COLOR, TERM=dumb or CI say so (see ui.Live). + Plain bool + // Term overrides UI(). Tests set it to see the live output. + Term *ui.UI } // New picks JSON when forced or when stdout is not a terminal. func New(forceJSON, quiet bool) *Printer { tty := isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) - return &Printer{JSON: forceJSON || !tty, Quiet: quiet, Out: os.Stdout, Err: os.Stderr} + return &Printer{JSON: forceJSON || !tty, Quiet: quiet, Out: os.Stdout, Err: os.Stderr, Plain: forceJSON || quiet} +} + +// UI is the live terminal output on stderr: spinner, colour, summary lines. +// It is off (every method a no-op) unless stderr is a terminal and neither +// --json, --quiet, NO_COLOR, TERM=dumb nor CI asked for plain output. Note +// that JSON picked only because stdout is piped does not turn it off: the +// spinner is stderr's, and a person watching a redirected run wants it. +func (p *Printer) UI() *ui.UI { + switch { + case p.Term != nil: + return p.Term + case p.Plain: + return nil + } + return ui.New(p.Err) } // IsStdinTerminal reports whether stdin is interactive. Commands must never @@ -89,10 +112,10 @@ func (p *Printer) Info(format string, a ...any) { if p.Quiet { return } - fmt.Fprintf(p.Err, format+"\n", a...) + ui.Fprintf(p.Err, format+"\n", a...) } // Warn writes a warning to stderr, even with --quiet. func (p *Printer) Warn(format string, a ...any) { - fmt.Fprintf(p.Err, "warning: "+format+"\n", a...) + ui.Fprintf(p.Err, "warning: "+format+"\n", a...) } diff --git a/internal/ui/ui.go b/internal/ui/ui.go new file mode 100644 index 0000000..7b7aa0b --- /dev/null +++ b/internal/ui/ui.go @@ -0,0 +1,364 @@ +// Package ui is the CLI's live terminal output: a spinner with a label and +// elapsed time, a one-line ✓/✗ summary, a readable error block and a small +// colour palette. It exists only for a person at a terminal. +// +// Everything here writes to the stream it is given (stderr, never stdout) and +// is OFF unless that stream is a terminal and nothing asked for plain output +// (NO_COLOR, TERM=dumb, CI). When it is off every method is a no-op, so a +// caller never branches on it and the plain output stays byte-for-byte what it +// was. A nil *UI and a nil *Spinner are valid and off. +// +// Callers that print to the same stream while a spinner runs must go through +// Fprintf (output.Printer.Info and Warn already do), so the line is not +// written into the middle of a frame. +package ui + +import ( + "bytes" + "fmt" + "io" + "os" + "runtime" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/mattn/go-isatty" + "golang.org/x/term" +) + +// Live reports whether w should get a spinner and colour: w is a terminal and +// nothing asked for plain output. NO_COLOR (any non-empty value), TERM=dumb +// and CI (anything but empty, 0 or false) all turn it off. +func Live(w io.Writer) bool { return live(w, os.Getenv, runtime.GOOS, isTTY) } + +func isTTY(f *os.File) bool { + return isatty.IsTerminal(f.Fd()) || isatty.IsCygwinTerminal(f.Fd()) +} + +func live(w io.Writer, getenv func(string) string, goos string, tty func(*os.File) bool) bool { + f, ok := w.(*os.File) + if !ok || !tty(f) { + return false + } + if getenv("NO_COLOR") != "" || getenv("TERM") == "dumb" { + return false + } + switch strings.ToLower(getenv("CI")) { + case "", "0", "false": + default: + return false + } + // A Windows console only understands escape codes when VT processing is + // on; Windows Terminal turns it on, the legacy console does not. + return goos != "windows" || getenv("WT_SESSION") != "" +} + +// UI draws on one stream. The zero value and nil are off. +type UI struct { + w io.Writer + on bool +} + +// New returns a UI for w that is on only when Live(w). +func New(w io.Writer) *UI { return &UI{w: w, on: Live(w)} } + +// Force returns a UI that is always on, whatever w is. For tests. +func Force(w io.Writer) *UI { return &UI{w: w, on: true} } + +// On reports whether this UI draws anything. +func (u *UI) On() bool { return u != nil && u.on } + +// ---- colour ---------------------------------------------------------------- + +// The palette is the terminal's own 8 colours plus bold and faint, so it +// follows the user's theme and stays readable on light and dark backgrounds. +// No white, black or yellow: those vanish on one of the two. + +func (u *UI) style(code, s string) string { + if !u.On() || s == "" { + return s + } + return "\x1b[" + code + "m" + s + "\x1b[0m" +} + +func (u *UI) Bold(s string) string { return u.style("1", s) } +func (u *UI) Dim(s string) string { return u.style("2", s) } +func (u *UI) Red(s string) string { return u.style("31", s) } +func (u *UI) Green(s string) string { return u.style("32", s) } +func (u *UI) Cyan(s string) string { return u.style("36", s) } + +// ---- lines ----------------------------------------------------------------- + +// Printf writes to the UI's stream without breaking a running spinner. It +// prints even when the UI is off (it is how plain lines are written). +func (u *UI) Printf(format string, a ...any) { + if u == nil || u.w == nil { + return + } + Fprintf(u.w, format, a...) +} + +// Done prints the summary line of one operation, only when the UI is on: +// +// ✓ https://example.com 200 · 1 credit · 4.2s +// +// parts[0] is the subject (a URL); the rest are the facts, dimmed. ok picks a +// green ✓ or a red ✗. +func (u *UI) Done(ok bool, parts ...string) { + if !u.On() || len(parts) == 0 { + return + } + mark := u.Red("✗") + if ok { + mark = u.Green("✓") + } + line := mark + " " + parts[0] + if len(parts) > 1 { + line += " " + u.Dim(strings.Join(parts[1:], " · ")) + } + u.Printf("%s\n", line) +} + +// Row is one "key value" line under a failure headline. +type Row struct{ Key, Val string } + +// Fail prints an error block, only when the UI is on: +// +// ✗ ERR::TARGET::BLOCKED: the site refused the request +// hint retry with --render +// request 01J9Z... (spicrawl logs get 01J9Z...) +// +// Rows with an empty value are skipped. +func (u *UI) Fail(headline string, rows ...Row) { + if !u.On() { + return + } + var b strings.Builder + b.WriteString(u.Red("✗") + " " + u.Bold(headline) + "\n") + pad := 0 + for _, r := range rows { + if r.Val != "" && len(r.Key) > pad { + pad = len(r.Key) + } + } + for _, r := range rows { + if r.Val != "" { + b.WriteString(" " + u.Dim(fmt.Sprintf("%-*s", pad, r.Key)) + " " + r.Val + "\n") + } + } + u.Printf("%s", b.String()) +} + +// Elapsed formats a duration for humans: 840ms is "0.8s", 12.14s is "12.1s", +// 64s is "1m04s". +func Elapsed(d time.Duration) string { + if d < time.Minute { + return fmt.Sprintf("%.1fs", d.Seconds()) + } + d = d.Round(time.Second) + return fmt.Sprintf("%dm%02ds", int(d/time.Minute), int(d%time.Minute/time.Second)) +} + +// ---- spinner --------------------------------------------------------------- + +// Delay is how long an operation runs before its spinner first appears, so a +// fast call never flashes one. Interval is the frame time. Tests change them. +var ( + Delay = 300 * time.Millisecond + Interval = 80 * time.Millisecond +) + +var frames = []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"} + +// Spinner is one live "⠹ label · 4.2s" line. At most one runs at a time: Spin +// while another is active returns nil, so the outermost operation (a whole +// batch wait) keeps its label over the calls inside it. +type Spinner struct { + u *UI + start time.Time + quit chan struct{} + done chan struct{} + + mu sync.Mutex + label string + frame int + drawn bool // something is on the line and the cursor is hidden + stopped bool + elapsed time.Duration +} + +var ( + curMu sync.Mutex + cur *Spinner +) + +// Spin starts a spinner with the given label, or returns nil when the UI is +// off or a spinner already runs. Always Stop it (defer is fine; Stop is +// idempotent and nil-safe). +func (u *UI) Spin(format string, a ...any) *Spinner { + if !u.On() { + return nil + } + curMu.Lock() + defer curMu.Unlock() + if cur != nil { + return nil + } + s := &Spinner{u: u, label: fmt.Sprintf(format, a...), start: time.Now(), + quit: make(chan struct{}), done: make(chan struct{})} + cur = s + go s.run() + return s +} + +// Label changes the text shown; the next frame picks it up. +func (s *Spinner) Label(format string, a ...any) { + if s == nil { + return + } + l := fmt.Sprintf(format, a...) + s.mu.Lock() + s.label = l + s.mu.Unlock() +} + +// Around runs fn with the spinner line cleared, then redraws it. Use it for +// writes to another stream that shares the terminal (results on stdout). +func (s *Spinner) Around(fn func()) { + if s == nil { + fn() + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.clear() + fn() + if s.drawn { + s.draw() + } +} + +// Stop removes the line, shows the cursor again and returns the time since +// Spin. Do it before printing a result or an error. +func (s *Spinner) Stop() time.Duration { + if s == nil { + return 0 + } + s.mu.Lock() + if s.stopped { + defer s.mu.Unlock() + return s.elapsed + } + s.stopped = true + s.elapsed = time.Since(s.start) + close(s.quit) + s.mu.Unlock() + <-s.done // the drawing goroutine is gone: nothing writes after this + s.mu.Lock() + s.clear() + if s.drawn { + io.WriteString(s.u.w, "\x1b[?25h") + s.drawn = false + } + d := s.elapsed + s.mu.Unlock() + curMu.Lock() + if cur == s { + cur = nil + } + curMu.Unlock() + return d +} + +func (s *Spinner) run() { + defer close(s.done) + select { + case <-time.After(Delay): + case <-s.quit: + return + } + t := time.NewTicker(Interval) + defer t.Stop() + for { + s.mu.Lock() + s.draw() + s.mu.Unlock() + select { + case <-t.C: + case <-s.quit: + return + } + } +} + +// clear wipes the spinner line. Callers hold s.mu. +func (s *Spinner) clear() { + if s.drawn { + io.WriteString(s.u.w, "\r\x1b[K") + } +} + +// draw writes one frame in a single Write. Callers hold s.mu. +func (s *Spinner) draw() { + if s.stopped { + return + } + suffix := " · " + Elapsed(time.Since(s.start)) + // The visible line is 2 cells of glyph, the label and the suffix, and must + // stay one short of the width: a line that wraps cannot be cleared by \r. + label := truncate(s.label, width(s.u.w)-1-2-utf8.RuneCountInString(suffix)) + var b bytes.Buffer + if !s.drawn { + b.WriteString("\x1b[?25l") // hide the cursor; Stop shows it again + s.drawn = true + } + b.WriteString("\r\x1b[K" + s.u.Cyan(frames[s.frame%len(frames)]) + " " + label + s.u.Dim(suffix)) + s.frame++ + s.u.w.Write(b.Bytes()) +} + +// width is the terminal's column count, 80 when it cannot be read. +func width(w io.Writer) int { + if f, ok := w.(*os.File); ok { + if cols, _, err := term.GetSize(int(f.Fd())); err == nil && cols > 0 { + return cols + } + } + return 80 +} + +// truncate shortens s to n cells with a trailing …. A rune is one cell; wide +// East Asian characters are not special-cased (ponytail: they may overshoot). +func truncate(s string, n int) string { + n = max(n, 4) + if utf8.RuneCountInString(s) <= n { + return s + } + r := []rune(s) + return string(r[:n-1]) + "…" +} + +// Fprintf writes to w like fmt.Fprintf, but when a spinner is drawing on w it +// clears the spinner line first and redraws it after, so a retry note or a +// warning lands on its own line above the spinner. +func Fprintf(w io.Writer, format string, a ...any) { + curMu.Lock() + s := cur + curMu.Unlock() + if s != nil && s.u.w == w { + s.Around(func() { fmt.Fprintf(w, format, a...) }) + return + } + fmt.Fprintf(w, format, a...) +} + +// StopAll stops the running spinner, if any. Execute calls it once the command +// returns so a cancelled run (Ctrl-C) cannot leave the cursor hidden. +func StopAll() { + curMu.Lock() + s := cur + curMu.Unlock() + s.Stop() +} diff --git a/internal/ui/ui_test.go b/internal/ui/ui_test.go new file mode 100644 index 0000000..d6da4e1 --- /dev/null +++ b/internal/ui/ui_test.go @@ -0,0 +1,226 @@ +package ui + +import ( + "bytes" + "io" + "os" + "strings" + "sync" + "testing" + "time" +) + +type syncBuf struct { + mu sync.Mutex + b bytes.Buffer +} + +func (s *syncBuf) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.Write(p) +} + +func (s *syncBuf) String() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.String() +} + +func fast(t *testing.T) { + t.Helper() + d, i := Delay, Interval + Delay, Interval = 20*time.Millisecond, 5*time.Millisecond + t.Cleanup(func() { Delay, Interval = d, i }) +} + +func TestLiveGating(t *testing.T) { + tty := func(*os.File) bool { return true } + notTTY := func(*os.File) bool { return false } + env := func(kv ...string) func(string) string { + m := map[string]string{} + for i := 0; i < len(kv); i += 2 { + m[kv[i]] = kv[i+1] + } + return func(k string) string { return m[k] } + } + cases := []struct { + name string + w io.Writer + env func(string) string + goos string + tty func(*os.File) bool + want bool + }{ + {"terminal", os.Stderr, env(), "linux", tty, true}, + {"not a terminal", os.Stderr, env(), "linux", notTTY, false}, + {"buffer is never live", &bytes.Buffer{}, env(), "linux", tty, false}, + {"NO_COLOR", os.Stderr, env("NO_COLOR", "1"), "linux", tty, false}, + {"empty NO_COLOR is ignored", os.Stderr, env("NO_COLOR", ""), "linux", tty, true}, + {"TERM=dumb", os.Stderr, env("TERM", "dumb"), "linux", tty, false}, + {"TERM=xterm-256color", os.Stderr, env("TERM", "xterm-256color"), "linux", tty, true}, + {"CI=true", os.Stderr, env("CI", "true"), "linux", tty, false}, + {"CI=1", os.Stderr, env("CI", "1"), "linux", tty, false}, + {"CI=false", os.Stderr, env("CI", "false"), "linux", tty, true}, + {"legacy windows console", os.Stderr, env(), "windows", tty, false}, + {"windows terminal", os.Stderr, env("WT_SESSION", "x"), "windows", tty, true}, + } + for _, c := range cases { + if got := live(c.w, c.env, c.goos, c.tty); got != c.want { + t.Errorf("%s: live = %v, want %v", c.name, got, c.want) + } + } +} + +func TestOffIsInert(t *testing.T) { + for _, u := range []*UI{nil, {}, New(&bytes.Buffer{})} { + if u.On() { + t.Fatal("expected off") + } + if got := u.Green("x") + u.Bold("y") + u.Dim("z"); got != "xyz" { + t.Errorf("off UI styled text: %q", got) + } + s := u.Spin("work") + if s != nil { + t.Fatal("off UI returned a spinner") + } + s.Label("x") // nil-safe + s.Around(func() {}) + if s.Stop() != 0 { + t.Error("nil Stop") + } + u.Done(true, "a", "b") + u.Fail("boom", Row{"k", "v"}) + } + var buf bytes.Buffer + New(&buf).Done(true, "x") + if buf.Len() != 0 { + t.Errorf("off UI wrote %q", buf.String()) + } +} + +func TestSpinnerLifecycle(t *testing.T) { + fast(t) + var buf syncBuf + u := Force(&buf) + s := u.Spin("Scraping %s", "example.com") + if s == nil { + t.Fatal("no spinner") + } + if u.Spin("nested") != nil { + t.Error("a second spinner started while one runs") + } + time.Sleep(60 * time.Millisecond) + s.Label("Scraping example.com · rendering JavaScript") + time.Sleep(30 * time.Millisecond) + if d := s.Stop(); d < 60*time.Millisecond { + t.Errorf("Stop returned %s", d) + } + out := buf.String() + for _, want := range []string{"\x1b[?25l", "Scraping example.com", "rendering JavaScript", "s\x1b[0m"} { + if !strings.Contains(out, want) { + t.Errorf("output lacks %q: %q", want, out) + } + } + if !strings.HasSuffix(out, "\r\x1b[K\x1b[?25h") { + t.Errorf("output must end by clearing the line and showing the cursor: %q", out) + } + if strings.Contains(out, "\n") { + t.Errorf("a spinner must never emit a newline: %q", out) + } + if strings.Count(out, "\x1b[?25l") != 1 { + t.Errorf("cursor hidden more than once: %q", out) + } + n := len(out) + s.Stop() // idempotent + time.Sleep(30 * time.Millisecond) + if got := buf.String(); len(got) != n { + t.Errorf("wrote after Stop: %q", got[n:]) + } + if u.Spin("again") == nil { + t.Error("cannot start a spinner after Stop") + } else { + StopAll() + } +} + +func TestFastCallShowsNothing(t *testing.T) { + var buf syncBuf + s := Force(&buf).Spin("quick") + time.Sleep(20 * time.Millisecond) // well under the 300 ms Delay + s.Stop() + if buf.String() != "" { + t.Errorf("a call under the delay drew %q", buf.String()) + } +} + +func TestFprintfClearsTheSpinnerLine(t *testing.T) { + fast(t) + var buf syncBuf + s := Force(&buf).Spin("waiting") + time.Sleep(50 * time.Millisecond) + Fprintf(&buf, "retrying in 1s\n") + s.Stop() + if !strings.Contains(buf.String(), "\r\x1b[Kretrying in 1s\n") { + t.Errorf("the note was not written on a cleared line: %q", buf.String()) + } + // With no spinner it is a plain write. + var plain bytes.Buffer + Fprintf(&plain, "x %d\n", 1) + if plain.String() != "x 1\n" { + t.Errorf("plain Fprintf = %q", plain.String()) + } +} + +func TestStopAllRestoresTheCursor(t *testing.T) { + fast(t) + var buf syncBuf + Force(&buf).Spin("work") + time.Sleep(40 * time.Millisecond) + StopAll() + if !strings.HasSuffix(buf.String(), "\x1b[?25h") { + t.Errorf("cursor not restored: %q", buf.String()) + } +} + +func TestDoneAndFail(t *testing.T) { + var buf bytes.Buffer + u := Force(&buf) + u.Done(true, "https://example.com", "200", "1 credit", "4.2s") + u.Done(false, "https://example.com", "503") + u.Fail("ERR::TARGET::BLOCKED: refused", Row{"hint", "use --render"}, Row{"skipped", ""}, Row{"request", "01J9"}) + out := buf.String() + for _, want := range []string{ + "\x1b[32m✓\x1b[0m https://example.com \x1b[2m200 · 1 credit · 4.2s\x1b[0m\n", + "\x1b[31m✗\x1b[0m https://example.com \x1b[2m503\x1b[0m\n", + "\x1b[31m✗\x1b[0m \x1b[1mERR::TARGET::BLOCKED: refused\x1b[0m\n", + " \x1b[2mhint \x1b[0m use --render\n", + " \x1b[2mrequest\x1b[0m 01J9\n", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q in %q", want, out) + } + } + if strings.Contains(out, "skipped") { + t.Error("an empty row was printed") + } +} + +func TestElapsedAndTruncate(t *testing.T) { + for d, want := range map[time.Duration]string{ + 840 * time.Millisecond: "0.8s", + 12140 * time.Millisecond: "12.1s", + 64 * time.Second: "1m04s", + 601 * time.Second: "10m01s", + } { + if got := Elapsed(d); got != want { + t.Errorf("Elapsed(%s) = %q, want %q", d, got, want) + } + } + if got := truncate("https://example.com/a/very/long/path", 12); got != "https://exa…" { + t.Errorf("truncate = %q", got) + } + if got := truncate("short", 12); got != "short" { + t.Errorf("truncate = %q", got) + } +}