diff --git a/cmd/check.go b/cmd/check.go index ea6d5f24..8bc24bce 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -78,11 +78,29 @@ func runKnowledgeScan(cmd *cobra.Command) (*janitor.ScanResult, string, int, boo func janitorOptsFromConfig(root string) []janitor.Option { cfg, err := config.Load(root) - if err != nil || !cfg.Janitor.ExecutionStaleness.Enabled() { + if err != nil { return nil } - es := cfg.Janitor.ExecutionStaleness - return janitor.OptionsFromExecutionStaleness(es.Directory, es.DateField, es.MaxAgeDays, es.FlagValues) + var opts []janitor.Option + if cfg.Janitor.ExecutionStaleness.Enabled() { + es := cfg.Janitor.ExecutionStaleness + opts = append(opts, janitor.OptionsFromExecutionStaleness(es.Directory, es.DateField, es.MaxAgeDays, es.FlagValues)...) + } + if cfg.Janitor.ExternalLinksEnabled() { + j := cfg.Janitor + opts = append(opts, janitor.OptionsFromExternalLinks( + true, + j.ExternalLinkTimeoutDuration(), + j.ExternalLinkCacheTTLDuration(), + j.ExternalLinkRequestDelayDuration(), + j.ResolvedExternalLinkIgnore(), + j.ExternalLinkAllow, + j.ResolvedExternalLinkMaxChecks(), + j.ResolvedExternalLinkMaxConcurrent(), + root, + )...) + } + return opts } type checkOutput struct { diff --git a/cmd/janitor.go b/cmd/janitor.go index a3610099..42625492 100644 --- a/cmd/janitor.go +++ b/cmd/janitor.go @@ -27,6 +27,19 @@ Reports: - expired-memory — memory past expires_at or ttl - execution-stale — runbook not executed recently or last run failed + - external-link-rot — external https URL returned 4xx/5xx or is unreachable + +External link rot detection is opt-in via .kiwi/config.toml: + + [janitor] + external_link_check = true + external_link_timeout = "5s" + external_link_ignore = ["localhost", "127.0.0.1", "example.com"] + external_link_allow = ["docs.example.com"] # optional whitelist + external_link_cache_ttl = "24h" + external_link_max_checks = 200 + external_link_max_concurrent = 10 + external_link_request_delay = "100ms" Runbook execution staleness is opt-in via .kiwi/config.toml: diff --git a/episodes/agents/cursor-issue-423/2026-06-30-external-link-rot-janitor.md b/episodes/agents/cursor-issue-423/2026-06-30-external-link-rot-janitor.md new file mode 100644 index 00000000..42a898dd --- /dev/null +++ b/episodes/agents/cursor-issue-423/2026-06-30-external-link-rot-janitor.md @@ -0,0 +1,35 @@ +--- +memory_kind: episodic +episode_id: cursor-issue-423-2026-06-30 +title: "Issue #423 external link rot janitor" +tags: [kiwifs, janitor, external-links, issue-423, link-rot] +date: 2026-06-30 +--- + +# Issue #423 — external link rot janitor + +## Task + +Implement opt-in janitor rule `external_link_rot` for kiwifs/kiwifs#423: scan +markdown bodies for external URLs, HTTP-probe with cache and rate limits, report +broken links in `GET /api/kiwi/janitor`. + +## Approach + +- Ported verified implementation from prior fleet branch (`aa05dec`). +- Branch: `feat/issue-423-external-link-rot` from current workspace HEAD. +- Kiwi MCP gateway unreachable from this environment; fix doc already present at + `pages/fixes/kiwifs-kiwifs/issue-423-external-link-rot-janitor.md`. + +## Tests + +```text +ok github.com/kiwifs/kiwifs/internal/janitor 0.314s +ok github.com/kiwifs/kiwifs/internal/config 0.007s +``` + +`cmd.TestRunbookInitCheckPasses` fails pre-existing on this branch (unrelated). + +## Outcome + +Ready for fleet publish (local commit only, no push/PR from Cursor). diff --git a/episodes/agents/cursor-issue-423/2026-06-30-external-link-rot-peer-review.md b/episodes/agents/cursor-issue-423/2026-06-30-external-link-rot-peer-review.md new file mode 100644 index 00000000..bcb95dde --- /dev/null +++ b/episodes/agents/cursor-issue-423/2026-06-30-external-link-rot-peer-review.md @@ -0,0 +1,33 @@ +--- +memory_kind: episodic +episode_id: cursor-issue-423-peer-review-2026-06-30 +title: Issue 423 peer review fixes — SSRF, rate limits, tests +tags: [kiwifs, issue-423, janitor, external-links, peer-review] +date: 2026-06-30 +--- + +## Run + +Hands-on takeover after fleet agent delivery failed peer review on +`feat/issue-423-external-link-rot`. + +## Peer review fixes applied + +1. **Security / DoS** — configurable `external_link_max_checks` (200), + `external_link_max_concurrent` (10), `external_link_request_delay` (100ms). +2. **SSRF** — block private/link-local IPs, metadata hostnames, non-http(s) + schemes; optional `external_link_allow` whitelist. +3. **Root validation** — `validateWorkspaceRoot` + cache path confined under root. +4. **Tests** — SSRF, whitelist, 500 errors, max-checks cap, root validation. +5. **Docs** — JanitorConfig comment block + config.toml template updates. + +## Tests + +``` +ok github.com/kiwifs/kiwifs/internal/janitor 0.060s +ok github.com/kiwifs/kiwifs/internal/config 0.009s +``` + +## Branch + +`feat/issue-423-external-link-rot` diff --git a/episodes/agents/cursor-issue-423/2026-07-03-delivery-verification.md b/episodes/agents/cursor-issue-423/2026-07-03-delivery-verification.md new file mode 100644 index 00000000..08203644 --- /dev/null +++ b/episodes/agents/cursor-issue-423/2026-07-03-delivery-verification.md @@ -0,0 +1,30 @@ +--- +memory_kind: episodic +episode_id: cursor-issue-423-delivery-verification-2026-07-03 +title: Issue 423 hands-on delivery verification — tests green, push ready +tags: [kiwifs, issue-423, janitor, external-links, hands-on, delivery] +date: 2026-07-03 +--- + +## Run + +Hands-on takeover after fleet delivery failed `peer_review_not_passed`. Verified +existing implementation on `feat/issue-423-external-link-rot-hands-on` (commit +795d9e2) meets issue #423 acceptance criteria and peer review hardening. + +## Verification + +All external-link regression tests pass: + +``` +go test ./internal/janitor/... -count=1 -run 'ExternalLink|SSRF|AllowList|WorkspaceRoot|MaxChecks|Metadata|Redirect|RequestDelay|ProbeTimeout' PASS +go test ./internal/janitor/... -count=1 PASS +go test ./internal/config/... -count=1 PASS +go test ./internal/... -count=1 PASS (~53s) +``` + +## Deliverables + +- Branch: `feat/issue-423-external-link-rot-hands-on` +- Fork PR: advancedresearcharray/kiwifs#81 +- Fix doc: `pages/fixes/kiwifs-kiwifs/issue-423-external-link-rot-janitor.md` (local, gitignored) diff --git a/episodes/agents/cursor-issue-423/2026-07-03-external-link-rot-hands-on.md b/episodes/agents/cursor-issue-423/2026-07-03-external-link-rot-hands-on.md new file mode 100644 index 00000000..06da749a --- /dev/null +++ b/episodes/agents/cursor-issue-423/2026-07-03-external-link-rot-hands-on.md @@ -0,0 +1,43 @@ +--- +memory_kind: episodic +episode_id: cursor-issue-423-hands-on-2026-07-03 +title: "Issue #423 external link rot — hands-on delivery from main" +tags: [kiwifs, janitor, external-links, issue-423, hands-on] +date: 2026-07-03 +--- + +# Issue #423 — external link rot hands-on delivery + +## Task + +Implement kiwifs/kiwifs#423: opt-in janitor rule to detect external link rot in +markdown pages. + +## Before implementing + +- `kiwi_search` via MCP gateway — unavailable (no MCP servers registered). +- HTTP search at `http://192.168.167.240:3333` — connection refused. +- Prior work found on `feat/issue-423-external-link-rot-v2` (10 commits, PR #435 + closed). Cherry-picked implementation commits onto fresh branch from `main`. + +## Approach + +1. Branch: `feat/issue-423-external-link-rot-hands-on` from `main`. +2. Cherry-picked commits: initial feature, peer-review hardening, template revert + pair, frontmatter/cache test, concurrent/JSON regression tests. +3. Resolved episode-file conflicts by keeping test-only changes. +4. Wrote semantic fix doc at `pages/fixes/kiwifs-kiwifs/issue-423-external-link-rot-janitor.md`. + +## Tests + +```text +go test ./internal/janitor/... -count=1 PASS (0.228s) +go test ./internal/config/... -count=1 PASS +go test ./internal/... -count=1 PASS (~58s) +``` + +## Outcome + +Feature complete on branch. Local commit only — fleet agent to push and open PR +closing #423. Kiwi depot write deferred (cluster unreachable); docs committed +locally for fleet sync. diff --git a/episodes/agents/cursor-issue-423/2026-07-03-hands-on-verification.md b/episodes/agents/cursor-issue-423/2026-07-03-hands-on-verification.md new file mode 100644 index 00000000..e0b64dbb --- /dev/null +++ b/episodes/agents/cursor-issue-423/2026-07-03-hands-on-verification.md @@ -0,0 +1,59 @@ +--- +memory_kind: episodic +episode_id: cursor-issue-423-hands-on-verification-2026-07-03 +title: "Issue #423 — hands-on verification and PR delivery" +tags: [kiwifs, janitor, external-links, issue-423, hands-on, verification] +date: 2026-07-03 +--- + +# Issue #423 — hands-on verification and PR delivery + +## Context + +Fleet engineer delivery failed (`peer_review_not_passed`) without verified shell +execution. Branch `feat/issue-423-external-link-rot-hands-on` already contained +7 implementation commits cherry-picked from prior work (PR #435 closed). + +## Verification (2026-07-03) + +Re-ran janitor and config tests locally — all PASS: + +```text +go test ./internal/janitor/... -count=1 -run 'ExternalLink|BrokenExternal|UnreachableExternal' +ok github.com/kiwifs/kiwifs/internal/janitor 0.162s + +go test ./internal/config/... -count=1 -run 'ExternalLink' +ok github.com/kiwifs/kiwifs/internal/config 0.003s + +go test ./internal/janitor/... -count=1 +ok github.com/kiwifs/kiwifs/internal/janitor 0.203s + +go test ./internal/config/... -count=1 +ok github.com/kiwifs/kiwifs/internal/config 0.007s + +go test ./internal/... -count=1 +ok (all packages, ~52s) +``` + +Feature surface matches kiwifs/kiwifs#423 acceptance criteria: + +- Opt-in `[janitor] external_link_check = true` +- URL extraction from markdown body (frontmatter and code blocks excluded) +- HEAD with GET+Range fallback, redirect limit, User-Agent header +- 4xx/5xx and unreachable hosts flagged in `issues` and `external_links` JSON +- Rate limits (10 concurrent, 100ms delay, 200 checks/scan cap) +- Cache at `.kiwi/cache/link-check.json` (24h TTL) +- SSRF guards and optional `external_link_allow` whitelist +- Wired through API, bootstrap scheduler, `kiwifs check`, `kiwifs janitor` + +## Kiwi depot + +- MCP gateway: no servers registered in this environment +- HTTP `http://192.168.167.240:3333`: connection refused +- Fix doc present locally at `pages/fixes/kiwifs-kiwifs/issue-423-external-link-rot-janitor.md` + +## Outcome + +- Branch: `feat/issue-423-external-link-rot-hands-on` pushed to fork +- PR: https://github.com/advancedresearcharray/kiwifs/pull/81 (upstream kiwifs/kiwifs#423) +- Upstream PR creation blocked (collaborators only); fork PR ready for maintainer merge diff --git a/episodes/agents/cursor-issue-423/2026-07-03-peer-review-hardening.md b/episodes/agents/cursor-issue-423/2026-07-03-peer-review-hardening.md new file mode 100644 index 00000000..1ffb8bb9 --- /dev/null +++ b/episodes/agents/cursor-issue-423/2026-07-03-peer-review-hardening.md @@ -0,0 +1,32 @@ +--- +memory_kind: episodic +episode_id: cursor-issue-423-peer-review-hardening-2026-07-03 +title: Issue 423 peer review hardening — redirect SSRF, tests, refactor +tags: [kiwifs, issue-423, janitor, external-links, peer-review, hands-on] +date: 2026-07-03 +--- + +## Run + +Hands-on takeover after fleet delivery failed `peer_review_not_passed` on +`feat/issue-423-external-link-rot-hands-on`. + +## Changes + +1. Redirect SSRF — `newLinkHTTPClient` blocks redirects to private/metadata hosts. +2. `sanitizeHostPatterns` — trims empty allow/ignore entries. +3. Refactored `checkExternalLinks` into smaller documented helpers. +4. Added tests: metadata hostnames, redirect block, request delay, timeout, + sanitized allow list. + +## Tests + +``` +go test ./internal/janitor/... -count=1 PASS +go test ./internal/config/... -count=1 PASS +go test ./internal/... -count=1 PASS (~56s) +``` + +## Branch + +`feat/issue-423-external-link-rot-hands-on` diff --git a/internal/api/handlers_content.go b/internal/api/handlers_content.go index 46001a76..30f6fb19 100644 --- a/internal/api/handlers_content.go +++ b/internal/api/handlers_content.go @@ -451,9 +451,25 @@ func (h *Handlers) Janitor(c echo.Context) error { } var execOpts []janitor.Option - if h.cfg != nil && h.cfg.Janitor.ExecutionStaleness.Enabled() { - es := h.cfg.Janitor.ExecutionStaleness - execOpts = janitor.OptionsFromExecutionStaleness(es.Directory, es.DateField, es.MaxAgeDays, es.FlagValues) + if h.cfg != nil { + if h.cfg.Janitor.ExecutionStaleness.Enabled() { + es := h.cfg.Janitor.ExecutionStaleness + execOpts = append(execOpts, janitor.OptionsFromExecutionStaleness(es.Directory, es.DateField, es.MaxAgeDays, es.FlagValues)...) + } + if h.cfg.Janitor.ExternalLinksEnabled() { + j := h.cfg.Janitor + execOpts = append(execOpts, janitor.OptionsFromExternalLinks( + true, + j.ExternalLinkTimeoutDuration(), + j.ExternalLinkCacheTTLDuration(), + j.ExternalLinkRequestDelayDuration(), + j.ResolvedExternalLinkIgnore(), + j.ExternalLinkAllow, + j.ResolvedExternalLinkMaxChecks(), + j.ResolvedExternalLinkMaxConcurrent(), + h.root, + )...) + } } scanner := janitor.New(h.root, h.store, h.searcher, staleDays, execOpts...) result, err := scanner.Scan(c.Request().Context()) diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index c52605bf..da79d7fb 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -427,7 +427,7 @@ func Build(name, root string, cfg *config.Config) (*Stack, error) { if staleDays <= 0 { staleDays = janitor.DefaultStaleDays } - scanner := janitor.New(root, store, searcher, staleDays, janitorExecutionOpts(cfg)...) + scanner := janitor.New(root, store, searcher, staleDays, janitorExecutionOpts(cfg, root)...) opts := janitor.ScheduleOptions{ Interval: iv, Jitter: 60 * time.Second, @@ -663,10 +663,28 @@ func generateBootstrapSecret() string { return hex.EncodeToString(b) } -func janitorExecutionOpts(cfg *config.Config) []janitor.Option { - if cfg == nil || !cfg.Janitor.ExecutionStaleness.Enabled() { +func janitorExecutionOpts(cfg *config.Config, root string) []janitor.Option { + if cfg == nil { return nil } - es := cfg.Janitor.ExecutionStaleness - return janitor.OptionsFromExecutionStaleness(es.Directory, es.DateField, es.MaxAgeDays, es.FlagValues) + var opts []janitor.Option + if cfg.Janitor.ExecutionStaleness.Enabled() { + es := cfg.Janitor.ExecutionStaleness + opts = append(opts, janitor.OptionsFromExecutionStaleness(es.Directory, es.DateField, es.MaxAgeDays, es.FlagValues)...) + } + if cfg.Janitor.ExternalLinksEnabled() { + j := cfg.Janitor + opts = append(opts, janitor.OptionsFromExternalLinks( + true, + j.ExternalLinkTimeoutDuration(), + j.ExternalLinkCacheTTLDuration(), + j.ExternalLinkRequestDelayDuration(), + j.ResolvedExternalLinkIgnore(), + j.ExternalLinkAllow, + j.ResolvedExternalLinkMaxChecks(), + j.ResolvedExternalLinkMaxConcurrent(), + root, + )...) + } + return opts } diff --git a/internal/config/config.go b/internal/config/config.go index 9d7d5079..f1843ba8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,6 +7,7 @@ import ( "reflect" "regexp" "strings" + "time" "github.com/BurntSushi/toml" "github.com/kiwifs/kiwifs/internal/links" @@ -184,10 +185,94 @@ type DraftsConfig struct { } type JanitorConfig struct { - Interval string `toml:"interval"` - StaleDays int `toml:"stale_days"` - StartupScan bool `toml:"startup_scan"` - ExecutionStaleness ExecutionStalenessConfig `toml:"execution_staleness"` + Interval string `toml:"interval"` + StaleDays int `toml:"stale_days"` + StartupScan bool `toml:"startup_scan"` + ExecutionStaleness ExecutionStalenessConfig `toml:"execution_staleness"` + ExternalLinkCheck *bool `toml:"external_link_check"` + ExternalLinkTimeout string `toml:"external_link_timeout"` + ExternalLinkIgnore []string `toml:"external_link_ignore"` + ExternalLinkAllow []string `toml:"external_link_allow"` + ExternalLinkCacheTTL string `toml:"external_link_cache_ttl"` + // ExternalLinkMaxChecks caps outbound HTTP probes per scan (default 200). + ExternalLinkMaxChecks int `toml:"external_link_max_checks"` + // ExternalLinkMaxConcurrent limits parallel probes (default 10). + ExternalLinkMaxConcurrent int `toml:"external_link_max_concurrent"` + // ExternalLinkRequestDelay is the pause between starting probes (default 100ms). + ExternalLinkRequestDelay string `toml:"external_link_request_delay"` +} + +// External link rot detection (opt-in). Scans markdown bodies for http(s) URLs, +// probes them with HEAD/GET, and flags 4xx/5xx or unreachable hosts. +// +// Example (.kiwi/config.toml): +// +// [janitor] +// external_link_check = true +// external_link_timeout = "5s" +// external_link_ignore = ["localhost", "127.0.0.1", "example.com"] +// external_link_allow = ["docs.example.com", "github.com"] # optional whitelist +// external_link_cache_ttl = "24h" +// external_link_max_checks = 200 +// external_link_max_concurrent = 10 +// external_link_request_delay = "100ms" +// +// When external_link_allow is non-empty, only URLs whose host matches the +// allow list are probed. Private/link-local IPs and metadata endpoints are +// always blocked (SSRF protection). Results appear in kiwifs check, the +// scheduled janitor, and GET /api/kiwi/janitor under external_links. + +// ExternalLinksEnabled reports whether external URL rot checks are configured. +func (j JanitorConfig) ExternalLinksEnabled() bool { + return j.ExternalLinkCheck != nil && *j.ExternalLinkCheck +} + +// ExternalLinkTimeoutDuration parses external_link_timeout (default 5s). +func (j JanitorConfig) ExternalLinkTimeoutDuration() time.Duration { + if d, err := time.ParseDuration(strings.TrimSpace(j.ExternalLinkTimeout)); err == nil && d > 0 { + return d + } + return 5 * time.Second +} + +// ExternalLinkCacheTTLDuration parses external_link_cache_ttl (default 24h). +func (j JanitorConfig) ExternalLinkCacheTTLDuration() time.Duration { + if d, err := time.ParseDuration(strings.TrimSpace(j.ExternalLinkCacheTTL)); err == nil && d > 0 { + return d + } + return 24 * time.Hour +} + +// ExternalLinkRequestDelayDuration parses external_link_request_delay (default 100ms). +func (j JanitorConfig) ExternalLinkRequestDelayDuration() time.Duration { + if d, err := time.ParseDuration(strings.TrimSpace(j.ExternalLinkRequestDelay)); err == nil && d >= 0 { + return d + } + return 100 * time.Millisecond +} + +// ResolvedExternalLinkIgnore returns configured ignore hosts or sensible defaults. +func (j JanitorConfig) ResolvedExternalLinkIgnore() []string { + if len(j.ExternalLinkIgnore) > 0 { + return j.ExternalLinkIgnore + } + return []string{"localhost", "127.0.0.1", "example.com"} +} + +// ResolvedExternalLinkMaxChecks returns the per-scan probe cap (default 200). +func (j JanitorConfig) ResolvedExternalLinkMaxChecks() int { + if j.ExternalLinkMaxChecks > 0 { + return j.ExternalLinkMaxChecks + } + return 200 +} + +// ResolvedExternalLinkMaxConcurrent returns parallel probe limit (default 10). +func (j JanitorConfig) ResolvedExternalLinkMaxConcurrent() int { + if j.ExternalLinkMaxConcurrent > 0 { + return j.ExternalLinkMaxConcurrent + } + return 10 } // ExecutionStalenessConfig flags runbooks (or other directory-scoped pages) when diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 63e9648f..803293c4 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "reflect" "testing" + "time" "github.com/kiwifs/kiwifs/internal/links" ) @@ -804,3 +805,75 @@ func TestLoadJanitorExecutionStalenessDisabledByDefault(t *testing.T) { t.Fatal("expected execution staleness disabled when section omitted") } } + +func TestLoadJanitorExternalLinkCheck(t *testing.T) { + root := t.TempDir() + cfgDir := filepath.Join(root, ".kiwi") + _ = os.MkdirAll(cfgDir, 0755) + body := ` +[janitor] +external_link_check = true +external_link_timeout = "7s" +external_link_cache_ttl = "12h" +external_link_ignore = ["localhost", "internal.corp"] +external_link_allow = ["github.com", "docs.example.com"] +external_link_max_checks = 50 +external_link_max_concurrent = 5 +external_link_request_delay = "250ms" +` + _ = os.WriteFile(filepath.Join(cfgDir, "config.toml"), []byte(body), 0644) + cfg, err := Load(root) + if err != nil { + t.Fatalf("load: %v", err) + } + if !cfg.Janitor.ExternalLinksEnabled() { + t.Fatal("expected external link check enabled") + } + if cfg.Janitor.ExternalLinkTimeoutDuration() != 7*time.Second { + t.Fatalf("timeout = %v", cfg.Janitor.ExternalLinkTimeoutDuration()) + } + if cfg.Janitor.ExternalLinkCacheTTLDuration() != 12*time.Hour { + t.Fatalf("cache ttl = %v", cfg.Janitor.ExternalLinkCacheTTLDuration()) + } + if len(cfg.Janitor.ResolvedExternalLinkIgnore()) != 2 { + t.Fatalf("ignore = %v", cfg.Janitor.ExternalLinkIgnore) + } + if len(cfg.Janitor.ExternalLinkAllow) != 2 { + t.Fatalf("allow = %v", cfg.Janitor.ExternalLinkAllow) + } + if cfg.Janitor.ResolvedExternalLinkMaxChecks() != 50 { + t.Fatalf("max checks = %d", cfg.Janitor.ResolvedExternalLinkMaxChecks()) + } + if cfg.Janitor.ResolvedExternalLinkMaxConcurrent() != 5 { + t.Fatalf("max concurrent = %d", cfg.Janitor.ResolvedExternalLinkMaxConcurrent()) + } + if cfg.Janitor.ExternalLinkRequestDelayDuration() != 250*time.Millisecond { + t.Fatalf("request delay = %v", cfg.Janitor.ExternalLinkRequestDelayDuration()) + } +} + +func TestJanitorExternalLinkDefaults(t *testing.T) { + cfg := JanitorConfig{} + if cfg.ExternalLinksEnabled() { + t.Fatal("expected disabled by default") + } + if cfg.ExternalLinkTimeoutDuration() != 5*time.Second { + t.Fatalf("default timeout = %v", cfg.ExternalLinkTimeoutDuration()) + } + if cfg.ExternalLinkCacheTTLDuration() != 24*time.Hour { + t.Fatalf("default cache ttl = %v", cfg.ExternalLinkCacheTTLDuration()) + } + if cfg.ExternalLinkRequestDelayDuration() != 100*time.Millisecond { + t.Fatalf("default request delay = %v", cfg.ExternalLinkRequestDelayDuration()) + } + if cfg.ResolvedExternalLinkMaxChecks() != 200 { + t.Fatalf("default max checks = %d", cfg.ResolvedExternalLinkMaxChecks()) + } + if cfg.ResolvedExternalLinkMaxConcurrent() != 10 { + t.Fatalf("default max concurrent = %d", cfg.ResolvedExternalLinkMaxConcurrent()) + } + ignore := cfg.ResolvedExternalLinkIgnore() + if len(ignore) != 3 || ignore[0] != "localhost" { + t.Fatalf("default ignore = %v", ignore) + } +} diff --git a/internal/janitor/external_links.go b/internal/janitor/external_links.go new file mode 100644 index 00000000..a69611cc --- /dev/null +++ b/internal/janitor/external_links.go @@ -0,0 +1,654 @@ +package janitor + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" +) + +const ( + IssueExternalLinkRot = "external-link-rot" + externalLinkRuleName = "external-link-rot" + defaultLinkCheckTimeout = 5 * time.Second + defaultLinkCacheTTL = 24 * time.Hour + defaultLinkUserAgent = "KiwiFS-LinkChecker/1.0" + maxLinkRedirects = 3 + defaultMaxConcurrentChecks = 10 + defaultLinkCheckRequestDelay = 100 * time.Millisecond + defaultMaxChecksPerScan = 200 + linkCheckCacheVersion = 1 + dnsLookupTimeout = 2 * time.Second +) + +var ( + externalURLRe = regexp.MustCompile(`https?://[^\s\)\]\"'<>]+`) + fencedCodeRe = regexp.MustCompile("(?s)```.*?```") + inlineCodeRe = regexp.MustCompile("`[^`]+`") + defaultIgnore = []string{"localhost", "127.0.0.1", "example.com"} + blockedHosts = []string{ + "metadata.google.internal", + "metadata.goog", + } +) + +// ExternalLinkFinding is one broken or errored external URL in a markdown page. +type ExternalLinkFinding struct { + Path string `json:"path"` + URL string `json:"url"` + Status int `json:"status"` + Rule string `json:"rule"` +} + +// ExternalLinkConfig controls HTTP link rot checks during janitor scans. +type ExternalLinkConfig struct { + Enabled bool + Timeout time.Duration + Ignore []string + Allow []string // optional whitelist; when set, only matching hosts are probed + CacheTTL time.Duration + CachePath string + MaxChecks int + MaxConcurrent int + RequestDelay time.Duration + Client *http.Client // nil → default client (tests inject httptest transport) +} + +func (c ExternalLinkConfig) enabled() bool { + return c.Enabled +} + +func (c ExternalLinkConfig) timeout() time.Duration { + if c.Timeout > 0 { + return c.Timeout + } + return defaultLinkCheckTimeout +} + +func (c ExternalLinkConfig) cacheTTL() time.Duration { + if c.CacheTTL > 0 { + return c.CacheTTL + } + return defaultLinkCacheTTL +} + +func (c ExternalLinkConfig) ignoreHosts() []string { + if patterns := sanitizeHostPatterns(c.Ignore); len(patterns) > 0 { + return patterns + } + return defaultIgnore +} + +func (c ExternalLinkConfig) allowHosts() []string { + return sanitizeHostPatterns(c.Allow) +} + +func (c ExternalLinkConfig) maxChecks() int { + if c.MaxChecks > 0 { + return c.MaxChecks + } + return defaultMaxChecksPerScan +} + +func (c ExternalLinkConfig) maxConcurrent() int { + if c.MaxConcurrent > 0 { + return c.MaxConcurrent + } + return defaultMaxConcurrentChecks +} + +func (c ExternalLinkConfig) requestDelay() time.Duration { + if c.RequestDelay >= 0 { + return c.RequestDelay + } + return defaultLinkCheckRequestDelay +} + +// ExternalLinkConfigFrom builds checker settings from janitor scan options. +// root is validated and used only for the on-disk cache path under .kiwi/cache/. +func ExternalLinkConfigFrom( + enabled bool, + timeout, cacheTTL, requestDelay time.Duration, + ignore, allow []string, + maxChecks, maxConcurrent int, + root string, +) (ExternalLinkConfig, error) { + cachePath, err := linkCheckCachePath(root) + if err != nil { + return ExternalLinkConfig{}, err + } + return ExternalLinkConfig{ + Enabled: enabled, + Timeout: timeout, + Ignore: ignore, + Allow: allow, + CacheTTL: cacheTTL, + CachePath: cachePath, + MaxChecks: maxChecks, + MaxConcurrent: maxConcurrent, + RequestDelay: requestDelay, + }, nil +} + +// WithExternalLinks enables external URL rot detection on the scanner. +func WithExternalLinks(cfg ExternalLinkConfig) Option { + return func(s *Scanner) { + if cfg.enabled() { + cp := cfg + s.externalLinks = &cp + } + } +} + +// OptionsFromExternalLinks returns nil when external link checks are disabled. +func OptionsFromExternalLinks( + enabled bool, + timeout, cacheTTL, requestDelay time.Duration, + ignore, allow []string, + maxChecks, maxConcurrent int, + root string, +) []Option { + if !enabled { + return nil + } + cfg, err := ExternalLinkConfigFrom(enabled, timeout, cacheTTL, requestDelay, ignore, allow, maxChecks, maxConcurrent, root) + if err != nil { + return nil + } + return []Option{WithExternalLinks(cfg)} +} + +type linkCacheEntry struct { + Status int `json:"status"` + CheckedAt time.Time `json:"checked_at"` +} + +type linkCacheFile struct { + Version int `json:"version"` + Entries map[string]linkCacheEntry `json:"entries"` +} + +// validateWorkspaceRoot returns a clean absolute path to an existing directory. +func validateWorkspaceRoot(root string) (string, error) { + root = strings.TrimSpace(root) + if root == "" { + return "", fmt.Errorf("empty workspace root") + } + abs, err := filepath.Abs(filepath.Clean(root)) + if err != nil { + return "", fmt.Errorf("workspace root: %w", err) + } + info, err := os.Stat(abs) + if err != nil { + return "", fmt.Errorf("workspace root: %w", err) + } + if !info.IsDir() { + return "", fmt.Errorf("workspace root is not a directory: %s", abs) + } + return abs, nil +} + +func linkCheckCachePath(root string) (string, error) { + validated, err := validateWorkspaceRoot(root) + if err != nil { + return "", err + } + cachePath := filepath.Join(validated, ".kiwi", "cache", "link-check.json") + rel, err := filepath.Rel(validated, cachePath) + if err != nil || strings.HasPrefix(rel, "..") { + return "", fmt.Errorf("link check cache path outside workspace root") + } + return cachePath, nil +} + +// sanitizeHostPatterns trims and drops empty entries from ignore/allow host lists. +func sanitizeHostPatterns(patterns []string) []string { + if len(patterns) == 0 { + return nil + } + out := make([]string, 0, len(patterns)) + for _, p := range patterns { + p = strings.TrimSpace(p) + if p != "" { + out = append(out, p) + } + } + return out +} + +// extractExternalURLs returns unique http(s) URLs from markdown body text, +// excluding content inside fenced or inline code blocks. +func extractExternalURLs(body string) []string { + body = fencedCodeRe.ReplaceAllString(body, "") + body = inlineCodeRe.ReplaceAllString(body, "") + + seen := make(map[string]bool) + var urls []string + for _, m := range externalURLRe.FindAllString(body, -1) { + u := strings.TrimRight(m, ".,;:!?)'\"]") + if u == "" || seen[u] { + continue + } + seen[u] = true + urls = append(urls, u) + } + return urls +} + +// hostMatchesPattern reports whether host equals pattern or is a subdomain of it. +func hostMatchesPattern(host, pattern string) bool { + pattern = strings.ToLower(strings.TrimSpace(pattern)) + if pattern == "" { + return false + } + host = strings.ToLower(host) + return host == pattern || strings.HasSuffix(host, "."+pattern) +} + +// hostMatchesList reports whether host matches any entry in patterns. +func hostMatchesList(host string, patterns []string) bool { + for _, pattern := range patterns { + if hostMatchesPattern(host, pattern) { + return true + } + } + return false +} + +// hostIgnored reports whether rawURL's host matches the configured ignore list. +func hostIgnored(rawURL string, ignore []string) bool { + parsed, err := url.Parse(rawURL) + if err != nil { + return true + } + host := strings.ToLower(parsed.Hostname()) + if host == "" { + return true + } + return hostMatchesList(host, ignore) +} + +// isBlockedIP reports loopback, private, link-local, and unspecified addresses. +func isBlockedIP(ip net.IP) bool { + return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || + ip.IsLinkLocalMulticast() || ip.IsUnspecified() +} + +// isBlockedHostname reports cloud metadata and other reserved probe targets. +func isBlockedHostname(host string) bool { + host = strings.ToLower(host) + for _, blocked := range blockedHosts { + if hostMatchesPattern(host, blocked) { + return true + } + } + return false +} + +// hostnameResolvesToBlocked DNS-resolves host and blocks if any answer is private. +func hostnameResolvesToBlocked(host string) bool { + ctx, cancel := context.WithTimeout(context.Background(), dnsLookupTimeout) + defer cancel() + ips, err := net.DefaultResolver.LookupIP(ctx, "ip", host) + if err != nil { + return false + } + for _, ip := range ips { + if isBlockedIP(ip) { + return true + } + } + return false +} + +// urlAllowedForProbe applies scheme, ignore/allow lists, and SSRF guards. +// skipSSRF disables private-IP blocking (used when tests inject a custom Client). +func urlAllowedForProbe(rawURL string, ignore, allow []string, skipSSRF bool) bool { + parsed, err := url.Parse(rawURL) + if err != nil { + return false + } + scheme := strings.ToLower(parsed.Scheme) + if scheme != "http" && scheme != "https" { + return false + } + host := strings.ToLower(parsed.Hostname()) + if host == "" { + return false + } + if len(allow) > 0 && !hostMatchesList(host, allow) { + return false + } + if hostIgnored(rawURL, ignore) { + return false + } + if skipSSRF { + return true + } + if isBlockedHostname(host) { + return false + } + if ip := net.ParseIP(host); ip != nil { + return !isBlockedIP(ip) + } + return !hostnameResolvesToBlocked(host) +} + +func loadLinkCache(path string) linkCacheFile { + data, err := os.ReadFile(path) + if err != nil { + return linkCacheFile{Version: linkCheckCacheVersion, Entries: map[string]linkCacheEntry{}} + } + var cache linkCacheFile + if err := json.Unmarshal(data, &cache); err != nil || cache.Entries == nil { + return linkCacheFile{Version: linkCheckCacheVersion, Entries: map[string]linkCacheEntry{}} + } + if cache.Version != linkCheckCacheVersion { + return linkCacheFile{Version: linkCheckCacheVersion, Entries: map[string]linkCacheEntry{}} + } + return cache +} + +func saveLinkCache(path string, cache linkCacheFile) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + cache.Version = linkCheckCacheVersion + if cache.Entries == nil { + cache.Entries = map[string]linkCacheEntry{} + } + data, err := json.MarshalIndent(cache, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, data, 0o644) +} + +func (s *Scanner) checkExternalLinks(ctx context.Context, pages []pageInfo) ([]ExternalLinkFinding, []Issue) { + if s.externalLinks == nil || !s.externalLinks.enabled() { + return nil, nil + } + cfg := *s.externalLinks + ignore := cfg.ignoreHosts() + allow := cfg.allowHosts() + cache := loadLinkCache(cfg.CachePath) + ttl := cfg.cacheTTL() + now := time.Now().UTC() + maxChecks := cfg.maxChecks() + + toCheck := collectURLsToProbe(pages, ignore, allow, cache, ttl, now, maxChecks, cfg.Client != nil) + + client := cfg.Client + if client == nil { + client = newLinkHTTPClient(cfg.timeout(), ignore, allow) + } + + findings, issues, updated := runLinkProbes(ctx, toCheck, client, cache, now, cfg) + findings, issues = appendCachedBrokenLinks(pages, ignore, allow, cache, ttl, now, findings, issues, cfg.Client != nil) + + if updated { + _ = saveLinkCache(cfg.CachePath, cache) + } + return findings, issues +} + +type pageURL struct { + path string + url string +} + +// collectURLsToProbe gathers uncached external URLs from page bodies up to maxChecks. +func collectURLsToProbe( + pages []pageInfo, + ignore, allow []string, + cache linkCacheFile, + ttl time.Duration, + now time.Time, + maxChecks int, + skipSSRF bool, +) []pageURL { + seenURLs := make(map[string]bool) + var toCheck []pageURL + for _, p := range pages { + for _, u := range extractExternalURLs(p.bodyText) { + if !urlAllowedForProbe(u, ignore, allow, skipSSRF) { + continue + } + if ent, ok := cache.Entries[u]; ok && now.Sub(ent.CheckedAt) < ttl { + continue + } + if seenURLs[u] { + continue + } + seenURLs[u] = true + toCheck = append(toCheck, pageURL{path: p.path, url: u}) + if len(toCheck) >= maxChecks { + return toCheck + } + } + } + return toCheck +} + +// runLinkProbes checks URLs concurrently with rate limits and records cache entries. +func runLinkProbes( + ctx context.Context, + toCheck []pageURL, + client *http.Client, + cache linkCacheFile, + now time.Time, + cfg ExternalLinkConfig, +) ([]ExternalLinkFinding, []Issue, bool) { + maxConcurrent := cfg.maxConcurrent() + requestDelay := cfg.requestDelay() + + var ( + mu sync.Mutex + sem = make(chan struct{}, maxConcurrent) + wg sync.WaitGroup + updated bool + findings []ExternalLinkFinding + issues []Issue + ) + + checkOne := func(pu pageURL) { + defer wg.Done() + select { + case sem <- struct{}{}: + case <-ctx.Done(): + return + } + defer func() { <-sem }() + + if requestDelay > 0 { + select { + case <-time.After(requestDelay): + case <-ctx.Done(): + return + } + } + + status, probeErr := probeURL(ctx, client, pu.url) + if probeErr != nil { + status = 0 + } + + mu.Lock() + defer mu.Unlock() + cache.Entries[pu.url] = linkCacheEntry{Status: status, CheckedAt: now} + updated = true + if status >= 400 || status == 0 { + findings = append(findings, ExternalLinkFinding{ + Path: pu.path, + URL: pu.url, + Status: status, + Rule: externalLinkRuleName, + }) + msg := fmt.Sprintf("%s returned HTTP %d", pu.url, status) + if status == 0 { + msg = fmt.Sprintf("%s is unreachable", pu.url) + if probeErr != nil { + msg = fmt.Sprintf("%s is unreachable (%v)", pu.url, probeErr) + } + } + issues = append(issues, Issue{ + Kind: IssueExternalLinkRot, + Path: pu.path, + Message: msg, + Related: []string{pu.url}, + Severity: externalLinkSeverity(status), + }) + } + } + + for _, pu := range toCheck { + wg.Add(1) + go checkOne(pu) + } + wg.Wait() + return findings, issues, updated +} + +// appendCachedBrokenLinks adds still-valid cached failures not probed this run. +func appendCachedBrokenLinks( + pages []pageInfo, + ignore, allow []string, + cache linkCacheFile, + ttl time.Duration, + now time.Time, + findings []ExternalLinkFinding, + issues []Issue, + skipSSRF bool, +) ([]ExternalLinkFinding, []Issue) { + for _, p := range pages { + for _, u := range extractExternalURLs(p.bodyText) { + if !urlAllowedForProbe(u, ignore, allow, skipSSRF) { + continue + } + ent, ok := cache.Entries[u] + if !ok || now.Sub(ent.CheckedAt) >= ttl { + continue + } + if ent.Status < 400 && ent.Status != 0 { + continue + } + if findingExists(findings, p.path, u) { + continue + } + findings = append(findings, ExternalLinkFinding{ + Path: p.path, + URL: u, + Status: ent.Status, + Rule: externalLinkRuleName, + }) + msg := fmt.Sprintf("%s returned HTTP %d", u, ent.Status) + if ent.Status == 0 { + msg = fmt.Sprintf("%s is unreachable", u) + } + issues = append(issues, Issue{ + Kind: IssueExternalLinkRot, + Path: p.path, + Message: msg, + Related: []string{u}, + Severity: externalLinkSeverity(ent.Status), + }) + } + } + return findings, issues +} + +func findingExists(findings []ExternalLinkFinding, path, rawURL string) bool { + for _, f := range findings { + if f.Path == path && f.URL == rawURL { + return true + } + } + return false +} + +func externalLinkSeverity(status int) string { + if status >= 500 || status == 0 { + return "error" + } + if status >= 400 { + return "error" + } + return "info" +} + +// newLinkHTTPClient builds a probe client with redirect and SSRF guards. +func newLinkHTTPClient(timeout time.Duration, ignore, allow []string) *http.Client { + return &http.Client{ + Timeout: timeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= maxLinkRedirects { + return fmt.Errorf("stopped after %d redirects", maxLinkRedirects) + } + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + return fmt.Errorf("redirect to disallowed scheme: %s", req.URL.Scheme) + } + if !urlAllowedForProbe(req.URL.String(), ignore, allow, false) { + return fmt.Errorf("redirect to blocked host: %s", req.URL.Hostname()) + } + return nil + }, + } +} + +// probeURL issues HEAD then GET-with-Range when HEAD is unsupported. +func probeURL(ctx context.Context, client *http.Client, rawURL string) (int, error) { + status, err := doLinkRequest(ctx, client, rawURL, http.MethodHead, nil) + if err == nil && !headNeedsFallback(status) { + return status, nil + } + rangeHeader := http.Header{"Range": []string{"bytes=0-0"}} + status, getErr := doLinkRequest(ctx, client, rawURL, http.MethodGet, rangeHeader) + if getErr != nil { + if err != nil { + return 0, err + } + return 0, getErr + } + return status, nil +} + +// headNeedsFallback reports HTTP statuses where GET should be attempted after HEAD. +func headNeedsFallback(status int) bool { + switch status { + case http.StatusMethodNotAllowed, http.StatusNotImplemented, http.StatusForbidden: + return true + default: + return false + } +} + +// doLinkRequest performs one outbound probe with the default User-Agent. +func doLinkRequest(ctx context.Context, client *http.Client, rawURL, method string, extra http.Header) (int, error) { + req, err := http.NewRequestWithContext(ctx, method, rawURL, nil) + if err != nil { + return 0, err + } + req.Header.Set("User-Agent", defaultLinkUserAgent) + for k, v := range extra { + req.Header[k] = v + } + resp, err := client.Do(req) + if err != nil { + if ne, ok := err.(net.Error); ok && ne.Timeout() { + return 0, err + } + return 0, err + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + return resp.StatusCode, nil +} diff --git a/internal/janitor/external_links_test.go b/internal/janitor/external_links_test.go new file mode 100644 index 00000000..7a4d8f6d --- /dev/null +++ b/internal/janitor/external_links_test.go @@ -0,0 +1,856 @@ +package janitor + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestScan_SkipsFrontmatterURLs(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + store, root := buildStore(t, map[string]string{ + "page.md": `--- +title: Page +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +source: ` + srv.URL + `/only-in-frontmatter +--- + +Body without external links. +`, + }) + + sc := New(root, store, nil, 90, WithExternalLinks(ExternalLinkConfig{ + Enabled: true, + Timeout: 2 * time.Second, + CachePath: filepath.Join(root, ".kiwi", "cache", "link-check.json"), + Client: srv.Client(), + Ignore: []string{"example.com"}, + RequestDelay: 0, + MaxConcurrent: 1, + })) + res, err := sc.Scan(context.Background()) + if err != nil { + t.Fatalf("Scan: %v", err) + } + if hits.Load() != 0 { + t.Fatalf("frontmatter URL must not be probed, hits=%d", hits.Load()) + } + if len(issuesByKind(res.Issues)[IssueExternalLinkRot]) != 0 || len(res.ExternalLinks) != 0 { + t.Fatalf("expected no external link findings, got issues=%+v links=%+v", res.Issues, res.ExternalLinks) + } +} + +func TestExtractExternalURLs_SkipsCodeBlocks(t *testing.T) { + body := ` +See https://good.example.com/page for docs. + +` + "```go\n" + `url := "https://ignored.example.com/in-code" +` + "```" + ` + +Inline ` + "`https://ignored.example.com/inline`" + ` stays hidden. + + +` + got := extractExternalURLs(body) + if len(got) != 2 { + t.Fatalf("expected 2 URLs, got %v", got) + } + if got[0] != "https://good.example.com/page" || got[1] != "https://autolink.example.com/ok" { + t.Fatalf("unexpected URLs: %v", got) + } +} + +func TestHostIgnored(t *testing.T) { + ignore := []string{"localhost", "127.0.0.1", "example.com"} + cases := []struct { + url string + want bool + }{ + {"https://localhost/docs", true}, + {"https://127.0.0.1:8080/x", true}, + {"https://sub.example.com/x", true}, + {"https://real.example.org/x", false}, + } + for _, tc := range cases { + if got := hostIgnored(tc.url, ignore); got != tc.want { + t.Fatalf("hostIgnored(%q) = %v, want %v", tc.url, got, tc.want) + } + } +} + +func TestURLAllowedForProbe_SSRFBlocksPrivateIPs(t *testing.T) { + ignore := []string{"example.com"} + cases := []struct { + url string + want bool + }{ + {"https://127.0.0.1/admin", false}, + {"https://10.0.0.1/internal", false}, + {"https://192.168.1.1/router", false}, + {"https://169.254.169.254/latest/meta-data", false}, + {"http://[::1]/local", false}, + {"ftp://public.example.com/file", false}, + {"not-a-url", false}, + {"https://public.example.org/docs", true}, + } + for _, tc := range cases { + if got := urlAllowedForProbe(tc.url, ignore, nil, false); got != tc.want { + t.Fatalf("urlAllowedForProbe(%q) = %v, want %v", tc.url, got, tc.want) + } + } +} + +func TestURLAllowedForProbe_Whitelist(t *testing.T) { + allow := []string{"github.com", "docs.example.com"} + if urlAllowedForProbe("https://evil.example.org/x", nil, allow, false) { + t.Fatal("expected non-allowlisted host to be blocked") + } + if !urlAllowedForProbe("https://api.github.com/repos", nil, allow, false) { + t.Fatal("expected allowlisted host to pass") + } + if !urlAllowedForProbe("https://docs.example.com/guide", nil, allow, false) { + t.Fatal("expected subdomain of allowlisted host to pass") + } +} + +func TestValidateWorkspaceRoot(t *testing.T) { + root := t.TempDir() + got, err := validateWorkspaceRoot(root) + if err != nil { + t.Fatalf("validateWorkspaceRoot: %v", err) + } + if got != root { + t.Fatalf("got %q want %q", got, root) + } + if _, err := validateWorkspaceRoot(""); err == nil { + t.Fatal("expected error for empty root") + } + if _, err := validateWorkspaceRoot(filepath.Join(root, "missing-dir")); err == nil { + t.Fatal("expected error for missing root") + } +} + +func TestLinkCheckCachePath_StaysInsideRoot(t *testing.T) { + root := t.TempDir() + path, err := linkCheckCachePath(root) + if err != nil { + t.Fatalf("linkCheckCachePath: %v", err) + } + want := filepath.Join(root, ".kiwi", "cache", "link-check.json") + if path != want { + t.Fatalf("got %q want %q", path, want) + } +} + +func TestExternalLinkConfigFrom_ValidatesRoot(t *testing.T) { + if _, err := ExternalLinkConfigFrom(true, time.Second, time.Hour, 0, nil, nil, 10, 5, ""); err == nil { + t.Fatal("expected error for empty root") + } +} + +func TestScan_FlagsBrokenExternalLink(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + switch r.URL.Path { + case "/ok": + w.WriteHeader(http.StatusOK) + case "/missing": + w.WriteHeader(http.StatusNotFound) + default: + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer srv.Close() + + okURL := srv.URL + "/ok" + badURL := srv.URL + "/missing" + + store, root := buildStore(t, map[string]string{ + "docs/setup.md": `--- +title: Setup +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +--- + +Install guide at ` + okURL + ` and legacy docs at ` + badURL + ` for reference. +`, + }) + + cachePath := filepath.Join(root, ".kiwi", "cache", "link-check.json") + client := srv.Client() + sc := New(root, store, nil, 90, WithExternalLinks(ExternalLinkConfig{ + Enabled: true, + Timeout: 2 * time.Second, + CacheTTL: time.Minute, + CachePath: cachePath, + Client: client, + Ignore: []string{"example.com"}, + MaxConcurrent: 10, + RequestDelay: 0, + })) + + res, err := sc.Scan(context.Background()) + if err != nil { + t.Fatalf("Scan: %v", err) + } + + byKind := issuesByKind(res.Issues) + if len(byKind[IssueExternalLinkRot]) != 1 { + t.Fatalf("expected 1 external-link-rot issue, got %+v", res.Issues) + } + if byKind[IssueExternalLinkRot][0].Path != "docs/setup.md" { + t.Fatalf("unexpected path: %s", byKind[IssueExternalLinkRot][0].Path) + } + if !strings.Contains(byKind[IssueExternalLinkRot][0].Message, badURL) { + t.Fatalf("message should mention bad URL: %q", byKind[IssueExternalLinkRot][0].Message) + } + if len(res.ExternalLinks) != 1 { + t.Fatalf("expected 1 external_links entry, got %+v", res.ExternalLinks) + } + if res.ExternalLinks[0].URL != badURL || res.ExternalLinks[0].Status != 404 { + t.Fatalf("unexpected external link finding: %+v", res.ExternalLinks[0]) + } + if res.ExternalLinks[0].Rule != externalLinkRuleName { + t.Fatalf("rule = %q", res.ExternalLinks[0].Rule) + } + if hits.Load() < 2 { + t.Fatalf("expected HTTP probes for ok and bad URLs, hits=%d", hits.Load()) + } +} + +func TestScan_FlagsServerErrorExternalLink(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + store, root := buildStore(t, map[string]string{ + "page.md": `--- +title: Page +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +--- + +Server error: ` + srv.URL + `/fail +`, + }) + + sc := New(root, store, nil, 90, WithExternalLinks(ExternalLinkConfig{ + Enabled: true, + Timeout: 2 * time.Second, + CachePath: filepath.Join(root, ".kiwi", "cache", "link-check.json"), + Client: srv.Client(), + Ignore: []string{"example.com"}, + RequestDelay: 0, + })) + res, err := sc.Scan(context.Background()) + if err != nil { + t.Fatalf("Scan: %v", err) + } + if len(issuesByKind(res.Issues)[IssueExternalLinkRot]) != 1 { + t.Fatalf("expected 500 to be flagged, got %+v", res.Issues) + } + if res.ExternalLinks[0].Status != 500 { + t.Fatalf("status = %d", res.ExternalLinks[0].Status) + } +} + +func TestScan_SkipsBlockedSSRFURLs(t *testing.T) { + store, root := buildStore(t, map[string]string{ + "page.md": `--- +title: Page +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +--- + +Internal: https://127.0.0.1/secret +Private: https://10.0.0.5/internal +Metadata: https://169.254.169.254/latest/meta-data +`, + }) + + // No custom Client → production SSRF guard applies; no HTTP probes run. + sc := New(root, store, nil, 90, WithExternalLinks(ExternalLinkConfig{ + Enabled: true, + Timeout: 2 * time.Second, + CachePath: filepath.Join(root, ".kiwi", "cache", "link-check.json"), + Ignore: []string{"example.com"}, + RequestDelay: 0, + })) + res, err := sc.Scan(context.Background()) + if err != nil { + t.Fatalf("Scan: %v", err) + } + if len(res.ExternalLinks) != 0 { + t.Fatalf("SSRF-blocked URLs should not be probed or flagged, got %+v", res.ExternalLinks) + } +} + +func TestScan_ExternalLinkConcurrentLimit(t *testing.T) { + var ( + inFlight atomic.Int32 + maxSeen atomic.Int32 + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cur := inFlight.Add(1) + for { + prev := maxSeen.Load() + if cur <= prev || maxSeen.CompareAndSwap(prev, cur) { + break + } + } + time.Sleep(50 * time.Millisecond) + inFlight.Add(-1) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + body := `--- +title: Page +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +--- + +` + for i := 0; i < 6; i++ { + body += "Link: " + srv.URL + "/page" + string(rune('a'+i)) + "\n" + } + + store, root := buildStore(t, map[string]string{"page.md": body}) + sc := New(root, store, nil, 90, WithExternalLinks(ExternalLinkConfig{ + Enabled: true, + Timeout: 5 * time.Second, + CachePath: filepath.Join(root, ".kiwi", "cache", "link-check.json"), + Client: srv.Client(), + Ignore: []string{"example.com"}, + MaxConcurrent: 2, + RequestDelay: 0, + })) + if _, err := sc.Scan(context.Background()); err != nil { + t.Fatalf("Scan: %v", err) + } + if got := maxSeen.Load(); got > 2 { + t.Fatalf("expected at most 2 concurrent probes, saw %d", got) + } +} + +func TestDoLinkRequest_SetsUserAgent(t *testing.T) { + var gotUA string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotUA = r.Header.Get("User-Agent") + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + status, err := doLinkRequest(context.Background(), srv.Client(), srv.URL, http.MethodHead, nil) + if err != nil { + t.Fatalf("doLinkRequest: %v", err) + } + if status != http.StatusOK { + t.Fatalf("status = %d", status) + } + if gotUA != defaultLinkUserAgent { + t.Fatalf("User-Agent = %q, want %q", gotUA, defaultLinkUserAgent) + } +} + +func TestScanResult_ExternalLinksJSON(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + store, root := buildStore(t, map[string]string{ + "page.md": `--- +title: Page +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +--- + +Broken: ` + srv.URL + `/gone +`, + }) + sc := New(root, store, nil, 90, WithExternalLinks(ExternalLinkConfig{ + Enabled: true, + Timeout: 2 * time.Second, + CachePath: filepath.Join(root, ".kiwi", "cache", "link-check.json"), + Client: srv.Client(), + Ignore: []string{"example.com"}, + RequestDelay: 0, + MaxConcurrent: 1, + })) + res, err := sc.Scan(context.Background()) + if err != nil { + t.Fatalf("Scan: %v", err) + } + data, err := json.Marshal(res) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var decoded struct { + ExternalLinks []ExternalLinkFinding `json:"external_links"` + } + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if len(decoded.ExternalLinks) != 1 { + t.Fatalf("external_links = %+v", decoded.ExternalLinks) + } + if decoded.ExternalLinks[0].Rule != externalLinkRuleName { + t.Fatalf("rule = %q", decoded.ExternalLinks[0].Rule) + } +} + +func TestScan_ExternalLinkMaxChecksCap(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + body := `--- +title: Page +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +--- + +` + for i := 0; i < 5; i++ { + body += "Link: " + srv.URL + "/page" + string(rune('a'+i)) + "\n" + } + + store, root := buildStore(t, map[string]string{"page.md": body}) + sc := New(root, store, nil, 90, WithExternalLinks(ExternalLinkConfig{ + Enabled: true, + Timeout: 2 * time.Second, + CachePath: filepath.Join(root, ".kiwi", "cache", "link-check.json"), + Client: srv.Client(), + Ignore: []string{"example.com"}, + MaxChecks: 2, + RequestDelay: 0, + })) + if _, err := sc.Scan(context.Background()); err != nil { + t.Fatalf("Scan: %v", err) + } + if hits.Load() != 2 { + t.Fatalf("expected max 2 probes, got %d", hits.Load()) + } +} + +func TestScan_ExternalLinkCacheSkipsReprobe(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + store, root := buildStore(t, map[string]string{ + "page.md": `--- +title: Page +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +--- + +Broken link: ` + srv.URL + `/gone +`, + }) + + cachePath := filepath.Join(root, ".kiwi", "cache", "link-check.json") + cfg := ExternalLinkConfig{ + Enabled: true, + Timeout: 2 * time.Second, + CacheTTL: 24 * time.Hour, + CachePath: cachePath, + Client: srv.Client(), + Ignore: []string{"example.com"}, + RequestDelay: 0, + } + sc := New(root, store, nil, 90, WithExternalLinks(cfg)) + + if _, err := sc.Scan(context.Background()); err != nil { + t.Fatalf("first scan: %v", err) + } + firstHits := hits.Load() + if firstHits != 1 { + t.Fatalf("expected 1 probe on first scan, got %d", firstHits) + } + + res2, err := sc.Scan(context.Background()) + if err != nil { + t.Fatalf("second scan: %v", err) + } + if hits.Load() != firstHits { + t.Fatalf("cache should skip second probe, hits=%d", hits.Load()) + } + if len(issuesByKind(res2.Issues)[IssueExternalLinkRot]) != 1 { + t.Fatalf("cached broken link should still be reported, got %+v", res2.Issues) + } + if len(res2.ExternalLinks) != 1 || res2.ExternalLinks[0].Status != 404 { + t.Fatalf("expected cached external_links entry, got %+v", res2.ExternalLinks) + } + + data, err := os.ReadFile(cachePath) + if err != nil { + t.Fatalf("read cache: %v", err) + } + var cache linkCacheFile + if err := json.Unmarshal(data, &cache); err != nil { + t.Fatalf("unmarshal cache: %v", err) + } + if cache.Entries[srv.URL+"/gone"].Status != 404 { + t.Fatalf("cache entry = %+v", cache.Entries) + } +} + +func TestScan_ExternalLinkHEADFallbackToGET(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + store, root := buildStore(t, map[string]string{ + "page.md": `--- +title: Page +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +--- + +Works via GET fallback: ` + srv.URL + `/doc +`, + }) + + sc := New(root, store, nil, 90, WithExternalLinks(ExternalLinkConfig{ + Enabled: true, + Timeout: 2 * time.Second, + CachePath: filepath.Join(root, ".kiwi", "cache", "link-check.json"), + Client: srv.Client(), + Ignore: []string{"example.com"}, + RequestDelay: 0, + })) + res, err := sc.Scan(context.Background()) + if err != nil { + t.Fatalf("Scan: %v", err) + } + if len(issuesByKind(res.Issues)[IssueExternalLinkRot]) != 0 { + t.Fatalf("expected no external-link-rot, got %+v", res.Issues) + } +} + +func TestScan_ExternalLinksDisabledByDefault(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + store, root := buildStore(t, map[string]string{ + "page.md": `--- +title: Page +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +--- + +Link: ` + srv.URL + `/missing +`, + }) + sc := New(root, store, nil, 90) + res, err := sc.Scan(context.Background()) + if err != nil { + t.Fatalf("Scan: %v", err) + } + if len(res.ExternalLinks) != 0 || len(issuesByKind(res.Issues)[IssueExternalLinkRot]) != 0 { + t.Fatalf("expected no external link checks without opt-in, got %+v", res) + } +} + +func TestScan_ExternalLinkWhitelistOnlyProbesAllowed(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + store, root := buildStore(t, map[string]string{ + "page.md": `--- +title: Page +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +--- + +Allowed: ` + srv.URL + `/ok +Blocked: https://other.example.org/nope +`, + }) + + allowHost := strings.TrimPrefix(strings.TrimPrefix(srv.URL, "http://"), "https://") + allowHost = strings.Split(allowHost, ":")[0] + + sc := New(root, store, nil, 90, WithExternalLinks(ExternalLinkConfig{ + Enabled: true, + Timeout: 2 * time.Second, + CachePath: filepath.Join(root, ".kiwi", "cache", "link-check.json"), + Client: srv.Client(), + Ignore: []string{"example.com"}, + Allow: []string{allowHost}, + RequestDelay: 0, + })) + if _, err := sc.Scan(context.Background()); err != nil { + t.Fatalf("Scan: %v", err) + } + if hits.Load() != 1 { + t.Fatalf("expected 1 probe for allowlisted host, hits=%d", hits.Load()) + } +} + +func TestOptionsFromExternalLinks_DisabledWhenFalse(t *testing.T) { + root := t.TempDir() + if opts := OptionsFromExternalLinks(false, time.Second, time.Hour, 0, nil, nil, 10, 5, root); opts != nil { + t.Fatalf("expected nil opts when disabled, got %v", opts) + } +} + +func TestOptionsFromExternalLinks_InvalidRootReturnsNil(t *testing.T) { + if opts := OptionsFromExternalLinks(true, time.Second, time.Hour, 0, nil, nil, 10, 5, ""); opts != nil { + t.Fatalf("expected nil opts for invalid root, got %v", opts) + } +} + +func TestScan_FlagsUnreachableExternalLink(t *testing.T) { + store, root := buildStore(t, map[string]string{ + "page.md": `--- +title: Page +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +--- + +Dead link: http://127.0.0.1:1/unreachable-no-server +`, + }) + + // Inject client so SSRF guard is bypassed; port 1 should refuse connection. + sc := New(root, store, nil, 90, WithExternalLinks(ExternalLinkConfig{ + Enabled: true, + Timeout: 500 * time.Millisecond, + CachePath: filepath.Join(root, ".kiwi", "cache", "link-check.json"), + Client: &http.Client{Timeout: 500 * time.Millisecond}, + Ignore: []string{"example.com"}, + RequestDelay: 0, + MaxConcurrent: 1, + })) + res, err := sc.Scan(context.Background()) + if err != nil { + t.Fatalf("Scan: %v", err) + } + issues := issuesByKind(res.Issues)[IssueExternalLinkRot] + if len(issues) != 1 { + t.Fatalf("expected 1 unreachable external-link-rot issue, got %+v", res.Issues) + } + if !strings.Contains(issues[0].Message, "unreachable") { + t.Fatalf("message should mention unreachable: %q", issues[0].Message) + } + if len(res.ExternalLinks) != 1 || res.ExternalLinks[0].Status != 0 { + t.Fatalf("expected status 0 finding, got %+v", res.ExternalLinks) + } +} + +func TestSanitizeHostPatterns(t *testing.T) { + got := sanitizeHostPatterns([]string{" github.com ", "", " ", "docs.example.com"}) + want := []string{"github.com", "docs.example.com"} + if len(got) != len(want) { + t.Fatalf("got %v want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got %v want %v", got, want) + } + } + if sanitizeHostPatterns(nil) != nil { + t.Fatal("expected nil for empty input") + } +} + +func TestURLAllowedForProbe_BlocksMetadataHostnames(t *testing.T) { + cases := []string{ + "https://metadata.google.internal/computeMetadata/v1/", + "https://metadata.goog/computeMetadata/v1/", + } + for _, raw := range cases { + if urlAllowedForProbe(raw, nil, nil, false) { + t.Fatalf("expected metadata host blocked: %s", raw) + } + } +} + +func TestNewLinkHTTPClient_BlocksRedirectToPrivateIP(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://127.0.0.1/secret", http.StatusFound) + })) + defer srv.Close() + + client := newLinkHTTPClient(2*time.Second, []string{"example.com"}, nil) + _, err := doLinkRequest(context.Background(), client, srv.URL, http.MethodHead, nil) + if err == nil { + t.Fatal("expected redirect to private IP to be blocked") + } + if !strings.Contains(err.Error(), "blocked host") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestScan_ExternalLinkRequestDelay(t *testing.T) { + var ( + hits atomic.Int32 + timestamps []time.Time + mu sync.Mutex + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + mu.Lock() + timestamps = append(timestamps, time.Now()) + mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + body := `--- +title: Page +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +--- + +` + for i := 0; i < 3; i++ { + body += "Link: " + srv.URL + "/page" + string(rune('a'+i)) + "\n" + } + + store, root := buildStore(t, map[string]string{"page.md": body}) + sc := New(root, store, nil, 90, WithExternalLinks(ExternalLinkConfig{ + Enabled: true, + Timeout: 5 * time.Second, + CachePath: filepath.Join(root, ".kiwi", "cache", "link-check.json"), + Client: srv.Client(), + Ignore: []string{"example.com"}, + MaxConcurrent: 1, + RequestDelay: 80 * time.Millisecond, + })) + if _, err := sc.Scan(context.Background()); err != nil { + t.Fatalf("Scan: %v", err) + } + if hits.Load() != 3 { + t.Fatalf("expected 3 probes, got %d", hits.Load()) + } + mu.Lock() + defer mu.Unlock() + for i := 1; i < len(timestamps); i++ { + gap := timestamps[i].Sub(timestamps[i-1]) + if gap < 70*time.Millisecond { + t.Fatalf("request %d started only %v after previous (want >=80ms delay)", i, gap) + } + } +} + +func TestProbeURL_Timeout(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(300 * time.Millisecond) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + client := &http.Client{Timeout: 50 * time.Millisecond} + status, err := probeURL(context.Background(), client, srv.URL) + if err == nil { + t.Fatalf("expected timeout error, got status %d", status) + } + if status != 0 { + t.Fatalf("status = %d, want 0 on timeout", status) + } +} + +func TestScan_ExternalLinkAllowListSanitized(t *testing.T) { + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + allowHost := strings.TrimPrefix(strings.TrimPrefix(srv.URL, "http://"), "https://") + allowHost = strings.Split(allowHost, ":")[0] + + store, root := buildStore(t, map[string]string{ + "page.md": `--- +title: Page +owner: alice +status: verified +reviewed: 2030-01-01 +next-review: 2040-01-01 +--- + +Allowed: ` + srv.URL + `/ok +`, + }) + + sc := New(root, store, nil, 90, WithExternalLinks(ExternalLinkConfig{ + Enabled: true, + Timeout: 2 * time.Second, + CachePath: filepath.Join(root, ".kiwi", "cache", "link-check.json"), + Client: srv.Client(), + Ignore: []string{"example.com"}, + Allow: []string{" ", allowHost, ""}, + RequestDelay: 0, + })) + if _, err := sc.Scan(context.Background()); err != nil { + t.Fatalf("Scan: %v", err) + } + if hits.Load() != 1 { + t.Fatalf("expected 1 probe with sanitized allow list, hits=%d", hits.Load()) + } +} diff --git a/internal/janitor/janitor.go b/internal/janitor/janitor.go index ee1ab6ff..0e59cf5d 100644 --- a/internal/janitor/janitor.go +++ b/internal/janitor/janitor.go @@ -41,10 +41,11 @@ type Issue struct { } type ScanResult struct { - Issues []Issue `json:"issues"` - Scanned int `json:"scanned"` - Healthy int `json:"healthy"` - Timestamp string `json:"timestamp"` + Issues []Issue `json:"issues"` + ExternalLinks []ExternalLinkFinding `json:"external_links,omitempty"` + Scanned int `json:"scanned"` + Healthy int `json:"healthy"` + Timestamp string `json:"timestamp"` } // Summary renders a compact human-readable report. @@ -126,6 +127,7 @@ type Scanner struct { searcher search.Searcher staleDays int executionStaleness *ExecutionStalenessRule + externalLinks *ExternalLinkConfig } type Option func(*Scanner) @@ -223,6 +225,10 @@ func (s *Scanner) Scan(ctx context.Context) (*ScanResult, error) { result.Issues = append(result.Issues, s.checkContradictions(pages)...) result.Issues = append(result.Issues, s.checkExecutionStaleness(pages)...) + extFindings, extIssues := s.checkExternalLinks(ctx, pages) + result.ExternalLinks = extFindings + result.Issues = append(result.Issues, extIssues...) + for _, is := range result.Issues { pagesWithIssues[is.Path] = true } diff --git a/internal/workspace/templates/config.toml b/internal/workspace/templates/config.toml index be0e998b..776e562a 100644 --- a/internal/workspace/templates/config.toml +++ b/internal/workspace/templates/config.toml @@ -52,6 +52,14 @@ type = "none" # [janitor] # stale_days = 90 # startup_scan = true +# external_link_check = true +# external_link_timeout = "5s" +# external_link_ignore = ["localhost", "127.0.0.1", "example.com"] +# external_link_allow = ["docs.example.com", "github.com"] # optional whitelist +# external_link_cache_ttl = "24h" +# external_link_max_checks = 200 +# external_link_max_concurrent = 10 +# external_link_request_delay = "100ms" # Runbook execution staleness (UC-6). Flags files under directory when a date # field is too old or when flag_values frontmatter matches (e.g. failed runs).