feat(api): external link rot detection in janitor

cmd/check.go

@@ -78,11 +78,29 @@ func runKnowledgeScan(cmd *cobra.Command) (*janitor.ScanResult, string, int, boo
 
 func janitorOptsFromConfig(root string) []janitor.Option {
 	cfg, err := config.Load(root)
-	if err != nil || !cfg.Janitor.ExecutionStaleness.Enabled() {
+	if err != nil {
 		return nil
 	}
-	es := cfg.Janitor.ExecutionStaleness
-	return janitor.OptionsFromExecutionStaleness(es.Directory, es.DateField, es.MaxAgeDays, es.FlagValues)
+	var opts []janitor.Option
+	if cfg.Janitor.ExecutionStaleness.Enabled() {
+		es := cfg.Janitor.ExecutionStaleness
+		opts = append(opts, janitor.OptionsFromExecutionStaleness(es.Directory, es.DateField, es.MaxAgeDays, es.FlagValues)...)
+	}
+	if cfg.Janitor.ExternalLinksEnabled() {
+		j := cfg.Janitor
+		opts = append(opts, janitor.OptionsFromExternalLinks(
+			true,
+			j.ExternalLinkTimeoutDuration(),
+			j.ExternalLinkCacheTTLDuration(),
+			j.ExternalLinkRequestDelayDuration(),
+			j.ResolvedExternalLinkIgnore(),
+			j.ExternalLinkAllow,
+			j.ResolvedExternalLinkMaxChecks(),
+			j.ResolvedExternalLinkMaxConcurrent(),
+			root,
+		)...)
+	}
+	return opts
 }
 
 type checkOutput struct {

cmd/janitor.go

@@ -27,6 +27,19 @@ Reports:
 
   - expired-memory — memory past expires_at or ttl
   - execution-stale — runbook not executed recently or last run failed
+  - external-link-rot — external https URL returned 4xx/5xx or is unreachable
+
+External link rot detection is opt-in via .kiwi/config.toml:
+
+  [janitor]
+  external_link_check = true
+  external_link_timeout = "5s"
+  external_link_ignore = ["localhost", "127.0.0.1", "example.com"]
+  external_link_allow = ["docs.example.com"]  # optional whitelist
+  external_link_cache_ttl = "24h"
+  external_link_max_checks = 200
+  external_link_max_concurrent = 10
+  external_link_request_delay = "100ms"
 
 Runbook execution staleness is opt-in via .kiwi/config.toml:
 

episodes/agents/cursor-issue-423/2026-06-30-external-link-rot-janitor.md

@@ -0,0 +1,35 @@
+---
+memory_kind: episodic
+episode_id: cursor-issue-423-2026-06-30
+title: "Issue #423 external link rot janitor"
+tags: [kiwifs, janitor, external-links, issue-423, link-rot]
+date: 2026-06-30
+---
+
+# Issue #423 — external link rot janitor
+
+## Task
+
+Implement opt-in janitor rule `external_link_rot` for kiwifs/kiwifs#423: scan
+markdown bodies for external URLs, HTTP-probe with cache and rate limits, report
+broken links in `GET /api/kiwi/janitor`.
+
+## Approach
+
+- Ported verified implementation from prior fleet branch (`aa05dec`).
+- Branch: `feat/issue-423-external-link-rot` from current workspace HEAD.
+- Kiwi MCP gateway unreachable from this environment; fix doc already present at
+  `pages/fixes/kiwifs-kiwifs/issue-423-external-link-rot-janitor.md`.
+
+## Tests
+
+```text
+ok  github.com/kiwifs/kiwifs/internal/janitor  0.314s
+ok  github.com/kiwifs/kiwifs/internal/config   0.007s
+```
+
+`cmd.TestRunbookInitCheckPasses` fails pre-existing on this branch (unrelated).
+
+## Outcome
+
+Ready for fleet publish (local commit only, no push/PR from Cursor).

episodes/agents/cursor-issue-423/2026-06-30-external-link-rot-peer-review.md

@@ -0,0 +1,33 @@
+---
+memory_kind: episodic
+episode_id: cursor-issue-423-peer-review-2026-06-30
+title: Issue 423 peer review fixes — SSRF, rate limits, tests
+tags: [kiwifs, issue-423, janitor, external-links, peer-review]
+date: 2026-06-30
+---
+
+## Run
+
+Hands-on takeover after fleet agent delivery failed peer review on
+`feat/issue-423-external-link-rot`.
+
+## Peer review fixes applied
+
+1. **Security / DoS** — configurable `external_link_max_checks` (200),
+   `external_link_max_concurrent` (10), `external_link_request_delay` (100ms).
+2. **SSRF** — block private/link-local IPs, metadata hostnames, non-http(s)
+   schemes; optional `external_link_allow` whitelist.
+3. **Root validation** — `validateWorkspaceRoot` + cache path confined under root.
+4. **Tests** — SSRF, whitelist, 500 errors, max-checks cap, root validation.
+5. **Docs** — JanitorConfig comment block + config.toml template updates.
+
+## Tests
+
+```
+ok  github.com/kiwifs/kiwifs/internal/janitor  0.060s
+ok  github.com/kiwifs/kiwifs/internal/config   0.009s
+```
+
+## Branch
+
+`feat/issue-423-external-link-rot`

episodes/agents/cursor-issue-423/2026-07-03-delivery-verification.md

@@ -0,0 +1,30 @@
+---
+memory_kind: episodic
+episode_id: cursor-issue-423-delivery-verification-2026-07-03
+title: Issue 423 hands-on delivery verification — tests green, push ready
+tags: [kiwifs, issue-423, janitor, external-links, hands-on, delivery]
+date: 2026-07-03
+---
+
+## Run
+
+Hands-on takeover after fleet delivery failed `peer_review_not_passed`. Verified
+existing implementation on `feat/issue-423-external-link-rot-hands-on` (commit
+795d9e2) meets issue #423 acceptance criteria and peer review hardening.
+
+## Verification
+
+All external-link regression tests pass:
+
+```
+go test ./internal/janitor/... -count=1 -run 'ExternalLink|SSRF|AllowList|WorkspaceRoot|MaxChecks|Metadata|Redirect|RequestDelay|ProbeTimeout'  PASS
+go test ./internal/janitor/... -count=1   PASS
+go test ./internal/config/... -count=1    PASS
+go test ./internal/... -count=1           PASS (~53s)
+```
+
+## Deliverables
+
+- Branch: `feat/issue-423-external-link-rot-hands-on`
+- Fork PR: advancedresearcharray/kiwifs#81
+- Fix doc: `pages/fixes/kiwifs-kiwifs/issue-423-external-link-rot-janitor.md` (local, gitignored)

episodes/agents/cursor-issue-423/2026-07-03-external-link-rot-hands-on.md

@@ -0,0 +1,43 @@
+---
+memory_kind: episodic
+episode_id: cursor-issue-423-hands-on-2026-07-03
+title: "Issue #423 external link rot — hands-on delivery from main"
+tags: [kiwifs, janitor, external-links, issue-423, hands-on]
+date: 2026-07-03
+---
+
+# Issue #423 — external link rot hands-on delivery
+
+## Task
+
+Implement kiwifs/kiwifs#423: opt-in janitor rule to detect external link rot in
+markdown pages.
+
+## Before implementing
+
+- `kiwi_search` via MCP gateway — unavailable (no MCP servers registered).
+- HTTP search at `http://192.168.167.240:3333` — connection refused.
+- Prior work found on `feat/issue-423-external-link-rot-v2` (10 commits, PR #435
+  closed). Cherry-picked implementation commits onto fresh branch from `main`.
+
+## Approach
+
+1. Branch: `feat/issue-423-external-link-rot-hands-on` from `main`.
+2. Cherry-picked commits: initial feature, peer-review hardening, template revert
+   pair, frontmatter/cache test, concurrent/JSON regression tests.
+3. Resolved episode-file conflicts by keeping test-only changes.
+4. Wrote semantic fix doc at `pages/fixes/kiwifs-kiwifs/issue-423-external-link-rot-janitor.md`.
+
+## Tests
+
+```text
+go test ./internal/janitor/... -count=1   PASS (0.228s)
+go test ./internal/config/... -count=1    PASS
+go test ./internal/... -count=1             PASS (~58s)
+```
+
+## Outcome
+
+Feature complete on branch. Local commit only — fleet agent to push and open PR
+closing #423. Kiwi depot write deferred (cluster unreachable); docs committed
+locally for fleet sync.

episodes/agents/cursor-issue-423/2026-07-03-hands-on-verification.md

@@ -0,0 +1,59 @@
+---
+memory_kind: episodic
+episode_id: cursor-issue-423-hands-on-verification-2026-07-03
+title: "Issue #423 — hands-on verification and PR delivery"
+tags: [kiwifs, janitor, external-links, issue-423, hands-on, verification]
+date: 2026-07-03
+---
+
+# Issue #423 — hands-on verification and PR delivery
+
+## Context
+
+Fleet engineer delivery failed (`peer_review_not_passed`) without verified shell
+execution. Branch `feat/issue-423-external-link-rot-hands-on` already contained
+7 implementation commits cherry-picked from prior work (PR #435 closed).
+
+## Verification (2026-07-03)
+
+Re-ran janitor and config tests locally — all PASS:
+
+```text
+go test ./internal/janitor/... -count=1 -run 'ExternalLink|BrokenExternal|UnreachableExternal'
+ok  github.com/kiwifs/kiwifs/internal/janitor  0.162s
+
+go test ./internal/config/... -count=1 -run 'ExternalLink'
+ok  github.com/kiwifs/kiwifs/internal/config   0.003s
+
+go test ./internal/janitor/... -count=1
+ok  github.com/kiwifs/kiwifs/internal/janitor  0.203s
+
+go test ./internal/config/... -count=1
+ok  github.com/kiwifs/kiwifs/internal/config   0.007s
+
+go test ./internal/... -count=1
+ok  (all packages, ~52s)
+```
+
+Feature surface matches kiwifs/kiwifs#423 acceptance criteria:
+
+- Opt-in `[janitor] external_link_check = true`
+- URL extraction from markdown body (frontmatter and code blocks excluded)
+- HEAD with GET+Range fallback, redirect limit, User-Agent header
+- 4xx/5xx and unreachable hosts flagged in `issues` and `external_links` JSON
+- Rate limits (10 concurrent, 100ms delay, 200 checks/scan cap)
+- Cache at `.kiwi/cache/link-check.json` (24h TTL)
+- SSRF guards and optional `external_link_allow` whitelist
+- Wired through API, bootstrap scheduler, `kiwifs check`, `kiwifs janitor`
+
+## Kiwi depot
+
+- MCP gateway: no servers registered in this environment
+- HTTP `http://192.168.167.240:3333`: connection refused
+- Fix doc present locally at `pages/fixes/kiwifs-kiwifs/issue-423-external-link-rot-janitor.md`
+
+## Outcome
+
+- Branch: `feat/issue-423-external-link-rot-hands-on` pushed to fork
+- PR: https://github.com/advancedresearcharray/kiwifs/pull/81 (upstream kiwifs/kiwifs#423)
+- Upstream PR creation blocked (collaborators only); fork PR ready for maintainer merge

episodes/agents/cursor-issue-423/2026-07-03-peer-review-hardening.md

@@ -0,0 +1,32 @@
+---
+memory_kind: episodic
+episode_id: cursor-issue-423-peer-review-hardening-2026-07-03
+title: Issue 423 peer review hardening — redirect SSRF, tests, refactor
+tags: [kiwifs, issue-423, janitor, external-links, peer-review, hands-on]
+date: 2026-07-03
+---
+
+## Run
+
+Hands-on takeover after fleet delivery failed `peer_review_not_passed` on
+`feat/issue-423-external-link-rot-hands-on`.
+
+## Changes
+
+1. Redirect SSRF — `newLinkHTTPClient` blocks redirects to private/metadata hosts.
+2. `sanitizeHostPatterns` — trims empty allow/ignore entries.
+3. Refactored `checkExternalLinks` into smaller documented helpers.
+4. Added tests: metadata hostnames, redirect block, request delay, timeout,
+   sanitized allow list.
+
+## Tests
+
+```
+go test ./internal/janitor/... -count=1   PASS
+go test ./internal/config/... -count=1    PASS
+go test ./internal/... -count=1           PASS (~56s)
+```
+
+## Branch
+
+`feat/issue-423-external-link-rot-hands-on`

internal/api/handlers_content.go

@@ -451,9 +451,25 @@ func (h *Handlers) Janitor(c echo.Context) error {
 	}
 
 	var execOpts []janitor.Option
-	if h.cfg != nil && h.cfg.Janitor.ExecutionStaleness.Enabled() {
-		es := h.cfg.Janitor.ExecutionStaleness
-		execOpts = janitor.OptionsFromExecutionStaleness(es.Directory, es.DateField, es.MaxAgeDays, es.FlagValues)
+	if h.cfg != nil {
+		if h.cfg.Janitor.ExecutionStaleness.Enabled() {
+			es := h.cfg.Janitor.ExecutionStaleness
+			execOpts = append(execOpts, janitor.OptionsFromExecutionStaleness(es.Directory, es.DateField, es.MaxAgeDays, es.FlagValues)...)
+		}
+		if h.cfg.Janitor.ExternalLinksEnabled() {
+			j := h.cfg.Janitor
+			execOpts = append(execOpts, janitor.OptionsFromExternalLinks(
+				true,
+				j.ExternalLinkTimeoutDuration(),
+				j.ExternalLinkCacheTTLDuration(),
+				j.ExternalLinkRequestDelayDuration(),
+				j.ResolvedExternalLinkIgnore(),
+				j.ExternalLinkAllow,
+				j.ResolvedExternalLinkMaxChecks(),
+				j.ResolvedExternalLinkMaxConcurrent(),
+				h.root,
+			)...)
+		}
 	}
 	scanner := janitor.New(h.root, h.store, h.searcher, staleDays, execOpts...)
 	result, err := scanner.Scan(c.Request().Context())

internal/bootstrap/bootstrap.go

@@ -427,7 +427,7 @@ func Build(name, root string, cfg *config.Config) (*Stack, error) {
 		if staleDays <= 0 {
 			staleDays = janitor.DefaultStaleDays
 		}
-		scanner := janitor.New(root, store, searcher, staleDays, janitorExecutionOpts(cfg)...)
+		scanner := janitor.New(root, store, searcher, staleDays, janitorExecutionOpts(cfg, root)...)
 		opts := janitor.ScheduleOptions{
 			Interval:    iv,
 			Jitter:      60 * time.Second,
@@ -663,10 +663,28 @@ func generateBootstrapSecret() string {
 	return hex.EncodeToString(b)
 }
 
-func janitorExecutionOpts(cfg *config.Config) []janitor.Option {
-	if cfg == nil || !cfg.Janitor.ExecutionStaleness.Enabled() {
+func janitorExecutionOpts(cfg *config.Config, root string) []janitor.Option {
+	if cfg == nil {
 		return nil
 	}
-	es := cfg.Janitor.ExecutionStaleness
-	return janitor.OptionsFromExecutionStaleness(es.Directory, es.DateField, es.MaxAgeDays, es.FlagValues)
+	var opts []janitor.Option
+	if cfg.Janitor.ExecutionStaleness.Enabled() {
+		es := cfg.Janitor.ExecutionStaleness
+		opts = append(opts, janitor.OptionsFromExecutionStaleness(es.Directory, es.DateField, es.MaxAgeDays, es.FlagValues)...)
+	}
+	if cfg.Janitor.ExternalLinksEnabled() {
+		j := cfg.Janitor
+		opts = append(opts, janitor.OptionsFromExternalLinks(
+			true,
+			j.ExternalLinkTimeoutDuration(),
+			j.ExternalLinkCacheTTLDuration(),
+			j.ExternalLinkRequestDelayDuration(),
+			j.ResolvedExternalLinkIgnore(),
+			j.ExternalLinkAllow,
+			j.ResolvedExternalLinkMaxChecks(),
+			j.ResolvedExternalLinkMaxConcurrent(),
+			root,
+		)...)
+	}
+	return opts
 }