diff --git a/hooks/harvest.js b/hooks/harvest.js index 9c3c500..b7412e8 100644 --- a/hooks/harvest.js +++ b/hooks/harvest.js @@ -151,9 +151,15 @@ function main() { fs.mkdirSync(queueDir, { recursive: true }); const queueFile = path.join(queueDir, `${sessionId}.jsonl`); + // Seed the dedupe set from the session queue AND the processed store — + // knowledge-flush empties the queue file when it retires rows, and the next + // Stop re-parses the unchanged transcript, so without the second source every + // flushed insight would be re-queued. Read-only: only the flush writes there. + const processedFile = path.join(queueDir, '.processed.jsonl'); const seen = new Set(); - if (fs.existsSync(queueFile)) { - for (const line of fs.readFileSync(queueFile, 'utf8').split('\n')) { + for (const src of [queueFile, processedFile]) { + if (!fs.existsSync(src)) continue; + for (const line of fs.readFileSync(src, 'utf8').split('\n')) { const o = safeJson(line); if (o.hash) seen.add(o.hash); } diff --git a/hooks/pre-flush-pr-gate.sh b/hooks/pre-flush-pr-gate.sh index 89e1c36..1362cd5 100755 --- a/hooks/pre-flush-pr-gate.sh +++ b/hooks/pre-flush-pr-gate.sh @@ -53,7 +53,12 @@ printf '%s' "$CMD" | grep -Eq -- 'INGEST_REPORT' && IS_FLUSH=1 # Locate the --body-file path referenced by the command. Portable (grep/sed only, # no node) so the enforcement path never depends on node being installed. -BODY_FILE="$(printf '%s' "$CMD" | grep -oE -- "--body-file[= ]+[^ '\"\`]+" | head -1 | sed -E 's/^--body-file[= ]+//')" +# The path may be bare, or wrapped in single or double quotes (the flush skill's +# own example quotes it) — accept all three and strip the quotes. +BODY_FILE="$(printf '%s' "$CMD" \ + | grep -oE -- "--body-file[= ]+(\"[^\"]+\"|'[^']+'|[^ '\"\`]+)" | head -1 \ + | sed -E "s/^--body-file[= ]+//" \ + | sed -E "s/^\"(.*)\"$/\\1/" | sed -E "s/^'(.*)'$/\\1/")" fail() { echo "dev-loop knowledge-flush gate: $1" >&2 @@ -69,8 +74,13 @@ fail() { } [ -n "$BODY_FILE" ] || fail "no --body-file found on the gh pr create command." -# Expand a leading ~ if present. -case "$BODY_FILE" in "~"/*) BODY_FILE="$HOME/${BODY_FILE#~/}" ;; esac +# Expand a leading ~, $HOME, or ${HOME} — the only variable forms the gate can +# resolve safely; anything else must be written as a literal path. +case "$BODY_FILE" in + "~"/*) BODY_FILE="$HOME/${BODY_FILE#~/}" ;; + '$HOME'/*) BODY_FILE="$HOME/${BODY_FILE#\$HOME/}" ;; + '${HOME}'/*) BODY_FILE="$HOME/${BODY_FILE#\$\{HOME\}/}" ;; +esac [ -f "$BODY_FILE" ] || fail "body file '$BODY_FILE' does not exist yet." miss="" diff --git a/plans/harvest-dedupe-processed/plan.md b/plans/harvest-dedupe-processed/plan.md new file mode 100644 index 0000000..fe22587 --- /dev/null +++ b/plans/harvest-dedupe-processed/plan.md @@ -0,0 +1,52 @@ +# harvest-dedupe-processed + +Goal: `hooks/harvest.js` must not re-queue an ★ Insight candidate that a previous +knowledge-flush already retired to `~/.dev-loop/queue/.processed.jsonl`. Today the +dedupe set is seeded only from the session's own queue file, so once a flush +empties that file the next Stop event re-parses the unchanged transcript and +re-appends the same hash. + +Acceptance criteria: +1. A hash present in `.processed.jsonl` is not appended to the session queue file, + even when that queue file is empty or absent. +2. Existing behavior is preserved: intra-file dedupe, normal harvesting of new + candidates, the `~/.dev-loop/repo` recursion guard, silent exit when there is + no transcript, and no error ever surfacing to the user. +3. `bats tests/` passes on the full suite. + +Stack: Node.js (CommonJS, no dependencies — `fs`/`path`/`os`/`crypto` only) for the +hook; bats for tests; CI runs `bats tests/` on ubuntu-latest and macos-latest +(`.github/workflows/test.yml`). + +Baseline: branch `fix/harvest-dedupe-processed` at HEAD `91409d0`, working tree clean. + +## Decisions + +| # | Decision | Choice | Wiki basis | +|---|----------|--------|------------| +| D1 | Where the dedupe set comes from | Seed `seen` from **both** the session queue file and `~/.dev-loop/queue/.processed.jsonl` | backend-common-jobs-idempotent-handlers — "Increment or append that must remain" row: a **processed-messages store keyed by message id**; a hit means already processed → skip. `.processed.jsonl` is that store; `hash` is the id | +| D2 | Why a re-read is required at all | Treat every Stop as a full re-execution of the same input; the handler must tolerate it | backend-common-jobs-idempotent-handlers directive 1 — "Every handler must tolerate full re-execution from any point it can crash at" | +| D3 | Processed-store path | `path.join(queueDir, '.processed.jsonl')` — same directory as the session queue files, resolved from `os.homedir()` exactly as `queueDir` already is (`harvest.js:150`) | `[no-wiki]` — fixed by the existing knowledge-flush contract (skills/knowledge-flush/SKILL.md step 5 writes there) | +| D4 | Hook's access mode to the processed store | **Read-only.** The hook never creates, writes, or prunes `.processed.jsonl`; only knowledge-flush moves rows there | `[no-wiki]` — single-writer keeps the flush the sole owner of retirement | +| D5 | Corrupt / missing store behavior | Each source is guarded by `fs.existsSync`, and every line goes through the existing `safeJson` (returns `{}` on parse failure) so a corrupt line is skipped, not fatal | backend-common-jobs-idempotent-handlers directive 1 (tolerate re-execution) + the file's existing `safeJson` + top-level `try/catch` convention (`harvest.js:34-40`, `189-193`) | +| D6 | Test isolation | `HOME="$BATS_TEST_TMPDIR/home"` per test (Node's `os.homedir()` reads `$HOME` on POSIX); transcript and payload files under `BATS_TEST_TMPDIR`; nothing written into the repo or the real `~/.dev-loop` | testing-data-test-data-and-isolation — "Filesystem / temp files" row (fresh per-test temp directory) and "Global config / environment variables" row (set in setup) | +| D7 | Case set | One normal, one error, and one boundary case per behavior; the bug gets a regression test | testing-quality-minimum-case-set directive 1 and directive 5 | +| D8 | Red before green | The regression test is written first and **observed failing** on unmodified `harvest.js`; that output is the evidence the test guards the bug | testing-quality-minimum-case-set directive 5 — "first write a regression test that reproduces the bug and fails on the current code" | +| D9 | What each test asserts | The observable outcome: the **number of lines** in the session queue file and **which hashes** it contains — never "the hook exited 0" alone, since the hook exits 0 unconditionally | testing-quality-minimum-case-set directive 2 + testing-quality-tests-that-cannot-fail (an assertion that cannot detect the defect is not a test) | +| D10 | Missing `node` in the test environment | Fail the test loudly in `setup()` rather than `skip` — a permanent skip is a test that cannot fail | testing-quality-tests-that-cannot-fail | +| D11 | Pruning `.processed.jsonl` | **Out of scope**, recorded as follow-up | backend-common-jobs-idempotent-handlers edge case "Dedupe table grows unbounded → prune rows older than the redelivery horizon" — real, but it is user data and the user deferred it | + +## Task order + +| Task | Depends on | Parallel-ok | +|------|-----------|-------------| +| 01-dedupe-against-processed-store | — | — | + +One task: the fix and its regression test are one concern (2 files, 4 wiki pages), +and splitting them would leave a task whose Verify is "the suite is red". + +## Follow-up (not in this branch) + +- Prune duplicate rows already in `~/.dev-loop/queue/.processed.jsonl` (103 rows / + 57 unique hashes as measured 2026-08-05). User data — separate, consented pass. +- Consider a retention horizon for `.processed.jsonl` per D11's wiki edge case. diff --git a/plans/harvest-dedupe-processed/tasks/01-dedupe-against-processed-store.md b/plans/harvest-dedupe-processed/tasks/01-dedupe-against-processed-store.md new file mode 100644 index 0000000..3090da5 --- /dev/null +++ b/plans/harvest-dedupe-processed/tasks/01-dedupe-against-processed-store.md @@ -0,0 +1,128 @@ +# Task 01: Dedupe harvested insights against the processed store + +## Objective + +`hooks/harvest.js` seeds its dedupe set from both the session queue file and +`~/.dev-loop/queue/.processed.jsonl`, so a hash a previous knowledge-flush retired +is never re-appended — even when the session queue file is empty or absent. +`tests/harvest.bats` covers this and the behaviors it must not break. + +## Wiki pages (read these first, only these) + +- wiki/backend/common/jobs/idempotent-handlers.md — use for: why the handler must + tolerate full re-execution (directive 1) and that the dedupe key is looked up in + a **processed-messages store** before the append (directive 2, the + "Increment or append that must remain" row). This is the row that decides D1. +- wiki/testing/quality/minimum-case-set.md — use for: which cases the test file + must hold (directive 1: normal + error + boundary per behavior), what each test + asserts (directive 2: observable outcome), and the red-first rule for a bug fix + (directive 5). +- wiki/testing/quality/tests-that-cannot-fail.md — use for: judging that each new + assertion can actually detect the defect; and D10 (never make a permanent skip). +- wiki/testing/data/test-data-and-isolation.md — use for: the "Filesystem / temp + files" row (fresh per-test temp directory) and the "Global config / environment + variables" row — how to point `HOME` at a per-test directory so the real + `~/.dev-loop/queue` is never touched. + +## Inputs + +- `hooks/harvest.js` (existing, 193 lines). The site to change is the `seen` + construction inside `main()` — the block that begins `const seen = new Set();` + and ends just before `const repo = path.basename(cwd);`. Anchor by that symbol, + not by line number. +- `hooks/harvest.js` helpers you must reuse, not re-implement: `safeJson(s)` + (returns `{}` on parse failure) and the existing `queueDir` /`queueFile` + computation. +- `tests/loop-gate.bats` — the style to match: `setup()` resolves the hook via + `${BATS_TEST_DIRNAME}/../hooks/...`, work happens under `BATS_TEST_TMPDIR`, the + payload is piped in with `printf`. +- Decisions that bind you: D1 (seed from both), D3 (`.processed.jsonl` in + `queueDir`), D4 (read-only), D5 (existsSync + safeJson), D6 (HOME isolation), + D7 (case set), D8 (red first), D9 (assert line count + hashes), D10 (no skip). + +## Steps + +1. Create `tests/harvest.bats`. In `setup()`: + - `HARVEST="${BATS_TEST_DIRNAME}/../hooks/harvest.js"` + - `export HOME="${BATS_TEST_TMPDIR}/home"`, `QDIR="$HOME/.dev-loop/queue"`, + `mkdir -p "$QDIR"` + - `WORK="${BATS_TEST_TMPDIR}/work"`, `mkdir -p "$WORK"` — this is the payload's + `cwd` (it must NOT be under `$HOME/.dev-loop/repo`, which the hook skips). + - Assert `node` is available and fail the test if not (D10): + `command -v node >/dev/null || { echo "node is required for these tests"; return 1; }` +2. Add a helper that writes a transcript containing one ★ Insight block and runs + the hook, so each test differs only in the queue/processed state: + + ```bash + _mk_transcript() { # + local body + body="★ Insight ─────\ntrigger: $2\ndirective: seed the dedupe set from the processed store too\nwhy: the transcript is re-parsed on every Stop\nevidence: measured 46 duplicate rows\ndomain: testing\n─────" + printf '{"message":{"role":"assistant","content":"%s"}}\n' "$body" > "$1" + } + + _run_harvest() { # + printf '{"cwd":"%s","session_id":"s1","transcript_path":"%s"}' "$WORK" "$1" | node "$HARVEST" + } + ``` + + The delimiter run must be U+2500 box-drawing characters (─), matching + `BLOCK_RE` in `harvest.js`. Write the `\n` escapes so they land as real + newlines inside the JSON string value. +3. Write these tests (each asserts the observable outcome per D9 — the queue + file's line count and, where it matters, the hash it holds): + + | Case | Setup | Assert | + |------|-------|--------| + | normal — a new candidate is harvested | empty `$QDIR`, no `.processed.jsonl` | `s1.jsonl` has exactly 1 line, and its `hash` field is non-empty | + | **regression** — a processed hash is not re-queued | run the hook once, capture `HASH` from `s1.jsonl`, then move that line into `.processed.jsonl` and truncate `s1.jsonl` (this is exactly what a flush does), then run the hook again | `s1.jsonl` has **0 lines** | + | preserved — intra-session dedupe still works | run the hook twice with the same transcript and no flush in between | `s1.jsonl` has exactly 1 line | + | error — corrupt line in the processed store | put `not json` plus the real processed row in `.processed.jsonl`, truncate `s1.jsonl`, run | hook exits 0 **and** `s1.jsonl` has 0 lines (the corrupt line is skipped, the valid one still dedupes) | + | error — no `.processed.jsonl` at all | remove it, empty queue, run | `s1.jsonl` has exactly 1 line (absence must not throw) | + | boundary — empty `.processed.jsonl` | `: > "$QDIR/.processed.jsonl"`, empty queue, run | `s1.jsonl` has exactly 1 line | + | boundary — transcript holds no ★ block | transcript with a plain assistant line | no `s1.jsonl` is created (or it has 0 lines) | + | preserved — recursion guard | payload `cwd` set to `$HOME/.dev-loop/repo` | no `s1.jsonl` is created | + +4. Run `bats tests/harvest.bats` on the **unmodified** `hooks/harvest.js` and + record the output. The regression row must FAIL and the others must pass + (D8). If the regression row passes here, the test does not reproduce the bug — + fix the test before touching the hook. +5. Edit `hooks/harvest.js`: replace the `seen` construction with a loop over both + sources, keeping `safeJson` and the `existsSync` guard: + + ```js + const processedFile = path.join(queueDir, '.processed.jsonl'); + const seen = new Set(); + for (const src of [queueFile, processedFile]) { + if (!fs.existsSync(src)) continue; + for (const line of fs.readFileSync(src, 'utf8').split('\n')) { + const o = safeJson(line); + if (o.hash) seen.add(o.hash); + } + } + ``` + + Do not change the append target (`queueFile`), the row shape, the recursion + guard, or the top-level `try/catch`. Do not write to `processedFile` (D4). +6. Re-run `bats tests/harvest.bats` — all rows green — then `bats tests/` for the + whole suite. + +## Deliverables + +- `tests/harvest.bats` (new) +- `hooks/harvest.js` (modified — only the `seen` construction inside `main()`) + +## Verify + +- `bats tests/harvest.bats` → all tests pass, and the run before step 5 showed the + regression test failing (paste both outputs). +- `bats tests/` → the full suite passes with no new failures against the `91409d0` + baseline. +- `git diff --stat` → exactly the two Deliverable files. + +## Out of scope + +- Pruning or rewriting `~/.dev-loop/queue/.processed.jsonl` (user data; follow-up + in `plan.md`). +- `hooks/auto-flush.sh`, `hooks/harvest-insights.sh`, any other hook. +- `skills/knowledge-flush/SKILL.md` and any wiki page. +- The installed plugin copy under `~/.claude/plugins/cache/` — read-only, never edit. diff --git a/plans/wiki-audit/findings/REPORT.md b/plans/wiki-audit/findings/REPORT.md new file mode 100644 index 0000000..809f451 --- /dev/null +++ b/plans/wiki-audit/findings/REPORT.md @@ -0,0 +1,36 @@ +# wiki-audit — Final report (2026-08-05) + +질문: "이 wiki는 LLM이 구현 계획을 세울 때 쓰는 프로덕션급 지식 베이스로서, +시멘틱하게 케이스를 확실히 찾을 수 있고, 내용이 충분·명확하며, harvest→flush +파이프라인이 재료를 잘 걸러 중복 없는 PR을 만드는가?" + +## Axis verdicts + +| 축 | 판정 | 근거 | +|----|------|------| +| 1. 구조 (기계적 건전성) | **PASS — 결함 0** | 141페이지·13인덱스 전수 스윕: frontmatter/id/related-link/인덱스 양방향/120줄/모호 한정어/소싱/신선도 전부 0건 (findings/structure.md). lint 규칙 2번의 문구-관행 불일치만 이슈화 (#36) | +| 2. 시멘틱 라우팅 | **PASS with 2 findings** | 15 프로브 중 in-charter 11/13 UNIQUE, out-of-charter 2건 clean MISS. AMBIGUOUS 2건(doc-gate 이중 소유, flaky 이중 소유) → #37 | +| 3. 카테고리 충분성 | **PASS, additive gaps** | 60개 카테고리 일관·명확. 16개 개발 관점 체크리스트에서 진짜 갭 G1–G6(아키텍처, CORS, API 버저닝, 피처 플래그, 백필, 실시간) + 하위 G7–G10 → #38. 재구조화 불요 | +| 4. harvest 필터링 | **PASS after fix** | "수집은 관대, 승격은 엄격" 설계 건전. trigger+directive 필수, 30자 하한, 템플릿 에코 차단, 해시 dedup. **processed-store dedup 누락 버그 수정**(red→green 회귀 테스트) | +| 5. flush 게이트/dedup | **PASS after 3 fixes** | INGEST_REPORT 3섹션 하드 게이트(13 테스트로 고정). 수정: 따옴표 경로 파싱, $HOME 확장, SKILL 리터럴 경로 지침, drop 후보 retire 명시(무한 오토플러시 루프 차단). 잔여 강화 → #39 | + +## Fixes applied on this branch + +- hooks/harvest.js — dedupe set seeded from `.processed.jsonl` (read-only) +- hooks/pre-flush-pr-gate.sh — quoted `--body-file` 인식 + `~`/`$HOME`/`${HOME}` 확장 +- skills/knowledge-flush/SKILL.md — 게이트-호환 리터럴 경로 지침(자체 wiki 페이지 인용), step 5 "모든 처리 후보 retire" +- tests/harvest.bats (8), tests/pre-flush-pr-gate.bats (13) — 전부 green; 독립 + test-quality-auditor VERDICT: PASS; 전체 스위트 `bats tests/` exit 0 (315 tests) + +## Issues filed + +- #36 wiki-lint check 2 vs corpus (규칙 정련) +- #37 라우팅 이중 소유 2건 + 마이너 문구 +- #38 카테고리 갭 G1–G10 (시드 페이지 목록 포함) +- #39 파이프라인 잔여 강화 (게이트 우회 창, 기계 검증 가능한 dedup 증빙, 수집 상한) + +## Overall + +프로덕션급 판정: **가깝다 — 구조·내용 규율은 이미 프로덕션 수준이고, 이번에 +수정한 파이프라인 결함 3건이 실사용 신뢰성의 실제 병목이었다.** 남은 것은 +전부 가산적(카테고리 확장, 트리거 문구 정련, 게이트 강화)이며 이슈로 추적된다. diff --git a/plans/wiki-audit/findings/categories.md b/plans/wiki-audit/findings/categories.md new file mode 100644 index 0000000..1e27b4d --- /dev/null +++ b/plans/wiki-audit/findings/categories.md @@ -0,0 +1,61 @@ +# Findings — Task 03: Category taxonomy sufficiency (all development perspectives) + +Inventory: 10 domains, 60 leaf categories, 141 pages (counts per category in the +sweep output; largest: testing/quality 7, databases/schema-design 8; 24 +categories hold exactly 1 page — healthy for a case-routed wiki, not a defect). + +## D3 checklist walk + +| Concern | Verdict | Where it lives today / what's missing | +|---------|---------|----------------------------------------| +| Requirements/planning | partial — acceptable | qa/process/acceptance-criteria + databases/requirements-to-tables; task decomposition is owned by the wiki-plan skill, not the wiki. No action. | +| Architecture/system design | **genuine gap (G1)** | No category owns "where does this logic live / sync call vs queue vs event / module boundaries". backend/common has point-patterns only. An implementation-planning wiki needs this: wiki-plan's design decisions currently go `[no-wiki]` here. | +| API design | partial → **gap slices (G2, G3)** | api-design has error-responses, idempotency, pagination-contract. Missing: **CORS & preflight** (top-frequency LLM implementation stumble) and **API versioning / breaking-change policy for public APIs** (call-site-enumeration covers internal contracts only). | +| Concurrency | covered | backend/common/concurrency + stack pages (kotlin coroutines, GIL, event loop) + databases/transactions + debugging/concurrency. | +| Distributed systems | partial → **gap slice (G10)** | distributed-locks + idempotent-handlers exist. Missing: multi-service write consistency (saga/outbox-beyond-enqueue, delivery-guarantee semantics: at-least-once vs exactly-once claims, event ordering). | +| Messaging/queues | mostly covered | jobs/idempotent-handlers (consumer side incl. DLQ/outbox) + scheduled-job-overlap. Producer-side topology/schema-evolution thin — fold into G10. | +| Caching | covered | caching/invalidation-and-stampede (keys, tenancy, stampede). HTTP/CDN caching partially in frontend/performance — acceptable. | +| Networking | partial → **gap slices (G2, G6)** | security/api-exposure owns edge/proxy. Missing: CORS (G2); **WebSocket/SSE/realtime lifecycle** (reconnect, backpressure, auth on long-lived connections) — graceful-shutdown only brushes it (G6). | +| Observability | covered | infrastructure/observability ×3 + debugging/signals. | +| Performance | partial — acceptable | frontend/performance, debugging/performance, mobile/startup-time, databases/query-optimization. Load/capacity testing absent (G8, low priority). | +| Docs & i18n | **gap (G7, lower priority)** | qa/document-verification is about *gating* docs, not writing them (fine). UI i18n/l10n (string externalization, pluralization, RTL) has no home; platforms/timezone-and-locale covers only OS locale mechanics. | +| Data engineering/ML | partial — acceptable | LLM *consumption* well covered (backend/common/llm ×2). ML training: out of charter (routing probe 15 confirms clean MISS). Missing slice: **data backfill/transform migrations** beyond DDL (G5) — online-schema-changes covers DDL locking, not batched backfill/dual-write verification. | +| Config mgmt/releases | partial → **gap slice (G4)** | infrastructure/config + deploy + mobile/release. **Feature-flag lifecycle** (naming, targeting, cleanup debt, kill-switch vs experiment) is referenced by rollout pages but owned by none. | +| Cost | out of charter | Cloud cost rarely decides code-level implementation cases this wiki routes. Revisit if orchestrate grows infra-provisioning tasks. | +| Accessibility | partial — acceptable | frontend/accessibility/interactive-elements. Forms-labeling/contrast/landmarks would fit as siblings when real cases arrive (ingest-driven growth is the charter). | +| Compliance/privacy | partial — acceptable | security/data/pii-handling (retention, erasure, test data). Audit-trail design (G9, low priority) unowned. | + +## Category-pair ambiguity (router confusion risk) + +| Pair | Evidence | Severity | +|------|----------|----------| +| testing/quality (doc-gate cluster: spec-artifact-checks, checks-that-cannot-pass, schema-additions-under-a-golden-gate, harness-reverse-controls) vs qa/document-verification (spec-document-gates, editing-a-gated-document) | Routing probe 6: both claim "automated checks that decide whether a spec document meets requirements" | high — 6 pages across 2 domains, one concern | +| testing/flaky vs debugging/concurrency/intermittent-failures | Routing probe 7: both triggers claim "flaky test / fails only in CI / passes on retry" verbatim | medium | +| infrastructure/data/backup-and-restore vs databases/operations | A DB operator looking for backups plausibly opens databases first; no cross-pointer in databases/index.md | low — add one cross-pointer line | +| backend/common/integrations (holds 1 page: externally-owned-defaults) | Category name promises third-party integration patterns broadly; content is one narrow case. Rename risk vs growth headroom | low — leave, revisit at 3+ pages | + +## Proposed seeds for genuine gaps (for issues) + +- **G1 backend/common/architecture**: `sync-vs-async-integration` (direct call vs + queue vs event — decision table by consistency/latency/failure-isolation), + `module-boundaries-and-layering`, `event-driven-adoption-criteria`. +- **G2 backend/common/api-design/cors-and-preflight**: browser-origin API calls + failing on CORS; wildcard-vs-allowlist; credentials mode; preflight caching. +- **G3 backend/common/api-design/api-versioning-and-breaking-changes**. +- **G4 infrastructure/deploy/feature-flag-lifecycle** (or backend/common). +- **G5 databases/operations/data-backfill-migrations**: batching, dual-write, + verification queries, resumability. +- **G6 backend/common/realtime/websocket-sse-lifecycle**: auth, reconnect, + heartbeat, backpressure, shutdown draining. +- **G7 frontend/i18n** (lower), **G8 testing/strategy/load-testing** (lower), + **G9 security/data/audit-trails** (lower), **G10 + backend/common/distributed/cross-service-writes** (saga/outbox/delivery + guarantees). + +## Verdict (axis 3) + +The 60 existing categories are coherent and, with two exceptions (doc-gates +split, flaky-test dual ownership), unambiguous. The taxonomy's real weakness for +a *planning* wiki is the missing architecture/design layer (G1) and a handful of +high-frequency implementation cases (CORS G2, versioning G3, flags G4, realtime +G6, backfill G5). All are additive — no restructuring required. diff --git a/plans/wiki-audit/findings/pipeline.md b/plans/wiki-audit/findings/pipeline.md new file mode 100644 index 0000000..b451d45 --- /dev/null +++ b/plans/wiki-audit/findings/pipeline.md @@ -0,0 +1,65 @@ +# Findings — Tasks 04+05: harvest filtering & flush-gate audit + +## Axis 4 — harvest (오토 플러시 수집단) admission filtering + +What the Stop-hook harvester admits into `~/.dev-loop/queue/` (hooks/harvest.js): + +| Filter | Evidence (line) | Verdict | +|--------|----------------|---------| +| Block must carry `trigger` AND `directive` | `if (!fields.trigger \|\| !fields.directive) continue` | good — unroutable/unactionable content never enters | +| Body ≥ 30 chars | `if (body.length < 30) continue` | good — rejects stubs | +| Template echo rejected | `/<[^>]+>/` test on trigger/directive | good — the instruction's own placeholders can't queue | +| Content-hash dedup, intra-session | `seen` from session queue file | good | +| Content-hash dedup vs already-flushed rows | **was missing — FIXED this run**: `seen` now also seeds from `.processed.jsonl` (read-only), regression-tested (tests/harvest.bats, red→green) | fixed | +| Recursion guard (flush checkout not harvested) | `startsWith(flushRepo)` | good, tested | +| Source-side quality bar | SessionStart instruction: "verified best-practice or real edge case… 0–3 per session… a guess is worse than nothing" | good — filtering starts at generation, not only collection | + +Design judgment: harvest is deliberately a **cheap offline collector**; the +*quality* bar is enforced downstream (flush research/verify/dedup) and at the +source (instruction). That split is sound — a Stop hook must never do network +research. 재료 필터링은 "수집은 관대하게, 승격은 엄격하게" 구조로 동작하며, +승격 단계가 게이트로 강제되므로 "아무 내용이나 PR로 가는" 경로는 없다. + +Residual (→ issue, low severity): +- harvest.js enforces no per-session block cap (instruction says 0–3); a + runaway session could queue dozens. Flush verification would drop junk, but + auto-flush would still spend a headless run on it. +- ~35 empty per-session queue files accumulate (cosmetic). + +## Axis 5 — knowledge-flush refinement / dedup / PR gates + +| Control | Enforcement level | Verdict | +|---------|-------------------|---------| +| Research+verify, existing-layer dedup check, routing decision before PR | **Hard gate**: hooks/pre-flush-pr-gate.sh denies `gh pr create` without an INGEST_REPORT containing the 3 filled sections — now covered by 13 bats tests | strong | +| Confidence honesty (never upgrade unverifiable to verified) | Prose (SKILL guardrails) + owner PR review | acceptable | +| Merge-before-create / related-links (dedup with existing pages) | Prose (wiki-ingest steps 4, 7) + INGEST_REPORT "Existing-layer check" section + owner PR review | acceptable — defense in depth; the gate can prove the section exists, not that pages were read. Final arbiter is the human PR review, by design | +| No auto-merge / PR-only | Prose + repo perms; auto-flush prompt repeats it | acceptable | +| Auto-flush guards (kill switch, recursion, threshold ≥3, 1h rate limit, TTL lock) | Shell, reviewed line-by-line | solid | + +### Defects found and FIXED this run +1. **Gate could not parse a quoted `--body-file` path** (`"[^ '\"\`]+"` stops at + the quote) — yet the skill's own example quotes the path → a correct flush + command was denied with a misleading "no --body-file found". Fixed + + regression test (`pre-flush-pr-gate.bats` test 12, red→green). +2. **Gate could not resolve `$HOME`-prefixed paths** (only `~`), while the + command text reaches PreToolUse unexpanded. Gate now expands `~`, `$HOME`, + `${HOME}`; test 13 added. SKILL example also switched `$REPO` → `$HOME` with + an explicit literal-path note citing + wiki/platforms/shells/command-text-inspected-before-execution.md (the wiki + already documented this exact failure class — the pipeline just wasn't + following its own page). +3. **Dropped candidates were never retired** — SKILL step 5 said to retire "the + flushed rows"; a candidate dropped as unverifiable stayed `pending`, kept the + queue over the auto-flush threshold, and would re-trigger a headless flush + every hour forever. SKILL step 5 rewritten: retire every handled row + (ingested, merged, or dropped). + +### Residual (→ issue) +- **Gate bypass window**: flush detection is command-marker-only (head + `knowledge/`, label, INGEST_REPORT). A `gh pr create --body inline` with no + label, head inferred from the current branch, engages nothing. Documented + tradeoff (global-install safety); tightening option: also match + `--title "knowledge:"` and/or have the flush prompt forbid `--body`. +- Existing-layer check verifiability: consider requiring the report to list the + page ids read (machine-checkable against `wiki/**`), giving the gate a + cross-reference to verify instead of free prose. diff --git a/plans/wiki-audit/findings/routing.md b/plans/wiki-audit/findings/routing.md new file mode 100644 index 0000000..24a4fde --- /dev/null +++ b/plans/wiki-audit/findings/routing.md @@ -0,0 +1,113 @@ +# Findings — Task 02: Semantic routing probe (15 scenarios) + +Method (plan D2): each probe is a realistic implementation intent, traced +INDEX.md "route here when" → domain index "load when" → page. Verdicts: +UNIQUE (one page wins), AMBIGUOUS (≥2 equally-matching rows), MISS (no row). +For the 2 out-of-charter probes, MISS is the *correct* outcome — recorded is +whether the router can tell cleanly. + +### Probe 1 — "리스트 API에 커서 기반 페이지네이션 추가" (backend+databases) +INDEX: backend "API contracts" (artifact = endpoint contract). Domain: +`pagination-contract` — "Designing a list endpoint's request/response contract — +cursor vs page-number … (the backing SQL/index → databases/query-optimization/ +keyset-pagination)". Cross-link to databases is inline in the load-when line. +**Verdict: UNIQUE** (exemplary cross-domain composition). + +### Probe 2 — "웹훅 중복 전달로 결제가 두 번 발생" +INDEX: backend. Candidates: `api-design/idempotency` ("An endpoint with side +effects (create, charge, send) can receive the same request twice") vs +`jobs/idempotent-handlers` ("queue consumer, background job"). Webhook = endpoint +→ idempotency matches; jobs line does not claim endpoints. **Verdict: UNIQUE.** + +### Probe 3 — "검색 자동완성에서 이전 검색어 결과가 늦게 도착해 최신 결과를 덮어씀" +INDEX: frontend. `data-fetching/race-conditions` — "search-as-you-type … UI +intermittently shows results for a previous input". **Verdict: UNIQUE.** + +### Probe 4 — "URL의 주문 ID를 바꾸면 남의 주문이 보임" +INDEX: security ("per-resource authorization (IDOR)"). `authz/resource-level-checks` +— "resource identified by a request-supplied id (IDOR risk)". **Verdict: UNIQUE.** + +### Probe 5 — "버그 수정에 회귀 테스트 추가 — 어떤 케이스를 커버해야 하나" +INDEX: testing. `quality/minimum-case-set` — "adding a regression test for a bug +fix". **Verdict: UNIQUE.** + +### Probe 6 — "스펙 문서가 요구사항을 충족하는지 검사하는 grep 게이트 작성" (qa+testing) +INDEX: qa "automated verification of document deliverables (spec/RFC gates)" AND +testing has `quality/spec-artifact-checks` ("automated check that a mapping table +covers every rule/field/enum case"), `quality/checks-that-cannot-pass` (gate on an +unwritten doc). qa's `spec-document-gates` load-when: "Writing or reviewing +automated checks (grep/script) that decide whether a spec/RFC/schema document +meets its requirements". Two domains claim near-identical scope; INDEX gives qa +the "document deliverables" phrase but testing's page triggers match the same +sentence. **Verdict: AMBIGUOUS — testing/quality doc-gate cluster (4 pages) vs +qa/document-verification (2 pages) split one concern across two domains.** +Mitigation exists (INDEX qa line says "automated verification of document +deliverables"; testing line says "writing automated test code") but the page-level +triggers overlap materially. + +### Probe 7 — "재시도하면 통과하는 간헐적 CI 테스트 실패" (testing+debugging) +testing `flaky/diagnosing-flaky-tests`: "A test fails intermittently with no code +change: on retry, in CI only". debugging `concurrency/intermittent-failures`: +"passes on retry, fails under load, fails only in CI, **flaky test**". Both +domain INDEX lines ("flaky tests" / "intermittent failures") and both page +triggers claim the same situation verbatim. **Verdict: AMBIGUOUS — dual ownership +of flaky-test diagnosis.** (INDEX debugging line does scope to "diagnosing", and +testing to policy/quarantine, but the page triggers don't respect that split.) + +### Probe 8 — "쿠버네티스 파드가 OOMKilled로 재시작" +INDEX: infrastructure. `containers/resource-limits-and-probes` — "pods OOMKilled, +evicted, or CPU-throttled". **Verdict: UNIQUE.** + +### Probe 9 — "macOS에선 되는데 리눅스 CI에서 sed -i가 실패" +INDEX: platforms ("BSD-vs-GNU CLI"). `tools/bsd-vs-gnu-cli` — names `sed -i` +explicitly. **Verdict: UNIQUE.** + +### Probe 10 — "앱 전환 후 돌아오면 작성 중이던 폼이 사라짐" +INDEX: mobile ("process death/state survival"). `lifecycle/process-death-and-state` +— "'app lost my data when I switched apps'". **Verdict: UNIQUE.** + +### Probe 11 — "로그인 유지 방식 설계: 세션 vs 토큰, 서버 발급, 클라 저장" (security+backend+frontend) +INDEX: security ("session-vs-token auth choice"). `authn/session-vs-token` owns +the choice and its load-when routes onward: "(implementation → wiki/backend/ +common/auth/, wiki/frontend/auth/)". Three-domain chain fully signposted. +**Verdict: UNIQUE** (chained). + +### Probe 12 — "외부 결제 API 호출에 타임아웃/재시도 설계" +INDEX: backend. `common/reliability/timeouts-and-retries` — exact match. +**Verdict: UNIQUE.** + +### Probe 13 — "S3 업로드 후 DB에 어떤 값을 저장해야 하나" +INDEX: backend ("object-storage references"). `common/storage/object-key-persistence` +— "choosing which response field goes in the DB column". **Verdict: UNIQUE.** + +### Probe 14 — [out-of-charter] "유니티 게임 셰이더 최적화" +INDEX: no "route here when" line mentions games/graphics/GPU. Router can tell +immediately; protocol step 5 applies (answer not-wiki-backed + gap log). +**Verdict: MISS — clean.** + +### Probe 15 — [out-of-charter] "ML 모델 학습 파이프라인의 하이퍼파라미터 튜닝" +INDEX: backend line contains "LLM completion validation & context budgeting" — a +weak attractor for ML-ish queries. Backend `llm` category pages are strictly about +*consuming* completions; their triggers reject the match (protocol step 3 drift +check catches it). **Verdict: MISS — clean, one hop wasted.** Minor: INDEX line +could say "consuming LLM APIs" to repel training/ML queries at the root. + +## Summary + +- Routing precision: **11/13 UNIQUE** on in-charter probes; both out-of-charter + probes MISS cleanly (correct behavior). +- **AMBIGUOUS #1 (probe 6):** doc-verification gates split across + testing/quality (spec-artifact-checks, checks-that-cannot-pass, + schema-additions-under-a-golden-gate, harness-reverse-controls) and + qa/document-verification (spec-document-gates, editing-a-gated-document). + → issue: unify ownership or sharpen the two clusters' load-when lines with + explicit mutual cross-pointers ("authoring the check code → testing; + release-gate policy → qa"). +- **AMBIGUOUS #2 (probe 7):** flaky-test diagnosis claimed verbatim by both + testing/flaky/diagnosing-flaky-tests and debugging/concurrency/ + intermittent-failures. → issue: give each trigger a disjoint scope + (test-suite-local diagnosis+policy → testing; general intermittent failures + incl. prod → debugging) and cross-link. +- Minor (probe 15): backend INDEX "LLM" phrasing → "consuming LLM APIs". +- Strengths worth keeping: inline cross-domain pointers in load-when lines + (probes 1, 11) make multi-domain work chain without bulk-loading. diff --git a/plans/wiki-audit/findings/structure.md b/plans/wiki-audit/findings/structure.md new file mode 100644 index 0000000..1362dfb --- /dev/null +++ b/plans/wiki-audit/findings/structure.md @@ -0,0 +1,49 @@ +# Findings — Task 01: Mechanical lint sweep + +Sweep tool: `.claude/tmp/lint-sweep.js` (Node, run from repo root, 2026-08-05). +Scope: 141 pages + 13 indexes (10 domain + 3 backend sub-indexes), 141 unique ids. + +## Results by check (command: `node .claude/tmp/lint-sweep.js`) + +| Check | Hits | Verdict | +|-------|------|---------| +| Frontmatter missing / field missing | 0 | clean | +| id duplicate / domain mismatch / slug mismatch | 0 | clean | +| C1 `verified` with empty sources | 0 | clean | +| C3 broken `related:` id | 0 | clean — all 141 ids resolve | +| C4 index dead link | 0 | clean | +| C4 page unlisted in its NEAREST index | 0 | clean (initial 16 hits were sweep-tool artifacts: backend routes via java/node/python sub-indexes; tool fixed to nearest-index semantics, re-run → 0) | +| C5 vague qualifiers (usually/consider/might want to/generally/as appropriate) | 0 | clean | +| C5b bare `might` sweep (`grep -rniE '\bmight\b' wiki --include='*.md'` excl. index) | 6 | 4 are React doc URLs ("you-might-not-need-an-effect"), 2 are situation-descriptions in trigger/edge prose, not directive sentences → compliant | +| C6 body > 120 lines | 0 | clean | +| C7 `confidence: unverified` pages | 0 | clean — every page verified or field-tested | +| C8 `verified` with last_verified > 12 months | 0 | clean | +| Section skeleton (When this applies / Do this) | 0 | clean | +| C2 don't/never/avoid in directive lines outside `Instead of` | 147 | manual-review class, see below | + +## C2 analysis (147 hits) + +Sampled 20+ hits: essentially all are decision-table rows where the prohibition is +**paired in the same cell with the replacement action and mechanism**, e.g. +`| 400/401/403/404/422 | Never retry — the request itself is wrong; the same bytes fail again |`. +This satisfies the *intent* of AGENTS.md rule 3 (no bare prohibition without a +replacement) but not its *letter* (anti-patterns only in the `Instead of` table). +No hit found where a prohibition dead-ends without a replacement (0 true +violations in sample; full-list scan found none of the form "never X." with no +alternative in the same row/step). + +→ issue: **wiki-lint check 2 wording vs. corpus practice** — refine the rule to +"a prohibition must be paired with its replacement in the same row/sentence, or +live in `Instead of`", so the lint is mechanically enforceable and the 147 +compliant rows stop being manual-review noise. + +## Fixes applied on this branch + +None required — zero true mechanical defects. (The only edit was to the +throwaway sweep tool itself, not the wiki.) + +## Verdict (axis 1, mechanical half) + +Structurally production-clean: ids, links, indexes, frontmatter, sourcing, size +and freshness discipline all hold across 141 pages. Semantic routing quality is +task 02's scope. diff --git a/plans/wiki-audit/plan.md b/plans/wiki-audit/plan.md new file mode 100644 index 0000000..3a573ea --- /dev/null +++ b/plans/wiki-audit/plan.md @@ -0,0 +1,48 @@ +# wiki-audit — LLM-perspective audit of the bundled wiki + knowledge pipeline + +Goal: Audit the dev-loop wiki as a production-grade knowledge base for LLM +implementation planning, across five axes, fixing defects found and filing +GitHub issues for enhancement-grade findings: +1. **Structure** — can an LLM route to the right page semantically and cheaply + (INDEX.md → domain index "load when" lines → page trigger)? +2. **Content** — do pages give unambiguous direction (no vague qualifiers, no + bare prohibitions, sourced claims, edge cases covered)? +3. **Category taxonomy** — are categories sufficient and unambiguous from ALL + development perspectives; which categories are missing? (→ GitHub issues) +4. **Harvest filtering** — does the Stop-hook queue admit only wiki-grade + material? (includes finishing the already-planned dedupe fix on this branch) +5. **Flush gates** — does knowledge-flush provably dedupe against the existing + wiki and refine before PR; can the pre-PR gate be bypassed? + +Acceptance criteria: findings documented per axis with evidence (commands + hit +counts, 0-hit stated); mechanical defects fixed on this branch with tests where +code changed; category/enhancement findings filed as GitHub issues on +choiyounggi/dev-loop; `bats tests/` green. + +Stack: Bash/Node (hooks), bats (tests), gh CLI (issues), Markdown wiki. +Baseline: branch `fix/harvest-dedup-processed`, HEAD 91409d0, untracked +`plans/` + `tests/harvest.bats` (red regression test from the adopted plan). + +## Decisions + +| # | Decision | Choice | Wiki basis | +|---|----------|--------|------------| +| D1 | Audit rubric for structure/content | The 10 checks in `skills/wiki-lint/SKILL.md` + `AGENTS.md` format rules, run mechanically (grep/script), each finding recorded with the command and hit count; gates assert structure (table shape, bidirectional index↔page match), not keyword presence alone | qa-document-verification-spec-document-gates — "what a doc gate must assert beyond keyword presence"; audit scripts live in `.claude/tmp/` (user security policy: no /tmp) | +| D2 | Routing probe method | 15 realistic implementation scenarios spanning all 10 domains + 3 cross-domain; for each, walk INDEX.md "route here when" → domain index "load when"; verdict per probe: UNIQUE / AMBIGUOUS (≥2 equally-matching rows) / MISS (no row); AMBIGUOUS+MISS are findings | `[no-wiki]` — semantic evaluation; method fixed here so the implementer doesn't design it | +| D3 | Category-gap reference taxonomy | Compare existing categories against a fixed checklist of development concerns: requirements/planning, architecture/design, API design, concurrency, distributed systems, messaging/queues, caching, networking, observability, performance, docs/i18n, data engineering/ML, config mgmt/releases, cost, accessibility, compliance/privacy. Each unmatched concern → judged "genuine gap" vs "out of wiki charter" with 1-line rationale | `[no-wiki]` — taxonomy fixed by planner per user request "모든 개발 관점" | +| D4 | Findings routing | Mechanical defects (broken links, index drift, vague qualifiers with statable conditions, stale dates) → fix on this branch. Structural/coverage/category enhancements → `gh issue create` on choiyounggi/dev-loop, one issue per coherent theme, label `dev-loop:knowledge` not used (that's for flush PRs); use plain issues with title prefix `wiki-audit:` | `[no-wiki]` — user instruction (fix gaps; issues for 고도화) | +| D5 | Harvest dedupe fix | Adopt the existing `plans/harvest-dedupe-processed/` plan verbatim (its D1–D11 are already wiki-grounded); execute its single task 01 in this run | backend-common-jobs-idempotent-handlers (via adopted plan D1/D2) | +| D6 | Gate tests | New `tests/pre-flush-pr-gate.bats` exercising the PreToolUse gate: pass path, each missing section, empty-stub body, non-flush command untouched, missing body-file; per-test $HOME/tmp isolation, no real ~/.dev-loop touched | testing-quality-minimum-case-set (normal+error+boundary); testing-data-test-data-and-isolation (per-test tmp/env); testing-quality-checks-that-cannot-pass (target-missing vs content-missing exit semantics) | +| D7 | Harvest filter verdict scope | Audit-only findings for filter-quality gaps beyond the dedupe fix (e.g., evidence-less rows admitted, no cross-session semantic dedup at harvest time); do NOT redesign the filter in this branch — file as issues, because filter policy is a schema-layer change needing owner approval per AGENTS.md layer table | `[no-wiki]` — AGENTS.md mutability table (Workflows layer = owner approval) | +| D8 | Issue filing mechanics | `gh issue create --repo choiyounggi/dev-loop` with body containing: finding, evidence, proposed fix; one issue per theme (category gaps may bundle related categories into one issue per domain-area) | `[no-wiki]` — gh CLI per user Tool Priority policy | + +## Task order + +| Task | Depends on | Parallel-ok | +|------|-----------|-------------| +| 01-mechanical-lint-sweep | — | parallel-ok with 04 | +| 02-routing-probe | 01 (uses its inventory) | — | +| 03-category-taxonomy-audit | 02 (uses probe misses) | — | +| 04-harvest-dedupe-fix | — | parallel-ok with 01 | +| 05-flush-gate-tests-and-audit | 04 (shares bats conventions) | — | +| 06-issues-and-final-report | 01,02,03,05 | — | diff --git a/plans/wiki-audit/tasks/01-mechanical-lint-sweep.md b/plans/wiki-audit/tasks/01-mechanical-lint-sweep.md new file mode 100644 index 0000000..0bf2fad --- /dev/null +++ b/plans/wiki-audit/tasks/01-mechanical-lint-sweep.md @@ -0,0 +1,42 @@ +# Task 01: Mechanical lint sweep of all wiki pages and indexes + +## Objective +A findings file `plans/wiki-audit/findings/structure.md` listing every mechanical +defect across the 154 wiki pages + 10 domain indexes + INDEX.md, each with the +command that found it and the hit count (0-hit sweeps stated explicitly); defects +that the wiki-lint fix protocol allows fixing directly are fixed on this branch. + +## Wiki pages (read these first, only these) +- wiki/qa/document-verification/spec-document-gates.md — use for: designing checks + that assert structure, not keyword presence + +## Inputs +- skills/wiki-lint/SKILL.md — the 10-check rubric (checks 1–10, severities, fix protocol) +- AGENTS.md — frontmatter schema, section skeleton, banned qualifiers list +- Decisions that bind you: D1 (rubric + scripts in .claude/tmp/), D4 (fix vs issue split) + +## Steps +1. Write `.claude/tmp/lint-sweep.sh` implementing mechanical projections of the + lint checks: frontmatter fields present (id/domain/category/confidence/sources/ + last_verified/related); id matches `--` and file path; + `related:` ids resolve to existing page ids; every page listed in its domain + index and vice versa; body ≤120 lines; banned qualifiers (usually, consider, + might, generally, as appropriate) in directive sentences; `verified` with empty + sources; `unverified` age; bare don't/never/avoid outside `Instead of` tables. +2. Run it; record every finding + command + count in findings/structure.md. +3. Apply direct fixes only for lint fix-protocol categories 3/4/6 (links, index + lines) and qualifier rewrites where the condition is already stated in the page. +4. Anything non-mechanical goes in the findings file marked `→ issue`. + +## Deliverables +- .claude/tmp/lint-sweep.sh (throwaway, not committed) +- plans/wiki-audit/findings/structure.md +- Direct fixes to wiki/**.md files where the fix protocol allows (list each in findings) + +## Verify +- `sh .claude/tmp/lint-sweep.sh` exits 0 and its final summary matches the counts + in findings/structure.md; re-run after fixes shows the fixed categories at 0. + +## Out of scope +- Semantic routing quality (task 02); category sufficiency (task 03); any edit to + AGENTS.md/templates/skills (schema layer — owner approval; file as issue). diff --git a/plans/wiki-audit/tasks/02-routing-probe.md b/plans/wiki-audit/tasks/02-routing-probe.md new file mode 100644 index 0000000..3fd9ce7 --- /dev/null +++ b/plans/wiki-audit/tasks/02-routing-probe.md @@ -0,0 +1,40 @@ +# Task 02: Semantic routing probe — 15 implementation scenarios + +## Objective +`plans/wiki-audit/findings/routing.md` containing 15 probe scenarios (one per +domain + 3 two-domain/cross-domain + 2 deliberately out-of-charter), each traced +INDEX.md → domain index → page, with verdict UNIQUE / AMBIGUOUS / MISS and, for +each non-UNIQUE probe, which index line or trigger wording caused it. + +## Wiki pages (read these first, only these) +- (none — the wiki itself is the audit object; the routing protocol under test is + AGENTS.md "Routing protocol" steps 1–6) + +## Inputs +- plans/wiki-audit/findings/structure.md (task 01 — inventory of pages per domain) +- INDEX.md, all 10 wiki//index.md files +- Decisions that bind you: D2 (probe method + verdict definitions), D4 (findings routing) + +## Steps +1. Author the 15 probes as realistic first-person implementation intents (e.g. + "add cursor pagination to a listing endpoint", "my session cookie works locally + but not in prod", "schedule a nightly cleanup job on a VM"). Cover all 10 + domains; 3 probes must legitimately touch ≥2 domains; 2 probes must be + out-of-charter (e.g. "write a game shader") to test that routing FAILS cleanly + (MISS is the correct verdict there — record whether the reader can tell). +2. For each probe, quote the exact "route here when" / "load when" text that + matched or tied; verdict + cause for AMBIGUOUS/MISS. +3. Summarize: routing precision (UNIQUE / applicable probes), the specific index + lines needing rewording, and pages whose "When this applies" contradicts their + index line (AGENTS.md drift). + +## Deliverables +- plans/wiki-audit/findings/routing.md + +## Verify +- findings/routing.md has exactly 15 probes, every one carries a verdict and a + quoted matched line; `grep -c '^### Probe' plans/wiki-audit/findings/routing.md` → 15. + +## Out of scope +- Fixing index wording (fold into task 06 issue list or task 01-style direct fix + only if the reword is purely mechanical); category gaps (task 03). diff --git a/plans/wiki-audit/tasks/03-category-taxonomy-audit.md b/plans/wiki-audit/tasks/03-category-taxonomy-audit.md new file mode 100644 index 0000000..dd1231a --- /dev/null +++ b/plans/wiki-audit/tasks/03-category-taxonomy-audit.md @@ -0,0 +1,41 @@ +# Task 03: Category taxonomy sufficiency audit (all development perspectives) + +## Objective +`plans/wiki-audit/findings/categories.md`: the full category inventory (domain × +category × page count), each D3 checklist concern matched to where it lives today +or judged "genuine gap" vs "out of charter" with a 1-line rationale, plus overlap/ +ambiguity findings between existing categories (two categories a router could +confuse), ready to be filed as issues in task 06. + +## Wiki pages (read these first, only these) +- (none — the taxonomy itself is the audit object) + +## Inputs +- plans/wiki-audit/findings/routing.md (task 02 — MISS probes are gap evidence) +- All 10 wiki//index.md files; INDEX.md +- Decisions that bind you: D3 (fixed reference taxonomy), D4 (enhancements → issues) + +## Steps +1. Build the inventory table: `find wiki -mindepth 2 -maxdepth 2 -type d` + page + counts per category. +2. Walk the D3 checklist concern by concern: requirements/planning, architecture/ + design, API design, concurrency, distributed systems, messaging/queues, caching, + networking, observability, performance, docs/i18n, data engineering/ML, config + mgmt/releases, cost, accessibility, compliance/privacy. For each: covered-where + (exact domain/category), partially-covered (name the missing slice), or gap. +3. Ambiguity pass: list category pairs whose "load when" scopes overlap enough to + misroute (e.g. testing/quality vs qa/document-verification), citing the + overlapping line text. +4. For every "genuine gap": propose target domain, category name, and 2–3 seed + page titles — concrete enough for a future ingest. + +## Deliverables +- plans/wiki-audit/findings/categories.md + +## Verify +- Every D3 checklist concern appears exactly once in findings/categories.md with a + verdict; `grep -c '^| ' ` on the concern table ≥ 16 rows. + +## Out of scope +- Creating any new category or page (owner-approval scope — issues only, task 06); + routing-line rewording (task 02's findings). diff --git a/plans/wiki-audit/tasks/04-harvest-dedupe-fix.md b/plans/wiki-audit/tasks/04-harvest-dedupe-fix.md new file mode 100644 index 0000000..f5326a8 --- /dev/null +++ b/plans/wiki-audit/tasks/04-harvest-dedupe-fix.md @@ -0,0 +1,37 @@ +# Task 04: Execute the adopted harvest-dedupe-processed fix + +## Objective +`hooks/harvest.js` seeds its dedupe set from `~/.dev-loop/queue/.processed.jsonl` +in addition to the session queue file, so a flushed insight is never re-queued; +the existing red regression test in `tests/harvest.bats` turns green with the full +suite passing. + +## Wiki pages (read these first, only these) +- wiki/backend/common/jobs/idempotent-handlers.md — use for: processed-store + dedupe pattern (adopted plan D1/D2) +- wiki/testing/data/test-data-and-isolation.md — use for: per-test $HOME isolation (D6) +- wiki/testing/quality/minimum-case-set.md — use for: case coverage check (D7/D8) + +## Inputs +- plans/harvest-dedupe-processed/plan.md and its task + plans/harvest-dedupe-processed/tasks/01-dedupe-against-processed-store.md — + execute THAT task file's steps verbatim; its D1–D11 bind you +- tests/harvest.bats (existing, expected red on the regression case) + +## Steps +1. Run `bats tests/harvest.bats`; confirm and record the red case(s). +2. Apply the adopted task's change to hooks/harvest.js: read-only seed of `seen` + from `.processed.jsonl` (existsSync-guarded, safeJson per line), placed with + the existing queue-file seeding. +3. Re-run the suite to green. + +## Deliverables +- hooks/harvest.js (modified) +- tests/harvest.bats (only if the adopted task file says it is incomplete) + +## Verify +- `bats tests/` → all pass; the previously red regression case is named in output. + +## Out of scope +- Pruning `.processed.jsonl` (adopted plan D11 follow-up); filter-policy changes + (task 05 audits, issues only). diff --git a/plans/wiki-audit/tasks/05-flush-gate-tests-and-audit.md b/plans/wiki-audit/tasks/05-flush-gate-tests-and-audit.md new file mode 100644 index 0000000..6529bc5 --- /dev/null +++ b/plans/wiki-audit/tasks/05-flush-gate-tests-and-audit.md @@ -0,0 +1,43 @@ +# Task 05: Flush-gate bats tests + harvest/flush pipeline audit + +## Objective +(a) `tests/pre-flush-pr-gate.bats` proving the PreToolUse gate's decision table; +(b) `plans/wiki-audit/findings/pipeline.md` auditing, with quoted line evidence, +whether harvest admits only wiki-grade material and whether knowledge-flush's +dedup/verification steps are enforced vs merely instructed. + +## Wiki pages (read these first, only these) +- wiki/testing/quality/minimum-case-set.md — use for: gate test case selection +- wiki/testing/quality/checks-that-cannot-pass.md — use for: distinguishing + "body-file missing" vs "section missing" vs "content empty" exit semantics +- wiki/testing/data/test-data-and-isolation.md — use for: per-test tmp isolation + +## Inputs +- hooks/pre-flush-pr-gate.sh, hooks/auto-flush.sh, hooks/harvest.js (as fixed by + task 04), hooks/insight-instruction.sh, skills/knowledge-flush/SKILL.md, + skills/wiki-ingest/SKILL.md +- Decisions that bind you: D6 (test cases), D7 (audit-only for filter policy), D4 + +## Steps +1. Write tests/pre-flush-pr-gate.bats: (normal) flush command with complete + INGEST_REPORT passes; (error) each of the 3 sections missing → exit 2 with the + section named; body under 40 non-header chars → exit 2; --body-file absent from + command → exit 2; body file path nonexistent → exit 2; (boundary) non-flush + `gh pr create` passes untouched; non-pr command passes; flush markers detected + from each of the 3 alternatives (--head knowledge/, label, INGEST_REPORT). +2. Audit for enforcement gaps, recording each as finding + evidence line: e.g. + is the dedup step (SKILL step 2b) enforced by any gate or only prose? Can + auto-flush PENDING count be satisfied by rows that will fail verification? + Does anything verify wiki-ingest's merge-before-create actually ran? +3. Classify each finding per D7: mechanical fix on-branch vs `→ issue`. + +## Deliverables +- tests/pre-flush-pr-gate.bats +- plans/wiki-audit/findings/pipeline.md + +## Verify +- `bats tests/` → all pass (harvest + gate suites together). + +## Out of scope +- Rewriting gate/skill policy (owner-approval layer — issues); auto-flush spawn + mechanics (claude/gh availability paths are environment-dependent, note only). diff --git a/plans/wiki-audit/tasks/06-issues-and-final-report.md b/plans/wiki-audit/tasks/06-issues-and-final-report.md new file mode 100644 index 0000000..0b0fad4 --- /dev/null +++ b/plans/wiki-audit/tasks/06-issues-and-final-report.md @@ -0,0 +1,34 @@ +# Task 06: File GitHub issues + final audit report + +## Objective +Every `→ issue` finding from tasks 01/02/03/05 exists as a GitHub issue on +choiyounggi/dev-loop titled `wiki-audit: `, and +`plans/wiki-audit/findings/REPORT.md` summarizes the whole audit: per-axis +verdict, fixes applied on this branch, issues filed (with numbers), and the +overall production-readiness judgment the user asked for. + +## Wiki pages (read these first, only these) +- (none — reporting task) + +## Inputs +- plans/wiki-audit/findings/structure.md, routing.md, categories.md, pipeline.md +- Decisions that bind you: D4, D8 (one issue per coherent theme; body = finding + + evidence + proposed fix) + +## Steps +1. Group `→ issue` findings into themes; draft each issue body; create via + `gh issue create --repo choiyounggi/dev-loop --title "wiki-audit: " --body-file `. +2. Write REPORT.md: axis-by-axis verdict (구조/내용/카테고리/harvest/flush), what + was fixed (files), what was filed (issue #s), what is genuinely fine as-is. + +## Deliverables +- plans/wiki-audit/findings/REPORT.md +- GitHub issues (numbers recorded in REPORT.md) + +## Verify +- `gh issue list --repo choiyounggi/dev-loop --search "wiki-audit in:title" --json number,title` + lists every theme named in REPORT.md. + +## Out of scope +- Merging/committing beyond this branch's scope decisions; opening PRs (the user + reviews the branch). diff --git a/skills/knowledge-flush/SKILL.md b/skills/knowledge-flush/SKILL.md index 16b6866..a275331 100644 --- a/skills/knowledge-flush/SKILL.md +++ b/skills/knowledge-flush/SKILL.md @@ -103,15 +103,24 @@ an `INGEST_REPORT.md` with three filled sections exists. So do the work first: git -C "$REPO" push -u origin "$BR" gh pr create --repo choiyounggi/dev-loop --base main --head "$BR" \ --title "knowledge: " \ - --body-file "$REPO/.dev-loop/INGEST_REPORT.md" \ + --body-file "$HOME/.dev-loop/repo/.dev-loop/INGEST_REPORT.md" \ --label dev-loop:knowledge ``` + Write the `--body-file` path so the gate can resolve it as text: the + PreToolUse gate inspects the command string before execution and cannot + expand skill-local variables like `$REPO` (see + wiki/platforms/shells/command-text-inspected-before-execution.md) — use a + literal absolute path or `$HOME`/`~` (which the gate expands). Do NOT `gh pr merge`. The owner reviews open `dev-loop:knowledge` PRs and merges or rejects each one. -5. **Retire processed candidates.** Move the flushed rows out of the active queue - (e.g. append them to `~/.dev-loop/queue/.processed.jsonl` and rewrite the - session file without them) so the next flush doesn't re-ingest them. +5. **Retire processed candidates — every one you handled, not only the ingested.** + Move each handled row out of the active queue (append it to + `~/.dev-loop/queue/.processed.jsonl` and rewrite the session file without it): + rows you ingested, rows you merged into existing pages, AND rows you dropped + as unverifiable or duplicate. A dropped row left `pending` re-crosses the + auto-flush threshold forever — the headless flush would re-run hourly on + candidates that can never be promoted. ## Guardrails - PR-only. Never auto-merge, never push to `main`, never force-push `main`. diff --git a/tests/harvest.bats b/tests/harvest.bats new file mode 100644 index 0000000..510aa3f --- /dev/null +++ b/tests/harvest.bats @@ -0,0 +1,143 @@ +#!/usr/bin/env bats +# Tests for hooks/harvest.js (the Stop-hook insight harvester). +# +# The harvester re-parses the whole session transcript on every Stop event, so it +# must dedupe against BOTH the session queue file and the processed store that +# knowledge-flush retires rows into. Without the second source, a flush that +# empties the session queue lets the next Stop re-append the same insight. +# +# Every test runs against a per-test HOME so the real ~/.dev-loop/queue is never +# touched. + +setup() { + HARVEST="${BATS_TEST_DIRNAME}/../hooks/harvest.js" + + command -v node >/dev/null || { + echo "node is required to run the harvester tests" + return 1 + } + + # Node's os.homedir() reads $HOME on POSIX, so this redirects the queue path. + export HOME="${BATS_TEST_TMPDIR}/home" + QDIR="$HOME/.dev-loop/queue" + QFILE="$QDIR/s1.jsonl" + PFILE="$QDIR/.processed.jsonl" + mkdir -p "$QDIR" + + # The payload's cwd. Must not sit under $HOME/.dev-loop/repo, which the + # harvester skips to avoid harvesting its own flush checkout. + WORK="${BATS_TEST_TMPDIR}/work" + mkdir -p "$WORK" + + TRANSCRIPT="${BATS_TEST_TMPDIR}/transcript.jsonl" +} + +# Write a transcript holding one well-formed insight block. The \n sequences stay +# literal here and become real newlines when the harvester JSON-parses the line. +# The delimiter run is U+2500, matching BLOCK_RE in harvest.js. +_mk_transcript() { + local body + body="★ Insight ─────\ntrigger: a Stop hook re-parses the same transcript every turn\ndirective: seed the dedupe set from the processed store as well as the queue file\nwhy: an emptied queue file makes the parser look new again\nevidence: measured 46 duplicate rows out of 103\ndomain: testing\ntags: hooks, dedupe\n─────" + printf '{"message":{"role":"assistant","content":"%s"}}\n' "$body" > "$TRANSCRIPT" +} + +_mk_transcript_without_insight() { + printf '{"message":{"role":"assistant","content":"Just a normal reply with no block."}}\n' > "$TRANSCRIPT" +} + +_run_harvest() { # [cwd] + printf '{"cwd":"%s","session_id":"s1","transcript_path":"%s"}' \ + "${1:-$WORK}" "$TRANSCRIPT" | node "$HARVEST" +} + +# Non-blank line count of the session queue file; 0 when it does not exist. +# awk is used rather than `grep -c` so an empty file yields "0" on a single line +# with exit status 0 — `grep -c` exits 1 there, which would print 0 twice. +_queue_lines() { + [ -f "$QFILE" ] || { echo 0; return; } + awk 'NF { c++ } END { print c + 0 }' "$QFILE" +} + +# Retire the queue file's rows the way knowledge-flush does: append to the +# processed store, then truncate the session file. +_flush_queue() { + cat "$QFILE" >> "$PFILE" + : > "$QFILE" +} + +@test "normal: a new insight block is harvested into the session queue" { + _mk_transcript + run _run_harvest + [ "$status" -eq 0 ] + [ "$(_queue_lines)" -eq 1 ] + + # The row must carry a usable dedupe key, not just exist. + hash="$(node -e 'const l=require("fs").readFileSync(process.argv[1],"utf8").trim();process.stdout.write(JSON.parse(l).hash||"")' "$QFILE")" + [ -n "$hash" ] + [ "${#hash}" -eq 16 ] +} + +@test "regression: a hash already in the processed store is not re-queued" { + _mk_transcript + _run_harvest + [ "$(_queue_lines)" -eq 1 ] + + # A flush retires the row and empties the session file... + _flush_queue + [ "$(_queue_lines)" -eq 0 ] + + # ...and the next Stop must not resurrect it from the unchanged transcript. + run _run_harvest + [ "$status" -eq 0 ] + [ "$(_queue_lines)" -eq 0 ] +} + +@test "preserved: the same insight is not duplicated within one session" { + _mk_transcript + _run_harvest + _run_harvest + [ "$(_queue_lines)" -eq 1 ] +} + +@test "error: a corrupt line in the processed store is skipped, valid rows still dedupe" { + _mk_transcript + _run_harvest + printf 'not json at all\n' > "$PFILE" + cat "$QFILE" >> "$PFILE" + printf '{"no_hash_field":true}\n' >> "$PFILE" + : > "$QFILE" + + run _run_harvest + [ "$status" -eq 0 ] + [ "$(_queue_lines)" -eq 0 ] +} + +@test "error: an absent processed store does not stop a new insight being harvested" { + _mk_transcript + rm -f "$PFILE" + run _run_harvest + [ "$status" -eq 0 ] + [ "$(_queue_lines)" -eq 1 ] +} + +@test "boundary: an empty processed store harvests normally" { + _mk_transcript + : > "$PFILE" + run _run_harvest + [ "$status" -eq 0 ] + [ "$(_queue_lines)" -eq 1 ] +} + +@test "boundary: a transcript with no insight block queues nothing" { + _mk_transcript_without_insight + run _run_harvest + [ "$status" -eq 0 ] + [ "$(_queue_lines)" -eq 0 ] +} + +@test "preserved: a session inside the flush checkout is not harvested" { + _mk_transcript + run _run_harvest "$HOME/.dev-loop/repo" + [ "$status" -eq 0 ] + [ "$(_queue_lines)" -eq 0 ] +} diff --git a/tests/pre-flush-pr-gate.bats b/tests/pre-flush-pr-gate.bats new file mode 100644 index 0000000..047f65d --- /dev/null +++ b/tests/pre-flush-pr-gate.bats @@ -0,0 +1,135 @@ +#!/usr/bin/env bats +# Tests for hooks/pre-flush-pr-gate.sh (PreToolUse gate on `gh pr create`). +# +# The gate must block a knowledge-flush PR until an INGEST_REPORT with three +# filled sections exists, while letting every non-flush command pass untouched. +# Stdin is the nested Claude Code PreToolUse payload: {"tool_input":{"command":..}}. +# Exit 0 = allow, exit 2 = deny (message on stderr). + +setup() { + GATE="${BATS_TEST_DIRNAME}/../hooks/pre-flush-pr-gate.sh" + + command -v node >/dev/null || { + echo "node is required to run the gate tests" + return 1 + } + + REPORT="${BATS_TEST_TMPDIR}/INGEST_REPORT.md" +} + +# A complete, filled report that must satisfy the gate. +_mk_full_report() { + cat > "$REPORT" <<'EOF' +# Knowledge flush — 1 insight + +## Verified best-practice +Claim verified against the official PostgreSQL docs (real URL checked); the +directive reproduces locally; confidence: verified. + +## Existing-layer check +Read databases/index.md and both indexing pages; no duplicate trigger found; +added related-links both ways. + +## Routing decision +Target databases/indexing — existing category fits; no new category needed. +EOF +} + +# Report with one required section removed. $1 = section heading to drop. +_mk_report_missing() { + _mk_full_report + grep -v "^## $1\$" "$REPORT" > "$REPORT.tmp" && mv "$REPORT.tmp" "$REPORT" +} + +_run_gate() { # + printf '{"tool_input":{"command":%s}}' \ + "$(node -e 'process.stdout.write(JSON.stringify(process.argv[1]))' "$1")" \ + | sh "$GATE" +} + +@test "normal: a flush PR with a complete INGEST_REPORT passes" { + _mk_full_report + run _run_gate "gh pr create --head knowledge/me-1 --label dev-loop:knowledge --body-file $REPORT" + [ "$status" -eq 0 ] +} + +@test "normal: a non-pr command passes untouched" { + run _run_gate "git status" + [ "$status" -eq 0 ] +} + +@test "normal: a non-flush gh pr create passes untouched" { + run _run_gate "gh pr create --title 'fix: unrelated' --body-file /nonexistent" + [ "$status" -eq 0 ] +} + +@test "error: flush PR with no --body-file at all is denied" { + run _run_gate "gh pr create --head knowledge/me-1 --label dev-loop:knowledge --body inline" + [ "$status" -eq 2 ] + [[ "$output" == *"no --body-file"* ]] +} + +@test "error: flush PR whose body file does not exist is denied" { + run _run_gate "gh pr create --head knowledge/me-1 --body-file ${BATS_TEST_TMPDIR}/missing.md" + [ "$status" -eq 2 ] + [[ "$output" == *"does not exist"* ]] +} + +@test "error: missing 'Verified best-practice' section is denied and named" { + _mk_report_missing "Verified best-practice" + run _run_gate "gh pr create --head knowledge/me-1 --body-file $REPORT" + [ "$status" -eq 2 ] + [[ "$output" == *"Verified best-practice"* ]] +} + +@test "error: missing 'Existing-layer check' section is denied and named" { + _mk_report_missing "Existing-layer check" + run _run_gate "gh pr create --head knowledge/me-1 --body-file $REPORT" + [ "$status" -eq 2 ] + [[ "$output" == *"Existing-layer check"* ]] +} + +@test "error: missing 'Routing decision' section is denied and named" { + _mk_report_missing "Routing decision" + run _run_gate "gh pr create --head knowledge/me-1 --body-file $REPORT" + [ "$status" -eq 2 ] + [[ "$output" == *"Routing decision"* ]] +} + +@test "boundary: headers-only report (empty stubs) is denied" { + printf '## Verified best-practice\n## Existing-layer check\n## Routing decision\n' > "$REPORT" + run _run_gate "gh pr create --head knowledge/me-1 --body-file $REPORT" + [ "$status" -eq 2 ] + [[ "$output" == *"empty"* ]] +} + +@test "boundary: each flush marker alone engages the gate (head / label / INGEST_REPORT)" { + for marker in \ + "--head knowledge/me-1 --body-file ${BATS_TEST_TMPDIR}/missing.md" \ + "--label dev-loop:knowledge --body-file ${BATS_TEST_TMPDIR}/missing.md" \ + "--body-file ${BATS_TEST_TMPDIR}/missing_INGEST_REPORT.md"; do + run _run_gate "gh pr create $marker" + [ "$status" -eq 2 ] + done +} + +@test "boundary: prose mentioning gh pr create does not trip the gate" { + run _run_gate "echo 'docs say: gh pr create --head knowledge/x needs a report'" + [ "$status" -eq 0 ] +} + +@test "regression: a double-quoted --body-file path is still recognized" { + _mk_full_report + run _run_gate "gh pr create --head knowledge/me-1 --body-file \"$REPORT\"" + [ "$status" -eq 0 ] +} + +@test "regression: a \$HOME-prefixed --body-file path is expanded by the gate" { + # The skill's example command writes the path with $HOME; the gate sees the + # command text unexpanded and must resolve that prefix itself. + export HOME="$BATS_TEST_TMPDIR" + REPORT="$HOME/INGEST_REPORT.md" + _mk_full_report + run _run_gate 'gh pr create --head knowledge/me-1 --body-file "$HOME/INGEST_REPORT.md"' + [ "$status" -eq 0 ] +}