From d0f092bdcb3c1fc537cfce6b0e486d66c2fa2627 Mon Sep 17 00:00:00 2001 From: Barrett Holien Date: Thu, 9 Jul 2026 18:02:51 -0500 Subject: [PATCH 1/2] feat: relevance ranking, evals, chronicle, embeddings, skill drift-check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Borrow the strongest ideas from tobi/qmd and garrytan/gbrain and graft them onto graphify's structural spine (five features, each: module + CLI + tests): 1. Ranking (graphify/ranking.py): rank query results by Reciprocal Rank Fusion over lexical + structural (proximity/centrality/community) + optional semantic backends, replacing the degree-only render order. New query flags --top-k, --explain, --semantic; also on the MCP query_graph tool. Shared core factored into serve._resolve_and_rank / serve.rank_query_nodes. 2. Evals (graphify/evals.py): `graphify bench` scores P@k/recall/MRR/nDCG against a JSONL fixture using the real ranking pipeline. --init scaffolds, --save/ --replay gate regressions (exit 1) via .graphify-evals/eval-results.jsonl. 3. Embeddings (graphify/embed.py): opt-in `graphify embed` builds a local embedding sidecar (Ollama / sentence-transformers, no API). --semantic fuses cosine similarity into ranking and seeds traversal when lexical finds nothing. 4. Chronicle (graphify/chronicle.py): `graphify chronicle OLD NEW` (or --rev via `git show`) diffs two graph snapshots — nodes/edges, god-nodes emerged/ vanished, community shifts. Generalizes prs.py across time, API-free. 5. Skill drift (graphify/skill_migrations.py): `graphify skill status|check-update` reports installed-skill-vs-package drift (exit 1 for CI) with a migration registry. Skill: document the new query flags and a new analysis.md reference (bench/embed/ chronicle/skill) via tools/skillgen fragments; regenerated + blessed all artifacts. skillgen --check/--audit-coverage/roundtrips all pass. --- .graphify-evals/eval-results.jsonl | 1 + graphify/__main__.py | 361 +++++++++++++++++- graphify/chronicle.py | 185 +++++++++ graphify/embed.py | 293 ++++++++++++++ graphify/evals.py | 348 +++++++++++++++++ graphify/ranking.py | 172 +++++++++ graphify/serve.py | 188 ++++++++- graphify/skill-agents.md | 6 + graphify/skill-amp.md | 6 + graphify/skill-claw.md | 6 + graphify/skill-codex.md | 6 + graphify/skill-copilot.md | 6 + graphify/skill-droid.md | 6 + graphify/skill-kilo.md | 6 + graphify/skill-kiro.md | 6 + graphify/skill-opencode.md | 6 + graphify/skill-pi.md | 6 + graphify/skill-trae.md | 6 + graphify/skill-vscode.md | 6 + graphify/skill-windows.md | 6 + graphify/skill.md | 6 + graphify/skill_migrations.py | 90 +++++ graphify/skills/agents/references/analysis.md | 75 ++++ graphify/skills/agents/references/query.md | 14 + graphify/skills/amp/references/analysis.md | 75 ++++ graphify/skills/amp/references/query.md | 14 + graphify/skills/claude/references/analysis.md | 75 ++++ graphify/skills/claude/references/query.md | 14 + graphify/skills/claw/references/analysis.md | 75 ++++ graphify/skills/claw/references/query.md | 14 + graphify/skills/codex/references/analysis.md | 75 ++++ graphify/skills/codex/references/query.md | 14 + .../skills/copilot/references/analysis.md | 75 ++++ graphify/skills/copilot/references/query.md | 14 + graphify/skills/droid/references/analysis.md | 75 ++++ graphify/skills/droid/references/query.md | 14 + graphify/skills/kilo/references/analysis.md | 75 ++++ graphify/skills/kilo/references/query.md | 14 + graphify/skills/kiro/references/analysis.md | 75 ++++ graphify/skills/kiro/references/query.md | 14 + .../skills/opencode/references/analysis.md | 75 ++++ graphify/skills/opencode/references/query.md | 14 + graphify/skills/pi/references/analysis.md | 75 ++++ graphify/skills/pi/references/query.md | 14 + graphify/skills/trae/references/analysis.md | 75 ++++ graphify/skills/trae/references/query.md | 14 + graphify/skills/vscode/references/analysis.md | 75 ++++ graphify/skills/vscode/references/query.md | 14 + .../skills/windows/references/analysis.md | 75 ++++ graphify/skills/windows/references/query.md | 14 + tests/test_chronicle.py | 100 +++++ tests/test_embed.py | 127 ++++++ tests/test_evals.py | 201 ++++++++++ tests/test_install_references.py | 3 +- tests/test_ranking.py | 138 +++++++ tests/test_serve.py | 73 ++++ tests/test_skill_migrations.py | 62 +++ tests/test_skillgen.py | 6 +- .../expected/graphify__skill-agents.md | 6 + .../skillgen/expected/graphify__skill-amp.md | 6 + .../skillgen/expected/graphify__skill-claw.md | 6 + .../expected/graphify__skill-codex.md | 6 + .../expected/graphify__skill-copilot.md | 6 + .../expected/graphify__skill-droid.md | 6 + .../skillgen/expected/graphify__skill-kilo.md | 6 + .../skillgen/expected/graphify__skill-kiro.md | 6 + .../expected/graphify__skill-opencode.md | 6 + tools/skillgen/expected/graphify__skill-pi.md | 6 + .../skillgen/expected/graphify__skill-trae.md | 6 + .../expected/graphify__skill-vscode.md | 6 + .../expected/graphify__skill-windows.md | 6 + tools/skillgen/expected/graphify__skill.md | 6 + ...y__skills__agents__references__analysis.md | 75 ++++ ...hify__skills__agents__references__query.md | 14 + ...hify__skills__amp__references__analysis.md | 75 ++++ ...raphify__skills__amp__references__query.md | 14 + ...y__skills__claude__references__analysis.md | 75 ++++ ...hify__skills__claude__references__query.md | 14 + ...ify__skills__claw__references__analysis.md | 75 ++++ ...aphify__skills__claw__references__query.md | 14 + ...fy__skills__codex__references__analysis.md | 75 ++++ ...phify__skills__codex__references__query.md | 14 + ...__skills__copilot__references__analysis.md | 75 ++++ ...ify__skills__copilot__references__query.md | 14 + ...fy__skills__droid__references__analysis.md | 75 ++++ ...phify__skills__droid__references__query.md | 14 + ...ify__skills__kilo__references__analysis.md | 75 ++++ ...aphify__skills__kilo__references__query.md | 14 + ...ify__skills__kiro__references__analysis.md | 75 ++++ ...aphify__skills__kiro__references__query.md | 14 + ..._skills__opencode__references__analysis.md | 75 ++++ ...fy__skills__opencode__references__query.md | 14 + ...phify__skills__pi__references__analysis.md | 75 ++++ ...graphify__skills__pi__references__query.md | 14 + ...ify__skills__trae__references__analysis.md | 75 ++++ ...aphify__skills__trae__references__query.md | 14 + ...y__skills__vscode__references__analysis.md | 75 ++++ ...hify__skills__vscode__references__query.md | 14 + ...__skills__windows__references__analysis.md | 75 ++++ ...ify__skills__windows__references__query.md | 14 + tools/skillgen/fragments/core/core.md | 6 + .../fragments/references/query/default.md | 14 + .../fragments/references/shared/analysis.md | 75 ++++ tools/skillgen/gen.py | 3 +- 104 files changed, 5090 insertions(+), 16 deletions(-) create mode 100644 .graphify-evals/eval-results.jsonl create mode 100644 graphify/chronicle.py create mode 100644 graphify/embed.py create mode 100644 graphify/evals.py create mode 100644 graphify/ranking.py create mode 100644 graphify/skill_migrations.py create mode 100644 graphify/skills/agents/references/analysis.md create mode 100644 graphify/skills/amp/references/analysis.md create mode 100644 graphify/skills/claude/references/analysis.md create mode 100644 graphify/skills/claw/references/analysis.md create mode 100644 graphify/skills/codex/references/analysis.md create mode 100644 graphify/skills/copilot/references/analysis.md create mode 100644 graphify/skills/droid/references/analysis.md create mode 100644 graphify/skills/kilo/references/analysis.md create mode 100644 graphify/skills/kiro/references/analysis.md create mode 100644 graphify/skills/opencode/references/analysis.md create mode 100644 graphify/skills/pi/references/analysis.md create mode 100644 graphify/skills/trae/references/analysis.md create mode 100644 graphify/skills/vscode/references/analysis.md create mode 100644 graphify/skills/windows/references/analysis.md create mode 100644 tests/test_chronicle.py create mode 100644 tests/test_embed.py create mode 100644 tests/test_evals.py create mode 100644 tests/test_ranking.py create mode 100644 tests/test_skill_migrations.py create mode 100644 tools/skillgen/expected/graphify__skills__agents__references__analysis.md create mode 100644 tools/skillgen/expected/graphify__skills__amp__references__analysis.md create mode 100644 tools/skillgen/expected/graphify__skills__claude__references__analysis.md create mode 100644 tools/skillgen/expected/graphify__skills__claw__references__analysis.md create mode 100644 tools/skillgen/expected/graphify__skills__codex__references__analysis.md create mode 100644 tools/skillgen/expected/graphify__skills__copilot__references__analysis.md create mode 100644 tools/skillgen/expected/graphify__skills__droid__references__analysis.md create mode 100644 tools/skillgen/expected/graphify__skills__kilo__references__analysis.md create mode 100644 tools/skillgen/expected/graphify__skills__kiro__references__analysis.md create mode 100644 tools/skillgen/expected/graphify__skills__opencode__references__analysis.md create mode 100644 tools/skillgen/expected/graphify__skills__pi__references__analysis.md create mode 100644 tools/skillgen/expected/graphify__skills__trae__references__analysis.md create mode 100644 tools/skillgen/expected/graphify__skills__vscode__references__analysis.md create mode 100644 tools/skillgen/expected/graphify__skills__windows__references__analysis.md create mode 100644 tools/skillgen/fragments/references/shared/analysis.md diff --git a/.graphify-evals/eval-results.jsonl b/.graphify-evals/eval-results.jsonl new file mode 100644 index 000000000..ed8416b01 --- /dev/null +++ b/.graphify-evals/eval-results.jsonl @@ -0,0 +1 @@ +{"aggregate": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.4322, "recall_at_k": 1.0}, "cases": [{"expect": ["main()"], "matched": {"main()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.3, "recall_at_k": 1.0}, "query": "main"}, {"expect": ["extract()"], "matched": {"extract()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.1, "recall_at_k": 1.0}, "query": "extract"}, {"expect": ["build_from_json()"], "matched": {"build_from_json()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.2, "recall_at_k": 1.0}, "query": "build_from_json"}, {"expect": ["Graph"], "matched": {"Graph": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.125, "recall_at_k": 1.0}, "query": "Graph"}, {"expect": ["Path"], "matched": {"Path": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 1.0, "recall_at_k": 1.0}, "query": "Path"}, {"expect": ["ingest_scip_json()"], "matched": {"ingest_scip_json()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.1, "recall_at_k": 1.0}, "query": "ingest_scip_json"}, {"expect": ["_read_text()"], "matched": {"_read_text()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 1.0, "recall_at_k": 1.0}, "query": "_read_text"}, {"expect": ["_make_id()"], "matched": {"_make_id()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.5, "recall_at_k": 1.0}, "query": "_make_id"}, {"expect": ["_labels()"], "matched": {"_labels()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 1.0, "recall_at_k": 1.0}, "query": "_labels"}, {"expect": ["extract_js()"], "matched": {"extract_js()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.1111, "recall_at_k": 1.0}, "query": "extract_js"}, {"expect": ["_file_stem()"], "matched": {"_file_stem()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.5, "recall_at_k": 1.0}, "query": "_file_stem"}, {"expect": ["cluster()"], "matched": {"cluster()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.25, "recall_at_k": 1.0}, "query": "cluster"}], "fixture": "/private/tmp/claude-501/-Users-barrett-code-graphify/7a133a61-c1e4-44a7-8365-8c9e0318ba14/scratchpad/verify/ev.jsonl", "k": 10} diff --git a/graphify/__main__.py b/graphify/__main__.py index 2bad90a15..b9fb6fde8 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -164,6 +164,59 @@ def _refresh_all_version_stamps() -> None: vf.write_text(__version__, encoding="utf-8") +def _installed_skill_report() -> list[dict]: + """One record per platform whose graphify skill is installed on this machine. + + Each record: platform, path, installed version, drift state vs the running + package (current/behind/ahead), sidecar health, and the migrations that a + re-install would apply. Powers `graphify skill status|check-update`. + """ + from graphify.skill_migrations import applicable_migrations, drift_state + + seen: set[Path] = set() + report: list[dict] = [] + for name in [*_PLATFORM_CONFIG, "gemini"]: + try: + skill_dst = _platform_skill_destination(name) + except Exception: + continue + if skill_dst in seen: + continue + try: + if not skill_dst.exists(): + continue + except OSError: + continue + seen.add(skill_dst) + version_file = skill_dst.parent / ".graphify_version" + try: + installed = version_file.read_text(encoding="utf-8").strip() if version_file.exists() else "unknown" + except OSError: + installed = "unknown" + try: + body = skill_dst.read_text(encoding="utf-8") + except OSError: + body = "" + sidecar_ok = True + if "references/" in body: + sidecar_ok = (skill_dst.parent / "references").exists() + state = drift_state(installed, __version__) + migrations = [m.summary for m in applicable_migrations(installed, __version__)] + report.append( + { + "platform": name, + "path": str(skill_dst), + "installed": installed, + "package": __version__, + "state": state, + "sidecar_ok": sidecar_ok, + "migrations": migrations, + "needs_update": state != "current" or not sidecar_ok, + } + ) + return report + + def _platform_skill_destination(platform_name: str, *, project: bool = False, project_dir: Path | None = None) -> Path: """Return the skill destination for a platform and scope.""" if platform_name == "gemini": @@ -2267,11 +2320,30 @@ def main() -> None: print(" --model= model to use for community naming") print(" --max-concurrency=N parallel labeling LLM calls (default 4; forced to 1 for ollama/claude-cli)") print(" --batch-size=N communities per labeling LLM call (default 100)") - print(" query \"\" BFS traversal of graph.json for a question") + print(" query \"\" BFS traversal of graph.json, results ranked by relevance") print(" --dfs use depth-first instead of breadth-first") print(" --context C explicit edge-context filter (repeatable)") print(" --budget N cap output at N tokens (default 2000)") + print(" --top-k N keep only the N most relevant (ranked) nodes") + print(" --explain show the per-node ranking breakdown (RRF backends)") + print(" --semantic fuse local-embedding similarity into ranking (needs `graphify embed`)") + print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" bench [FIXTURE] relevance eval: P@k / recall / MRR / nDCG vs a fixture") + print(" --init scaffold a starter fixture from prominent nodes") + print(" --k N rank cutoff (default 10)") + print(" --save append the run to .graphify-evals/eval-results.jsonl") + print(" --replay diff metrics vs the last saved run (exit 1 on regression)") + print(" --semantic include the embedding backend in the ranking under test") + print(" --json machine-readable output") + print(" embed build the optional semantic-embedding sidecar next to graph.json") + print(" --force rebuild even if the sidecar is current") print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" skill [status|check-update] report installed-skill vs package drift + migrations") + print(" --json machine-readable output (check-update exits 1 on drift)") + print(" chronicle OLD [NEW] structural diff of two graph.json snapshots") + print(" --rev REV [--rev2 REV2] read snapshots from git history (git show REV:graph.json)") + print(" --top-god N god-node ranking depth to compare (default 15)") + print(" --json machine-readable diff") print(" affected \"X\" reverse traversal to find nodes impacted by X") print(" --relation R edge relation to traverse in reverse (repeatable)") print(" --depth N reverse traversal depth (default 2)") @@ -2808,7 +2880,7 @@ def main() -> None: sys.exit(1) elif cmd == "query": if len(sys.argv) < 3: - print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) + print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--top-k N] [--explain] [--semantic] [--graph path]", file=sys.stderr) sys.exit(1) from graphify.serve import _query_graph_text from graphify.security import sanitize_label @@ -2817,12 +2889,31 @@ def main() -> None: question = sys.argv[2] use_dfs = "--dfs" in sys.argv + explain = "--explain" in sys.argv + use_semantic = "--semantic" in sys.argv + top_k: int | None = None budget = 2000 graph_path = _default_graph_path() context_filters: list[str] = [] args = sys.argv[3:] i = 0 while i < len(args): + if args[i] in ("--top-k", "--topk") and i + 1 < len(args): + try: + top_k = int(args[i + 1]) + except ValueError: + print("error: --top-k must be an integer", file=sys.stderr) + sys.exit(1) + i += 2 + continue + if args[i].startswith("--top-k=") or args[i].startswith("--topk="): + try: + top_k = int(args[i].split("=", 1)[1]) + except ValueError: + print("error: --top-k must be an integer", file=sys.stderr) + sys.exit(1) + i += 1 + continue if args[i] == "--budget" and i + 1 < len(args): try: budget = int(args[i + 1]) @@ -2884,6 +2975,13 @@ def main() -> None: import time as _time _t0 = _time.perf_counter() _mode = "dfs" if use_dfs else "bfs" + _semantic_scores = None + if use_semantic: + try: + from graphify.embed import semantic_scores_for_query + _semantic_scores = semantic_scores_for_query(G, question, graph_path=str(gp)) + except Exception as exc: # embeddings are strictly optional + print(f"[graphify] --semantic unavailable ({exc}); ranking without it.", file=sys.stderr) _result = _query_graph_text( G, question, @@ -2891,6 +2989,9 @@ def main() -> None: depth=2, token_budget=budget, context_filters=context_filters, + explain=explain, + top_k=top_k, + semantic_scores=_semantic_scores, ) querylog.log_query( kind="query", @@ -2963,6 +3064,262 @@ def main() -> None: depth=depth, ) ) + elif cmd == "chronicle": + # graphify chronicle OLD.json [NEW.json] + # graphify chronicle --rev REV [--rev2 REV2] [--graph PATH] + import json as _json + import subprocess as _sp + + args = sys.argv[2:] + graph_path = _default_graph_path() + rev: str | None = None + rev2: str | None = None + as_json = "--json" in args + top_god = 15 + positional: list[str] = [] + i = 0 + while i < len(args): + a = args[i] + if a == "--graph" and i + 1 < len(args): + graph_path = args[i + 1]; i += 2; continue + if a.startswith("--graph="): + graph_path = a.split("=", 1)[1]; i += 1; continue + if a == "--rev" and i + 1 < len(args): + rev = args[i + 1]; i += 2; continue + if a == "--rev2" and i + 1 < len(args): + rev2 = args[i + 1]; i += 2; continue + if a == "--top-god" and i + 1 < len(args): + try: + top_god = int(args[i + 1]) + except ValueError: + print("error: --top-god must be an integer", file=sys.stderr); sys.exit(1) + i += 2; continue + if a.startswith("--"): + i += 1; continue + positional.append(a); i += 1 + + from graphify import chronicle as _chron + + def _read_file(p: Path) -> str: + if not p.exists(): + print(f"error: graph file not found: {p}", file=sys.stderr); sys.exit(1) + _enforce_graph_size_cap_or_exit(p) + return p.read_text(encoding="utf-8") + + if rev is not None: + gp = Path(graph_path).resolve() + try: + top = _sp.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=str(gp.parent), capture_output=True, text=True, check=True, + ).stdout.strip() + relpath = str(gp.relative_to(Path(top).resolve())) + except Exception as exc: + print(f"error: need a git repo containing {gp} for --rev ({exc})", file=sys.stderr); sys.exit(1) + + def _show(r: str) -> str: + res = _sp.run(["git", "show", f"{r}:{relpath}"], cwd=top, capture_output=True, text=True) + if res.returncode != 0: + print(f"error: could not read {relpath} at {r}: {res.stderr.strip()}", file=sys.stderr) + sys.exit(1) + return res.stdout + + old_text = _show(rev) + new_text = _show(rev2) if rev2 else _read_file(gp) + else: + if not positional: + print( + "Usage: graphify chronicle OLD.json [NEW.json] | --rev REV [--rev2 REV2] [--graph PATH]", + file=sys.stderr, + ) + sys.exit(1) + old_text = _read_file(Path(positional[0]).resolve()) + new_path = Path(positional[1]).resolve() if len(positional) > 1 else Path(graph_path).resolve() + new_text = _read_file(new_path) + + try: + old_G = _chron.load_graph_from_text(old_text) + new_G = _chron.load_graph_from_text(new_text) + except Exception as exc: + print(f"error: could not parse a graph snapshot: {exc}", file=sys.stderr); sys.exit(1) + diff = _chron.diff_graphs(old_G, new_G, top_god=top_god) + if as_json: + print(_json.dumps(diff, indent=2)) + else: + print(_chron.format_diff(diff)) + elif cmd == "skill": + # graphify skill [status|check-update] [--json] + sub = sys.argv[2] if len(sys.argv) > 2 else "" + as_json = "--json" in sys.argv[2:] + if sub not in ("status", "check-update"): + print("Usage: graphify skill [status|check-update] [--json]", file=sys.stderr) + sys.exit(1) + report = _installed_skill_report() + if as_json: + import json as _json + print(_json.dumps({"package": __version__, "skills": report}, indent=2)) + elif not report: + print("No graphify skills installed. Run `graphify install` to add one.") + else: + print(f"graphify package: {__version__}") + for r in report: + if not r["sidecar_ok"]: + status = "references/ sidecar MISSING — reinstall to repair" + elif r["state"] == "current": + status = f"up to date ({r['installed']})" + elif r["state"] == "ahead": + status = ( + f"skill {r['installed']} is AHEAD of package {__version__} " + f"— this graphify may be older than the installed skill" + ) + else: + status = f"behind: skill {r['installed']} < package {__version__}" + print(f" [{r['platform']}] {status}") + print(f" {r['path']}") + for m in r["migrations"]: + print(f" apply: {m}") + needing = [r for r in report if r["needs_update"]] + if needing: + print( + f"\n{len(needing)} skill(s) need attention. " + f"Run `graphify install` to re-render them from this package." + ) + else: + print("\nAll installed skills are in sync with this package.") + if sub == "check-update" and any(r["needs_update"] for r in report): + sys.exit(1) # cron/CI-friendly: nonzero exit signals drift + elif cmd == "embed": + # graphify embed [--graph P] [--force] (build the semantic sidecar) + args = sys.argv[2:] + graph_path = _default_graph_path() + force = "--force" in args + i = 0 + while i < len(args): + a = args[i] + if a == "--graph" and i + 1 < len(args): + graph_path = args[i + 1]; i += 2; continue + if a.startswith("--graph="): + graph_path = a.split("=", 1)[1]; i += 1; continue + i += 1 + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + from graphify import evals as _evals + from graphify import embed as _embed + + G = _evals.load_graph(str(gp)) + try: + summary = _embed.build_embeddings(G, str(gp), force=force) + except RuntimeError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + if summary["status"] == "cached": + print(f"Embeddings already current for {summary['count']} nodes ({summary['model']}). Use --force to rebuild.") + else: + print(f"Embedded {summary['count']} nodes with {summary['model']} -> {summary['path']}") + print("Semantic ranking is now available: add --semantic to `graphify query` or `graphify bench`.") + elif cmd == "bench": + # graphify bench [FIXTURE] [--graph P] [--k N] [--json] [--save] [--replay] + # [--init] [--semantic] + import json as _json + + args = sys.argv[2:] + fixture: str | None = None + graph_path = _default_graph_path() + k = 10 + as_json = "--json" in args + do_save = "--save" in args + do_replay = "--replay" in args + do_init = "--init" in args + use_semantic = "--semantic" in args + default_fixture = str(Path(_GRAPHIFY_OUT) / "evals.jsonl") + i = 0 + while i < len(args): + a = args[i] + if a in ("--graph",) and i + 1 < len(args): + graph_path = args[i + 1]; i += 2; continue + if a.startswith("--graph="): + graph_path = a.split("=", 1)[1]; i += 1; continue + if a in ("--k",) and i + 1 < len(args): + try: + k = int(args[i + 1]) + except ValueError: + print("error: --k must be an integer", file=sys.stderr); sys.exit(1) + i += 2; continue + if a.startswith("--k="): + try: + k = int(a.split("=", 1)[1]) + except ValueError: + print("error: --k must be an integer", file=sys.stderr); sys.exit(1) + i += 1; continue + if a.startswith("--"): + i += 1; continue + if fixture is None: + fixture = a + i += 1 + from graphify import evals as _evals + + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + + if do_init: + target = Path(fixture or default_fixture) + if target.exists(): + print(f"error: {target} already exists (refusing to overwrite)", file=sys.stderr) + sys.exit(1) + G = _evals.load_graph(str(gp)) + cases = _evals.scaffold_fixture(G) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + "".join(_json.dumps(c) + "\n" for c in cases), encoding="utf-8" + ) + print(f"Wrote {len(cases)} starter eval cases to {target}") + print("Edit them — each line is {\"query\": ..., \"expect\": [labels/files]} — then run `graphify bench`.") + sys.exit(0) + + fixture_path = Path(fixture or default_fixture) + if not fixture_path.exists(): + print( + f"error: no eval fixture at {fixture_path}. " + f"Create one with `graphify bench --init`, or pass a fixture path.", + file=sys.stderr, + ) + sys.exit(1) + try: + cases = _evals.load_cases(fixture_path) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + G = _evals.load_graph(str(gp)) + report = _evals.run_evals(G, cases, k=k, semantic=use_semantic, graph_path=str(gp)) + baseline = _evals.load_last_report() if do_replay else None + + if as_json: + out = report.to_dict() + if baseline is not None: + out["baseline_aggregate"] = baseline.get("aggregate") + out["delta"] = _evals.diff_aggregates(report.aggregate, baseline.get("aggregate", {})) + print(_json.dumps(out, indent=2, sort_keys=True)) + else: + print(_evals.format_report(report, baseline=baseline)) + + if do_save: + saved = _evals.save_report(report, fixture=str(fixture_path)) + if not as_json: + print(f"\nSaved run to {saved}") + + if do_replay and baseline is not None: + delta = _evals.diff_aggregates(report.aggregate, baseline.get("aggregate", {})) + regressed = {kk: dv for kk, dv in delta.items() if dv < -_evals.REPLAY_TOLERANCE} + if regressed: + print( + "\nREGRESSION: " + ", ".join(f"{kk} {dv:+.4f}" for kk, dv in sorted(regressed.items())), + file=sys.stderr, + ) + sys.exit(1) elif cmd == "save-result": # graphify save-result --question Q --answer A [--type T] [--nodes N1 N2 ...] # [--outcome useful|dead_end|corrected] [--correction TEXT] diff --git a/graphify/chronicle.py b/graphify/chronicle.py new file mode 100644 index 000000000..58c3ab3fd --- /dev/null +++ b/graphify/chronicle.py @@ -0,0 +1,185 @@ +"""Chronicle — diff two graph snapshots to see how a codebase's structure moved. + +`prs.py` answers "what does *this* PR touch?". Chronicle generalizes that across +time: given two ``graph.json`` snapshots (two files, or the same file at two git +revisions), it reports how the *structure* changed — nodes and edges added or +removed, which god-nodes (high-degree hubs) emerged or vanished, and how the +community structure shifted. That surfaces architectural drift a line diff never +shows: a hub forming, a module fragmenting, a subsystem appearing. + +The git path is deliberately cheap and API-free: graphify users commit +``graphify-out/graph.json``, so a historical snapshot is just +``git show :graphify-out/graph.json`` — no re-extraction, no model calls. + +Communities are compared by ``community_name`` rather than numeric id, because +ids are reassigned on every rebuild while the human label ("auth", "py") is +stable enough to track a subsystem growing or splitting across versions. +""" +from __future__ import annotations + +import json +from collections import Counter + +import networkx as nx +from networkx.readwrite import json_graph + + +def load_graph_from_text(text: str) -> nx.Graph: + """Parse a graph.json payload (edges- or links-keyed) into a DiGraph.""" + data = json.loads(text) + if "links" not in data and "edges" in data: + data = dict(data, links=data["edges"]) + data = {**data, "directed": True} + try: + return json_graph.node_link_graph(data, edges="links") + except TypeError: + return json_graph.node_link_graph(data) + + +def _label(G: nx.Graph, nid: str) -> str: + return str(G.nodes[nid].get("label", nid)) + + +def _god_node_ids(G: nx.Graph, top_n: int) -> list[str]: + """Top-N node ids by degree (graphify's god-node definition).""" + ranked = sorted(G.nodes(), key=lambda n: (-G.degree(n), str(n))) + return ranked[:top_n] + + +def _edge_key(G: nx.Graph, u: str, v: str, data: dict) -> tuple[str, str, str]: + return (str(u), str(v), str(data.get("relation", ""))) + + +def _edge_set(G: nx.Graph) -> set[tuple[str, str, str]]: + out: set[tuple[str, str, str]] = set() + if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)): + for u, v, data in G.edges(data=True): + out.add(_edge_key(G, u, v, data)) + else: + for u, v, data in G.edges(data=True): + out.add(_edge_key(G, u, v, data)) + return out + + +def _community_sizes(G: nx.Graph) -> Counter: + sizes: Counter = Counter() + for _nid, data in G.nodes(data=True): + name = data.get("community_name") + if name is None and data.get("community") is not None: + name = f"#{data.get('community')}" + if name is not None: + sizes[str(name)] += 1 + return sizes + + +def diff_graphs(old: nx.Graph, new: nx.Graph, *, top_god: int = 15) -> dict: + """Structural diff between two snapshots. Returns a JSON-friendly dict.""" + old_ids, new_ids = set(old.nodes()), set(new.nodes()) + added_ids = new_ids - old_ids + removed_ids = old_ids - new_ids + + old_edges, new_edges = _edge_set(old), _edge_set(new) + added_edges = new_edges - old_edges + removed_edges = old_edges - new_edges + + old_gods, new_gods = _god_node_ids(old, top_god), _god_node_ids(new, top_god) + emerged = [n for n in new_gods if n not in set(old_gods)] + vanished = [n for n in old_gods if n not in set(new_gods)] + + old_sizes, new_sizes = _community_sizes(old), _community_sizes(new) + appeared = sorted(set(new_sizes) - set(old_sizes)) + disappeared = sorted(set(old_sizes) - set(new_sizes)) + resized = [] + for name in sorted(set(old_sizes) & set(new_sizes)): + delta = new_sizes[name] - old_sizes[name] + if delta: + resized.append({"name": name, "old": old_sizes[name], "new": new_sizes[name], "delta": delta}) + resized.sort(key=lambda r: (-abs(r["delta"]), r["name"])) + + return { + "nodes": { + "old_count": len(old_ids), + "new_count": len(new_ids), + "delta": len(new_ids) - len(old_ids), + "added": sorted( + ({"id": n, "label": _label(new, n)} for n in added_ids), + key=lambda d: d["label"], + ), + "removed": sorted( + ({"id": n, "label": _label(old, n)} for n in removed_ids), + key=lambda d: d["label"], + ), + }, + "edges": { + "old_count": len(old_edges), + "new_count": len(new_edges), + "delta": len(new_edges) - len(old_edges), + "added": sorted(f"{u} --{r}--> {v}" for (u, v, r) in added_edges), + "removed": sorted(f"{u} --{r}--> {v}" for (u, v, r) in removed_edges), + }, + "god_nodes": { + "top_n": top_god, + "emerged": [{"id": n, "label": _label(new, n), "degree": new.degree(n)} for n in emerged], + "vanished": [{"id": n, "label": _label(old, n), "degree": old.degree(n)} for n in vanished], + }, + "communities": { + "old_count": len(old_sizes), + "new_count": len(new_sizes), + "appeared": appeared, + "disappeared": disappeared, + "resized": resized, + }, + } + + +def _plural(n: int, word: str) -> str: + return f"{n} {word}" + ("" if n == 1 else "s") + + +def format_diff(diff: dict, *, limit: int = 15) -> str: + n, e, g, c = diff["nodes"], diff["edges"], diff["god_nodes"], diff["communities"] + lines: list[str] = [] + lines.append("graphify chronicle — structural diff") + lines.append("=" * 48) + lines.append( + f"Nodes: {n['old_count']} -> {n['new_count']} ({n['delta']:+d}) " + f"Edges: {e['old_count']} -> {e['new_count']} ({e['delta']:+d})" + ) + lines.append( + f"Communities: {c['old_count']} -> {c['new_count']} " + f"({len(c['appeared'])} appeared, {len(c['disappeared'])} disappeared)" + ) + lines.append("") + + if g["emerged"]: + lines.append("God-nodes emerged (new structural hubs):") + for d in g["emerged"][:limit]: + lines.append(f" + {d['label']} (degree {d['degree']})") + if g["vanished"]: + lines.append("God-nodes vanished:") + for d in g["vanished"][:limit]: + lines.append(f" - {d['label']} (was degree {d['degree']})") + if g["emerged"] or g["vanished"]: + lines.append("") + + if c["appeared"]: + lines.append("Communities appeared: " + ", ".join(c["appeared"][:limit])) + if c["disappeared"]: + lines.append("Communities disappeared: " + ", ".join(c["disappeared"][:limit])) + if c["resized"]: + lines.append("Largest community shifts:") + for r in c["resized"][:limit]: + lines.append(f" {r['name']}: {r['old']} -> {r['new']} ({r['delta']:+d})") + lines.append("") + + lines.append( + f"Detail: {_plural(len(n['added']), 'node')} added, " + f"{_plural(len(n['removed']), 'node')} removed, " + f"{_plural(len(e['added']), 'edge')} added, " + f"{_plural(len(e['removed']), 'edge')} removed." + ) + if n["added"]: + lines.append(" new nodes: " + ", ".join(d["label"] for d in n["added"][:limit]) + (" …" if len(n["added"]) > limit else "")) + if n["removed"]: + lines.append(" gone nodes: " + ", ".join(d["label"] for d in n["removed"][:limit]) + (" …" if len(n["removed"]) > limit else "")) + return "\n".join(lines) diff --git a/graphify/embed.py b/graphify/embed.py new file mode 100644 index 000000000..2457d87a7 --- /dev/null +++ b/graphify/embed.py @@ -0,0 +1,293 @@ +"""Optional local-embedding backend — a *semantic* ranking signal for queries. + +graphify's default query path is structural and deterministic: no model, no API, +$0. This module adds an **opt-in** semantic backend that fuses into the ranking +alongside the structural signals (it never replaces them). It answers fuzzy +questions — "where do we handle auth timeouts" — that no exact-token match can. + +Design constraints: + + * Strictly optional. Nothing here runs unless the user passes ``--semantic`` + or calls ``graphify embed``. Import stays cheap; heavy deps are lazy. + * Local-first. The backend is Ollama's embeddings endpoint or a + ``sentence-transformers`` model — both run on the user's machine, no API key. + * Cached. Node embeddings are computed once by ``graphify embed`` and written + to a sidecar next to ``graph.json``; queries embed only the question and + cosine-compare against the cached matrix. + +The embedder is injectable (``embed_texts(texts, embedder=...)``) so tests run +with a deterministic fake and never need a model server. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Callable, Sequence + +import numpy as np + +from graphify.paths import out_path + +# An embedder maps a list of texts to an (N, D) float matrix. +Embedder = Callable[[Sequence[str]], "np.ndarray"] + +_DEFAULT_OLLAMA_MODEL = "nomic-embed-text" +_DEFAULT_ST_MODEL = "all-MiniLM-L6-v2" +_SIDECAR_NAME = "embeddings.npz" +_META_NAME = "embeddings.meta.json" + + +# --------------------------------------------------------------------------- # +# node text +# --------------------------------------------------------------------------- # + +def node_text(data: dict) -> str: + """The text used to represent a node to the embedder. + + Label carries the identifier; source_file and community add topical context + so two ``handle()`` functions in different subsystems embed differently. Any + LLM-derived summary/docstring, when present, is the richest signal. + """ + parts = [ + str(data.get("label", "")), + str(data.get("summary") or data.get("docstring") or data.get("description") or ""), + str(data.get("source_file", "")), + str(data.get("community_name") or ""), + ] + return " — ".join(p for p in parts if p).strip() or str(data.get("label", "")) + + +# --------------------------------------------------------------------------- # +# backends +# --------------------------------------------------------------------------- # + +def _ollama_embedder(model: str, host: str) -> Embedder: + import urllib.request + + url = host.rstrip("/") + "/api/embeddings" + + def embed(texts: Sequence[str]) -> np.ndarray: + vecs: list[list[float]] = [] + for text in texts: + payload = json.dumps({"model": model, "prompt": text}).encode("utf-8") + req = urllib.request.Request(url, data=payload, headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=60) as resp: # noqa: S310 (local host) + body = json.loads(resp.read().decode("utf-8")) + vec = body.get("embedding") + if not vec: + raise RuntimeError(f"ollama returned no embedding for model {model!r}") + vecs.append(vec) + return np.asarray(vecs, dtype=np.float32) + + return embed + + +def _sentence_transformers_embedder(model: str) -> Embedder: + from sentence_transformers import SentenceTransformer # type: ignore[import-untyped] + + st_model = SentenceTransformer(model) + + def embed(texts: Sequence[str]) -> np.ndarray: + return np.asarray(st_model.encode(list(texts), show_progress_bar=False), dtype=np.float32) + + return embed + + +def get_embedder() -> tuple[Embedder, str]: + """Resolve a local embedder, returning ``(embedder, model_tag)``. + + Honours ``GRAPHIFY_EMBED_BACKEND`` (``ollama`` | ``sentence-transformers``) + and ``GRAPHIFY_EMBED_MODEL``; otherwise auto-detects Ollama, then + sentence-transformers. Raises with actionable guidance if neither is usable. + """ + backend = os.environ.get("GRAPHIFY_EMBED_BACKEND", "").strip().lower() + model = os.environ.get("GRAPHIFY_EMBED_MODEL", "").strip() + host = os.environ.get("OLLAMA_HOST", "http://localhost:11434") + + def _try_ollama() -> tuple[Embedder, str] | None: + import urllib.request + + try: + with urllib.request.urlopen(host.rstrip("/") + "/api/tags", timeout=3): + pass + except Exception: + return None + m = model or _DEFAULT_OLLAMA_MODEL + return _ollama_embedder(m, host), f"ollama:{m}" + + def _try_st() -> tuple[Embedder, str] | None: + try: + import sentence_transformers # noqa: F401 + except Exception: + return None + m = model or _DEFAULT_ST_MODEL + return _sentence_transformers_embedder(m), f"st:{m}" + + if backend == "ollama": + got = _try_ollama() + if got: + return got + raise RuntimeError(f"GRAPHIFY_EMBED_BACKEND=ollama but no Ollama server at {host}") + if backend in ("sentence-transformers", "st"): + got = _try_st() + if got: + return got + raise RuntimeError("GRAPHIFY_EMBED_BACKEND=sentence-transformers but the package is not installed") + + for probe in (_try_ollama, _try_st): + got = probe() + if got: + return got + raise RuntimeError( + "no local embedding backend available. Start Ollama (and `ollama pull " + f"{_DEFAULT_OLLAMA_MODEL}`) or `pip install sentence-transformers`, " + "then re-run with --semantic." + ) + + +def embed_texts(texts: Sequence[str], *, embedder: Embedder | None = None) -> np.ndarray: + if not texts: + return np.zeros((0, 0), dtype=np.float32) + if embedder is None: + embedder, _ = get_embedder() + return embedder(texts) + + +# --------------------------------------------------------------------------- # +# sidecar IO +# --------------------------------------------------------------------------- # + +def sidecar_paths(graph_path: str | None) -> tuple[Path, Path]: + """(vectors.npz, meta.json) paths beside the graph (or under GRAPHIFY_OUT).""" + if graph_path: + base = Path(graph_path).resolve().parent + else: + base = out_path().resolve() + return base / _SIDECAR_NAME, base / _META_NAME + + +def _normalize(matrix: np.ndarray) -> np.ndarray: + if matrix.size == 0: + return matrix + norms = np.linalg.norm(matrix, axis=1, keepdims=True) + norms[norms == 0] = 1.0 + return matrix / norms + + +def build_embeddings( + G, + graph_path: str | None, + *, + embedder: Embedder | None = None, + model_tag: str | None = None, + force: bool = False, +) -> dict: + """Compute (or refresh) the node-embedding sidecar. Returns a small summary. + + Skips work when a sidecar already covers the current node set and model tag, + unless ``force``. Vectors are L2-normalized on write so query-time scoring is + a plain dot product. + """ + if embedder is None: + embedder, model_tag = get_embedder() + model_tag = model_tag or "custom" + + node_ids = list(G.nodes()) + texts = [node_text(G.nodes[n]) for n in node_ids] + content_hash = _hash_ids_model(node_ids, model_tag) + + vec_path, meta_path = sidecar_paths(graph_path) + if not force and meta_path.exists(): + try: + meta = json.loads(meta_path.read_text(encoding="utf-8")) + if meta.get("content_hash") == content_hash and vec_path.exists(): + return {"status": "cached", "count": len(node_ids), "model": model_tag, "path": str(vec_path)} + except Exception: + pass + + matrix = _normalize(embed_texts(texts, embedder=embedder).astype(np.float32)) + vec_path.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(vec_path, vectors=matrix, ids=np.asarray(node_ids, dtype=object)) + meta_path.write_text( + json.dumps( + { + "model": model_tag, + "count": len(node_ids), + "dim": int(matrix.shape[1]) if matrix.size else 0, + "content_hash": content_hash, + }, + indent=2, + ), + encoding="utf-8", + ) + return {"status": "built", "count": len(node_ids), "model": model_tag, "path": str(vec_path)} + + +def _hash_ids_model(node_ids: Sequence[str], model_tag: str) -> str: + h = hashlib.sha256() + h.update(model_tag.encode("utf-8")) + for nid in node_ids: # order-sensitive: a reordered graph is a different build + h.update(b"\0") + h.update(str(nid).encode("utf-8")) + return h.hexdigest() + + +def load_embeddings(graph_path: str | None) -> tuple[list[str], np.ndarray, dict] | None: + """Load (ids, normalized-matrix, meta) from the sidecar, or None if absent.""" + vec_path, meta_path = sidecar_paths(graph_path) + if not vec_path.exists(): + return None + with np.load(vec_path, allow_pickle=True) as data: + ids = [str(x) for x in data["ids"].tolist()] + matrix = np.asarray(data["vectors"], dtype=np.float32) + meta = {} + if meta_path.exists(): + try: + meta = json.loads(meta_path.read_text(encoding="utf-8")) + except Exception: + meta = {} + return ids, matrix, meta + + +# --------------------------------------------------------------------------- # +# query-time scoring (the ranking backend) +# --------------------------------------------------------------------------- # + +def semantic_scores_for_query( + G, + question: str, + *, + graph_path: str | None = None, + embedder: Embedder | None = None, +) -> dict[str, float]: + """Cosine similarity between the question and every embedded node. + + Returns ``{node_id: score}`` for nodes present in both the sidecar and ``G``. + Raises if no sidecar exists (the caller — query/bench — catches this and + degrades to structural-only ranking with a note). Query embedding uses the + same model tag the sidecar was built with when possible. + """ + loaded = load_embeddings(graph_path) + if loaded is None: + raise RuntimeError( + "no embeddings sidecar found. Run `graphify embed` first to enable --semantic." + ) + ids, matrix, meta = loaded + if matrix.size == 0: + return {} + if embedder is None: + # Pin the query embedder to the sidecar's model so vectors are comparable. + model = meta.get("model", "") + if model and ":" in model and not os.environ.get("GRAPHIFY_EMBED_MODEL"): + backend, _, name = model.partition(":") + os.environ.setdefault("GRAPHIFY_EMBED_BACKEND", "ollama" if backend == "ollama" else "sentence-transformers") + os.environ["GRAPHIFY_EMBED_MODEL"] = name + embedder, _ = get_embedder() + q = _normalize(embed_texts([question], embedder=embedder).astype(np.float32)) + if q.size == 0: + return {} + sims = matrix @ q[0] # both L2-normalized -> dot product is cosine + present = set(G.nodes()) + return {nid: float(sims[i]) for i, nid in enumerate(ids) if nid in present} diff --git a/graphify/evals.py b/graphify/evals.py new file mode 100644 index 000000000..8462abdf1 --- /dev/null +++ b/graphify/evals.py @@ -0,0 +1,348 @@ +"""Relevance evals for `graphify query` — does the ranked result actually contain +the nodes that answer the question? + +`benchmark.py` measures how many *tokens* a query saves. This module measures +whether the query is *right*: given a fixture of (question -> expected nodes) +cases, it runs the real ranking pipeline (``serve.rank_query_nodes``) and scores +precision@k, recall@k, MRR, and nDCG@k. Metrics are defined once, up front, in +``METRIC_GLOSSARY`` (contract-first) so a number always means the same thing. + +A fixture is JSONL, one case per line:: + + {"query": "how does community detection work", "expect": ["cluster()", "graphify/cluster.py"]} + {"query": "what renders the wiki", "expect": ["to_wiki()"], "depth": 2, "mode": "bfs"} + +Each ``expect`` string matches a result node when it equals the node's label or +id, or names its ``source_file`` (full path or basename) — so fixtures can be +authored by naming the function or file you expect to surface, no node ids +required. + +``--save`` appends the run to ``.graphify-evals/eval-results.jsonl`` (repo-local, +git-trackable); ``--replay`` re-runs the fixture and diffs every metric against +the most recent saved run, flagging regressions. That is the safety net for +tuning the ranking: a change that helps one query but quietly hurts three others +shows up as a negative aggregate delta. +""" +from __future__ import annotations + +import json +import math +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import networkx as nx +from networkx.readwrite import json_graph + +from graphify.paths import default_graph_json as _default_graph_json + + +# Contract-first: every metric is defined here, once. key -> (label, description). +METRIC_GLOSSARY: dict[str, tuple[str, str]] = { + "p_at_k": ( + "P@k", + "Fraction of the top-k ranked nodes that match some expected node.", + ), + "recall_at_k": ( + "Recall@k", + "Fraction of expected nodes that appear within the top-k ranked nodes.", + ), + "mrr": ( + "MRR", + "Reciprocal rank (1/r) of the first ranked node matching any expected node.", + ), + "ndcg_at_k": ( + "nDCG@k", + "Binary-relevance normalized discounted cumulative gain over the top-k.", + ), + "hit_at_k": ( + "Hit@k", + "1 if any expected node appears in the top-k, else 0.", + ), +} + +DEFAULT_K = 10 +# A saved metric may wobble slightly across runs on identical inputs (it won't, +# since ranking is deterministic — but tolerance guards against float noise and +# lets --replay ignore trivial deltas). Regressions beyond this fail --replay. +REPLAY_TOLERANCE = 1e-9 + +_RESULTS_DIR = ".graphify-evals" +_RESULTS_FILE = "eval-results.jsonl" + + +@dataclass +class EvalCase: + query: str + expect: list[str] + mode: str = "bfs" + depth: int = 2 + context: list[str] = field(default_factory=list) + + @classmethod + def from_dict(cls, d: dict) -> "EvalCase": + query = d.get("query") or d.get("question") + if not query: + raise ValueError(f"eval case missing 'query': {d!r}") + expect = d.get("expect") or d.get("expected") or [] + if isinstance(expect, str): + expect = [expect] + if not expect: + raise ValueError(f"eval case for {query!r} has no 'expect' entries") + return cls( + query=str(query), + expect=[str(e) for e in expect], + mode=str(d.get("mode", "bfs")), + depth=int(d.get("depth", 2)), + context=[str(c) for c in (d.get("context") or [])], + ) + + +@dataclass +class CaseResult: + query: str + expect: list[str] + matched: dict[str, int] # expected string -> 1-based rank of first match (absent if unmatched) + metrics: dict[str, float] + + +@dataclass +class EvalReport: + k: int + cases: list[CaseResult] + aggregate: dict[str, float] + + def to_dict(self) -> dict: + return { + "k": self.k, + "aggregate": self.aggregate, + "cases": [ + { + "query": c.query, + "expect": c.expect, + "matched": c.matched, + "metrics": c.metrics, + } + for c in self.cases + ], + } + + +def load_cases(path: Path) -> list[EvalCase]: + """Load eval cases from a JSONL file (or a JSON array of cases).""" + text = path.read_text(encoding="utf-8") + cases: list[EvalCase] = [] + stripped = text.lstrip() + if stripped.startswith("["): + for d in json.loads(text): + cases.append(EvalCase.from_dict(d)) + else: + for line_no, line in enumerate(text.splitlines(), start=1): + line = line.strip() + if not line or line.startswith("#"): + continue + try: + cases.append(EvalCase.from_dict(json.loads(line))) + except json.JSONDecodeError as exc: + raise ValueError(f"{path}:{line_no}: invalid JSON ({exc})") from exc + if not cases: + raise ValueError(f"no eval cases found in {path}") + return cases + + +def _node_matches(G: nx.Graph, node_id: str, expected: str) -> bool: + """True when `expected` names this node by label, id, or source_file.""" + exp = expected.strip().casefold() + if not exp: + return False + if node_id.casefold() == exp: + return True + data = G.nodes[node_id] + label = str(data.get("label", "")).casefold() + if label == exp or label.rstrip("()") == exp.rstrip("()"): + return True + src = str(data.get("source_file", "")).casefold() + if src == exp: + return True + # Basename / suffix match so "cluster.py" hits "graphify/cluster.py". + if src and (src.endswith("/" + exp) or Path(src).name == exp): + return True + return False + + +def _score_case(G: nx.Graph, ranked: list[str], expect: list[str], k: int) -> tuple[dict[str, int], dict[str, float]]: + top = ranked[:k] + # First rank (1-based, within top-k) at which each expected item matches. + matched: dict[str, int] = {} + for exp in expect: + for rank, nid in enumerate(top, start=1): + if _node_matches(G, nid, exp): + matched[exp] = rank + break + # A top-k node is "relevant" if it matches any expected item. + relevant_positions = [ + i for i, nid in enumerate(top, start=1) + if any(_node_matches(G, nid, exp) for exp in expect) + ] + denom_k = max(1, min(k, len(top))) + p_at_k = len(relevant_positions) / denom_k + recall_at_k = len(matched) / len(expect) if expect else 0.0 + first_rank = min(matched.values()) if matched else 0 + mrr = 1.0 / first_rank if first_rank else 0.0 + hit_at_k = 1.0 if matched else 0.0 + # Binary-relevance nDCG@k. The ideal ranking puts all `num_relevant` relevant + # nodes at the top, so IDCG normalizes against how many were actually found + # (not len(expect) — one expected label can match several nodes, which would + # otherwise push nDCG above 1.0). Result stays in [0, 1]. + num_relevant = len(relevant_positions) + dcg = sum(1.0 / math.log2(pos + 1) for pos in relevant_positions) + idcg = sum(1.0 / math.log2(i + 1) for i in range(1, min(num_relevant, k) + 1)) + ndcg = dcg / idcg if idcg else 0.0 + metrics = { + "p_at_k": round(p_at_k, 4), + "recall_at_k": round(recall_at_k, 4), + "mrr": round(mrr, 4), + "ndcg_at_k": round(ndcg, 4), + "hit_at_k": hit_at_k, + } + return matched, metrics + + +def run_evals( + G: nx.Graph, + cases: list[EvalCase], + *, + k: int = DEFAULT_K, + semantic: bool = False, + graph_path: str | None = None, +) -> EvalReport: + """Run every case through the real ranking pipeline and score it.""" + from graphify.serve import rank_query_nodes + + results: list[CaseResult] = [] + for case in cases: + semantic_scores = None + if semantic: + try: + from graphify.embed import semantic_scores_for_query + semantic_scores = semantic_scores_for_query(G, case.query, graph_path=graph_path) + except Exception: + semantic_scores = None + ranked = rank_query_nodes( + G, + case.query, + mode=case.mode, + depth=case.depth, + context_filters=case.context or None, + semantic_scores=semantic_scores, + ) + matched, metrics = _score_case(G, ranked, case.expect, k) + results.append(CaseResult(case.query, case.expect, matched, metrics)) + + aggregate: dict[str, float] = {} + if results: + for key in METRIC_GLOSSARY: + aggregate[key] = round( + sum(c.metrics[key] for c in results) / len(results), 4 + ) + return EvalReport(k=k, cases=results, aggregate=aggregate) + + +def load_graph(graph_path: str) -> nx.Graph: + from graphify.security import check_graph_file_size_cap + + p = Path(graph_path) + check_graph_file_size_cap(p) + data = json.loads(p.read_text(encoding="utf-8")) + if "links" not in data and "edges" in data: + data = dict(data, links=data["edges"]) + data = {**data, "directed": True} + try: + return json_graph.node_link_graph(data, edges="links") + except TypeError: + return json_graph.node_link_graph(data) + + +def results_path(base: Path | None = None) -> Path: + return (base or Path.cwd()) / _RESULTS_DIR / _RESULTS_FILE + + +def save_report(report: EvalReport, *, base: Path | None = None, fixture: str | None = None) -> Path: + """Append the run to the repo-local results log and return its path.""" + path = results_path(base) + path.parent.mkdir(parents=True, exist_ok=True) + record = report.to_dict() + record["fixture"] = fixture + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(record, sort_keys=True) + "\n") + return path + + +def load_last_report(*, base: Path | None = None) -> dict | None: + """The most recent saved run, or None if there is no history.""" + path = results_path(base) + if not path.exists(): + return None + last = None + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line: + try: + last = json.loads(line) + except json.JSONDecodeError: + continue + return last + + +def diff_aggregates(current: dict[str, float], baseline: dict[str, float]) -> dict[str, float]: + """current - baseline per metric (only keys present in current).""" + return {key: round(current.get(key, 0.0) - baseline.get(key, 0.0), 4) for key in current} + + +def scaffold_fixture(G: nx.Graph, *, limit: int = 12) -> list[dict]: + """Generate a starter fixture: query each prominent symbol, expect itself. + + A sanity baseline — "searching for X surfaces X near the top" — that a user + then edits into real cases. Prominent = highest-degree non-file nodes with a + callable-looking label, so the queries read naturally. + """ + ranked = sorted(G.nodes(data=True), key=lambda nd: G.degree(nd[0]), reverse=True) + cases: list[dict] = [] + seen: set[str] = set() + for nid, data in ranked: + label = str(data.get("label", "")).strip() + if not label or label in seen: + continue + # Prefer symbol-ish labels (functions/classes) over bare file nodes. + if str(data.get("source_location", "")) == "L1" and "." in label: + continue + seen.add(label) + cases.append({"query": label.rstrip("()"), "expect": [label]}) + if len(cases) >= limit: + break + return cases + + +def format_report(report: EvalReport, *, baseline: dict | None = None) -> str: + lines: list[str] = [] + lines.append(f"graphify relevance eval — {len(report.cases)} cases @ k={report.k}") + lines.append("-" * 56) + for c in report.cases: + m = c.metrics + flag = "ok " if m["hit_at_k"] else "MISS" + lines.append( + f" [{flag}] P@k={m['p_at_k']:.2f} R@k={m['recall_at_k']:.2f} " + f"MRR={m['mrr']:.2f} nDCG={m['ndcg_at_k']:.2f} {c.query[:44]}" + ) + lines.append("-" * 56) + agg = report.aggregate + base_agg = (baseline or {}).get("aggregate") if baseline else None + for key, (label, _desc) in METRIC_GLOSSARY.items(): + cur = agg.get(key, 0.0) + if base_agg is not None: + delta = round(cur - base_agg.get(key, 0.0), 4) + arrow = "▲" if delta > 0 else ("▼" if delta < 0 else "=") + lines.append(f" {label:<10} {cur:.4f} {arrow} {delta:+.4f} vs baseline") + else: + lines.append(f" {label:<10} {cur:.4f}") + return "\n".join(lines) diff --git a/graphify/ranking.py b/graphify/ranking.py new file mode 100644 index 000000000..a405da0f7 --- /dev/null +++ b/graphify/ranking.py @@ -0,0 +1,172 @@ +"""Relevance ranking of query subgraphs via Reciprocal Rank Fusion (RRF). + +`graphify query` seeds a BFS/DFS from the best lexical matches and expands the +neighbourhood. The raw traversal result was then rendered in *degree* order, so +a broad question could surface hundreds of high-degree hubs ahead of the handful +of nodes that actually answer it. This module re-orders the traversal result by +a fused relevance score so the token budget spends itself on the nodes that +matter. + +Several independent rankings vote on each node: + + * ``lexical`` - query-term match strength (from ``serve._score_nodes``) + * ``proximity`` - graph distance from the seed nodes (closer answers first) + * ``centrality`` - degree / hub-ness (a mild prior toward important nodes) + * ``community`` - membership in a seed's community (topical cohesion) + * ``semantic`` - optional local-embedding cosine similarity (opt-in feature) + +Rankings are combined with Reciprocal Rank Fusion: a node at rank ``r`` in a +backend contributes ``1 / (k + r)``. RRF needs no per-backend weight tuning and +is robust to backends that score on wildly different scales (BM25 vs. cosine +vs. hop count). The structural backends (proximity / centrality / community) are +graphify's edge — a pure-retrieval tool can't compute them because it has no +typed graph. + +Everything here is pure and deterministic: ties break on node id, so the same +graph + query always yields the same order. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Iterable, Mapping, Sequence + +# Standard RRF damping constant (Cormack, Clarke & Buettcher, SIGIR 2009). Large +# k flattens the contribution curve so no single backend dominates on rank alone. +RRF_K = 60 + +# Backends are listed most-informative-first purely so --explain reads naturally. +_BACKEND_ORDER = ("lexical", "semantic", "proximity", "community", "centrality") + +_FAR = 1_000_000 # proximity sentinel for nodes with no path back to a seed + + +@dataclass +class RankedNode: + """A node's fused relevance and its per-backend breakdown (for --explain).""" + + node_id: str + score: float = 0.0 + ranks: dict[str, int] = field(default_factory=dict) + contributions: dict[str, float] = field(default_factory=dict) + + def explain(self) -> str: + """One-line breakdown, backends sorted by their contribution.""" + parts = [ + f"{b}#{self.ranks[b]}(+{self.contributions[b]:.4f})" + for b in sorted(self.contributions, key=lambda b: -self.contributions[b]) + ] + return f"rank score={self.score:.4f} :: " + " ".join(parts) + + +def rrf_fuse( + rankings: Mapping[str, Sequence[str]], k: int = RRF_K +) -> dict[str, RankedNode]: + """Fuse named rankings into per-node RRF scores. + + ``rankings`` maps a backend name to an ordered list of node ids (best first). + A node absent from a backend simply receives no contribution from it. The + first (best) occurrence of a node within a backend wins, so a caller may pass + a ranking with duplicates without double-counting. + """ + out: dict[str, RankedNode] = {} + for backend, ordered in rankings.items(): + for rank, nid in enumerate(ordered, start=1): + rn = out.get(nid) + if rn is None: + rn = RankedNode(nid) + out[nid] = rn + if backend in rn.ranks: + continue + contrib = 1.0 / (k + rank) + rn.ranks[backend] = rank + rn.contributions[backend] = contrib + rn.score += contrib + return out + + +def _lexical_ranking(lexical_scores: Mapping[str, float], nodes: set[str]) -> list[str]: + scored = [(s, n) for n, s in lexical_scores.items() if n in nodes and s > 0] + scored.sort(key=lambda t: (-t[0], t[1])) + return [n for _, n in scored] + + +def _proximity_ranking(nodes: set[str], distances: Mapping[str, int]) -> list[str]: + # Nodes with no known distance (never reached from a seed) sink to the back. + return sorted(nodes, key=lambda n: (distances.get(n, _FAR), n)) + + +def _centrality_ranking(nodes: set[str], degrees: Mapping[str, int]) -> list[str]: + return sorted(nodes, key=lambda n: (-degrees.get(n, 0), n)) + + +def _community_ranking( + nodes: set[str], + node_community: Mapping[str, object], + seed_communities: set[object], + degrees: Mapping[str, int], +) -> list[str]: + in_comm = [n for n in nodes if node_community.get(n) in seed_communities] + out_comm = [n for n in nodes if node_community.get(n) not in seed_communities] + in_comm.sort(key=lambda n: (-degrees.get(n, 0), n)) + out_comm.sort(key=lambda n: (-degrees.get(n, 0), n)) + return in_comm + out_comm + + +def _semantic_ranking(semantic_scores: Mapping[str, float], nodes: set[str]) -> list[str]: + scored = [(s, n) for n, s in semantic_scores.items() if n in nodes] + scored.sort(key=lambda t: (-t[0], t[1])) + return [n for _, n in scored] + + +def rank_nodes( + nodes: Iterable[str], + seeds: Sequence[str], + *, + lexical_scores: Mapping[str, float] | None = None, + distances: Mapping[str, int] | None = None, + degrees: Mapping[str, int] | None = None, + node_community: Mapping[str, object] | None = None, + semantic_scores: Mapping[str, float] | None = None, + k: int = RRF_K, +) -> list[RankedNode]: + """Rank ``nodes`` by fused relevance, returning them best-first. + + ``seeds`` (the exact query matches the traversal started from) are always + pinned to the top in seed order — they are the answer's anchor and must never + be pushed below expanded neighbours. Every other node is ordered by its RRF + score. Each returned ``RankedNode`` carries its per-backend breakdown so the + caller can render ``--explain``. + """ + node_set = set(nodes) + degrees = degrees or {} + lexical_scores = lexical_scores or {} + distances = distances or {} + node_community = node_community or {} + + seed_communities = { + node_community.get(s) + for s in seeds + if node_community.get(s) is not None + } + + rankings: dict[str, list[str]] = { + "lexical": _lexical_ranking(lexical_scores, node_set), + "proximity": _proximity_ranking(node_set, distances), + "centrality": _centrality_ranking(node_set, degrees), + } + if seed_communities: + rankings["community"] = _community_ranking( + node_set, node_community, seed_communities, degrees + ) + if semantic_scores: + rankings["semantic"] = _semantic_ranking(semantic_scores, node_set) + + fused = rrf_fuse(rankings, k=k) + + seed_order = [s for s in seeds if s in node_set] + seed_set = set(seed_order) + rest = [n for n in node_set if n not in seed_set] + rest.sort(key=lambda n: (-(fused[n].score if n in fused else 0.0), n)) + + ordered_ids = seed_order + rest + return [fused.get(n) or RankedNode(n) for n in ordered_ids] diff --git a/graphify/serve.py b/graphify/serve.py index f096f1075..e884f1aaf 100644 --- a/graphify/serve.py +++ b/graphify/serve.py @@ -495,17 +495,45 @@ def _dfs(G: nx.Graph, start_nodes: list[str], depth: int) -> tuple[set[str], lis return visited, edges_seen -def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_budget: int = 2000, *, seeds: list[str] | None = None) -> str: +def _subgraph_to_text( + G: nx.Graph, + nodes: set[str], + edges: list[tuple], + token_budget: int = 2000, + *, + seeds: list[str] | None = None, + order: list[str] | None = None, + explain: dict[str, str] | None = None, + top_k: int | None = None, +) -> str: """Render subgraph as text, cutting at token_budget (approx 3 chars/token). seeds: exact-match nodes rendered first before the degree-sorted expansion, so the queried symbol always appears at the top of the output. + order: an explicit relevance order (from graphify.ranking). When given it + replaces the default degree sort, so the token budget spends itself on the + highest-ranked nodes instead of the highest-degree hubs. Any node missing + from `order` is appended by degree so nothing is silently dropped. + explain: node_id -> per-node ranking breakdown appended to each NODE line. + top_k: keep only the first N nodes of the (ranked) order before rendering. """ char_budget = token_budget * 3 lines = [] seed_set = set(seeds or []) - ordered = [n for n in (seeds or []) if n in nodes] + \ - sorted(nodes - seed_set, key=lambda n: G.degree(n), reverse=True) + if order is not None: + order_set = set(order) + ordered = [n for n in order if n in nodes] + ordered += sorted( + (n for n in nodes if n not in order_set), + key=lambda n: G.degree(n), + reverse=True, + ) + else: + ordered = [n for n in (seeds or []) if n in nodes] + \ + sorted(nodes - seed_set, key=lambda n: G.degree(n), reverse=True) + if top_k is not None and top_k >= 0: + ordered = ordered[:top_k] + render_nodes = set(ordered) for nid in ordered: d = G.nodes[nid] # Every LLM-derived field passes through sanitize_label before being @@ -519,9 +547,14 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu f"loc={sanitize_label(str(d.get('source_location', '')))} " f"community={sanitize_label(str(d.get('community_name') or d.get('community', '')))}]" ) + # explain values are graphify-generated numeric breakdowns (no corpus + # data), but sanitize anyway to keep the "all rendered text is sanitized" + # invariant trivially true. + if explain and nid in explain: + line += f" {sanitize_label(explain[nid])}" lines.append(line) for u, v in edges: - if u in nodes and v in nodes: + if u in render_nodes and v in render_nodes: raw = G[u][v] d = next(iter(raw.values()), {}) if isinstance(G, (nx.MultiGraph, nx.MultiDiGraph)) else raw context = d.get("context") @@ -548,32 +581,159 @@ def _subgraph_to_text(G: nx.Graph, nodes: set[str], edges: list[tuple], token_bu return output -def _query_graph_text( +def _seed_distances(G: nx.Graph, nodes: set[str], seeds: list[str]) -> dict[str, int]: + """Hop distance from the nearest seed to every node, ignoring edge direction. + + Proximity is a relevance signal (a node one call away from the queried symbol + is more relevant than one five hops out), so it is measured undirected on the + induced subgraph of the traversal result. + """ + try: + sub = G.subgraph(nodes) + if sub.is_directed(): + sub = sub.to_undirected(as_view=True) + present = [s for s in seeds if s in sub] + if not present: + return {} + return dict(nx.multi_source_shortest_path_length(sub, present)) + except Exception: + return {} + + +def _resolve_and_rank( G: nx.Graph, question: str, *, mode: str = "bfs", depth: int = 3, - token_budget: int = 2000, context_filters: list[str] | None = None, -) -> str: + semantic_scores: dict[str, float] | None = None, +) -> dict | None: + """Shared query core: seed → traverse → fuse-rank. + + Returns everything both the text renderer and the eval harness need, or None + when no seed node matched. Keeping this in one place means `graphify bench` + scores the exact ranking `graphify query` serves — the eval can't drift from + production. + """ + from graphify.ranking import rank_nodes + terms = _query_terms(question) scored = _score_nodes(G, terms) start_nodes = _pick_seeds(scored) + seed_source = "lexical" + if not start_nodes and semantic_scores: + # Semantic rescue: a fuzzy question ("where do we handle expired logins") + # may share no token with any label, so lexical seeding finds nothing — + # exactly when embeddings should take over. Seed the traversal from the + # top cosine matches instead of giving up. + top_sem = sorted(semantic_scores.items(), key=lambda kv: (-kv[1], kv[0]))[:3] + start_nodes = [nid for nid, _ in top_sem if nid in G] + seed_source = "semantic" if not start_nodes: - return "No matching nodes found." + return None resolved_filters, filter_source = _resolve_context_filters(question, context_filters) traversal_graph = _filter_graph_by_context(G, resolved_filters) nodes, edges = _dfs(traversal_graph, start_nodes, depth) if mode == "dfs" else _bfs(traversal_graph, start_nodes, depth) + + # Fuse lexical + structural (+ optional semantic) signals into one relevance + # order so the token budget renders the nodes that answer the question, not + # the highest-degree hubs that merely sit near them. + lexical_scores = {nid: sc for sc, nid in scored} + degrees = {n: traversal_graph.degree(n) for n in nodes} + distances = _seed_distances(traversal_graph, nodes, start_nodes) + node_community = {n: traversal_graph.nodes[n].get("community") for n in nodes} + ranked = rank_nodes( + nodes, + start_nodes, + lexical_scores=lexical_scores, + distances=distances, + degrees=degrees, + node_community=node_community, + semantic_scores=semantic_scores, + ) + return { + "start_nodes": start_nodes, + "ranked": ranked, + "traversal_graph": traversal_graph, + "nodes": nodes, + "edges": edges, + "resolved_filters": resolved_filters, + "filter_source": filter_source, + "seed_source": seed_source, + } + + +def rank_query_nodes( + G: nx.Graph, + question: str, + *, + mode: str = "bfs", + depth: int = 3, + context_filters: list[str] | None = None, + semantic_scores: dict[str, float] | None = None, +) -> list[str]: + """Return the query's result node ids in fused-relevance order (best first).""" + result = _resolve_and_rank( + G, question, mode=mode, depth=depth, + context_filters=context_filters, semantic_scores=semantic_scores, + ) + if result is None: + return [] + return [rn.node_id for rn in result["ranked"]] + + +def _query_graph_text( + G: nx.Graph, + question: str, + *, + mode: str = "bfs", + depth: int = 3, + token_budget: int = 2000, + context_filters: list[str] | None = None, + explain: bool = False, + top_k: int | None = None, + semantic_scores: dict[str, float] | None = None, +) -> str: + result = _resolve_and_rank( + G, question, mode=mode, depth=depth, + context_filters=context_filters, semantic_scores=semantic_scores, + ) + if result is None: + return "No matching nodes found." + start_nodes = result["start_nodes"] + ranked = result["ranked"] + traversal_graph = result["traversal_graph"] + nodes, edges = result["nodes"], result["edges"] + resolved_filters, filter_source = result["resolved_filters"], result["filter_source"] + + order = [r.node_id for r in ranked] + explain_map = {r.node_id: r.explain() for r in ranked} if explain else None + + shown = min(len(nodes), top_k) if top_k is not None else len(nodes) header_parts = [ f"Traversal: {mode.upper()} depth={depth}", f"Start: {[G.nodes[n].get('label', n) for n in start_nodes]}", ] if resolved_filters: header_parts.append(f"Context: {', '.join(resolved_filters)} ({filter_source})") - header_parts.append(f"{len(nodes)} nodes found") + if semantic_scores: + header_parts.append("semantic: on" + (" (seeded)" if result.get("seed_source") == "semantic" else "")) + if top_k is not None and shown < len(nodes): + header_parts.append(f"top {shown} of {len(nodes)} nodes (ranked)") + else: + header_parts.append(f"{len(nodes)} nodes found (ranked)") header = " | ".join(header_parts) + "\n\n" - return header + _subgraph_to_text(traversal_graph, nodes, edges, token_budget) + return header + _subgraph_to_text( + traversal_graph, + nodes, + edges, + token_budget, + seeds=start_nodes, + order=order, + explain=explain_map, + top_k=top_k, + ) def _find_node(G: nx.Graph, label: str) -> list[str]: @@ -760,6 +920,9 @@ async def list_tools() -> list[types.Tool]: "items": {"type": "string"}, "description": "Optional explicit edge-context filter, e.g. ['call', 'field']", }, + "top_k": {"type": "integer", "description": "Keep only the N most relevant (ranked) nodes"}, + "explain": {"type": "boolean", "default": False, + "description": "Append the per-node relevance-ranking breakdown to each node"}, }, "required": ["question"], }, @@ -873,6 +1036,9 @@ def _tool_query_graph(arguments: dict) -> str: depth = min(int(arguments.get("depth", 3)), 6) budget = int(arguments.get("token_budget", 2000)) context_filter = arguments.get("context_filter") + top_k = arguments.get("top_k") + top_k = int(top_k) if top_k is not None else None + explain = bool(arguments.get("explain", False)) _t0 = _time.perf_counter() result = _query_graph_text( G, @@ -881,6 +1047,8 @@ def _tool_query_graph(arguments: dict) -> str: depth=depth, token_budget=budget, context_filters=context_filter, + top_k=top_k, + explain=explain, ) querylog.log_query( kind="mcp_query", diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index 3735e8a24..6384ef942 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -653,6 +653,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 3735e8a24..6384ef942 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -653,6 +653,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index 8ec9ad22a..652dde40a 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -656,6 +656,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 4a956e725..c08336f30 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -653,6 +653,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index 8ec9ad22a..652dde40a 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -656,6 +656,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 480ef9294..b8c2569ad 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -653,6 +653,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index df53e1477..cca34f103 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -656,6 +656,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index 8ec9ad22a..652dde40a 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -656,6 +656,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index b023ff8ee..64c94fd08 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -648,6 +648,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index 8ec9ad22a..652dde40a 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -656,6 +656,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index a643daa8c..f2a578a44 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -654,6 +654,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index f0719bf8b..dd9285092 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -652,6 +652,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index f6c83fb81..ae90015dd 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -678,6 +678,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill.md b/graphify/skill.md index 8ec9ad22a..652dde40a 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -656,6 +656,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/graphify/skill_migrations.py b/graphify/skill_migrations.py new file mode 100644 index 000000000..c05ed6600 --- /dev/null +++ b/graphify/skill_migrations.py @@ -0,0 +1,90 @@ +"""Skill version reconciliation — detect installed-vs-package drift and name the +migrations needed to close it. + +graphify ships its agent instructions as a *skill* (SKILL.md + references/) that +is copied into each platform's skill dir at ``graphify install`` time, with a +``.graphify_version`` stamp beside it. Nothing keeps that copy in lockstep with +the installed Python package afterwards, so the two drift — e.g. a skill stamped +0.9.2 sitting next to a 0.9.0 package. ``graphify skill check-update`` surfaces +that drift; this module holds the version math and the migration registry that +explains what a given jump entails. + +A ``Migration`` records a version whose skill format changed and how to adopt it +(almost always: re-run ``graphify install``, which re-renders the skill + sidecar +from the packaged artifacts). Append an entry here whenever the skill's on-disk +contract changes so ``check-update`` can tell users *why* a re-install matters, +not just that versions differ. +""" +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Migration: + version: str # the package version that introduced the skill change + summary: str # what changed in the on-disk skill contract + action: str = "reinstall" # how to adopt it + + +# Oldest -> newest. Keep sorted by version. +MIGRATIONS: list[Migration] = [ + Migration( + "0.9.0", + "Baseline progressive skill: SKILL.md + references/ sidecar with the " + "graphify-query-first workflow.", + "reinstall", + ), +] + + +def parse_version(value: str) -> tuple[int, ...]: + """Lenient dotted-numeric parse. Non-numeric or 'unknown' -> (0,). + + Only the leading digit run of each dotted segment is read, so pre-release + tags ('1.2.0rc1') compare by their numeric core without raising. + """ + if not value or value == "unknown": + return (0,) + parts: list[int] = [] + for segment in str(value).split("."): + digits = "" + for ch in segment: + if ch.isdigit(): + digits += ch + else: + break + parts.append(int(digits) if digits else 0) + return tuple(parts) or (0,) + + +def compare_versions(a: str, b: str) -> int: + """-1 if ab (by parsed numeric tuple).""" + pa, pb = parse_version(a), parse_version(b) + # Pad to equal length so (0,9) and (0,9,0) compare equal. + width = max(len(pa), len(pb)) + pa += (0,) * (width - len(pa)) + pb += (0,) * (width - len(pb)) + return (pa > pb) - (pa < pb) + + +def applicable_migrations(installed: str, target: str) -> list[Migration]: + """Migrations introduced after `installed` and up to (including) `target`. + + Empty when the skill is up to date or ahead of the package (a downgrade — + ``check-update`` reports that separately; there is nothing to migrate *to*). + """ + return [ + m + for m in MIGRATIONS + if compare_versions(m.version, installed) > 0 + and compare_versions(m.version, target) <= 0 + ] + + +def drift_state(installed: str, package: str) -> str: + """Classify the relationship: 'current' | 'behind' | 'ahead'.""" + cmp = compare_versions(installed, package) + if cmp == 0: + return "current" + return "behind" if cmp < 0 else "ahead" diff --git a/graphify/skills/agents/references/analysis.md b/graphify/skills/agents/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/agents/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/agents/references/query.md b/graphify/skills/agents/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/agents/references/query.md +++ b/graphify/skills/agents/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/graphify/skills/amp/references/analysis.md b/graphify/skills/amp/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/amp/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/amp/references/query.md b/graphify/skills/amp/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/amp/references/query.md +++ b/graphify/skills/amp/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/graphify/skills/claude/references/analysis.md b/graphify/skills/claude/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/claude/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/claude/references/query.md b/graphify/skills/claude/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/claude/references/query.md +++ b/graphify/skills/claude/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/graphify/skills/claw/references/analysis.md b/graphify/skills/claw/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/claw/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/claw/references/query.md b/graphify/skills/claw/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/claw/references/query.md +++ b/graphify/skills/claw/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/graphify/skills/codex/references/analysis.md b/graphify/skills/codex/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/codex/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/codex/references/query.md b/graphify/skills/codex/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/codex/references/query.md +++ b/graphify/skills/codex/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/graphify/skills/copilot/references/analysis.md b/graphify/skills/copilot/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/copilot/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/copilot/references/query.md b/graphify/skills/copilot/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/copilot/references/query.md +++ b/graphify/skills/copilot/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/graphify/skills/droid/references/analysis.md b/graphify/skills/droid/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/droid/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/droid/references/query.md b/graphify/skills/droid/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/droid/references/query.md +++ b/graphify/skills/droid/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/graphify/skills/kilo/references/analysis.md b/graphify/skills/kilo/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/kilo/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/kilo/references/query.md b/graphify/skills/kilo/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/kilo/references/query.md +++ b/graphify/skills/kilo/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/graphify/skills/kiro/references/analysis.md b/graphify/skills/kiro/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/kiro/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/kiro/references/query.md b/graphify/skills/kiro/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/kiro/references/query.md +++ b/graphify/skills/kiro/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/graphify/skills/opencode/references/analysis.md b/graphify/skills/opencode/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/opencode/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/opencode/references/query.md b/graphify/skills/opencode/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/opencode/references/query.md +++ b/graphify/skills/opencode/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/graphify/skills/pi/references/analysis.md b/graphify/skills/pi/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/pi/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/pi/references/query.md b/graphify/skills/pi/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/pi/references/query.md +++ b/graphify/skills/pi/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/graphify/skills/trae/references/analysis.md b/graphify/skills/trae/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/trae/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/trae/references/query.md b/graphify/skills/trae/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/trae/references/query.md +++ b/graphify/skills/trae/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/graphify/skills/vscode/references/analysis.md b/graphify/skills/vscode/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/vscode/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/vscode/references/query.md b/graphify/skills/vscode/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/vscode/references/query.md +++ b/graphify/skills/vscode/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/graphify/skills/windows/references/analysis.md b/graphify/skills/windows/references/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/graphify/skills/windows/references/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/graphify/skills/windows/references/query.md b/graphify/skills/windows/references/query.md index 52321f101..69ea13fa6 100644 --- a/graphify/skills/windows/references/query.md +++ b/graphify/skills/windows/references/query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tests/test_chronicle.py b/tests/test_chronicle.py new file mode 100644 index 000000000..ea7a31107 --- /dev/null +++ b/tests/test_chronicle.py @@ -0,0 +1,100 @@ +"""Tests for graphify.chronicle — structural diff between two graph snapshots.""" +from __future__ import annotations + +import json + +from graphify.chronicle import diff_graphs, format_diff, load_graph_from_text + + +def _snapshot(nodes, links, *, edges_key="links") -> str: + return json.dumps({"directed": False, "multigraph": False, "graph": {}, "nodes": nodes, edges_key: links}) + + +def _old() -> str: + return _snapshot( + [ + {"id": "a", "label": "a()", "community": 0, "community_name": "core"}, + {"id": "b", "label": "b()", "community": 0, "community_name": "core"}, + {"id": "c", "label": "c()", "community": 1, "community_name": "legacy"}, + ], + [ + {"source": "a", "target": "b", "relation": "calls"}, + {"source": "b", "target": "c", "relation": "calls"}, + ], + ) + + +def _new() -> str: + return _snapshot( + [ + {"id": "a", "label": "a()", "community": 0, "community_name": "core"}, + {"id": "b", "label": "b()", "community": 0, "community_name": "core"}, + {"id": "d", "label": "d()", "community": 2, "community_name": "auth"}, + {"id": "e", "label": "e()", "community": 0, "community_name": "core"}, + ], + [ + {"source": "a", "target": "b", "relation": "calls"}, + {"source": "a", "target": "d", "relation": "calls"}, + {"source": "a", "target": "e", "relation": "calls"}, + ], + ) + + +def test_load_graph_from_text_edges_key(): + G = load_graph_from_text(_snapshot( + [{"id": "x", "label": "x"}], [{"source": "x", "target": "x", "relation": "self"}], edges_key="edges" + )) + assert G.number_of_nodes() == 1 + + +def test_diff_node_counts_and_membership(): + diff = diff_graphs(load_graph_from_text(_old()), load_graph_from_text(_new())) + assert diff["nodes"]["old_count"] == 3 + assert diff["nodes"]["new_count"] == 4 + assert diff["nodes"]["delta"] == 1 + added = {d["id"] for d in diff["nodes"]["added"]} + removed = {d["id"] for d in diff["nodes"]["removed"]} + assert added == {"d", "e"} + assert removed == {"c"} + + +def test_diff_edges(): + diff = diff_graphs(load_graph_from_text(_old()), load_graph_from_text(_new())) + assert any("--calls-->" in s and s.startswith("a ") for s in diff["edges"]["added"]) + # b->c existed before and is gone now. + assert any(s.startswith("b ") for s in diff["edges"]["removed"]) + + +def test_diff_communities(): + diff = diff_graphs(load_graph_from_text(_old()), load_graph_from_text(_new())) + c = diff["communities"] + assert "auth" in c["appeared"] + assert "legacy" in c["disappeared"] + core = next(r for r in c["resized"] if r["name"] == "core") + assert core["old"] == 2 and core["new"] == 3 and core["delta"] == 1 + + +def test_diff_god_nodes_emerged(): + # In _new, 'a' has degree 3 (a hub); in _old the top hub is different. + diff = diff_graphs(load_graph_from_text(_old()), load_graph_from_text(_new()), top_god=1) + emerged = {d["id"] for d in diff["god_nodes"]["emerged"]} + vanished = {d["id"] for d in diff["god_nodes"]["vanished"]} + assert "a" in emerged + assert "b" in vanished # b was the top-degree node in _old (degree 2), not in _new top-1 + + +def test_diff_identical_is_empty(): + diff = diff_graphs(load_graph_from_text(_old()), load_graph_from_text(_old())) + assert diff["nodes"]["delta"] == 0 + assert diff["nodes"]["added"] == [] and diff["nodes"]["removed"] == [] + assert diff["edges"]["added"] == [] and diff["edges"]["removed"] == [] + assert diff["god_nodes"]["emerged"] == [] and diff["god_nodes"]["vanished"] == [] + assert diff["communities"]["appeared"] == [] and diff["communities"]["disappeared"] == [] + + +def test_format_diff_readable(): + diff = diff_graphs(load_graph_from_text(_old()), load_graph_from_text(_new()), top_god=1) + text = format_diff(diff) + assert "structural diff" in text + assert "God-nodes emerged" in text + assert "auth" in text # appeared community named diff --git a/tests/test_embed.py b/tests/test_embed.py new file mode 100644 index 000000000..db2b7afb6 --- /dev/null +++ b/tests/test_embed.py @@ -0,0 +1,127 @@ +"""Tests for graphify.embed — optional semantic ranking backend. + +A deterministic letter-count fake embedder stands in for a real model: texts +that share letters get similar vectors, so semantic scores are meaningful and +reproducible without Ollama or sentence-transformers. +""" +from __future__ import annotations + +import numpy as np +import pytest + +import networkx as nx + +from graphify import embed +from graphify.embed import ( + build_embeddings, + get_embedder, + load_embeddings, + node_text, + semantic_scores_for_query, + sidecar_paths, +) + + +def fake_embedder(texts): + """26-d letter-count vectors: shared letters -> higher cosine.""" + out = [] + for t in texts: + v = np.zeros(26, dtype=np.float32) + for ch in str(t).lower(): + if "a" <= ch <= "z": + v[ord(ch) - 97] += 1.0 + out.append(v) + return np.asarray(out, dtype=np.float32) + + +def _graph() -> nx.Graph: + G = nx.Graph() + G.add_node("n1", label="extract()", source_file="graphify/extract.py", community_name="io") + G.add_node("n2", label="cluster()", source_file="graphify/cluster.py", community_name="graph") + G.add_node("n3", label="wxyz()", source_file="q/jkq.py", community_name="misc") + return G + + +def test_node_text_combines_fields(): + text = node_text({"label": "foo()", "source_file": "a/b.py", "community_name": "core"}) + assert "foo()" in text and "a/b.py" in text and "core" in text + + +def test_node_text_falls_back_to_label(): + assert node_text({"label": "solo"}) == "solo" + + +def test_build_and_load_roundtrip(tmp_path): + G = _graph() + gp = str(tmp_path / "graph.json") + summary = build_embeddings(G, gp, embedder=fake_embedder, model_tag="fake") + assert summary["status"] == "built" + assert summary["count"] == 3 + loaded = load_embeddings(gp) + assert loaded is not None + ids, matrix, meta = loaded + assert set(ids) == {"n1", "n2", "n3"} + assert matrix.shape == (3, 26) + assert meta["model"] == "fake" + # Vectors are L2-normalized on write. + norms = np.linalg.norm(matrix, axis=1) + assert np.allclose(norms, 1.0) + + +def test_build_is_cached_then_forced(tmp_path): + G = _graph() + gp = str(tmp_path / "graph.json") + assert build_embeddings(G, gp, embedder=fake_embedder, model_tag="fake")["status"] == "built" + assert build_embeddings(G, gp, embedder=fake_embedder, model_tag="fake")["status"] == "cached" + assert build_embeddings(G, gp, embedder=fake_embedder, model_tag="fake", force=True)["status"] == "built" + + +def test_build_rebuilds_when_nodes_change(tmp_path): + gp = str(tmp_path / "graph.json") + G = _graph() + build_embeddings(G, gp, embedder=fake_embedder, model_tag="fake") + G.add_node("n4", label="new()", source_file="n.py") + # Different node set -> different content hash -> not cached. + assert build_embeddings(G, gp, embedder=fake_embedder, model_tag="fake")["status"] == "built" + + +def test_semantic_scores_rank_by_overlap(tmp_path): + G = _graph() + gp = str(tmp_path / "graph.json") + build_embeddings(G, gp, embedder=fake_embedder, model_tag="fake") + scores = semantic_scores_for_query(G, "extract", graph_path=gp, embedder=fake_embedder) + assert set(scores) == {"n1", "n2", "n3"} + # 'extract' shares more letters with extract() than with wxyz(). + assert scores["n1"] > scores["n3"] + + +def test_semantic_scores_only_for_present_nodes(tmp_path): + G = _graph() + gp = str(tmp_path / "graph.json") + build_embeddings(G, gp, embedder=fake_embedder, model_tag="fake") + G.remove_node("n3") # sidecar still has n3, but it's no longer in the graph + scores = semantic_scores_for_query(G, "cluster", graph_path=gp, embedder=fake_embedder) + assert "n3" not in scores + assert set(scores) == {"n1", "n2"} + + +def test_semantic_scores_without_sidecar_raises(tmp_path): + G = _graph() + gp = str(tmp_path / "nope.json") + with pytest.raises(RuntimeError): + semantic_scores_for_query(G, "extract", graph_path=gp, embedder=fake_embedder) + + +def test_sidecar_paths_beside_graph(tmp_path): + gp = str(tmp_path / "sub" / "graph.json") + vec, meta = sidecar_paths(gp) + assert vec.parent == (tmp_path / "sub") + assert vec.name.endswith(".npz") + assert meta.name.endswith(".json") + + +def test_get_embedder_errors_when_backend_forced_and_absent(monkeypatch): + monkeypatch.setenv("GRAPHIFY_EMBED_BACKEND", "ollama") + monkeypatch.setenv("OLLAMA_HOST", "http://127.0.0.1:1") # nothing listening + with pytest.raises(RuntimeError): + get_embedder() diff --git a/tests/test_evals.py b/tests/test_evals.py new file mode 100644 index 000000000..efd6756c5 --- /dev/null +++ b/tests/test_evals.py @@ -0,0 +1,201 @@ +"""Tests for graphify.evals — relevance eval harness (P@k / recall / MRR / nDCG).""" +from __future__ import annotations + +import json + +import networkx as nx +import pytest + +from graphify import evals +from graphify.evals import ( + METRIC_GLOSSARY, + EvalCase, + _node_matches, + _score_case, + diff_aggregates, + load_cases, + load_last_report, + run_evals, + save_report, + scaffold_fixture, +) + + +def _graph() -> nx.Graph: + G = nx.Graph() + G.add_node("n1", label="extract()", source_file="graphify/extract.py", source_location="L10", community=0) + G.add_node("n2", label="cluster()", source_file="graphify/cluster.py", source_location="L5", community=0) + G.add_node("n3", label="build()", source_file="graphify/build.py", source_location="L1", community=1) + G.add_edge("n1", "n2", relation="calls", confidence="INFERRED", context="call") + G.add_edge("n2", "n3", relation="imports", confidence="EXTRACTED", context="import") + return G + + +# --- EvalCase parsing --- + +def test_evalcase_from_dict_basic(): + c = EvalCase.from_dict({"query": "q", "expect": ["a", "b"]}) + assert c.query == "q" and c.expect == ["a", "b"] + assert c.mode == "bfs" and c.depth == 2 + + +def test_evalcase_accepts_question_and_expected_aliases(): + c = EvalCase.from_dict({"question": "q", "expected": "solo"}) + assert c.query == "q" + assert c.expect == ["solo"] # string coerced to single-item list + + +def test_evalcase_requires_query(): + with pytest.raises(ValueError): + EvalCase.from_dict({"expect": ["a"]}) + + +def test_evalcase_requires_expect(): + with pytest.raises(ValueError): + EvalCase.from_dict({"query": "q"}) + + +# --- node matching --- + +def test_node_matches_label_and_callable_variants(): + G = _graph() + assert _node_matches(G, "n1", "extract()") + assert _node_matches(G, "n1", "extract") # decoration-insensitive + assert _node_matches(G, "n1", "EXTRACT") # case-insensitive + + +def test_node_matches_id_and_source_file(): + G = _graph() + assert _node_matches(G, "n1", "n1") + assert _node_matches(G, "n1", "graphify/extract.py") + assert _node_matches(G, "n1", "extract.py") # basename + assert not _node_matches(G, "n1", "cluster.py") + + +# --- scoring --- + +def test_score_case_perfect_hit_at_rank_one(): + G = _graph() + matched, m = _score_case(G, ["n1", "n2", "n3"], ["extract()"], k=10) + assert matched["extract()"] == 1 + assert m["mrr"] == 1.0 + assert m["recall_at_k"] == 1.0 + assert m["hit_at_k"] == 1.0 + assert 0.0 <= m["ndcg_at_k"] <= 1.0 + + +def test_score_case_miss(): + G = _graph() + matched, m = _score_case(G, ["n1", "n2"], ["nonexistent"], k=10) + assert matched == {} + assert m["mrr"] == 0.0 + assert m["recall_at_k"] == 0.0 + assert m["hit_at_k"] == 0.0 + assert m["ndcg_at_k"] == 0.0 + + +def test_score_case_respects_k_cutoff(): + G = _graph() + # Expected node is at rank 3 but k=2 excludes it. + matched, m = _score_case(G, ["n1", "n2", "n3"], ["build()"], k=2) + assert matched == {} + assert m["hit_at_k"] == 0.0 + + +def test_score_case_mrr_reflects_rank(): + G = _graph() + _matched, m = _score_case(G, ["n1", "n2", "n3"], ["build()"], k=10) + assert m["mrr"] == pytest.approx(1.0 / 3, abs=1e-4) # metrics round to 4 places + + +def test_ndcg_never_exceeds_one_with_duplicate_matches(): + # Two distinct nodes share the same label; one expected item matches both. + G = nx.Graph() + G.add_node("a", label="dup()", source_file="a.py", community=0) + G.add_node("b", label="dup()", source_file="b.py", community=0) + G.add_node("c", label="other()", source_file="c.py", community=0) + _matched, m = _score_case(G, ["a", "b", "c"], ["dup()"], k=10) + assert 0.0 <= m["ndcg_at_k"] <= 1.0 + + +# --- end to end --- + +def test_run_evals_end_to_end(): + G = _graph() + cases = [ + EvalCase(query="extract", expect=["extract()"]), + EvalCase(query="cluster", expect=["cluster()"]), + ] + report = run_evals(G, cases, k=5) + assert len(report.cases) == 2 + assert set(report.aggregate) == set(METRIC_GLOSSARY) + assert report.aggregate["hit_at_k"] == 1.0 + assert report.aggregate["recall_at_k"] == 1.0 + + +def test_run_evals_is_deterministic(): + G = _graph() + cases = [EvalCase(query="extract", expect=["extract()"])] + a = run_evals(G, cases, k=5).aggregate + b = run_evals(G, cases, k=5).aggregate + assert a == b + + +# --- fixture IO --- + +def test_load_cases_jsonl(tmp_path): + p = tmp_path / "f.jsonl" + p.write_text( + '# a comment\n' + '{"query": "a", "expect": ["x"]}\n' + '\n' + '{"query": "b", "expect": ["y", "z"]}\n', + encoding="utf-8", + ) + cases = load_cases(p) + assert [c.query for c in cases] == ["a", "b"] + + +def test_load_cases_json_array(tmp_path): + p = tmp_path / "f.json" + p.write_text(json.dumps([{"query": "a", "expect": ["x"]}]), encoding="utf-8") + cases = load_cases(p) + assert cases[0].expect == ["x"] + + +def test_load_cases_empty_raises(tmp_path): + p = tmp_path / "f.jsonl" + p.write_text("\n# only comments\n", encoding="utf-8") + with pytest.raises(ValueError): + load_cases(p) + + +def test_save_and_load_last_report(tmp_path): + G = _graph() + report = run_evals(G, [EvalCase(query="extract", expect=["extract()"])], k=5) + save_report(report, base=tmp_path, fixture="f.jsonl") + # A second run appends; load_last_report returns the latest. + report2 = run_evals(G, [EvalCase(query="cluster", expect=["cluster()"])], k=5) + save_report(report2, base=tmp_path, fixture="f.jsonl") + last = load_last_report(base=tmp_path) + assert last is not None + assert last["cases"][0]["query"] == "cluster" + + +def test_load_last_report_missing(tmp_path): + assert load_last_report(base=tmp_path) is None + + +def test_diff_aggregates(): + delta = diff_aggregates({"p_at_k": 0.5, "mrr": 0.8}, {"p_at_k": 0.4, "mrr": 0.9}) + assert delta["p_at_k"] == pytest.approx(0.1) + assert delta["mrr"] == pytest.approx(-0.1) + + +def test_scaffold_fixture_generates_cases(): + G = _graph() + cases = scaffold_fixture(G, limit=5) + assert cases + assert all("query" in c and "expect" in c for c in cases) + # Each scaffolded case expects the symbol it queries. + assert all(c["expect"] for c in cases) diff --git a/tests/test_install_references.py b/tests/test_install_references.py index 0f6061a5b..81bdbbcc5 100644 --- a/tests/test_install_references.py +++ b/tests/test_install_references.py @@ -281,10 +281,11 @@ def test_claude_install_ships_lean_core_and_references(tmp_path): assert len(body.splitlines()) < 800 # The version stamp covers SKILL.md + references/ together. assert (skill_dir / ".graphify_version").read_text() == mainmod.__version__ - # The eight on-demand fragments all landed. + # The nine on-demand fragments all landed. names = sorted(p.name for p in refs.glob("*.md")) assert names == [ "add-watch.md", + "analysis.md", "exports.md", "extraction-spec.md", "github-and-merge.md", diff --git a/tests/test_ranking.py b/tests/test_ranking.py new file mode 100644 index 000000000..405cc6619 --- /dev/null +++ b/tests/test_ranking.py @@ -0,0 +1,138 @@ +"""Tests for graphify.ranking — Reciprocal Rank Fusion over query subgraphs.""" +from __future__ import annotations + +import networkx as nx + +from graphify.ranking import RRF_K, RankedNode, rank_nodes, rrf_fuse + + +def test_rrf_fuse_sums_reciprocal_ranks(): + fused = rrf_fuse({"a": ["x", "y"], "b": ["y", "x"]}, k=RRF_K) + # x: 1/(60+1) + 1/(60+2); y: 1/(60+2) + 1/(60+1) -> equal, both backends agree symmetrically + assert abs(fused["x"].score - fused["y"].score) < 1e-12 + assert fused["x"].ranks == {"a": 1, "b": 2} + assert fused["y"].ranks == {"a": 2, "b": 1} + + +def test_rrf_fuse_first_occurrence_wins_within_backend(): + fused = rrf_fuse({"a": ["x", "x", "y"]}, k=RRF_K) + # duplicate x must not double-count; it keeps its best (rank 1) contribution + assert fused["x"].ranks == {"a": 1} + assert abs(fused["x"].contributions["a"] - 1.0 / (RRF_K + 1)) < 1e-12 + + +def test_rrf_fuse_agreement_beats_single_backend_top(): + # y is #1 in one backend only; x is #2 in both. Consensus should win. + fused = rrf_fuse({"a": ["y", "x"], "b": ["z", "x"]}, k=RRF_K) + assert fused["x"].score > fused["y"].score + assert fused["x"].score > fused["z"].score + + +def test_rank_nodes_pins_seeds_first(): + nodes = ["seed", "a", "b"] + ranked = rank_nodes( + nodes, + ["seed"], + lexical_scores={"a": 100.0, "b": 50.0, "seed": 1.0}, + degrees={"a": 9, "b": 9, "seed": 0}, + ) + # Even though 'a' dominates every non-seed signal, the seed renders first. + assert ranked[0].node_id == "seed" + + +def test_rank_nodes_orders_rest_by_fused_score(): + nodes = ["seed", "a", "b"] + ranked = rank_nodes( + nodes, + ["seed"], + lexical_scores={"a": 100.0, "b": 1.0, "seed": 1.0}, + distances={"seed": 0, "a": 1, "b": 3}, + degrees={"a": 5, "b": 1, "seed": 0}, + ) + order = [r.node_id for r in ranked] + assert order[0] == "seed" + assert order.index("a") < order.index("b") + + +def test_rank_nodes_proximity_matters(): + # Isolate the proximity backend: the closer node must earn a strictly larger + # proximity contribution than the far node (other backends are held equal, so + # asserting final order would just measure the id tie-break, not proximity). + ranked = rank_nodes( + ["seed", "near", "far"], + ["seed"], + lexical_scores={"near": 5.0, "far": 5.0}, + distances={"seed": 0, "near": 1, "far": 6}, + degrees={"near": 2, "far": 2}, + ) + near = next(r for r in ranked if r.node_id == "near") + far = next(r for r in ranked if r.node_id == "far") + assert near.contributions["proximity"] > far.contributions["proximity"] + + +def test_rank_nodes_community_backend_only_with_seed_community(): + # No community info -> no 'community' backend contributes. + ranked = rank_nodes( + ["seed", "a"], + ["seed"], + lexical_scores={"a": 1.0}, + ) + a = next(r for r in ranked if r.node_id == "a") + assert "community" not in a.contributions + + +def test_rank_nodes_community_boost(): + common = dict( + lexical_scores={"same": 5.0, "other": 5.0}, + degrees={"same": 2, "other": 2}, + ) + with_comm = rank_nodes( + ["seed", "same", "other"], + ["seed"], + node_community={"seed": 7, "same": 7, "other": 99}, + **common, + ) + without = rank_nodes(["seed", "same", "other"], ["seed"], **common) + same_with = next(r for r in with_comm if r.node_id == "same") + same_without = next(r for r in without if r.node_id == "same") + # Sharing the seed's community adds a backend vote the node otherwise lacks. + assert "community" in same_with.contributions + assert same_with.score > same_without.score + + +def test_rank_nodes_semantic_backend_optional_and_used(): + without = rank_nodes(["s", "a", "b"], ["s"], lexical_scores={"a": 1.0, "b": 1.0}) + assert all("semantic" not in r.contributions for r in without) + with_sem = rank_nodes( + ["s", "a", "b"], + ["s"], + lexical_scores={"a": 1.0, "b": 1.0}, + semantic_scores={"a": 0.9, "b": 0.1}, + ) + a = next(r for r in with_sem if r.node_id == "a") + b = next(r for r in with_sem if r.node_id == "b") + assert "semantic" in a.contributions + assert a.score > b.score + + +def test_rank_nodes_is_deterministic(): + kwargs = dict( + lexical_scores={"a": 1.0, "b": 1.0, "c": 1.0}, + degrees={"a": 1, "b": 1, "c": 1}, + ) + first = [r.node_id for r in rank_nodes(["s", "a", "b", "c"], ["s"], **kwargs)] + second = [r.node_id for r in rank_nodes(["c", "b", "a", "s"], ["s"], **kwargs)] + assert first == second # ties break on node id regardless of input order + + +def test_ranked_node_explain_is_readable(): + rn = RankedNode("x", 0.05, {"lexical": 1, "proximity": 4}, {"lexical": 0.03, "proximity": 0.02}) + text = rn.explain() + assert "score=0.0500" in text + assert "lexical#1" in text + # Sorted by contribution: lexical (0.03) before proximity (0.02). + assert text.index("lexical") < text.index("proximity") + + +def test_rank_nodes_empty(): + assert rank_nodes([], []) == [] diff --git a/tests/test_serve.py b/tests/test_serve.py index 4cd477d65..1a71c2c25 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -392,6 +392,79 @@ def test_query_graph_text_heuristic_context_filter_changes_traversal(): text = _query_graph_text(G, "who calls extract", mode="bfs", depth=2, token_budget=2000) assert "Context: call (heuristic)" in text assert "cluster" in text + + +# --- ranking integration --- + +def test_query_graph_text_header_marks_ranked(): + G = _make_graph() + text = _query_graph_text(G, "extract", mode="bfs", depth=2) + assert "(ranked)" in text + + +def test_query_graph_text_top_k_limits_and_reports(): + G = _make_graph() + text = _query_graph_text(G, "extract", mode="bfs", depth=3, top_k=2) + assert "top 2 of" in text + assert text.count("\nNODE ") == 2 # exactly two node lines survive the cap + + +def test_query_graph_text_explain_shows_breakdown(): + G = _make_graph() + text = _query_graph_text(G, "extract", mode="bfs", depth=2, explain=True) + assert "rank score=" in text + assert "lexical#" in text + + +def test_query_graph_text_no_explain_by_default(): + G = _make_graph() + text = _query_graph_text(G, "extract", mode="bfs", depth=2) + assert "rank score=" not in text + + +def test_query_graph_text_semantic_scores_are_used(): + G = _make_graph() + # Force 'build' (n3) to the top of the non-seed order via a semantic signal. + sem = {"n3": 0.99, "n2": 0.01, "n4": 0.01} + text = _query_graph_text(G, "extract", mode="bfs", depth=3, semantic_scores=sem, explain=True) + assert "semantic: on" in text + assert "semantic#" in text + + +def test_query_graph_text_semantic_seed_rescue_when_lexical_misses(): + G = _make_graph() + # 'zzznomatch' matches no label lexically; without semantics this returns + # "No matching nodes found." With a semantic signal it seeds from the top + # cosine node instead of giving up. + lexical_only = _query_graph_text(G, "zzznomatch", mode="bfs", depth=2) + assert "No matching nodes found." in lexical_only + rescued = _query_graph_text( + G, "zzznomatch", mode="bfs", depth=2, + semantic_scores={"n3": 0.9, "n4": 0.4}, + ) + assert "No matching nodes found." not in rescued + assert "semantic: on (seeded)" in rescued + assert "build" in rescued # n3 -> 'build', the top semantic match + + +def test_subgraph_to_text_respects_explicit_order(): + G = _make_graph() + text = _subgraph_to_text(G, {"n1", "n2", "n3"}, [], order=["n3", "n1", "n2"]) + # n3 -> 'build' renders before n1 -> 'extract' + assert text.index("build") < text.index("extract") + + +def test_subgraph_to_text_top_k_trims_nodes(): + G = _make_graph() + text = _subgraph_to_text(G, {"n1", "n2", "n3", "n4"}, [], order=["n1", "n2", "n3", "n4"], top_k=2) + assert "extract" in text and "cluster" in text + assert "report" not in text # n4 trimmed + + +def test_subgraph_to_text_explain_appended(): + G = _make_graph() + text = _subgraph_to_text(G, {"n1"}, [], order=["n1"], explain={"n1": "rank score=0.10"}) + assert "rank score=0.10" in text assert "build" not in text diff --git a/tests/test_skill_migrations.py b/tests/test_skill_migrations.py new file mode 100644 index 000000000..b4357d362 --- /dev/null +++ b/tests/test_skill_migrations.py @@ -0,0 +1,62 @@ +"""Tests for graphify.skill_migrations — version math + migration selection.""" +from __future__ import annotations + +from graphify.skill_migrations import ( + MIGRATIONS, + applicable_migrations, + compare_versions, + drift_state, + parse_version, +) + + +def test_parse_version_basic(): + assert parse_version("0.9.2") == (0, 9, 2) + assert parse_version("1.2.0") == (1, 2, 0) + + +def test_parse_version_handles_unknown_and_empty(): + assert parse_version("unknown") == (0,) + assert parse_version("") == (0,) + + +def test_parse_version_strips_prerelease_tags(): + assert parse_version("1.2.0rc1") == (1, 2, 0) + + +def test_compare_versions_numeric_not_lexical(): + # 0.10 > 0.9 numerically (a string compare would get this wrong). + assert compare_versions("0.10.0", "0.9.9") == 1 + assert compare_versions("0.9.0", "0.9.2") == -1 + + +def test_compare_versions_pads_lengths(): + assert compare_versions("0.9", "0.9.0") == 0 + + +def test_drift_state(): + assert drift_state("0.9.0", "0.9.0") == "current" + assert drift_state("0.9.0", "0.9.2") == "behind" + assert drift_state("0.9.2", "0.9.0") == "ahead" + + +def test_applicable_migrations_current_is_empty(): + # Installed == package: nothing to apply. + assert applicable_migrations("0.9.0", "0.9.0") == [] + + +def test_applicable_migrations_fresh_install_includes_baseline(): + # unknown -> 0.9.0 should surface the baseline migration. + got = applicable_migrations("unknown", "0.9.0") + assert [m.version for m in got] == ["0.9.0"] + + +def test_applicable_migrations_ahead_is_empty(): + # Installed skill newer than package: nothing to migrate *to*. + assert applicable_migrations("0.9.2", "0.9.0") == [] + + +def test_migrations_registry_is_sorted(): + versions = [m.version for m in MIGRATIONS] + ordered = sorted(versions, key=parse_version) + assert versions == ordered diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py index 25b40a914..d676c9512 100644 --- a/tests/test_skillgen.py +++ b/tests/test_skillgen.py @@ -190,10 +190,11 @@ def test_query_heading_is_homed_in_core_stub_only(): def test_eight_references_render_for_claude(): - """claude renders exactly the eight on-demand fragments from the design.""" + """claude renders exactly the nine on-demand fragments from the design.""" _, refs = _claude_artifacts() assert sorted(refs) == [ "add-watch.md", + "analysis.md", "exports.md", "extraction-spec.md", "github-and-merge.md", @@ -437,10 +438,11 @@ def test_compact_extraction_hosts_use_the_compact_spec(): def test_every_split_host_renders_eight_references(): - """All twelve split hosts render exactly the eight on-demand references.""" + """All twelve split hosts render exactly the nine on-demand references.""" platforms = gen.load_platforms() expected = [ "add-watch.md", + "analysis.md", "exports.md", "extraction-spec.md", "github-and-merge.md", diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index 3735e8a24..6384ef942 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -653,6 +653,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 3735e8a24..6384ef942 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -653,6 +653,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index 8ec9ad22a..652dde40a 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -656,6 +656,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index 4a956e725..c08336f30 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -653,6 +653,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index 8ec9ad22a..652dde40a 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -656,6 +656,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index 480ef9294..b8c2569ad 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -653,6 +653,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index df53e1477..cca34f103 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -656,6 +656,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index 8ec9ad22a..652dde40a 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -656,6 +656,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index b023ff8ee..64c94fd08 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -648,6 +648,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index 8ec9ad22a..652dde40a 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -656,6 +656,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index a643daa8c..f2a578a44 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -654,6 +654,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index f0719bf8b..dd9285092 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -652,6 +652,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index f6c83fb81..ae90015dd 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -678,6 +678,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index 8ec9ad22a..652dde40a 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -656,6 +656,12 @@ Before traversal, expand the question against the graph's own vocabulary so a wo --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/expected/graphify__skills__agents__references__analysis.md b/tools/skillgen/expected/graphify__skills__agents__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__agents__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__agents__references__query.md b/tools/skillgen/expected/graphify__skills__agents__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__query.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__analysis.md b/tools/skillgen/expected/graphify__skills__amp__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__amp__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__amp__references__query.md b/tools/skillgen/expected/graphify__skills__amp__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__query.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__analysis.md b/tools/skillgen/expected/graphify__skills__claude__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claude__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__claude__references__query.md b/tools/skillgen/expected/graphify__skills__claude__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__query.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__analysis.md b/tools/skillgen/expected/graphify__skills__claw__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__claw__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__claw__references__query.md b/tools/skillgen/expected/graphify__skills__claw__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__query.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__analysis.md b/tools/skillgen/expected/graphify__skills__codex__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__codex__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__codex__references__query.md b/tools/skillgen/expected/graphify__skills__codex__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__query.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__analysis.md b/tools/skillgen/expected/graphify__skills__copilot__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__copilot__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__query.md b/tools/skillgen/expected/graphify__skills__copilot__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__query.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__analysis.md b/tools/skillgen/expected/graphify__skills__droid__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__droid__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__droid__references__query.md b/tools/skillgen/expected/graphify__skills__droid__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__query.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__analysis.md b/tools/skillgen/expected/graphify__skills__kilo__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kilo__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__query.md b/tools/skillgen/expected/graphify__skills__kilo__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__query.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__analysis.md b/tools/skillgen/expected/graphify__skills__kiro__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__kiro__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__query.md b/tools/skillgen/expected/graphify__skills__kiro__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__query.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__analysis.md b/tools/skillgen/expected/graphify__skills__opencode__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__opencode__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__query.md b/tools/skillgen/expected/graphify__skills__opencode__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__query.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__analysis.md b/tools/skillgen/expected/graphify__skills__pi__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__pi__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__pi__references__query.md b/tools/skillgen/expected/graphify__skills__pi__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__query.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__analysis.md b/tools/skillgen/expected/graphify__skills__trae__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__trae__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__trae__references__query.md b/tools/skillgen/expected/graphify__skills__trae__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__query.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__analysis.md b/tools/skillgen/expected/graphify__skills__vscode__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__vscode__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__query.md b/tools/skillgen/expected/graphify__skills__vscode__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__query.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__analysis.md b/tools/skillgen/expected/graphify__skills__windows__references__analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/expected/graphify__skills__windows__references__analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/expected/graphify__skills__windows__references__query.md b/tools/skillgen/expected/graphify__skills__windows__references__query.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__query.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__query.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index a1189478c..6955c2670 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -585,6 +585,12 @@ Both are non-default subcommands. `--update` re-extracts only new or changed fil --- +## For evals, semantic search, history, and skill health + +None of these run in a default build. When the user wants to measure retrieval quality (`graphify bench`), enable semantic query matching (`graphify embed` + `graphify query --semantic`), diff the graph's structure across git history (`graphify chronicle`), or check whether the installed skill is current (`graphify skill check-update`), see `references/analysis.md`. + +--- + ## For /graphify add and --watch Neither is part of the default build. When the user runs `/graphify add ` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`. diff --git a/tools/skillgen/fragments/references/query/default.md b/tools/skillgen/fragments/references/query/default.md index 52321f101..69ea13fa6 100644 --- a/tools/skillgen/fragments/references/query/default.md +++ b/tools/skillgen/fragments/references/query/default.md @@ -68,6 +68,20 @@ graphify query "QUESTION" # or: graphify query "QUESTION" --dfs --budget 3000 ``` +Query results are **ranked by relevance** (Reciprocal Rank Fusion over lexical +match, graph proximity to the matched symbols, hub centrality, and community +cohesion), so the most relevant nodes render first and survive the token budget. +Useful flags: + +| Flag | Effect | +|------|--------| +| `--top-k N` | Keep only the N most relevant ranked nodes — a tight, high-signal answer. | +| `--explain` | Append each node's per-signal ranking breakdown (which backends ranked it and how). Use when a result surprises you. | +| `--semantic` | Also fuse local-embedding cosine similarity, and — when no label matches the wording — **seed** the search from the closest embeddings. This is what answers fuzzy questions ("where do we handle expired logins") whose words appear in no label. Requires a one-time `graphify embed` first (see the analysis reference); silently degrades to structural-only ranking if embeddings aren't built. | + +Prefer `--top-k` + `--explain` when you want to justify an answer; add `--semantic` +when literal + vocab-expanded matching (Step 0) still returns nothing on-point. + If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline: 1. Find the 1-3 nodes whose label best matches the expanded tokens. diff --git a/tools/skillgen/fragments/references/shared/analysis.md b/tools/skillgen/fragments/references/shared/analysis.md new file mode 100644 index 000000000..4a94f6bf6 --- /dev/null +++ b/tools/skillgen/fragments/references/shared/analysis.md @@ -0,0 +1,75 @@ +# graphify reference: evals, embeddings, history, skill health + +Load this when the user wants to **measure retrieval quality**, enable **semantic +search**, **diff the graph across time**, or **check skill freshness**. None of +these run in a default build; each is an explicit command against an existing +`graphify-out/graph.json`. All are deterministic and API-free except `embed`, +which uses a local embedding model. + +## Relevance evals — `graphify bench` + +Measures whether `graphify query` returns the *right* nodes, against a fixture of +`{question -> expected nodes}` cases. Metrics: P@k, Recall@k, MRR, nDCG@k. + +```bash +graphify bench --init # scaffold graphify-out/evals.jsonl from prominent nodes +# edit the fixture: each line is {"query": "...", "expect": ["label_or_file", ...]} +graphify bench # run + print the metrics table +graphify bench --save # append the run to .graphify-evals/eval-results.jsonl +graphify bench --replay # diff metrics vs the last saved run; exit 1 on regression +``` + +An `expect` entry matches a result node by its label, id, or source file (full +path or basename). Use `--replay` in CI or a pre-commit hook to catch a change +that quietly degrades retrieval. `--k N` sets the rank cutoff (default 10); +`--json` emits machine-readable output; `--semantic` includes the embedding +backend in the ranking under test. + +## Semantic search — `graphify embed` + +Builds a one-time embedding sidecar next to `graph.json` so `graphify query +--semantic` can match on meaning, not just tokens. Local-only: uses Ollama +(`nomic-embed-text` by default) or `sentence-transformers` if installed — no API +key, no network beyond localhost. + +```bash +graphify embed # build (or refresh) graphify-out/embeddings.npz +graphify embed --force # rebuild even if the sidecar is current +graphify query "fuzzy question" --semantic # now fuses cosine similarity into ranking +``` + +`--semantic` also **seeds** the traversal from the nearest embeddings when no +label matches the wording, which is how it answers questions whose vocabulary +appears nowhere in the graph. Set `GRAPHIFY_EMBED_BACKEND` (`ollama` | +`sentence-transformers`) and `GRAPHIFY_EMBED_MODEL` to override auto-detection. + +## Structural history — `graphify chronicle` + +Diffs two graph snapshots to show how the codebase's *structure* moved: nodes and +edges added/removed, god-nodes (high-degree hubs) that emerged or vanished, and +community shifts. Surfaces architectural drift a line diff can't. + +```bash +graphify chronicle OLD.json NEW.json # diff two snapshot files +graphify chronicle --rev HEAD~20 # vs the working-tree graph +graphify chronicle --rev v1.0 --rev2 v2.0 # commit vs commit +``` + +The `--rev` form reads snapshots straight from git history (`git show +REV:graphify-out/graph.json`) — cheap and API-free — so it needs the graph to be +committed. `--top-god N` sets the hub-ranking depth to compare (default 15); +`--json` emits the full diff. + +## Skill freshness — `graphify skill check-update` + +Reports whether the installed graphify skill (SKILL.md + references) matches the +running package, across every platform it's installed on, and names any migration +a re-install would apply. + +```bash +graphify skill status # human-readable per-platform report +graphify skill check-update # same, but exit 1 when any skill has drifted (cron/CI) +``` + +When it flags drift, run `graphify install` to re-render the skill from the +current package. `--json` emits machine-readable output. diff --git a/tools/skillgen/gen.py b/tools/skillgen/gen.py index e369dd7fc..aa3d6c78f 100644 --- a/tools/skillgen/gen.py +++ b/tools/skillgen/gen.py @@ -98,7 +98,7 @@ def _v8_baseline_ref(platform_key: str) -> str: ENUM_VALUES = "code|document|paper|image|rationale|concept" ENUM_PROSE = "`code`, `document`, `paper`, `image`, `rationale`, `concept`" -# The eight on-demand references every split platform renders. Six are +# The nine on-demand references every split platform renders. Seven are # shared-verbatim; two (extraction-spec, hooks) are variant-selected and resolved # per platform from the extraction/hooks_variant fields. _SHARED_REFERENCES = { @@ -107,6 +107,7 @@ def _v8_baseline_ref(platform_key: str) -> str: "github-and-merge": "references/shared/github-and-merge.md", "transcribe": "references/shared/transcribe.md", "add-watch": "references/shared/add-watch.md", + "analysis": "references/shared/analysis.md", } _EXTRACTION_SOURCE = { "verbose": "references/shared/extraction-spec.md", From 86e0e8014d66828be43029dd68c5a9858f953dc8 Mon Sep 17 00:00:00 2001 From: Barrett Holien Date: Thu, 9 Jul 2026 18:51:02 -0500 Subject: [PATCH 2/2] chore: bump to 0.10.0; nomic task prefixes; gitignore eval log - Bump version 0.9.0 -> 0.10.0 (minor: five new features) + CHANGELOG entry, and register the 0.10.0 skill migration so `skill check-update` explains it. - Fix embedding quality: nomic-embed-text is trained with `search_query:` / `search_document:` instruction prefixes; apply them (model-keyed, so non-nomic backends are unaffected). Without them nomic's query/doc vectors are mis-calibrated and retrieval degrades. - Stop tracking the local `.graphify-evals/` eval-results log (gitignore + drop the file that `git add -A` swept into the feature commit). --- .gitignore | 1 + .graphify-evals/eval-results.jsonl | 1 - CHANGELOG.md | 8 ++++++++ graphify/embed.py | 22 ++++++++++++++++++++-- graphify/skill_migrations.py | 7 +++++++ pyproject.toml | 2 +- tests/test_embed.py | 12 ++++++++++++ uv.lock | 2 +- 8 files changed, 50 insertions(+), 5 deletions(-) delete mode 100644 .graphify-evals/eval-results.jsonl diff --git a/.gitignore b/.gitignore index 0a6775b2a..0c0567370 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ build/ *.egg .graphify/ graphify-out/ +.graphify-evals/ .graphify_*.json .graphify_python .claude/ diff --git a/.graphify-evals/eval-results.jsonl b/.graphify-evals/eval-results.jsonl deleted file mode 100644 index ed8416b01..000000000 --- a/.graphify-evals/eval-results.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"aggregate": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.4322, "recall_at_k": 1.0}, "cases": [{"expect": ["main()"], "matched": {"main()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.3, "recall_at_k": 1.0}, "query": "main"}, {"expect": ["extract()"], "matched": {"extract()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.1, "recall_at_k": 1.0}, "query": "extract"}, {"expect": ["build_from_json()"], "matched": {"build_from_json()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.2, "recall_at_k": 1.0}, "query": "build_from_json"}, {"expect": ["Graph"], "matched": {"Graph": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.125, "recall_at_k": 1.0}, "query": "Graph"}, {"expect": ["Path"], "matched": {"Path": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 1.0, "recall_at_k": 1.0}, "query": "Path"}, {"expect": ["ingest_scip_json()"], "matched": {"ingest_scip_json()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.1, "recall_at_k": 1.0}, "query": "ingest_scip_json"}, {"expect": ["_read_text()"], "matched": {"_read_text()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 1.0, "recall_at_k": 1.0}, "query": "_read_text"}, {"expect": ["_make_id()"], "matched": {"_make_id()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.5, "recall_at_k": 1.0}, "query": "_make_id"}, {"expect": ["_labels()"], "matched": {"_labels()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 1.0, "recall_at_k": 1.0}, "query": "_labels"}, {"expect": ["extract_js()"], "matched": {"extract_js()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.1111, "recall_at_k": 1.0}, "query": "extract_js"}, {"expect": ["_file_stem()"], "matched": {"_file_stem()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.5, "recall_at_k": 1.0}, "query": "_file_stem"}, {"expect": ["cluster()"], "matched": {"cluster()": 1}, "metrics": {"hit_at_k": 1.0, "mrr": 1.0, "ndcg_at_k": 1.0, "p_at_k": 0.25, "recall_at_k": 1.0}, "query": "cluster"}], "fixture": "/private/tmp/claude-501/-Users-barrett-code-graphify/7a133a61-c1e4-44a7-8365-8c9e0318ba14/scratchpad/verify/ev.jsonl", "k": 10} diff --git a/CHANGELOG.md b/CHANGELOG.md index dc8e14d00..84c1f63f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.10.0 (2026-07-09) + +- **Feat: query results are now ranked by relevance** instead of raw node degree. `graphify query` fuses lexical match, graph proximity to the matched symbols, hub centrality, and community cohesion via Reciprocal Rank Fusion (`graphify/ranking.py`), so the nodes that answer the question render first and survive the token budget. New flags: `--top-k N` (keep the N most relevant nodes), `--explain` (per-node ranking breakdown), and `--semantic` (see below). The MCP `query_graph` tool gains `top_k` and `explain`. Traversal, seeds, and the graph itself are unchanged — only the render order and the new flags are new. +- **Feat: `graphify bench` — relevance evals.** Scores `graphify query` against a JSONL fixture (`{query, expect}`) with P@k / Recall@k / MRR / nDCG@k, running the real ranking pipeline (`graphify/evals.py`). `--init` scaffolds a starter fixture from prominent nodes; `--save` appends the run to `.graphify-evals/eval-results.jsonl`; `--replay` diffs metrics against the last saved run and exits non-zero on regression (CI/pre-commit safe). Deterministic, no API. +- **Feat: optional local semantic search — `graphify embed` + `graphify query --semantic`.** Builds a local embedding sidecar next to `graph.json` (`graphify/embed.py`) using Ollama (`nomic-embed-text` by default) or `sentence-transformers` — no API key, localhost only. `--semantic` fuses cosine similarity into the ranking and, when no label matches the wording, *seeds* the traversal from the nearest embeddings, so fuzzy questions ("where do we handle expired logins") resolve. Strictly opt-in; the default query path stays $0 and deterministic. Configure with `GRAPHIFY_EMBED_BACKEND` / `GRAPHIFY_EMBED_MODEL`. +- **Feat: `graphify chronicle` — structural diff over time.** Diffs two `graph.json` snapshots (`graphify chronicle OLD NEW`) or two git revisions (`--rev REV [--rev2 REV2]`, read via `git show`, no re-extraction) and reports nodes/edges added or removed, god-nodes that emerged or vanished, and community shifts (`graphify/chronicle.py`). Generalizes the single-PR impact view across the whole history; API-free. +- **Feat: `graphify skill status` / `graphify skill check-update` — skill drift detection.** Reports whether the installed skill (SKILL.md + references) matches the running package across every platform, and names any migration a re-install would apply (`graphify/skill_migrations.py`). `check-update` exits non-zero on drift for cron/CI. The agent skill documents the new query flags and a new `references/analysis.md` covering bench / embed / chronicle / skill. + ## 0.9.0 (2026-06-28) - **Breaking — node IDs now include the full repo-relative path** (#1504, #1509). The node-ID stem was the immediate parent dir + filename, so same-named files in different directories collided into one last-writer-wins node and silently dropped graph content (`docs/v1/api/README.md` and `docs/v2/api/README.md` both → `api_readme`). The stem is now the full repo-relative path (`docs_v1_api_readme` vs `docs_v2_api_readme`); top-level files are unchanged (`setup.py` → `setup`). The AST extractor, the LLM system prompt, the extraction-spec, and the two hand-copied stem helpers are all aligned to this one rule (fixing the #1509 AST↔LLM divergence that produced ghost duplicates), and `build_from_json` deterministically re-keys any cached/older semantic fragment onto the new IDs from its `source_file` so the unversioned semantic cache survives without ghosts or a re-bill. **Existing graphs migrate to the new ID format automatically on the next `build`/`update`** (no re-bill). Note: same-named files in different directories that previously collided into one node are only *recovered as distinct nodes* by a fresh extraction — run `graphify extract --force` to rebuild and gain them (migrating an already-collided graph/cache can't resurrect the nodes that were already dropped). If you push to a persisted **Neo4j** store, re-import after upgrading (re-exported IDs change); saved Gephi/yEd (GraphML) layouts go stale; MCP/cypher consumers should query by label rather than persisting node IDs across rebuilds. diff --git a/graphify/embed.py b/graphify/embed.py index 2457d87a7..7e4174c4e 100644 --- a/graphify/embed.py +++ b/graphify/embed.py @@ -43,6 +43,21 @@ # node text # --------------------------------------------------------------------------- # +def _text_prefix(model_tag: str, kind: str) -> str: + """Task-instruction prefix for asymmetric retrieval, keyed to the model. + + Nomic embed models are trained with ``search_document:`` / ``search_query:`` + instruction prefixes; omitting them measurably degrades retrieval quality + (the query and document embeddings land in different regions than intended). + Models that don't use instruction prefixes get raw text. ``kind`` is + ``"query"`` or ``"document"``. + """ + tag = (model_tag or "").lower() + if "nomic" in tag: + return "search_query: " if kind == "query" else "search_document: " + return "" + + def node_text(data: dict) -> str: """The text used to represent a node to the embedder. @@ -195,7 +210,8 @@ def build_embeddings( model_tag = model_tag or "custom" node_ids = list(G.nodes()) - texts = [node_text(G.nodes[n]) for n in node_ids] + doc_prefix = _text_prefix(model_tag, "document") + texts = [doc_prefix + node_text(G.nodes[n]) for n in node_ids] content_hash = _hash_ids_model(node_ids, model_tag) vec_path, meta_path = sidecar_paths(graph_path) @@ -285,7 +301,9 @@ def semantic_scores_for_query( os.environ.setdefault("GRAPHIFY_EMBED_BACKEND", "ollama" if backend == "ollama" else "sentence-transformers") os.environ["GRAPHIFY_EMBED_MODEL"] = name embedder, _ = get_embedder() - q = _normalize(embed_texts([question], embedder=embedder).astype(np.float32)) + # Prefix the query to match how the sidecar's documents were embedded. + query_prefix = _text_prefix(meta.get("model", ""), "query") + q = _normalize(embed_texts([query_prefix + question], embedder=embedder).astype(np.float32)) if q.size == 0: return {} sims = matrix @ q[0] # both L2-normalized -> dot product is cosine diff --git a/graphify/skill_migrations.py b/graphify/skill_migrations.py index c05ed6600..067273ac7 100644 --- a/graphify/skill_migrations.py +++ b/graphify/skill_migrations.py @@ -35,6 +35,13 @@ class Migration: "graphify-query-first workflow.", "reinstall", ), + Migration( + "0.10.0", + "Ranked query results (--top-k / --explain / --semantic) and a new " + "references/analysis.md covering bench, embed, chronicle, and skill " + "check-update. Re-install to pick up the new query flags and reference.", + "reinstall", + ), ] diff --git a/pyproject.toml b/pyproject.toml index 46a4b518a..67b7212b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "graphifyy" -version = "0.9.0" +version = "0.10.0" description = "AI coding assistant skill (Claude Code, CodeBuddy, Codex, OpenCode, Kilo Code, Cursor, Gemini CLI, Aider, OpenClaw, Factory Droid, Trae, Hermes, Kiro, Pi, Devin CLI, Google Antigravity) - turn any folder of code, docs, papers, images, or videos into a queryable knowledge graph" readme = "README.md" license = { file = "LICENSE" } diff --git a/tests/test_embed.py b/tests/test_embed.py index db2b7afb6..897ac4f71 100644 --- a/tests/test_embed.py +++ b/tests/test_embed.py @@ -13,6 +13,7 @@ from graphify import embed from graphify.embed import ( + _text_prefix, build_embeddings, get_embedder, load_embeddings, @@ -22,6 +23,17 @@ ) +def test_text_prefix_nomic_is_asymmetric(): + assert _text_prefix("ollama:nomic-embed-text", "query") == "search_query: " + assert _text_prefix("ollama:nomic-embed-text", "document") == "search_document: " + + +def test_text_prefix_non_nomic_is_empty(): + assert _text_prefix("st:all-MiniLM-L6-v2", "query") == "" + assert _text_prefix("fake", "document") == "" + assert _text_prefix("", "query") == "" + + def fake_embedder(texts): """26-d letter-count vectors: shared letters -> higher cosine.""" out = [] diff --git a/uv.lock b/uv.lock index 807f0ef27..ccbecd281 100644 --- a/uv.lock +++ b/uv.lock @@ -1150,7 +1150,7 @@ wheels = [ [[package]] name = "graphifyy" -version = "0.8.51" +version = "0.10.0" source = { editable = "." } dependencies = [ { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },