From c8b1268dd35771d978e84bd75f41554ad5f136a0 Mon Sep 17 00:00:00 2001 From: Muthukumaran Navaneethakrishnan Date: Fri, 10 Jul 2026 07:59:55 +0530 Subject: [PATCH] =?UTF-8?q?feat:=20central=20graph=20store=20=E2=80=94=20g?= =?UTF-8?q?raphify-out=20links=20into=20a=20store=20folder=20+=20repo-loca?= =?UTF-8?q?l=20sync=20hooks=20(graphify=20remote)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A committed .graphify/config.json ({"store": "~/graphify-store"}) makes every graphify run keep ./graphify-out as a link (POSIX symlink / Windows directory junction, no admin) into ////graphify-out. Writes physically land outside the repo; every literal graphify-out/... path — CLI defaults, the agent skill, plain cat — keeps working through the link, so the skill and skillgen artifacts stay byte-identical to upstream. One command group: graphify remote - remote init: bootstrap the committed .graphify/ (config.json + starter S3 push/pull hooks; swap for .sh/.js/.ps1/.cmd of the same name or point config {"push": ...} anywhere; creds stay in env, never in the repo) - remote push/pull: run the repo's hook with GRAPHIFY_STORE_DIR/PREFIX/CONFIG in the env (resolution: config path -> .graphify/.* -> ~/.config/graphify/hooks/); pull then fans a link out for every module in the store — one pull recreates a fresh clone's whole working set - remote delete: leave the store — every link becomes a real local folder holding a copy of its store data (store untouched) Adoption/exit are both safe: pre-existing local graphify-out dirs (even ones tracked in git) migrate into the store once; a bare 'graphify-out' .gitignore entry is maintained (trailing-slash misses symlinks; bare also stops git descending junctions on Windows). Repo key from the origin remote basename so all clones share it; branch switches retarget the link. No config -> behaviour identical to upstream. --- docs/graph-store.md | 131 +++++++ graphify/__main__.py | 17 + graphify/cli.py | 4 + graphify/remote.py | 215 +++++++++++ graphify/remote_hook_templates.py | 82 ++++ graphify/store.py | 317 ++++++++++++++++ tests/test_remote_sync.py | 608 ++++++++++++++++++++++++++++++ 7 files changed, 1374 insertions(+) create mode 100644 docs/graph-store.md create mode 100644 graphify/remote.py create mode 100644 graphify/remote_hook_templates.py create mode 100644 graphify/store.py create mode 100644 tests/test_remote_sync.py diff --git a/docs/graph-store.md b/docs/graph-store.md new file mode 100644 index 000000000..29e7d7203 --- /dev/null +++ b/docs/graph-store.md @@ -0,0 +1,131 @@ +# Central graph store + push/pull hooks + +By default graphify writes `graphify-out/` in the current directory. For teams — +especially monorepos where graphs run 300MB–1GB per module — you can keep **all** +graph data in a central **store folder** instead: the repo never holds the bytes, +only a link. Sharing happens through **push/pull hooks committed in the repo**, so +graphify stays backend-agnostic (S3, MinIO, git-lfs, rsync, a network share, +anything) and a clone carries its own sync behaviour. + +## One-command setup + +```bash +graphify remote init +``` + +creates the repo's committed graphify home: + +``` +.graphify/ +├── config.json { "store": "~/graphify-store" } ← edit if you want +├── push.py starter S3/MinIO hooks — swap for push.sh/.js/.ps1/.cmd +└── pull.py (same name) or any script via config.json {"push": ""} +``` + +Commit `.graphify/` and the team is onboarded: teammates clone and run +`graphify remote pull`. Existing files are never overwritten; a hook already present in +another language is respected. + +## How the store works + +Any graphify command first makes sure `./graphify-out` is a **link** into the +store: + +``` +~/graphify-store/ ← the real data (outside the repo) +└── myrepo/ + └── mybranch/ + ├── graphify-out/ ← repo-root graphs + └── services/api/graphify-out/ ← that module's graphs + +myrepo/ ← the git repo +├── .graphify/ ← committed (config + hooks) +├── graphify-out → ~/graphify-store/myrepo/mybranch/graphify-out +└── services/api/graphify-out → …/mybranch/services/api/graphify-out +``` + +- POSIX: a symlink. Windows: a **directory junction** — no admin rights, but the + store must be a local path there (the hooks do the network leg). +- Every write physically lands in the store; every literal `graphify-out/...` + path — CLI defaults, the agent skill's code blocks, plain `cat` — keeps working + through the link. **The skill needs no changes.** +- `/` come from git — the repo name from the origin remote's + basename (so every clone keys the same, whatever its folder is called; no + remote → folder name; `{ "repo": "name" }` in the config overrides both). + Branches never collide, and switching branches retargets the link on the next + graphify run. +- A monorepo gets one link per directory you build in, keyed by its repo-relative + path. `graphify remote pull` additionally fans links out for **every** module already + present in the store, so a fresh clone reads any module's graphs (e.g. + `merge-graphs ./services/api/graphify-out/graph.json` from the root) after one + pull. +- A pre-existing real `graphify-out/` folder is **migrated** into the store once + (contents moved, folder replaced by the link). +- No `.graphify/config.json` (or no `store` key) → the classic local + `./graphify-out`, nothing changes. An absolute `GRAPHIFY_OUT` env override also + disables linking. + +### git never sees the data + +When graphify creates a link it also ensures the repo-root `.gitignore` contains a +bare `graphify-out` entry (creating the file if needed). Bare on purpose: the +common `graphify-out/` pattern matches only *directories*, and git treats a POSIX +symlink as a file — with the trailing slash the links themselves would appear in +`git status`. The bare entry matches the symlink, the junction, and a plain folder +alike, at any depth, so one line covers every module. Even unignored, a symlink +commits as a ~100-byte target path — git never follows it into content. + +## Sync hooks + +`graphify remote push` / `graphify remote pull` run a hook — any executable: Python, Node, +shell, PowerShell, `.cmd`/`.bat`, or a binary. The contract: mirror +`GRAPHIFY_STORE_DIR` (the `//` tree) to/from your backend and +exit non-zero on failure. If the store folder is already shared (NFS, Dropbox, …), +you never need push/pull. To leave the store later, `graphify remote delete` turns every link back into a real local folder (a copy — the store is untouched); then delete `.graphify/` and commit. + +Resolution order: +1. an explicit path in `.graphify/config.json`: + `{ "store": "...", "push": "./scripts/push.py", "pull": "./scripts/pull.py" }` +2. `.graphify/.{py,js,mjs,cjs,sh,ps1,cmd,bat}` committed in the repo + +An executable hook runs directly (its shebang wins — e.g. +`#!/usr/bin/env -S uv run --with boto3 python3`); otherwise graphify picks the +interpreter by extension, so it also works on Windows where shebangs are ignored. + +**Secrets stay out of the repo.** The committed hooks read credentials from the +environment (env vars, `~/.aws`, …) — the starter S3 hooks use `GRAPHIFY_BUCKET` +/ `GRAPHIFY_S3_ENDPOINT` plus boto3's standard credential chain, and skip +`cache/` (a local-only accelerator). Extra non-secret keys you add to +`config.json` are yours to read — hooks get its path as `GRAPHIFY_CONFIG`. + +### Hook environment + +| Variable | Meaning | +|---|---| +| `GRAPHIFY_ACTION` | `push` or `pull` | +| `GRAPHIFY_STORE_DIR` | `//` — the tree to mirror | +| `GRAPHIFY_STORE` | the configured store base folder (expanded) | +| `GRAPHIFY_CONFIG` | path to `.graphify/config.json` (read your own extra keys) | +| `GRAPHIFY_REPO_ROOT` | repo top-level dir (context only — data is not here) | +| `GRAPHIFY_REPO` / `GRAPHIFY_BRANCH` | store key parts | +| `GRAPHIFY_PREFIX` | `"/"` — a natural object-key / path prefix | + +## Typical team flow + +```bash +# publisher (once) +graphify remote init # .graphify/ with config + hooks; edit, then commit it +graphify . # writes through the link into the store (per module: cd m && graphify .) +graphify remote push # hook uploads /// + +# teammate +git clone … && cd … +graphify remote pull # hook downloads the store tree AND recreates a graphify-out + # link for every module found in it — the whole working set +graphify query "how does auth work?" +``` + +Graphs never enter git; the repo carries only `.graphify/` and the auto-maintained +`.gitignore` entry. Note `graphify update .` / `watch` write through the link too — +keep the store on a local disk (hooks do the network leg) so incremental updates +stay fast. diff --git a/graphify/__main__.py b/graphify/__main__.py index 708c45819..ef3d62952 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -575,6 +575,10 @@ def main() -> None: print(" --cargo extract crate→crate deps from Cargo.toml") print(" --global also merge the resulting graph into the global graph") print(" --as repo tag for --global (default: target directory name)") + print(" remote init set up a central graph store: .graphify/ with config.json + push/pull hooks") + print(" remote push run the push hook — upload the central graph store (docs/graph-store.md)") + print(" remote pull run the pull hook — download the store + recreate graphify-out links") + print(" remote delete leave the store — turn graphify-out links back into real local folders") print(" global add add/update a project graph in the global graph (~/.graphify/global-graph.json)") print(" --as repo tag (default: parent directory name)") print(" global remove remove a repo's nodes from the global graph") @@ -664,6 +668,19 @@ def main() -> None: print(f"Run 'graphify --help' for full usage.") return + # Central graph store (graphify.store): when .graphify/config.json selects a + # store folder, keep ./graphify-out linked into it before any command runs, + # so writes land in the store and literal graphify-out/... reads keep + # working. Skipped for the silent hook commands (must not print) and never + # blocks the actual command. + if cmd not in _silent_cmds: + try: + from graphify.store import ensure_out_link + + ensure_out_link() + except Exception as exc: + print(f"warning: graph-store link upkeep failed: {exc}", file=sys.stderr) + if dispatch_install_cli(cmd): return dispatch_command(cmd) diff --git a/graphify/cli.py b/graphify/cli.py index 540a43510..243602e6a 100644 --- a/graphify/cli.py +++ b/graphify/cli.py @@ -1823,6 +1823,10 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph": result = run_benchmark(graph_path, corpus_words=corpus_words) print_benchmark(result) + elif cmd == "remote": + from graphify.remote import cmd_remote as _remote + _remote(sys.argv[2:]) + elif cmd == "global": subcmd = sys.argv[2] if len(sys.argv) > 2 else "" from graphify.global_graph import ( diff --git a/graphify/remote.py b/graphify/remote.py new file mode 100644 index 000000000..abcb4e8c0 --- /dev/null +++ b/graphify/remote.py @@ -0,0 +1,215 @@ +"""Sync the central graph store via a pluggable push/pull hook. + +With a ``.graphify/config.json`` store configured (see graphify.store), the +real graph data lives in ``///...`` and the repo only +holds links. ``graphify push`` / ``graphify pull`` hand that store tree to a +**hook** — a script (Python, JS, shell, PowerShell, ``.cmd``, or any +executable) that mirrors it to a backend (S3, git-lfs, rsync, a network +share, …). graphify itself has no backend dependency; if the store folder is +already shared (NFS/Dropbox/…) you never need push/pull at all. + +Hooks live **in the repo** by default — ``.graphify/push.py`` / +``.graphify/pull.py`` (any supported extension) committed next to the config, +so a clone carries its own sync behaviour and teammates need zero per-machine +setup (``graphify remote init`` scaffolds the folder). Secrets never enter the +repo: hooks read credentials from the environment (env vars, ``~/.aws``, …). + +Hook resolution for action ``push``/``pull``: + 1. an explicit path in ``.graphify/config.json`` — ``{"push": "...", "pull": "..."}`` + (relative paths resolve against the config's folder) + 2. ``.graphify/.{py,js,mjs,cjs,sh,ps1,cmd,bat}`` in the repo, or an + extension-less executable of the same name + +The hook receives (via environment): + GRAPHIFY_ACTION "push" | "pull" + GRAPHIFY_STORE_DIR // — the tree to mirror + GRAPHIFY_STORE the configured store base folder (expanded) + GRAPHIFY_CONFIG path to .graphify/config.json (extra keys are yours) + GRAPHIFY_REPO_ROOT the repo top-level dir (context; data is NOT here) + GRAPHIFY_REPO repo name (config "repo" override, else origin basename) + GRAPHIFY_BRANCH current branch + GRAPHIFY_PREFIX "/" (a natural object-key / path prefix) + +Interpreter selection: an executable hook is run directly (its shebang wins — +an author can pin ``#!/usr/bin/env -S uv run --with boto3 python``). Otherwise +graphify maps by extension (.py→python, .js→node, .sh→bash, .ps1→powershell, +.cmd/.bat→cmd) so it also works on Windows where shebangs are ignored. +""" +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +from graphify import store as _store + +_HOOK_EXTS = (".py", ".js", ".mjs", ".cjs", ".sh", ".ps1", ".cmd", ".bat", "") + + +def _context() -> dict: + ctx = _store.store_context() + if ctx is None: + sys.exit( + 'no store configured — add { "store": "~/graphify-store" } to ' + ".graphify/config.json at the repo root (try: graphify remote init)" + ) + return ctx + + +# ---------------------------------------------------------------- hook resolution + +def _find_hook(cfg: dict, cfg_dir: Path, action: str) -> Path | None: + explicit = cfg.get(action) + if explicit: + p = Path(os.path.expanduser(explicit)) + return p if p.is_absolute() else cfg_dir / p + for ext in _HOOK_EXTS: + cand = cfg_dir / _store.CONFIG_DIR / f"{action}{ext}" + if cand.is_file(): + return cand + return None + + +def _interpreter(hook: Path) -> list[str]: + # An executable hook runs directly so its shebang wins (custom envs, uv, etc.). + if os.name != "nt" and os.access(hook, os.X_OK): + return [] + ext = hook.suffix.lower() + if ext == ".py": + return [sys.executable] + if ext in (".js", ".mjs", ".cjs"): + return ["node"] + if ext == ".sh": + return ["bash"] + if ext == ".ps1": + return ["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File"] + if ext in (".cmd", ".bat"): + return ["cmd", "/c"] + return [] # last resort: let the OS try (shebang / association) + + +def _run_hook(action: str) -> dict: + ctx = _context() + hook = _find_hook(ctx["cfg"], ctx["cfg_dir"], action) + if not hook or not hook.is_file(): + sys.exit( + f"no {action} hook found — create .graphify/{action}.py (or .sh/.js/…) in the " + f"repo, or set \"{action}\": \"