diff --git a/docs/cli.md b/docs/cli.md index a5c3741..514f8d5 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -39,6 +39,7 @@ SUBCOMMANDS history Commits that changed one path init Create an empty bare repository log Commit history, newest first + maintenance Plan or run desired-state repository maintenance merge Three-way merge two revisions mv Move a tracked path, staging both halves pr Pull requests: open, review, discuss, check, merge diff --git a/docs/walgit.md b/docs/walgit.md new file mode 100644 index 0000000..eed89d2 --- /dev/null +++ b/docs/walgit.md @@ -0,0 +1,1096 @@ +# Bundle Delivery, Observable Operations, and Self-Healing Maintenance + +**Status:** Proposed +**Target:** `@chr33s/git` +**Priority:** P0/P1 +**Scope:** Git hosting performance and operability + +## 1. Summary + +Add three related capabilities: + +1. **Bundle URI delivery** — precompute immutable Git bundles so clones and stale fetches can obtain most repository data directly from static storage instead of generating and streaming a large pack through Smart HTTP. +2. **Observable operations** — represent slow server work as named operations with IDs, progress, logs, cancellation where safe, CLI output, Smart HTTP sideband narration, and SSE for the web UI. +3. **Desired-state maintenance** — replace ad-hoc scheduling of maintenance work with a deterministic planner that continually derives the next bounded unit of work from repository state and configuration. + +The implementation must preserve the existing architecture: + +- `Repository` remains the Git domain boundary. +- `ObjectStore`, `RefStore`, and `PackStore` remain authoritative for Git state. +- Durable Objects remain the Cloudflare serialization mechanism. +- No repository-wide WAL is introduced. +- Node, Cloudflare, CLI, and browser portability must not be weakened to support a server-only feature. + +## 2. Goals + +### 2.1 Bundle delivery + +- Reduce CPU, memory, and origin-server byte volume for large clones. +- Allow stale clients to catch up using precomputed immutable data. +- Let R2/CDN/static storage carry large byte streams. +- Remain compatible with stock Git protocol v2. +- Degrade transparently to ordinary Smart HTTP if bundle delivery fails. +- Support full and `blob:none` bundle families. + +### 2.2 Observable operations + +- Make long-running work visibly progress instead of appearing hung. +- Give CLI, Git clients, API clients, and the web UI one operation model. +- Keep operational jobs distinct from agent/hub tasks. +- Preserve enough diagnostic history to explain failures. + +### 2.3 Desired-state maintenance + +- Make maintenance idempotent and restart-safe. +- Recover naturally after missed schedules or process crashes. +- Bound work performed by each maintenance invocation. +- Prioritize correctness work ahead of optimization work. +- Support different scheduling mechanisms on Node and Cloudflare without putting scheduling policy in the Git core. + +## 3. Non-goals + +This project does **not** introduce: + +- an object-store WAL as repository source of truth; +- leader election or a distributed scheduler; +- another ref database; +- a replacement for Durable Object serialization; +- a new Git object or pack format; +- packfile-URI support; +- a generic workflow engine; +- a replacement for `refs/hub/task/*`; +- mandatory bundles for correctness. + +The ordinary Git server remains the ultimate source of truth. + +--- + +# 4. Feature A — Bundle URI Delivery + +## 4.1 User-visible behavior + +A compatible stock Git client cloning through protocol v2 may discover: + +```text +bundle-uri +``` + +Before the normal `fetch`, it requests the bundle list and downloads one or more immutable `.bundle` files. + +After those objects have been installed, normal fetch negotiation transfers only the remaining objects. + +A client that does not understand `bundle-uri` behaves exactly as it does today. + +If a bundle is missing, corrupt, unauthorized, stale, or unavailable, the client falls back to normal fetch. + +## 4.2 Architecture + +Add server-specific bundle modules without changing `Repository`'s storage contract. + +Suggested module shape: + +```text +src/server/ + Bundles.ts bundle list and protocol integration + BundleBuilder.ts repository snapshot -> bundle artifact + BundleStore.ts server-side bundle metadata/artifact port + Bundles.node.ts + Bundles.cloudflare.ts +``` + +`BundleStore` is intentionally separate from `ObjectStore`. + +Git objects and refs are repository truth. Bundles are derived artifacts. + +### BundleStore + +Conceptually: + +```ts +interface BundleStore { + list(repo: string): Effect; + publish(repo: string, manifest: BundleManifest): Effect; + + read(id: string): Effect>; + write(id: string, source: Stream): Effect; + delete(id: string): Effect; +} +``` + +Cloudflare implementation stores bundle bytes in R2. + +Node implementation stores bundle bytes beneath server-managed repository metadata. + +Browser implementations do not provide this service. + +## 4.3 Bundle metadata + +```ts +interface BundleArtifact { + readonly id: string; + readonly kind: "full" | "incremental"; + readonly filter: null | "blob:none"; + + readonly creationToken: bigint; + + readonly refs: Readonly>; + readonly prerequisites: ReadonlyArray; + + readonly objectId: string; + readonly bytes: number; + readonly createdAt: string; +} +``` + +`id` must be stable and safe for Git's bundle-list identifier restrictions. + +Artifact URLs are immutable. + +A content checksum should appear in the object key: + +```text +/bundles//-.bundle +``` + +Never overwrite bundle contents in place. + +## 4.4 Snapshot model + +Do **not** reproduce walgit's historical WAL reconstruction. + +A bundle is built from an explicit repository snapshot: + +```ts +interface BundleSnapshot { + readonly createdAt: Date; + readonly refs: Readonly>; + readonly filter: null | "blob:none"; +} +``` + +The builder first captures the ref OIDs it intends to bundle. + +All subsequent object traversal uses those OIDs. + +A concurrent push may advance the repository, but it cannot change the meaning of the already captured snapshot. + +The subsequent Smart HTTP fetch transfers the difference. + +## 4.5 Bundle families + +Support two parallel families: + +```text +full +blob:none +``` + +The `blob:none` family MUST advertise: + +```text +bundle..filter=blob:none +``` + +Do not attempt arbitrary filters in the first implementation. + +## 4.6 Initial chain policy + +Use a simple chain before adding calendar complexity. + +Each family maintains: + +- one recent full base; +- zero or more incrementals chained from previous published state. + +An incremental contains objects reachable from its captured refs that are not already covered by its prerequisite snapshot. + +Conceptually: + +```text +FULL A + ↓ +INC B + ↓ +INC C + ↓ +INC D +``` + +Each incremental's prerequisite OIDs must correspond to an earlier published state. + +### Compaction + +When the incremental chain exceeds configurable limits, generate a new full base and retire old entries from the advertised clone list. + +Initial configuration: + +```ts +interface BundlePolicy { + readonly enabled: boolean; + + readonly fullMaxAge: Duration; // default 7d + readonly incrementalMinAge: Duration; // default 1h + + readonly maxIncrementals: number; // default 24 + readonly minNewObjects: number; +} +``` + +These defaults are policy, not protocol. They may change after measurements. + +## 4.7 Clone list vs catch-up list + +Maintain two logical lists. + +### Clone list + +```text +/:repo.git/bundles/clone +``` + +Contains the current full base plus the incremental chain above it. + +It is intended to bootstrap a repository with no objects. + +### Catch-up list + +```text +/:repo.git/bundles/catchup +``` + +Contains incrementals only. + +It is intended for an existing clone and MUST NOT introduce a newly-created gigantic full bundle into a routine fetch. + +Both lists use: + +```ini +[bundle] + version = 1 + mode = all + heuristic = creationToken +``` + +### Protocol v2 behavior + +The protocol `bundle-uri` command returns the clone bundle-list entries directly. + +This allows stock Git clone discovery without requiring an additional configuration step. + +### `git+` client behavior + +When `git+ clone` or a future setup command controls the clone workflow, configure subsequent fetches to use: + +```text +/:repo.git/bundles/catchup +``` + +Do not configure the full clone list as the persistent fetch source. + +## 4.8 Static artifact HTTP contract + +Bundle artifacts MUST support: + +- `GET` +- `HEAD` +- byte `Range` +- `ETag` +- `If-None-Match` +- `If-Range` +- `Content-Length` +- `Accept-Ranges: bytes` + +Immutable artifact responses should include: + +```text +Cache-Control: public, max-age=31536000, immutable +``` + +where repository visibility permits public caching. + +Private repositories must preserve existing read authorization. + +A backend MAY later return a short-lived authorized direct-storage URL instead of proxying bytes through the application, but direct delivery is an optimization and not required for the first implementation. + +## 4.9 Authorization + +Bundle-list discovery requires the same read permission as clone/fetch. + +Artifact access MUST NOT create a weaker alternate path around `repo.read`. + +For authenticated repositories: + +- application-proxied artifact requests use normal repository authentication; or +- direct object URLs must be authorization-scoped and time-limited. + +Bundle identifiers and object keys contain no secrets. + +## 4.10 Generation + +Bundle construction must use the existing object traversal and pack writer where practical. + +It must not require collecting an entire large repository pack into one `Uint8Array`. + +Before enabling large-repository bundle generation on Cloudflare, the write path MUST be capable of streaming bundle bytes to backing storage. + +If the current pack writer cannot do this within the runtime's memory bound, streaming bundle output is a prerequisite. + +## 4.11 Atomic publication + +Artifact bytes are written first. + +Only after the artifact is complete and verified is the bundle manifest updated to advertise it. + +Order: + +```text +capture refs +→ generate artifact +→ verify artifact +→ store immutable artifact +→ atomically publish new manifest +``` + +A crash before manifest publication leaves an unreferenced artifact that maintenance may later collect. + +A client must never observe a manifest entry pointing to incomplete bytes. + +## 4.12 Verification + +Before publication: + +- parse the generated bundle header; +- verify expected refs; +- verify prerequisites; +- verify object format; +- ensure the pack is structurally readable. + +Interop tests should additionally run stock: + +```text +git bundle verify +``` + +against generated bundles. + +## 4.13 Bundle API + +The first release does not require a user-facing JSON management API beyond observability. + +The following read-only API is useful: + +```text +GET /api/bundles +``` + +Response: + +```json +{ + "enabled": true, + "families": [ + { + "filter": null, + "full": "...", + "incrementals": 4, + "latestCreationToken": "..." + } + ] +} +``` + +Administrative bundle creation should occur through the maintenance system rather than a synchronous HTTP request. + +--- + +# 5. Feature B — Observable Operations + +## 5.1 Terminology + +Use **Operation**, not **Task**. + +`Task` already describes agent/hub coordination. + +An Operation describes execution performed by this Git host. + +Examples: + +```text +bundle.build +maintenance.fsck +maintenance.gc +maintenance.repack +remote.fetch +remote.push +archive.build +``` + +## 5.2 Operation model + +```ts +type OperationState = "queued" | "running" | "succeeded" | "failed" | "cancelled"; + +interface Operation { + readonly id: string; + readonly repo: string; + readonly kind: string; + + readonly state: OperationState; + + readonly createdAt: string; + readonly startedAt?: string; + readonly finishedAt?: string; + + readonly progress?: { + readonly current?: number; + readonly total?: number; + readonly unit?: string; + }; + + readonly message?: string; + readonly result?: unknown; + readonly error?: OperationError; +} +``` + +## 5.3 Events + +Progress is an append-only sequence while the operation runs. + +```ts +interface OperationEvent { + readonly sequence: number; + readonly at: string; + readonly level: "debug" | "info" | "warning" | "error"; + + readonly message: string; + + readonly progress?: { + readonly current?: number; + readonly total?: number; + readonly unit?: string; + }; +} +``` + +Event history may be bounded after completion. + +The final state must remain available longer than individual progress events. + +## 5.4 Execution model + +The Operation abstraction is **not** initially a durable generic queue. + +It wraps work that another subsystem already knows how to execute. + +Examples: + +```text +maintenance planner chooses bundle.build +→ create Operation +→ execute BundleBuilder +→ emit progress +→ complete Operation +``` + +If the process disappears, desired-state maintenance determines whether the work still needs doing and creates a new operation. + +This avoids turning operation persistence into workflow correctness. + +## 5.5 API + +Add: + +```text +GET /api/operations/:id +GET /api/operations/:id/events +POST /api/operations/:id/cancel +``` + +`events` uses: + +```text +Content-Type: text/event-stream +``` + +Events: + +```text +event: progress +event: message +event: completed +event: failed +``` + +An optional repository list endpoint: + +```text +GET /api/operations?state=running +``` + +may be added for the UI. + +## 5.6 Cancellation + +Cancellation is cooperative. + +Every operation receives the Effect interruption signal. + +Operations define whether cancellation is allowed. + +Safe: + +- object walk; +- fsck; +- bundle generation before publication; +- remote fetch before ref application. + +Potentially unsafe or meaningless after a commit point: + +- atomic ref apply; +- manifest publication; +- completed pack replacement. + +After the commit point, cancellation may stop follow-up work but MUST NOT claim the committed action was undone. + +## 5.7 CLI presentation + +Long-running CLI commands show: + +```text +* repack: walking objects 18,220 / 42,912 +* repack: writing pack 61% +* repack: verifying objects +* repack: complete +``` + +Non-interactive output remains line oriented. + +Machine-readable CLI output may emit operation IDs but does not mix progress into JSON result bodies. + +## 5.8 Smart HTTP sideband + +When a Git protocol operation performs meaningful server work, progress may be emitted on sideband 2. + +Examples: + +```text +remote: bundle catch-up unavailable; generating delta normally +remote: validating 3 ref updates +remote: indexing received pack +``` + +Do not flood the channel with per-object messages. + +Progress messages should describe phases or meaningful thresholds. + +## 5.9 Web UI + +Add a reusable operation-progress component. + +It should support: + +- current phase; +- progress bar where total is known; +- textual stream where total is unknown; +- success/failure state; +- retry entry point when the originating feature supports retry. + +The component should be reusable by maintenance, remotes, and future server operations. + +--- + +# 6. Feature C — Desired-State Maintenance + +## 6.1 Principle + +Maintenance should answer: + +> Given repository state and configuration now, what is the single most valuable bounded unit of missing maintenance work? + +It should not depend on remembering that a timer fired. + +A missed timer therefore causes delay, not permanent missing state. + +## 6.2 Pure planner + +Introduce a planner separate from execution: + +```ts +planMaintenance( + snapshot: MaintenanceSnapshot, + config: MaintenanceConfig +): ReadonlyArray +``` + +A `MaintenanceUnit` is declarative: + +```ts +type MaintenanceUnit = BuildBundle | PruneBundles | RunFsck | RunGc | RunRepack; +``` + +Each unit contains everything necessary to identify the desired action but not mutable executor state. + +## 6.3 Snapshot + +```ts +interface MaintenanceSnapshot { + readonly now: Date; + + readonly refs: Readonly>; + + readonly bundles: BundleManifest | null; + + readonly lastFsck?: Date; + readonly lastGc?: Date; + readonly lastRepack?: Date; + + readonly looseObjectCount?: number; + readonly packCount?: number; +} +``` + +The planner does not perform I/O. + +A separate reader gathers the snapshot. + +## 6.4 Priority + +Initial priority: + +1. clean up invalid/incomplete derived artifacts; +2. build a required full bundle; +3. build a required incremental bundle; +4. prune retired bundle artifacts; +5. run overdue integrity check; +6. perform GC when its threshold is exceeded; +7. repack when pack/object thresholds justify it. + +Correctness work must always outrank performance optimization. + +If future `fsck` develops automatic repair through a configured remote, repair goes before bundle generation. + +## 6.5 Bounded units + +One scheduler tick executes one expensive unit by default. + +Example: + +```text +tick 1 → build incremental bundle +tick 2 → prune obsolete bundle +tick 3 → fsck +``` + +A unit may itself stream through many objects, but it has one externally meaningful outcome. + +Cheap cleanup actions may be grouped when measured cost is negligible. + +## 6.6 Scheduling adapters + +Scheduling stays outside the planner. + +### Node + +Use a host timer or explicit service loop: + +```text +git+ serve +→ maintenance scheduler +→ planner +→ one unit +``` + +### Cloudflare + +Use the platform's appropriate scheduled/alarm mechanism. + +The Cloudflare adapter merely invokes the same planner/executor. + +### CLI + +Expose: + +```text +git+ maintenance plan +git+ maintenance run +``` + +`plan` performs no writes. + +`run` executes the highest-priority planned unit. + +Optional: + +```text +git+ maintenance run --all +``` + +replans after each completed unit until no work remains. + +## 6.7 Restart behavior + +No maintenance step should rely on a transient "running" bit for correctness. + +Example bundle build: + +```text +artifact absent +→ planner says build +→ process crashes halfway +→ no manifest entry exists +→ next planner says build +``` + +Example successful build: + +```text +artifact exists + manifest publishes it +→ planner no longer says build +``` + +This is the governing design pattern for all new maintenance features. + +## 6.8 Failure behavior + +A failed unit: + +- marks its Operation failed; +- does not mark desired state satisfied; +- does not prevent unrelated later maintenance forever; +- is eligible to retry according to scheduler backoff. + +Repeated failures should become visible in the UI/API. + +The planner itself never returns a generic permanent "maintenance failed" state. + +--- + +# 7. Cross-cutting Static Cache Contract + +Bundle delivery should establish one reusable static-artifact HTTP helper. + +Later candidates include: + +- archive artifacts; +- SHA-addressed generated responses; +- immutable generated packs. + +The helper owns: + +```text +ETag +HEAD +Range +If-Range +Cache-Control +Content-Length +Content-Type +``` + +Do not apply `immutable` caching to ref-relative endpoints such as: + +```text +/main/files/... +/HEAD/... +``` + +An API response is immutable only when all data determining that response is itself immutable and explicitly addressed. + +--- + +# 8. Configuration + +Suggested server configuration: + +```text +bundles.enabled +bundles.fullMaxAge +bundles.incrementalMinAge +bundles.maxIncrementals +bundles.minNewObjects +bundles.filters + +maintenance.enabled +maintenance.interval +maintenance.fsckInterval +maintenance.gcInterval +maintenance.repackThreshold +``` + +Application logic should consume configuration through the repository's existing Effect configuration conventions rather than reading environment variables directly. + +Invalid bundle configuration fails closed at startup. + +--- + +# 9. Observability + +Add spans around: + +```text +Bundles.snapshot +Bundles.build +Bundles.verify +Bundles.publish + +Maintenance.plan +Maintenance.execute + +Operation.start +Operation.complete +``` + +Useful metrics: + +```text +bundle_build_seconds +bundle_bytes +bundle_objects +bundle_build_failures + +git_fetch_bytes +git_fetch_objects +bundle_http_bytes + +maintenance_unit_seconds +maintenance_failures + +operations_running +operations_failed +``` + +The important product metric is: + +```text +bytes served through Smart HTTP +vs +bytes served from static bundles +``` + +Bundle URI is successful only if it materially reduces the former. + +--- + +# 10. Testing + +## 10.1 Unit + +Test: + +- bundle-list serialization; +- creation-token ordering; +- prerequisite-chain construction; +- clone/catch-up list separation; +- planner determinism; +- planner priority; +- operation state transitions; +- cancellation state handling; +- static range parsing. + +## 10.2 Store contracts + +Add a `BundleStore` contract suite analogous to the existing storage contract. + +Run against: + +- memory/test backend; +- Node backend; +- Cloudflare/R2 backend. + +## 10.3 Git interoperability + +Use the real Git binary. + +Required cases: + +1. stock clone without bundle capability; +2. stock clone using server-advertised bundle URI; +3. clone followed by ordinary incremental fetch; +4. stale clone using catch-up bundle list; +5. unavailable bundle falls back successfully; +6. corrupt bundle falls back successfully; +7. private repository bundle requires authorization; +8. `blob:none` client selects only matching bundle family; +9. bundle plus final fetch produces the same reachable object graph as a normal clone. + +Generated artifacts must pass: + +```text +git bundle verify +``` + +## 10.4 Concurrency + +Test: + +```text +bundle snapshot captured +→ concurrent push occurs +→ bundle publishes old snapshot +→ subsequent fetch obtains new push +``` + +The result must be correct without rebuilding the bundle. + +Also test two maintenance invocations racing to build the same desired artifact. + +At most one manifest entry should become canonical; duplicate immutable bytes are harmless and collectible. + +## 10.5 Crash injection + +Inject failure: + +- before artifact write; +- during artifact write; +- after artifact write; +- before manifest publish; +- after manifest publish; +- during obsolete-artifact cleanup. + +After restart, the planner must converge to the correct state. + +--- + +# 11. Rollout Plan + +## Phase 1 — Static bundle foundation + +Implement: + +- `BundleStore`; +- bundle generation; +- immutable artifact serving; +- bundle verification; +- read-only bundle metadata API. + +No automatic scheduling yet. + +Success condition: + +```text +git+ maintenance run +``` + +can build a bundle that stock `git bundle verify` accepts. + +## Phase 2 — Protocol v2 clone acceleration + +Implement: + +- `bundle-uri` capability; +- `bundle-uri` command; +- clone bundle list; +- stock Git interoperability tests. + +Success condition: + +a fresh clone consumes the advertised bundle and the final Smart HTTP fetch transfers only the tail. + +## Phase 3 — Catch-up bundles + +Implement: + +- incremental chain; +- `creationToken`; +- `/bundles/catchup`; +- `git+` setup of persistent fetch bundle URI; +- `blob:none` parallel family. + +Success condition: + +a days-stale clone downloads incrementals rather than a new full base. + +## Phase 4 — Operations + +Implement: + +- Operation model; +- progress events; +- SSE; +- CLI rendering; +- operation API; +- cancellation. + +Convert bundle generation and existing maintenance commands first. + +## Phase 5 — Desired-state maintenance + +Implement: + +- pure planner; +- Node scheduler adapter; +- Cloudflare scheduler adapter; +- maintenance plan/run CLI; +- retry/backoff; +- UI presentation. + +Bundle generation becomes maintenance-driven. + +--- + +# 12. Acceptance Criteria + +The feature set is complete when: + +### Bundle URI + +- Stock Git can discover a bundle through protocol v2. +- A clone remains successful when all bundle infrastructure is unavailable. +- Bundle artifacts are immutable and range-readable. +- No bundle is advertised before verification and complete storage. +- Private repository bundles enforce normal read authorization. +- `blob:none` bundles interoperate with stock Git. +- Routine catch-up never downloads a newly-created full base through the `git+` configured catch-up path. + +### Operations + +- Every bundle build has an operation ID. +- Progress is visible through CLI and SSE. +- Failure contains an actionable error. +- Cancellation interrupts eligible Effect work. +- Agent/hub tasks remain semantically separate. + +### Maintenance + +- `maintenance plan` is deterministic for identical state/configuration. +- Interrupted work is rediscovered automatically. +- One failed maintenance unit does not permanently block unrelated maintenance. +- Repeated scheduler invocations eventually converge to no planned work. +- No new repository correctness state exists solely in an in-memory scheduler. + +--- + +# 13. Explicitly Deferred + +Do not include in the initial implementation: + +- WAL/time-travel repository snapshots; +- bundle generation for arbitrary historical wall-clock times; +- weekly/daily/hourly multi-tier scheduling; +- mandatory bundles for large clones; +- refusal of zero-have fetches; +- geographically distributed bundle mirrors; +- signed direct R2 URLs; +- automatic object repair; +- generic durable job queues; +- packfile URI. + +Those can be evaluated after bundle effectiveness is measured. + +## 14. Recommended First Slice + +The first PR should establish only the protocol-independent foundation: + +```text +BundleStore ++ BundleManifest ++ BundleBuilder ++ verification ++ static immutable artifact handler ++ unit/store/real-git tests +``` + +The second PR should add protocol v2 `bundle-uri`. + +Keeping those separate proves that the derived artifact model is correct before changing Git wire behavior. diff --git a/src/cli/main.test.ts b/src/cli/main.test.ts index cf9f2ff..f363bfd 100644 --- a/src/cli/main.test.ts +++ b/src/cli/main.test.ts @@ -627,6 +627,21 @@ describe("cli", () => { } }), ); + + it.effect("plans and runs desired-state maintenance", () => + Effect.promise(async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "cli-plan-")); + try { + await seed(path.join(root, "repo"), "kept"); + const planned = await cli(["maintenance", "plan", "--root", root, "repo"]); + assert.match(planned, /build-full|fsck|no maintenance needed/); + const ran = await cli(["maintenance", "run", "--root", root, "repo"]); + assert.match(ran, /complete|no maintenance needed/); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }), + ); }); /** diff --git a/src/cli/main.ts b/src/cli/main.ts index ac30dc8..8b27938 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -17,6 +17,7 @@ * unexpected defect prints a `Cause` with the fiber trace. */ import { createWriteStream } from "node:fs"; +import * as fs from "node:fs/promises"; import * as path from "node:path"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; @@ -61,6 +62,7 @@ import { withRepo, } from "./shared.ts"; import { sessionCommand } from "./session.ts"; +import { maintenanceCommand } from "./maintenance.ts"; import { serveCommand } from "./serve.ts"; import { fetchCommand, pullCommand } from "./transport.ts"; import { queueCommand } from "./queue.ts"; @@ -166,6 +168,14 @@ const clone = Command.make( yield* target.refs.setHead(`refs/heads/${result.defaultBranch}`); } yield* Console.log(`Cloned ${result.refs.length} ref(s) from ${url} into ${directory}`); + const catchup = `${url.replace(/\/+$/, "").replace(/\.git$/, "")}/bundles/catchup`; + yield* Effect.tryPromise({ + try: () => + fs.appendFile(path.join(directory, "config"), `\n[fetch]\n\tbundleURI = ${catchup}\n`), + catch: () => + new Invalid({ field: "clone", reason: "could not record catch-up bundle URI" }), + }).pipe(Effect.ignore); + yield* Console.log(`configured catch-up bundle URI ${catchup}`); // Refused rather than applied, and said out loud: a branch that stops // tracking silently is a divergence nobody sees until they compare tips. for (const ref of result.rejected) { @@ -1047,6 +1057,9 @@ const git = Command.make("git+").pipe( history.pipe(Command.withDescription("Commits that changed one path")), init.pipe(Command.withDescription("Create an empty bare repository")), log.pipe(Command.withDescription("Commit history, newest first")), + maintenanceCommand.pipe( + Command.withDescription("Plan or run desired-state repository maintenance"), + ), merge.pipe(Command.withDescription("Three-way merge two revisions")), work.mv.pipe(Command.withDescription("Move a tracked path, staging both halves")), prCommand.pipe(Command.withDescription("Pull requests: open, review, discuss, check, merge")), diff --git a/src/cli/maintenance.ts b/src/cli/maintenance.ts new file mode 100644 index 0000000..627685f --- /dev/null +++ b/src/cli/maintenance.ts @@ -0,0 +1,97 @@ +/** + * `git+ maintenance plan` and `git+ maintenance run`. + * + * `plan` is read-only and deterministic for identical state. `run` executes + * the highest-priority unit; `--all` replans until nothing remains. + */ +import { Console, Effect } from "effect"; +import { Command, Flag } from "effect/unstable/cli"; + +import * as Bundles from "../server/Bundles.ts"; +import { fileLayer as bundleFiles, fileMetaLayer } from "../server/Bundles.node.ts"; +import { defaultsLayer as featuresLayer } from "../server/Features.ts"; +import * as Maintenance from "../server/MaintenanceRun.ts"; +import { memoryLayer as operationsMemory, renderLine } from "../server/Operations.ts"; +import { repoArgument, rootFlag } from "./shared.ts"; + +import { Layer } from "effect"; +import * as path from "node:path"; + +import { stores } from "../git/Node.ts"; +import * as GitRepository from "../git/Repository.ts"; + +const extras = (root: string, repo: string) => { + const directory = path.join(root, repo); + const repoStores = stores(directory); + const bundleStore = bundleFiles(directory); + const live = GitRepository.layer.pipe( + Layer.provide(GitRepository.hooksNoop), + Layer.provideMerge(repoStores), + ); + return Layer.mergeAll( + live, + bundleStore, + fileMetaLayer(directory), + operationsMemory, + featuresLayer, + Bundles.layer.pipe(Layer.provide(bundleStore), Layer.provide(live)), + ); +}; + +const planCommand = Command.make("plan", { root: rootFlag, repo: repoArgument }, ({ repo, root }) => + Effect.gen(function* () { + const units = yield* Maintenance.plan(); + if (units.length === 0) { + yield* Console.log("no maintenance needed"); + return; + } + for (const unit of units) { + if (unit.kind === "build-full" || unit.kind === "build-incremental") { + yield* Console.log(`${unit.kind} filter=${unit.filter ?? "full"}`); + } else if (unit.kind === "prune-invalid" || unit.kind === "prune-retired") { + yield* Console.log(`${unit.kind} ${String(unit.ids.length)} artifact(s)`); + } else { + yield* Console.log(unit.kind); + } + } + }).pipe(Effect.provide(extras(root, repo))), +); + +const runCommand = Command.make( + "run", + { + root: rootFlag, + all: Flag.boolean("all").pipe( + Flag.withDefault(false), + Flag.withDescription("Replan after each unit until no work remains"), + ), + repo: repoArgument, + }, + ({ all, repo, root }) => + Effect.gen(function* () { + const ticks = all ? yield* Maintenance.runAll() : [yield* Maintenance.tick()]; + for (const tick of ticks) { + if (tick.ran === null) { + yield* Console.log("no maintenance needed"); + continue; + } + const label = tick.ran.kind; + yield* Console.log(`* ${label}: complete`); + if (tick.operationId !== undefined) { + yield* Console.log(`operation ${tick.operationId}`); + } + } + }).pipe(Effect.provide(extras(root, repo))), +); + +export const maintenanceCommand = Command.make("maintenance", {}, () => + Console.log("git+ maintenance — see --help"), +).pipe( + Command.withSubcommands([ + planCommand.pipe(Command.withDescription("Show the next maintenance units without writing")), + runCommand.pipe(Command.withDescription("Execute the highest-priority maintenance unit")), + ]), +); + +export { renderLine }; +export { extras as maintenanceLayer }; diff --git a/src/git/Durable.ts b/src/git/Durable.ts index a1ccc3a..a1de087 100644 --- a/src/git/Durable.ts +++ b/src/git/Durable.ts @@ -20,9 +20,17 @@ import * as Api from "../server/Api.ts"; import * as Auth from "../server/Auth.ts"; import * as Policy from "../server/Policy.ts"; import * as Archive from "../server/Archive.ts"; +import * as Bundles from "../server/Bundles.ts"; +import { r2Layer as bundleR2, r2MetaLayer } from "../server/Bundles.cloudflare.ts"; import * as CommitPack from "../server/CommitPack.ts"; +import { defaultsLayer as featuresLayer } from "../server/Features.ts"; import { r2 as lfsR2 } from "../server/Lfs.cloudflare.ts"; import * as Lfs from "../server/Lfs.ts"; +import { + handleEvents as operationEvents, + memoryLayer as operationsMemory, + Operations, +} from "../server/Operations.ts"; import * as Protocol from "../server/Protocol.ts"; import * as Remotes from "../server/Remotes.ts"; import * as Sending from "../server/Sending.ts"; @@ -34,6 +42,7 @@ import { collector } from "./Conformance.ts"; import { type GitError, statusOf } from "./Error.ts"; import * as GitRepository from "./Repository.ts"; import { Repository } from "./Repository.ts"; +import type { ObjectStore } from "./Store.ts"; import { storeContract } from "./Store.contract.ts"; /** @@ -57,7 +66,7 @@ interface TestEnv { } export class GitRepo extends DurableObject { - #layer: Layer.Layer | null = null; + #layer: Layer.Layer | null = null; #subscribers: Layer.Layer | null = null; #remotes: Layer.Layer | null = null; @@ -118,7 +127,7 @@ export class GitRepo extends DurableObject { } /** Built once per instance: the DO is the unit of isolation, not the request. */ - #live(repo: string): Layer.Layer { + #live(repo: string): Layer.Layer { // Delivery runs in `waitUntil`, and so does forwarding: a remote that is // down must not touch a push that has already been accepted. const detached = (effect: Effect.Effect) => @@ -180,6 +189,20 @@ export class GitRepo extends DurableObject { return this.#layer; } + /** Repository plus derived-artifact and operation services. */ + #services(repo: string) { + const live = this.#live(repo); + const bundles = bundleR2({ bucket: this.env.GIT_OBJECTS, repo }); + return Layer.mergeAll( + live, + bundles, + r2MetaLayer({ bucket: this.env.GIT_OBJECTS, repo }), + operationsMemory, + featuresLayer, + Bundles.layer.pipe(Layer.provide(bundles), Layer.provide(live)), + ); + } + /** * Response bodies still being read. * @@ -218,14 +241,14 @@ export class GitRepo extends DurableObject { #respond( repo: string, requester: Layer.Layer, - effect: Effect.Effect, + effect: Effect.Effect, ): Promise { return Effect.runPromise( effect.pipe( Effect.catch((error: GitError) => Effect.succeed(Response.json({ error: error._tag }, { status: statusOf(error) })), ), - Effect.provide(Layer.mergeAll(this.#live(repo), requester, this.#openWrites())), + Effect.provide(Layer.mergeAll(this.#services(repo), requester, this.#openWrites())), Effect.map((response) => this.#track(response)), ), ); @@ -301,6 +324,30 @@ export class GitRepo extends DurableObject { ); } + if (route === "bundles" && matched.rest !== "bundles") { + return this.#respond( + repo, + requester, + Bundles.tryHandle(request).pipe( + Effect.map( + (response) => response ?? Response.json({ error: "NotFound" }, { status: 404 }), + ), + ), + ); + } + + if (route === "operations" && matched.rest.endsWith("/events")) { + return this.#respond( + repo, + requester, + operationEvents(request).pipe( + Effect.map( + (response) => response ?? Response.json({ error: "NotFound" }, { status: 404 }), + ), + ), + ); + } + // The smart-HTTP endpoints; everything else is the JSON API. if (route === "info" || route === "git-upload-pack" || route === "git-receive-pack") { return this.#respond( @@ -324,7 +371,7 @@ export class GitRepo extends DurableObject { const api = (this.#api ??= (() => { const router = HttpRouter.toWebHandler( Api.layer(this.#remoteRegistry(repo)).pipe( - Layer.provideMerge(this.#live(repo)), + Layer.provideMerge(this.#services(repo)), Layer.provideMerge(this.#registry(repo)), Layer.provideMerge(this.#openWrites()), ), diff --git a/src/git/Error.ts b/src/git/Error.ts index fdc0d9c..3ce98a6 100644 --- a/src/git/Error.ts +++ b/src/git/Error.ts @@ -54,6 +54,19 @@ export class Invalid extends Schema.TaggedError()( { httpApiStatus: 400 }, ) {} +/** A generated bundle failed verification before it could be advertised. */ +export class BundleCorrupt extends Schema.TaggedError()( + "BundleCorrupt", + { reason: Schema.String }, + { httpApiStatus: 422 }, +) {} + +export class OperationNotFound extends Schema.TaggedError()( + "OperationNotFound", + { id: Schema.String }, + { httpApiStatus: 404 }, +) {} + /** A hook rejected a push; carried as a value, not thrown. */ export class HookRejected extends Schema.TaggedError()( "HookRejected", @@ -71,7 +84,9 @@ export type GitError = | PackCorrupt | StorageFailure | Invalid - | HookRejected; + | HookRejected + | BundleCorrupt + | OperationNotFound; export const GitError = Schema.Union([ ObjectNotFound, @@ -80,6 +95,8 @@ export const GitError = Schema.Union([ StorageFailure, Invalid, HookRejected, + BundleCorrupt, + OperationNotFound, ]); /** Status for a failure, for the paths that are not `HttpApi` (smart-HTTP). */ @@ -90,10 +107,12 @@ export const statusOf = (error: GitError): number => { case "HookRejected": return 403; case "ObjectNotFound": + case "OperationNotFound": return 404; case "RefConflict": return 409; case "PackCorrupt": + case "BundleCorrupt": return 422; case "StorageFailure": return 500; diff --git a/src/host/Cloudflare.ts b/src/host/Cloudflare.ts index 3a247f5..00f7b4f 100644 --- a/src/host/Cloudflare.ts +++ b/src/host/Cloudflare.ts @@ -27,12 +27,20 @@ import { type GitError, statusOf, StorageFailure } from "../git/Error.ts"; import type { Sql } from "../git/Sql.ts"; import * as GitRepository from "../git/Repository.ts"; import type { Repository } from "../git/Repository.ts"; +import type { ObjectStore } from "../git/Store.ts"; import * as Api from "../server/Api.ts"; import * as Auth from "../server/Auth.ts"; import * as Archive from "../server/Archive.ts"; +import * as Bundles from "../server/Bundles.ts"; +import { r2Layer as bundleR2, r2MetaLayer } from "../server/Bundles.cloudflare.ts"; import * as CommitPack from "../server/CommitPack.ts"; +import { defaultsLayer as featuresLayer } from "../server/Features.ts"; import { r2 as lfsR2 } from "../server/Lfs.cloudflare.ts"; import * as LfsCore from "../server/Lfs.ts"; +import { + handleEvents as operationEvents, + memoryLayer as operationsMemory, +} from "../server/Operations.ts"; import * as Policy from "../server/Policy.ts"; import * as Protocol from "../server/Protocol.ts"; import * as Snapshot from "../server/Snapshot.ts"; @@ -80,7 +88,7 @@ export default Repo.make( * from the stores, so storage has to resolve when the layer is built. * That correspondence is why a repository maps onto a DO at all. */ - const layers = new Map>(); + const layers = new Map>(); /** * The subscriber registry on this instance's own SQLite, beside the @@ -95,7 +103,7 @@ export default Repo.make( /** The remotes this repository fetches from, on that same SQLite. */ const remotes = (repo: string) => Remotes.sql(sql, repo); - const live = (repo: string): Layer.Layer => { + const live = (repo: string): Layer.Layer => { const existing = layers.get(repo); if (existing !== undefined) return existing; // Delivery runs in `waitUntil`, so a slow receiver never adds its @@ -159,6 +167,19 @@ export default Repo.make( return built; }; + const services = (repo: string) => { + const built = live(repo); + const bundles = bundleR2({ bucket: r2, repo }); + return Layer.mergeAll( + built, + bundles, + r2MetaLayer({ bucket: r2, repo }), + operationsMemory, + featuresLayer, + Bundles.layer.pipe(Layer.provide(bundles), Layer.provide(built)), + ); + }; + /** * Response bodies still being read, per repository. * @@ -216,7 +237,7 @@ export default Repo.make( if (existing !== undefined) return existing; const router = HttpRouter.toWebHandler( Api.layer(remotes(repo)).pipe( - Layer.provideMerge(live(repo)), + Layer.provideMerge(services(repo)), Layer.provideMerge(subscribers(repo)), Layer.provideMerge(openWrites), ), @@ -423,6 +444,31 @@ export default Repo.make( ); } + if (route === "bundles" && matched.rest !== "bundles") { + return Bundles.tryHandle(request).pipe( + Effect.map( + (response) => response ?? Response.json({ error: "NotFound" }, { status: 404 }), + ), + Effect.catch((error: GitError) => + Effect.succeed(Response.json({ _tag: error._tag }, { status: statusOf(error) })), + ), + Effect.provide(services(repo)), + Effect.map((response) => track(repo, response)), + ); + } + + if (route === "operations" && matched.rest.endsWith("/events")) { + return operationEvents(request).pipe( + Effect.map( + (response) => response ?? Response.json({ error: "NotFound" }, { status: 404 }), + ), + Effect.catch((error: GitError) => + Effect.succeed(Response.json({ _tag: error._tag }, { status: statusOf(error) })), + ), + Effect.provide(services(repo)), + ); + } + if (route === "info" || route === "git-upload-pack" || route === "git-receive-pack") { return Protocol.handle(request).pipe( Effect.map( @@ -431,7 +477,7 @@ export default Repo.make( Effect.catch((error: GitError) => Effect.succeed(Response.json({ _tag: error._tag }, { status: statusOf(error) })), ), - Effect.provide(Layer.mergeAll(live(repo), requester, openWrites)), + Effect.provide(Layer.mergeAll(services(repo), requester, openWrites)), // The pack is the body that outlives its handler. Effect.map((response) => track(repo, response)), ); diff --git a/src/host/Node.ts b/src/host/Node.ts index 8f56d1b..3c72348 100644 --- a/src/host/Node.ts +++ b/src/host/Node.ts @@ -24,6 +24,7 @@ import { statusOf } from "../git/Error.ts"; import { stores } from "../git/Node.ts"; import * as GitRepository from "../git/Repository.ts"; import type { Repository } from "../git/Repository.ts"; +import type { ObjectStore } from "../git/Store.ts"; import * as SocialLog from "../social/Log.ts"; import { SocialWeb } from "../social/Projection.ts"; import { readGenesis } from "../trust/Genesis.ts"; @@ -39,9 +40,18 @@ import * as Api from "../server/Api.ts"; import * as Auth from "../server/Auth.ts"; import * as Policy from "../server/Policy.ts"; import * as Archive from "../server/Archive.ts"; +import * as Bundles from "../server/Bundles.ts"; +import { fileLayer as bundleFiles, fileMetaLayer } from "../server/Bundles.node.ts"; import * as CommitPack from "../server/CommitPack.ts"; +import { configuration as featuresConfiguration, ServerFeatures } from "../server/Features.ts"; import { file as lfsFile } from "../server/Lfs.node.ts"; import * as Lfs from "../server/Lfs.ts"; +import { tick as maintenanceTick } from "../server/MaintenanceRun.ts"; +import { + handleEvents as operationEvents, + memoryLayer as operationsMemory, + Operations, +} from "../server/Operations.ts"; import * as Protocol from "../server/Protocol.ts"; import { file as remotesFile } from "../server/Remotes.node.ts"; import { collects, routeOf, settledWithin } from "../server/Route.ts"; @@ -131,12 +141,14 @@ const nodeHeaders = (headers: Headers): NodeHeaders => { }; export const serve = async (options: ServeOptions): Promise => { + const features = await Effect.runPromise(featuresConfiguration().pipe(Effect.orDie)); + const featuresLayer = Layer.succeed(ServerFeatures, ServerFeatures.of(features)); const hostname = options.hostname ?? "127.0.0.1"; /** Only reached by a client that sent no `Host`, which HTTP/1.1 requires. */ let fallbackAuthority = `${hostname}:${options.port ?? 0}`; interface RepoState { - readonly layer: Layer.Layer; + readonly layer: Layer.Layer; readonly lfs: Layer.Layer; readonly api: ( request: Request, @@ -391,13 +403,23 @@ export const serve = async (options: ServeOptions): Promise => { ? AfterPush.chain({ root: options.root, repo, wake: true }) : AfterPush.chain({ root: options.root, repo }); - const layer = GitRepository.layer.pipe( + const repoStores = stores(directory); + const bundleStore = bundleFiles(directory); + const live = GitRepository.layer.pipe( // Real hooks, not `hooksNoop`: this is what makes a push deliver. // `forkDetach` is the node stand-in for `waitUntil` — delivery outlives // the response without the push waiting on a slow receiver. Layer.provide(afterPush), // As `guardLayer` above: `provide` would swallow `Storage`. - Layer.provideMerge(stores(directory)), + Layer.provideMerge(repoStores), + ); + const layer = Layer.mergeAll( + live, + bundleStore, + fileMetaLayer(directory), + operationsMemory, + featuresLayer, + Bundles.layer.pipe(Layer.provide(bundleStore), Layer.provide(live)), ); // Built once per repository, not once per request. The requester stays @@ -493,6 +515,16 @@ export const serve = async (options: ServeOptions): Promise => { ); if (exported !== null) return exported; + const bundled = await Effect.runPromise( + Bundles.tryHandle(request).pipe(Effect.provide(state.layer)), + ); + if (bundled !== null) return bundled; + + const events = await Effect.runPromise( + operationEvents(request).pipe(Effect.provide(state.layer)), + ); + if (events !== null) return events; + const matched = await Effect.runPromise( Protocol.handle(request).pipe( Effect.catch((error) => @@ -673,9 +705,28 @@ export const serve = async (options: ServeOptions): Promise => { const bound = (server.address() as AddressInfo).port; fallbackAuthority = `${hostname}:${bound}`; + let maintenanceTimer: ReturnType | undefined; + const tickMaintenance = async () => { + const entries = await fs.readdir(options.root, { withFileTypes: true }).catch(() => []); + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const state = stateFor(entry.name); + await Effect.runPromise(maintenanceTick().pipe(Effect.provide(state.layer))).catch( + () => undefined, + ); + } + }; + maintenanceTimer = setInterval( + () => { + void tickMaintenance(); + }, + 60 * 60 * 1000, + ); + return { url: `http://${hostname}:${bound}`, close: async () => { + if (maintenanceTimer !== undefined) clearInterval(maintenanceTimer); await development?.close(); await new Promise((resolve, reject) => { server.close((error) => (error ? reject(error) : resolve())); diff --git a/src/server/Api.ts b/src/server/Api.ts index 7d49845..4f614d9 100644 --- a/src/server/Api.ts +++ b/src/server/Api.ts @@ -30,7 +30,13 @@ import { import { push as pushToRemote, type PushRef } from "../client/Push.ts"; import { isBinary, unified } from "../git/Diff.ts"; -import { Invalid, ObjectNotFound, PackCorrupt, RefConflict } from "../git/Error.ts"; +import { + Invalid, + ObjectNotFound, + OperationNotFound, + PackCorrupt, + RefConflict, +} from "../git/Error.ts"; import { EMPTY_TREE_OID, isGitlink, @@ -91,7 +97,10 @@ import { GrepRequest, GrepResponse, BisectAnswer, + BundlesResponse, HistoryPage, + OperationList, + OperationView, HubEventAppended, HubEventRequest, type HubMergeable, @@ -138,6 +147,8 @@ interface RemoteRequest { sync?: { mode: "manual" | "fetch" | "push" | "mirror"; refs: ReadonlyArray }; } import { NewSubscriberWire, redact, Subscribers } from "./Subscribers.ts"; +import { Bundles } from "./Bundles.ts"; +import { Operations } from "./Operations.ts"; import { fetchFrom, pull, remoteFor } from "./Sync.ts"; /** JSON has no `Date`: `at` crosses as an ISO string, `offset` in minutes. */ @@ -1005,6 +1016,33 @@ const repo = HttpApiGroup.make("repo") error: [ObjectNotFound, Invalid], }), ) + .add( + HttpApiEndpoint.get("bundles", "/bundles", { + params: RepoParam, + success: BundlesResponse, + }), + ) + .add( + HttpApiEndpoint.get("operations", "/operations", { + params: RepoParam, + query: { state: Schema.optional(Schema.String) }, + success: OperationList, + }), + ) + .add( + HttpApiEndpoint.get("operation", "/operations/:id", { + params: { ...RepoParam, id: Schema.String }, + success: OperationView, + error: [OperationNotFound], + }), + ) + .add( + HttpApiEndpoint.post("operationCancel", "/operations/:id/cancel", { + params: { ...RepoParam, id: Schema.String }, + success: OperationView, + error: [OperationNotFound, Invalid], + }), + ) .prefix("/:repo"); /** @@ -1259,6 +1297,46 @@ type PushRequest = { atomic?: boolean; }; +type OperationDraft = { + id: string; + repo: string; + kind: string; + state: OperationView["state"]; + created_at: string; + started_at?: string; + finished_at?: string; + progress?: OperationView["progress"]; + message?: string; + error?: OperationView["error"]; +}; + +const operationView = (operation: { + readonly id: string; + readonly repo: string; + readonly kind: string; + readonly state: OperationView["state"]; + readonly createdAt: string; + readonly startedAt?: string; + readonly finishedAt?: string; + readonly progress?: OperationView["progress"]; + readonly message?: string; + readonly error?: OperationView["error"]; +}): OperationView => { + const view: OperationDraft = { + id: operation.id, + repo: operation.repo, + kind: operation.kind, + state: operation.state, + created_at: operation.createdAt, + }; + if (operation.startedAt !== undefined) view.started_at = operation.startedAt; + if (operation.finishedAt !== undefined) view.finished_at = operation.finishedAt; + if (operation.progress !== undefined) view.progress = operation.progress; + if (operation.message !== undefined) view.message = operation.message; + if (operation.error !== undefined) view.error = operation.error; + return view; +}; + /** `StorageFailure` is a defect here: a 500 no caller can act on. */ export const handlers = HttpApiBuilder.group(api, "repo", (group) => group @@ -2097,6 +2175,44 @@ export const handlers = HttpApiBuilder.group(api, "repo", (group) => bisectNext({ bad: payload.bad, good: payload.good }).pipe( Effect.catchTag("StorageFailure", Effect.die), ), + ) + .handle("bundles", () => + Effect.gen(function* () { + const bundles = yield* Effect.serviceOption(Bundles); + if (Option.isNone(bundles)) return { enabled: false, families: [] }; + return yield* bundles.value.summary.pipe(Effect.catchTag("StorageFailure", Effect.die)); + }), + ) + .handle("operations", ({ query }) => + Effect.gen(function* () { + const operations = yield* Effect.serviceOption(Operations); + if (Option.isNone(operations)) return { operations: [] }; + const state = + query.state === "queued" || + query.state === "running" || + query.state === "succeeded" || + query.state === "failed" || + query.state === "cancelled" + ? query.state + : undefined; + const listed = yield* operations.value.list(state === undefined ? {} : { state }); + return { operations: listed.map(operationView) }; + }), + ) + .handle("operation", ({ params }) => + Effect.gen(function* () { + const operations = yield* Effect.serviceOption(Operations); + if (Option.isNone(operations)) return yield* new OperationNotFound({ id: params.id }); + return operationView(yield* operations.value.get(params.id)); + }), + ) + .handle("operationCancel", ({ params }) => + Effect.gen(function* () { + yield* requireCapability("repo.admin"); + const operations = yield* Effect.serviceOption(Operations); + if (Option.isNone(operations)) return yield* new OperationNotFound({ id: params.id }); + return operationView(yield* operations.value.cancel(params.id)); + }), ), ); diff --git a/src/server/ApiContract.ts b/src/server/ApiContract.ts index a6df982..8ec3f65 100644 --- a/src/server/ApiContract.ts +++ b/src/server/ApiContract.ts @@ -706,3 +706,66 @@ export const BisectAnswer = Schema.Struct({ steps: Schema.Finite, }); export type BisectAnswer = (typeof BisectAnswer)["Type"]; + +export const BundleFamilySummary = Schema.Struct({ + filter: Schema.NullOr(Schema.Literals(["blob:none"])), + full: Schema.NullOr(Schema.String), + incrementals: Schema.Finite, + latestCreationToken: Schema.NullOr(Schema.String), +}); +export type BundleFamilySummary = (typeof BundleFamilySummary)["Type"]; + +export const BundlesResponse = Schema.Struct({ + enabled: Schema.Boolean, + families: Schema.Array(BundleFamilySummary), +}); +export type BundlesResponse = (typeof BundlesResponse)["Type"]; + +export const OperationProgress = Schema.Struct({ + current: Schema.optional(Schema.Finite), + total: Schema.optional(Schema.Finite), + unit: Schema.optional(Schema.String), +}); + +export const OperationError = Schema.Struct({ + tag: Schema.String, + message: Schema.String, +}); + +export const OperationState = Schema.Literals([ + "queued", + "running", + "succeeded", + "failed", + "cancelled", +]); + +export const OperationView = Schema.Struct({ + id: Schema.String, + repo: Schema.String, + kind: Schema.String, + state: OperationState, + created_at: Schema.String, + started_at: Schema.optional(Schema.String), + finished_at: Schema.optional(Schema.String), + progress: Schema.optional(OperationProgress), + message: Schema.optional(Schema.String), + error: Schema.optional(OperationError), +}); +export type OperationView = (typeof OperationView)["Type"]; + +export const OperationList = Schema.Struct({ + operations: Schema.Array(OperationView), +}); +export type OperationList = (typeof OperationList)["Type"]; + +export const MaintenanceUnitView = Schema.Struct({ + kind: Schema.String, + filter: Schema.optional(Schema.NullOr(Schema.Literals(["blob:none"]))), + ids: Schema.optional(Schema.Array(Schema.String)), +}); + +export const MaintenancePlan = Schema.Struct({ + units: Schema.Array(MaintenanceUnitView), +}); +export type MaintenancePlan = (typeof MaintenancePlan)["Type"]; diff --git a/src/server/Artifact.test.ts b/src/server/Artifact.test.ts new file mode 100644 index 0000000..9f87db2 --- /dev/null +++ b/src/server/Artifact.test.ts @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import { describe, it } from "@effect/vitest"; + +import { Effect } from "effect"; + +import { decide, parseRange } from "./Artifact.ts"; + +describe("Artifact range parsing", () => { + it.effect("treats a missing header as the whole object", () => + Effect.sync(() => { + assert.deepEqual(parseRange(null, 100), { kind: "all" }); + assert.deepEqual(parseRange("", 100), { kind: "all" }); + }), + ); + + it.effect("parses closed, open, and suffix ranges", () => + Effect.sync(() => { + assert.deepEqual(parseRange("bytes=0-49", 100), { + kind: "range", + range: { offset: 0, length: 50 }, + }); + assert.deepEqual(parseRange("bytes=50-", 100), { + kind: "range", + range: { offset: 50, length: 50 }, + }); + assert.deepEqual(parseRange("bytes=-20", 100), { + kind: "range", + range: { offset: 80, length: 20 }, + }); + }), + ); + + it.effect("rejects unsatisfiable and multi ranges", () => + Effect.sync(() => { + assert.equal(parseRange("bytes=100-200", 100).kind, "unsatisfiable"); + assert.equal(parseRange("bytes=0-10,20-30", 100).kind, "unsatisfiable"); + assert.equal(parseRange("items=0-1", 100).kind, "unsatisfiable"); + }), + ); + + it.effect("honours If-None-Match and If-Range", () => + Effect.sync(() => { + const etag = "abc"; + const none = decide( + new Request("http://host/a", { headers: { "if-none-match": '"abc"' } }), + 10, + etag, + ); + assert.equal(none.kind, "notModified"); + + const ranged = decide( + new Request("http://host/a", { + headers: { range: "bytes=0-3", "if-range": '"abc"' }, + }), + 10, + etag, + ); + assert.equal(ranged.kind, "range"); + + const stale = decide( + new Request("http://host/a", { + headers: { range: "bytes=0-3", "if-range": '"old"' }, + }), + 10, + etag, + ); + assert.equal(stale.kind, "all"); + }), + ); +}); diff --git a/src/server/Artifact.ts b/src/server/Artifact.ts new file mode 100644 index 0000000..8a4dddc --- /dev/null +++ b/src/server/Artifact.ts @@ -0,0 +1,165 @@ +/** + * Immutable static-artifact HTTP: Range, ETag, and cache headers in one place. + * + * Bundle bytes are the first caller. Later archives or SHA-addressed packs + * should come through here rather than grow a second copy of the same rules. + * Ref-relative endpoints (`/main/files/…`, `/HEAD/…`) must not use this — + * they are not immutable, however cacheable they look. + */ +import { Effect, Stream } from "effect"; + +import { StorageFailure } from "../git/Error.ts"; + +export interface ArtifactBody { + readonly bytes: number; + readonly etag: string; + readonly contentType: string; + /** + * `public` only when the repository itself is anonymously readable. + * Private artifacts stay `private` so a shared cache cannot leak them. + */ + readonly cache: "public" | "private"; + readonly read: ( + range?: ByteRange, + ) => Effect.Effect, StorageFailure>; +} + +export interface ByteRange { + readonly offset: number; + readonly length: number; +} + +export type RangeRequest = + | { readonly kind: "all" } + | { readonly kind: "range"; readonly range: ByteRange } + | { readonly kind: "unsatisfiable" } + | { readonly kind: "notModified" }; + +const IMMUTABLE = "max-age=31536000, immutable"; + +/** + * One RFC 7233 byte range, or unsatisfiable. + * + * Only `bytes=` is accepted, and only a single closed or suffix range — + * multi-range is a different response shape this helper does not implement. + */ +export const parseRange = (header: string | null, size: number): RangeRequest => { + if (header === null || header === "") return { kind: "all" }; + if (!header.startsWith("bytes=")) return { kind: "unsatisfiable" }; + const spec = header.slice("bytes=".length); + if (spec.includes(",")) return { kind: "unsatisfiable" }; + + const dash = spec.indexOf("-"); + if (dash === -1) return { kind: "unsatisfiable" }; + const startText = spec.slice(0, dash); + const endText = spec.slice(dash + 1); + + if (startText === "") { + const suffix = Number.parseInt(endText, 10); + if (!Number.isInteger(suffix) || suffix <= 0) return { kind: "unsatisfiable" }; + const length = Math.min(suffix, size); + return { kind: "range", range: { offset: size - length, length } }; + } + + const start = Number.parseInt(startText, 10); + if (!Number.isInteger(start) || start < 0) return { kind: "unsatisfiable" }; + if (start >= size) return { kind: "unsatisfiable" }; + + const end = endText === "" ? size - 1 : Number.parseInt(endText, 10); + if (!Number.isInteger(end) || end < start) return { kind: "unsatisfiable" }; + const last = Math.min(end, size - 1); + return { kind: "range", range: { offset: start, length: last - start + 1 } }; +}; + +const quoted = (etag: string): string => (etag.startsWith('"') ? etag : `"${etag}"`); + +const matches = (header: string | null, etag: string): boolean => { + if (header === null || header === "") return false; + const wanted = quoted(etag); + return header.split(",").some((part) => { + const tag = part.trim(); + return tag === "*" || tag === wanted || tag === `W/${wanted}`; + }); +}; + +/** Decide which bytes, if any, this request should receive. */ +export const decide = (request: Request, bytes: number, etag: string): RangeRequest => { + const tagged = quoted(etag); + if (matches(request.headers.get("if-none-match"), etag)) { + const range = request.headers.get("range"); + // If-None-Match wins on a GET without a range; a ranged GET with a + // matching If-Range is still a range, and a matching If-None-Match on a + // range is 304 the same as an unconditioned one. + if (range === null || range === "") return { kind: "notModified" }; + if (matches(request.headers.get("if-range"), etag)) return parseRange(range, bytes); + return { kind: "notModified" }; + } + + const range = request.headers.get("range"); + if (range === null || range === "") return { kind: "all" }; + + const ifRange = request.headers.get("if-range"); + if (ifRange !== null && ifRange !== "" && !matches(ifRange, tagged) && ifRange !== tagged) { + return { kind: "all" }; + } + return parseRange(range, bytes); +}; + +const cacheControl = (cache: "public" | "private"): string => `${cache}, ${IMMUTABLE}`; + +/** + * GET/HEAD answer for an immutable artifact, or `null` when the method is + * something else so a router can fall through. + */ +export const respond = Effect.fn("Artifact.respond")(function* ( + request: Request, + artifact: ArtifactBody, +) { + if (request.method !== "GET" && request.method !== "HEAD") return null; + + const etag = quoted(artifact.etag); + const decision = decide(request, artifact.bytes, artifact.etag); + const common = { + etag, + "accept-ranges": "bytes", + "cache-control": cacheControl(artifact.cache), + "content-type": artifact.contentType, + }; + + if (decision.kind === "notModified") { + return new Response(null, { status: 304, headers: common }); + } + if (decision.kind === "unsatisfiable") { + return new Response(null, { + status: 416, + headers: { + ...common, + "content-range": `bytes */${artifact.bytes}`, + }, + }); + } + + const range = decision.kind === "range" ? decision.range : undefined; + const length = range === undefined ? artifact.bytes : range.length; + const headers = new Headers({ + ...common, + "content-length": String(length), + }); + if (range !== undefined) { + const last = range.offset + range.length - 1; + headers.set("content-range", `bytes ${range.offset}-${last}/${artifact.bytes}`); + } + + if (request.method === "HEAD") { + return new Response(null, { + status: range === undefined ? 200 : 206, + headers, + }); + } + + const body = yield* artifact.read(range); + return new Response(Stream.toReadableStream(body), { + status: range === undefined ? 200 : 206, + headers, + }); +}); diff --git a/src/server/BundleBuilder.interop.test.ts b/src/server/BundleBuilder.interop.test.ts new file mode 100644 index 0000000..a7e7a78 --- /dev/null +++ b/src/server/BundleBuilder.interop.test.ts @@ -0,0 +1,84 @@ +/** + * Generated bundles must be what `git bundle verify` accepts. + */ +import assert from "node:assert/strict"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, it } from "@effect/vitest"; + +import { Effect, Layer, Stream } from "effect"; + +import { type Signature } from "../git/Format.ts"; +import { stores } from "../git/Memory.ts"; +import * as GitRepository from "../git/Repository.ts"; +import { Repository } from "../git/Repository.ts"; +import { gitIn, hasGit } from "../testing/Git.ts"; +import * as BundleBuilder from "./BundleBuilder.ts"; + +const alice: Signature = { + name: "Alice", + email: "alice@example.com", + at: new Date(1_700_000_000_000), + offset: 0, +}; + +const live = GitRepository.layer.pipe( + Layer.provide(GitRepository.hooksNoop), + Layer.provideMerge(stores), +); + +describe.skipIf(!hasGit)("BundleBuilder interop", () => { + it.live("produces a bundle git bundle verify accepts", () => + Effect.gen(function* () { + const bytes = yield* Effect.gen(function* () { + const repository = yield* Repository; + const blob = yield* repository.writeBlob(new TextEncoder().encode("hello from bundle\n")); + const tree = yield* repository.writeTree([ + { mode: "100644", name: "hello.txt", oid: blob }, + ]); + const commit = yield* repository.commit({ + branch: "main", + tree, + message: "hello", + author: alice, + }); + const built = yield* BundleBuilder.build({ + snapshot: { + createdAt: new Date(), + refs: { "refs/heads/main": commit }, + filter: null, + }, + kind: "full", + }); + const chunks = yield* Stream.runCollect(built.stream); + let total = 0; + for (const chunk of chunks) total += chunk.length; + const out = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + yield* BundleBuilder.verifyBundle(out, { + refs: { "refs/heads/main": commit }, + prerequisites: [], + filter: null, + }); + return out; + }).pipe(Effect.provide(live)); + + const root = yield* Effect.promise(() => mkdtemp(join(tmpdir(), "bundle-verify-"))); + try { + const file = join(root, "repo.bundle"); + yield* Effect.promise(() => writeFile(file, bytes)); + const git = gitIn(root); + git("init", "--bare", "--initial-branch=main", "verify.git"); + const output = git("-C", "verify.git", "bundle", "verify", file); + assert.match(output, /The bundle contains|refs\/heads\/main/i); + } finally { + yield* Effect.promise(() => rm(root, { recursive: true, force: true })); + } + }), + ); +}); diff --git a/src/server/BundleBuilder.test.ts b/src/server/BundleBuilder.test.ts new file mode 100644 index 0000000..adaa8fd --- /dev/null +++ b/src/server/BundleBuilder.test.ts @@ -0,0 +1,151 @@ +import assert from "node:assert/strict"; +import { describe, it } from "@effect/vitest"; + +import { Effect, Layer, Result, Stream } from "effect"; + +import { type Signature } from "../git/Format.ts"; +import { stores } from "../git/Memory.ts"; +import * as GitRepository from "../git/Repository.ts"; +import { Repository } from "../git/Repository.ts"; +import { ObjectStore } from "../git/Store.ts"; +import * as BundleBuilder from "./BundleBuilder.ts"; +import { parseHeader } from "./BundleFormat.ts"; + +const alice: Signature = { + name: "Alice", + email: "alice@example.com", + at: new Date(1_700_000_000_000), + offset: 0, +}; + +const live = GitRepository.layer.pipe( + Layer.provide(GitRepository.hooksNoop), + Layer.provideMerge(stores), +); + +const scenario = (effect: Effect.Effect) => + Effect.runPromise(effect.pipe(Effect.provide(live))); + +describe("BundleBuilder", () => { + it.effect("emits a header whose refs match the captured snapshot", () => + Effect.promise(async () => { + const { header, oids } = await scenario( + Effect.gen(function* () { + const repository = yield* Repository; + const blob = yield* repository.writeBlob(new TextEncoder().encode("hello\n")); + const tree = yield* repository.writeTree([{ mode: "100644", name: "hi.txt", oid: blob }]); + yield* repository.commit({ + branch: "main", + tree, + message: "hello", + author: alice, + }); + const head = (yield* repository.resolve("refs/heads/main"))!; + const snapshot = { + createdAt: new Date(), + refs: { "refs/heads/main": head }, + filter: null, + }; + const built = yield* BundleBuilder.build({ snapshot, kind: "full" }); + const bytes = yield* Stream.runCollect(built.stream).pipe( + Effect.map((chunks) => { + let total = 0; + for (const chunk of chunks) total += chunk.length; + const out = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; + }), + ); + yield* BundleBuilder.verifyBundle(bytes, { + refs: snapshot.refs, + prerequisites: [], + filter: null, + }); + return { header: parseHeader(bytes), oids: built.oids }; + }), + ); + assert.equal(Result.isSuccess(header), true); + if (Result.isFailure(header)) return; + assert.ok(oids.length >= 1); + assert.equal(header.success.header.refs["refs/heads/main"] !== undefined, true); + }), + ); + + it.effect("omits blobs from a blob:none family", () => + Effect.promise(async () => { + const types = await scenario( + Effect.gen(function* () { + const repository = yield* Repository; + const blob = yield* repository.writeBlob(new TextEncoder().encode("payload\n")); + const tree = yield* repository.writeTree([{ mode: "100644", name: "p.txt", oid: blob }]); + yield* repository.commit({ + branch: "main", + tree, + message: "payload", + author: alice, + }); + const head = (yield* repository.resolve("refs/heads/main"))!; + const objects = yield* ObjectStore; + const oids = yield* BundleBuilder.oidsFor({ + snapshot: { + createdAt: new Date(), + refs: { "refs/heads/main": head }, + filter: "blob:none", + }, + }); + const found: string[] = []; + for (const oid of oids) found.push((yield* objects.read(oid)).type); + return found; + }), + ); + assert.ok(types.includes("commit")); + assert.ok(types.includes("tree")); + assert.ok(!types.includes("blob")); + }), + ); + + it.effect("incremental oids skip the prerequisite snapshot", () => + Effect.promise(async () => { + const built = await scenario( + Effect.gen(function* () { + const repository = yield* Repository; + const firstBlob = yield* repository.writeBlob(new TextEncoder().encode("one\n")); + const firstTree = yield* repository.writeTree([ + { mode: "100644", name: "a.txt", oid: firstBlob }, + ]); + const first = yield* repository.commit({ + branch: "main", + tree: firstTree, + message: "one", + author: alice, + }); + const secondBlob = yield* repository.writeBlob(new TextEncoder().encode("two\n")); + const secondTree = yield* repository.writeTree([ + { mode: "100644", name: "a.txt", oid: secondBlob }, + ]); + const second = yield* repository.commit({ + branch: "main", + tree: secondTree, + message: "two", + author: alice, + }); + const incremental = yield* BundleBuilder.oidsFor({ + snapshot: { + createdAt: new Date(), + refs: { "refs/heads/main": second }, + filter: null, + }, + prerequisiteRefs: { "refs/heads/main": first }, + }); + return { first, second, incremental }; + }), + ); + assert.ok(!built.incremental.includes(built.first)); + assert.ok(built.incremental.includes(built.second)); + }), + ); +}); diff --git a/src/server/BundleBuilder.ts b/src/server/BundleBuilder.ts new file mode 100644 index 0000000..607ee9c --- /dev/null +++ b/src/server/BundleBuilder.ts @@ -0,0 +1,292 @@ +/** + * Repository snapshot → bundle bytes, then verification. + * + * The builder captures the ref OIDs it intends to bundle and walks only + * those. A concurrent push can advance the repository; it cannot change the + * meaning of an already captured snapshot. The pack is streamed — nothing + * here collects a repository into one `Uint8Array` on the write path. + */ +import { Effect, Result, Schema, Stream } from "effect"; + +import { BundleCorrupt, type ObjectNotFound, type StorageFailure } from "../git/Error.ts"; +import { bytesToHex, concatBytes } from "../git/Format.ts"; +import { inflate, InflateError, type ByteSource } from "../git/Inflate.ts"; +import { reachable } from "../git/Maintenance.ts"; +import * as Pack from "../git/Pack.ts"; +import { Sha1 } from "../git/Sha1.ts"; +import { ObjectStore, type Oid } from "../git/Store.ts"; +import { + type BundleArtifact, + type BundleFilter, + type BundleHeader, + type BundleKind, + type BundleSnapshot, + encodeHeader, + parseHeader, +} from "./BundleFormat.ts"; + +const readU32 = (bytes: Uint8Array, at: number): number => + ((bytes[at]! << 24) | (bytes[at + 1]! << 16) | (bytes[at + 2]! << 8) | bytes[at + 3]!) >>> 0; + +const OFS_DELTA = 6; +const REF_DELTA = 7; + +class Cursor implements ByteSource { + #pending: Uint8Array[] = []; + #iterator: AsyncIterator; + readonly hash = new Sha1(); + hashing = true; + + constructor(input: AsyncIterable) { + this.#iterator = input[Symbol.asyncIterator](); + } + + async next(): Promise { + const head = this.#pending.shift(); + if (head !== undefined) return head; + const result = await this.#iterator.next(); + if (result.done === true) return null; + return result.value.length === 0 ? this.next() : result.value; + } + + pushBack(bytes: Uint8Array): void { + if (bytes.length > 0) this.#pending.unshift(bytes); + } + + async take(n: number): Promise { + const out = new Uint8Array(n); + let filled = 0; + while (filled < n) { + const chunk = await this.next(); + if (chunk === null) { + throw new BundleCorrupt({ reason: `truncated pack: needed ${n - filled} more bytes` }); + } + const use = Math.min(chunk.length, n - filled); + out.set(chunk.subarray(0, use), filled); + this.pushBack(chunk.subarray(use)); + filled += use; + } + if (this.hashing) this.hash.update(out); + return out; + } + + async byte(): Promise { + return (await this.take(1))[0]!; + } +} + +const skipObject = async (source: Cursor): Promise => { + let byte = await source.byte(); + const code = (byte >> 4) & 0x7; + let size = byte & 0x0f; + let shift = 4; + while (byte & 0x80) { + byte = await source.byte(); + size += (byte & 0x7f) * 2 ** shift; + shift += 7; + } + if (code === OFS_DELTA) { + byte = await source.byte(); + while (byte & 0x80) byte = await source.byte(); + } else if (code === REF_DELTA) { + await source.take(20); + } else if (code < 1 || code > 4) { + throw new BundleCorrupt({ reason: `unknown pack object type ${code}` }); + } + try { + const data = await inflate(source, size + 1); + if (data.length !== size) { + throw new BundleCorrupt({ + reason: `pack object header says ${size} bytes, inflated to ${data.length}`, + }); + } + } catch (cause) { + if (Schema.is(BundleCorrupt)(cause)) throw cause; + const reason = cause instanceof InflateError ? cause.reason : String(cause); + throw new BundleCorrupt({ reason: `pack object inflate failed: ${reason}` }); + } +}; + +const verifyPack = async (bytes: Uint8Array): Promise => { + if (bytes.length < 32) { + throw new BundleCorrupt({ reason: "pack is shorter than a header and trailer" }); + } + if (bytes[0] !== 0x50 || bytes[1] !== 0x41 || bytes[2] !== 0x43 || bytes[3] !== 0x4b) { + throw new BundleCorrupt({ reason: "bundle payload is not a packfile" }); + } + const version = readU32(bytes, 4); + if (version !== 2) { + throw new BundleCorrupt({ reason: `unsupported pack version ${version}` }); + } + const count = readU32(bytes, 8); + const body = bytes.subarray(0, bytes.length - 20); + const trailer = bytes.subarray(bytes.length - 20); + const source = new Cursor( + (async function* () { + yield body.subarray(12); + })(), + ); + for (let index = 0; index < count; index++) await skipObject(source); + const leftover = await source.next(); + if (leftover !== null && leftover.length > 0) { + throw new BundleCorrupt({ reason: "pack has trailing bytes before its trailer" }); + } + const expected = new Sha1().update(body).digest(); + if (bytesToHex(expected) !== bytesToHex(trailer)) { + throw new BundleCorrupt({ reason: "pack trailer does not match the SHA-1 of its contents" }); + } +}; + +const sameRefs = ( + actual: Readonly>, + expected: Readonly>, +): string | null => { + const names = new Set([...Object.keys(actual), ...Object.keys(expected)]); + for (const name of names) { + if (actual[name] !== expected[name]) { + return `ref '${name}' is ${actual[name] ?? "missing"}, expected ${expected[name] ?? "missing"}`; + } + } + return null; +}; + +const sameOids = (actual: ReadonlyArray, expected: ReadonlyArray): string | null => { + if (actual.length !== expected.length) { + return `prerequisite count ${actual.length} != ${expected.length}`; + } + for (let index = 0; index < expected.length; index++) { + if (actual[index] !== expected[index]) { + return `prerequisite ${index} is ${actual[index]}, expected ${expected[index]}`; + } + } + return null; +}; + +/** Parse and check a generated bundle against the snapshot it claims to hold. */ +export const verifyBundle = ( + bytes: Uint8Array, + expected: { + readonly refs: Readonly>; + readonly prerequisites: ReadonlyArray; + readonly filter: BundleFilter; + }, +): Effect.Effect => + Effect.gen(function* () { + const parsed = parseHeader(bytes); + if (Result.isFailure(parsed)) return yield* parsed.failure; + const { header, packOffset } = parsed.success; + if (header.filter !== expected.filter) { + return yield* new BundleCorrupt({ + reason: `filter is ${header.filter ?? "none"}, expected ${expected.filter ?? "none"}`, + }); + } + const refs = sameRefs(header.refs, expected.refs); + if (refs !== null) return yield* new BundleCorrupt({ reason: refs }); + const prerequisites = sameOids(header.prerequisites, expected.prerequisites); + if (prerequisites !== null) return yield* new BundleCorrupt({ reason: prerequisites }); + yield* Effect.tryPromise({ + try: () => verifyPack(bytes.subarray(packOffset)), + catch: (cause) => + Schema.is(BundleCorrupt)(cause) + ? cause + : new BundleCorrupt({ reason: cause instanceof Error ? cause.message : String(cause) }), + }); + return header; + }); + +/** + * Reachability for a snapshot. `blob:none` still walks trees so the client + * can reconstruct structure; blobs are dropped from the pack. + */ +export const oidsFor = Effect.fn("BundleBuilder.oidsFor")(function* (input: { + readonly snapshot: BundleSnapshot; + readonly prerequisiteRefs?: Readonly>; +}) { + const objects = yield* ObjectStore; + const roots = Object.values(input.snapshot.refs); + const skip = + input.prerequisiteRefs === undefined + ? undefined + : (yield* reachable(objects, Object.values(input.prerequisiteRefs), { + ignoreMissing: true, + })).seen; + + const walked = yield* reachable(objects, roots, { + ignoreMissing: true, + skip, + classify: input.snapshot.filter === "blob:none", + }); + + const oids: Oid[] = []; + for (const oid of walked.order) { + if (input.snapshot.filter === "blob:none") { + const kind = walked.classified.kinds.get(oid); + if (kind?.type === "blob") continue; + } + oids.push(oid); + } + return oids; +}); + +export const headerFor = (input: { + readonly snapshot: BundleSnapshot; + readonly prerequisites: ReadonlyArray; +}): BundleHeader => ({ + version: input.snapshot.filter === null ? 2 : 3, + filter: input.snapshot.filter, + refs: input.snapshot.refs, + prerequisites: input.prerequisites, +}); + +/** Header bytes followed by a lazily written pack. */ +export const bundleStream = ( + oids: ReadonlyArray, + header: BundleHeader, +): Stream.Stream => + Stream.fromIterable([encodeHeader(header)]).pipe(Stream.concat(Pack.pack(oids))); + +export interface BuiltBundle { + readonly header: BundleHeader; + readonly oids: ReadonlyArray; + readonly stream: Stream.Stream; +} + +export const build = Effect.fn("BundleBuilder.build")(function* (input: { + readonly snapshot: BundleSnapshot; + readonly kind: BundleKind; + readonly prerequisiteRefs?: Readonly>; +}) { + const prerequisites = + input.kind === "incremental" && input.prerequisiteRefs !== undefined + ? uniqueOids(Object.values(input.prerequisiteRefs)) + : []; + const oids = + input.prerequisiteRefs === undefined + ? yield* oidsFor({ snapshot: input.snapshot }) + : yield* oidsFor({ snapshot: input.snapshot, prerequisiteRefs: input.prerequisiteRefs }); + const header = headerFor({ snapshot: input.snapshot, prerequisites }); + return { + header, + oids, + stream: bundleStream(oids, header), + } satisfies BuiltBundle; +}); + +const uniqueOids = (oids: ReadonlyArray): ReadonlyArray => { + const seen = new Set(); + const out: Oid[] = []; + for (const oid of oids) { + if (seen.has(oid)) continue; + seen.add(oid); + out.push(oid); + } + return out; +}; + +/** Concatenate a just-written artifact for verification. Streaming write already happened. */ +export const collect = ( + source: Stream.Stream, +): Effect.Effect => + Stream.runCollect(source).pipe(Effect.map((chunks) => concatBytes(chunks))); + +export type { BundleArtifact }; diff --git a/src/server/BundleFormat.test.ts b/src/server/BundleFormat.test.ts new file mode 100644 index 0000000..11de782 --- /dev/null +++ b/src/server/BundleFormat.test.ts @@ -0,0 +1,155 @@ +import assert from "node:assert/strict"; +import { describe, it } from "@effect/vitest"; + +import { Effect, Result } from "effect"; + +import type { Oid } from "../git/Oid.ts"; +import { + artifactId, + artifactPath, + catchupArtifacts, + cloneArtifacts, + decodeManifest, + encodeConfigList, + encodeHeader, + encodeManifest, + encodeProtocolList, + familyState, + isBundleId, + listEntries, + nextToken, + objectIdOf, + parseHeader, + publishArtifact, +} from "./BundleFormat.ts"; + +// SAFETY: forty hex characters by construction. +const oid = (char: string): Oid => char.repeat(40) as Oid; + +describe("BundleFormat identifiers", () => { + it.effect("accepts git's bundle-list identifier grammar", () => + Effect.sync(() => { + assert.equal(isBundleId("full-1"), true); + assert.equal(isBundleId("ninc-1700"), true); + assert.equal(isBundleId("1full"), false); + assert.equal(isBundleId("full_1"), false); + assert.equal(artifactId("full", null, 7n), "full-7"); + assert.equal(artifactId("incremental", "blob:none", 8n), "ninc-8"); + assert.equal(objectIdOf(null, 7n, "abc"), "full/7-abc.bundle"); + }), + ); + + it.effect("issues strictly increasing creation tokens", () => + Effect.sync(() => { + const now = new Date(1_700_000_000_000); + assert.equal(nextToken(null, now), 1_700_000_000_000n); + assert.equal(nextToken(1_700_000_000_000n, now), 1_700_000_000_001n); + assert.equal(nextToken(1n, now), 1_700_000_000_000n); + }), + ); +}); + +describe("BundleFormat header", () => { + it.effect("round-trips v2 refs and prerequisites", () => + Effect.sync(() => { + const bytes = encodeHeader({ + version: 2, + filter: null, + refs: { "refs/heads/main": oid("a") }, + prerequisites: [oid("b")], + }); + const parsed = parseHeader(bytes); + assert.equal(Result.isSuccess(parsed), true); + if (Result.isFailure(parsed)) return; + assert.equal(parsed.success.header.version, 2); + assert.equal(parsed.success.header.refs["refs/heads/main"], oid("a")); + assert.deepEqual(parsed.success.header.prerequisites, [oid("b")]); + assert.equal(bytes[parsed.success.packOffset], undefined); + }), + ); + + it.effect("writes a v3 filter capability for blob:none", () => + Effect.sync(() => { + const bytes = encodeHeader({ + version: 3, + filter: "blob:none", + refs: { "refs/heads/main": oid("a") }, + prerequisites: [], + }); + const text = new TextDecoder().decode(bytes); + assert.match(text, /^# v3 git bundle\n/); + assert.match(text, /@filter=blob:none\n/); + const parsed = parseHeader(bytes); + assert.equal(Result.isSuccess(parsed) && parsed.success.header.filter === "blob:none", true); + }), + ); +}); + +describe("BundleFormat lists", () => { + const artifact = { + id: "full-1", + kind: "full" as const, + filter: null, + creationToken: 1n, + refs: { "refs/heads/main": oid("a") }, + prerequisites: [], + objectId: "full/1-aa.bundle", + bytes: 10, + checksum: "aa", + createdAt: "2024-01-01T00:00:00.000Z", + }; + const incremental = { + ...artifact, + id: "inc-2", + kind: "incremental" as const, + creationToken: 2n, + objectId: "full/2-bb.bundle", + checksum: "bb", + }; + + it.effect("keeps the full base off the catch-up list", () => + Effect.sync(() => { + const published = publishArtifact(null, artifact); + const chained = publishArtifact(published, incremental); + const family = familyState(chained, null); + assert.deepEqual( + cloneArtifacts(family).map((entry) => entry.id), + ["full-1", "inc-2"], + ); + assert.deepEqual( + catchupArtifacts(family).map((entry) => entry.id), + ["inc-2"], + ); + }), + ); + + it.effect("orders protocol and config lists by creationToken", () => + Effect.sync(() => { + const entries = listEntries([incremental, artifact], (item) => `http://h/${item.objectId}`); + assert.deepEqual( + entries.map((entry) => entry.id), + ["full-1", "inc-2"], + ); + const protocol = encodeProtocolList(entries); + assert.ok(protocol.includes("bundle.version=1")); + assert.ok(protocol.includes("bundle.full-1.uri=http://h/full/1-aa.bundle")); + assert.ok(protocol.includes("bundle.inc-2.creationToken=2")); + const ini = encodeConfigList(entries); + assert.match(ini, /\[bundle "full-1"\]/); + assert.match(ini, /heuristic = creationToken/); + }), + ); + + it.effect("round-trips a manifest and parses artifact paths", () => + Effect.sync(() => { + const manifest = publishArtifact(null, artifact); + const again = decodeManifest(encodeManifest(manifest)); + assert.equal(again?.families[0]?.full?.id, "full-1"); + assert.deepEqual(artifactPath("/repo/bundles/full/1-aa.bundle"), { + family: "full", + file: "1-aa.bundle", + }); + assert.equal(artifactPath("/repo/bundles/clone"), null); + }), + ); +}); diff --git a/src/server/BundleFormat.ts b/src/server/BundleFormat.ts new file mode 100644 index 0000000..be11183 --- /dev/null +++ b/src/server/BundleFormat.ts @@ -0,0 +1,420 @@ +/** + * Bundle bytes and bundle-list text — the codecs, not the store. + * + * A bundle is a small header plus a pack. The header is the only part this + * module invents; the pack is `Pack.pack`. The list formats are what protocol + * v2 `bundle-uri` and the HTTP clone/catch-up endpoints speak. + */ +import { Result, Schema } from "effect"; + +import { BundleCorrupt } from "../git/Error.ts"; +import { isOid, type Oid } from "../git/Oid.ts"; + +export type BundleFilter = null | "blob:none"; +export type BundleKind = "full" | "incremental"; +export type BundleFamily = "full" | "blobnone"; + +export interface BundleHeader { + readonly version: 2 | 3; + readonly filter: BundleFilter; + readonly refs: Readonly>; + readonly prerequisites: ReadonlyArray; +} + +export interface BundleArtifact { + readonly id: string; + readonly kind: BundleKind; + readonly filter: BundleFilter; + readonly creationToken: bigint; + readonly refs: Readonly>; + readonly prerequisites: ReadonlyArray; + readonly objectId: string; + readonly bytes: number; + readonly checksum: string; + readonly createdAt: string; +} + +export interface BundleFamilyState { + readonly filter: BundleFilter; + readonly full: BundleArtifact | null; + readonly incrementals: ReadonlyArray; +} + +export interface BundleManifest { + readonly version: 1; + readonly families: ReadonlyArray; +} + +export interface BundleSnapshot { + readonly createdAt: Date; + readonly refs: Readonly>; + readonly filter: BundleFilter; +} + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +/** Git's bundle-list identifier: a letter, then letters, digits or hyphens. */ +export const isBundleId = (value: string): boolean => /^[a-zA-Z][a-zA-Z0-9-]*$/.test(value); + +export const familyOf = (filter: BundleFilter): BundleFamily => + filter === "blob:none" ? "blobnone" : "full"; + +export const filterOf = (family: BundleFamily): BundleFilter => + family === "blobnone" ? "blob:none" : null; + +/** + * Stable list identifier. Token is decimal so ordering is visible in the + * name; the prefix keeps full and incremental, and the two families, apart. + */ +export const artifactId = (kind: BundleKind, filter: BundleFilter, token: bigint): string => { + const family = filter === "blob:none" ? (kind === "full" ? "nfull" : "ninc") : kind; + return `${family}-${token.toString()}`; +}; + +/** Object key suffix: `/-.bundle`. */ +export const objectIdOf = (filter: BundleFilter, token: bigint, checksum: string): string => + `${familyOf(filter)}/${token.toString()}-${checksum}.bundle`; + +/** + * The next creation token: strictly greater than any already published, and + * at least the current millisecond so a clock-based heuristic stays honest. + */ +export const nextToken = (previous: bigint | null, now: Date): bigint => { + const millis = BigInt(now.getTime()); + if (previous === null || previous < millis) return millis; + return previous + 1n; +}; + +export const latestToken = (manifest: BundleManifest | null): bigint | null => { + if (manifest === null) return null; + let latest: bigint | null = null; + for (const family of manifest.families) { + const artifacts = + family.full === null ? family.incrementals : [family.full, ...family.incrementals]; + for (const artifact of artifacts) { + if (latest === null || artifact.creationToken > latest) latest = artifact.creationToken; + } + } + return latest; +}; + +export const familyState = ( + manifest: BundleManifest | null, + filter: BundleFilter, +): BundleFamilyState => { + const found = manifest?.families.find((family) => family.filter === filter); + return found ?? { filter, full: null, incrementals: [] }; +}; + +/** Clone list: current full base plus the incrementals chained from it. */ +export const cloneArtifacts = (state: BundleFamilyState): ReadonlyArray => + state.full === null ? [] : [state.full, ...state.incrementals]; + +/** Catch-up list: incrementals only — never a newly-created full base. */ +export const catchupArtifacts = (state: BundleFamilyState): ReadonlyArray => + state.incrementals; + +const headerLine = (line: string): Uint8Array => encoder.encode(`${line}\n`); + +/** Bundle header bytes, including the terminating blank line. */ +export const encodeHeader = (header: BundleHeader): Uint8Array => { + const lines: Uint8Array[] = []; + if (header.version === 3 || header.filter !== null) { + lines.push(headerLine("# v3 git bundle")); + lines.push(headerLine("@object-format=sha1")); + if (header.filter !== null) lines.push(headerLine(`@filter=${header.filter}`)); + } else { + lines.push(headerLine("# v2 git bundle")); + } + for (const oid of header.prerequisites) { + lines.push(headerLine(`- ${oid} prerequisite`)); + } + const names = Object.keys(header.refs).sort(); + for (const name of names) { + const oid = header.refs[name]; + if (oid === undefined) continue; + lines.push(headerLine(`${oid} ${name}`)); + } + lines.push(encoder.encode("\n")); + let total = 0; + for (const line of lines) total += line.length; + const out = new Uint8Array(total); + let offset = 0; + for (const line of lines) { + out.set(line, offset); + offset += line.length; + } + return out; +}; + +const parseOid = (text: string): Oid | null => (isOid(text) ? text : null); + +/** + * Split a bundle's leading header from the pack that follows. + * + * The header ends at the first blank line. Anything after that is pack bytes + * and is not interpreted here. + */ +export const parseHeader = ( + bytes: Uint8Array, +): Result.Result<{ readonly header: BundleHeader; readonly packOffset: number }, BundleCorrupt> => { + const text = decoder.decode(bytes); + const blank = text.indexOf("\n\n"); + if (blank === -1) { + return Result.fail(new BundleCorrupt({ reason: "bundle header is missing a blank line" })); + } + const head = text.slice(0, blank); + const lines = head.split("\n"); + const first = lines[0]; + if (first !== "# v2 git bundle" && first !== "# v3 git bundle") { + return Result.fail( + new BundleCorrupt({ reason: `unrecognised bundle signature '${first ?? ""}'` }), + ); + } + const version: 2 | 3 = first === "# v3 git bundle" ? 3 : 2; + let filter: BundleFilter = null; + const refs: Record = {}; + const prerequisites: Oid[] = []; + + for (const line of lines.slice(1)) { + if (line.startsWith("@")) { + if (line === "@filter=blob:none") filter = "blob:none"; + continue; + } + if (line.startsWith("- ")) { + const oid = parseOid(line.slice(2, 42)); + if (oid === null) { + return Result.fail(new BundleCorrupt({ reason: `bad prerequisite line '${line}'` })); + } + prerequisites.push(oid); + continue; + } + if (line.length < 42 || line[40] !== " ") { + return Result.fail(new BundleCorrupt({ reason: `bad ref line '${line}'` })); + } + const oid = parseOid(line.slice(0, 40)); + if (oid === null) { + return Result.fail(new BundleCorrupt({ reason: `bad ref oid in '${line}'` })); + } + refs[line.slice(41)] = oid; + } + + return Result.succeed({ + header: { version, filter, refs, prerequisites }, + packOffset: blank + 2, + }); +}; + +export interface BundleListEntry { + readonly id: string; + readonly uri: string; + readonly creationToken: bigint; + readonly filter: BundleFilter; +} + +const byToken = (left: BundleListEntry, right: BundleListEntry): number => + left.creationToken < right.creationToken ? -1 : left.creationToken > right.creationToken ? 1 : 0; + +export const listEntries = ( + artifacts: ReadonlyArray, + uriOf: (artifact: BundleArtifact) => string, +): ReadonlyArray => + artifacts + .map((artifact) => ({ + id: artifact.id, + uri: uriOf(artifact), + creationToken: artifact.creationToken, + filter: artifact.filter, + })) + .sort(byToken); + +/** Protocol v2 `bundle-uri` body: flattened `bundle.=` lines. */ +export const encodeProtocolList = ( + entries: ReadonlyArray, +): ReadonlyArray => { + const lines = ["bundle.version=1", "bundle.mode=all", "bundle.heuristic=creationToken"]; + for (const entry of entries) { + lines.push(`bundle.${entry.id}.uri=${entry.uri}`); + lines.push(`bundle.${entry.id}.creationToken=${entry.creationToken.toString()}`); + if (entry.filter !== null) lines.push(`bundle.${entry.id}.filter=${entry.filter}`); + } + return lines; +}; + +/** HTTP clone/catch-up body: gitconfig INI. */ +export const encodeConfigList = (entries: ReadonlyArray): string => { + const lines = ["[bundle]", "\tversion = 1", "\tmode = all", "\theuristic = creationToken"]; + for (const entry of entries) { + lines.push("", `[bundle "${entry.id}"]`, `\turi = ${entry.uri}`); + lines.push(`\tcreationToken = ${entry.creationToken.toString()}`); + if (entry.filter !== null) lines.push(`\tfilter = ${entry.filter}`); + } + return `${lines.join("\n")}\n`; +}; + +export interface ArtifactPointer { + readonly family: BundleFamily; + readonly file: string; +} + +/** Parse `/bundles//.bundle` out of a request path. */ +export const artifactPath = (pathname: string): ArtifactPointer | null => { + const segments = pathname.split("/").filter((segment) => segment !== ""); + const at = segments.lastIndexOf("bundles"); + if (at === -1) return null; + const family = segments[at + 1]; + const file = segments[at + 2]; + if ((family !== "full" && family !== "blobnone") || file === undefined) return null; + if (!file.endsWith(".bundle") || segments.length !== at + 3) return null; + if (file.includes("..") || file.includes("/")) return null; + return { family, file }; +}; + +export const objectIdFromPath = (pointer: ArtifactPointer): string => + `${pointer.family}/${pointer.file}`; + +const artifactJson = (artifact: BundleArtifact) => ({ + id: artifact.id, + kind: artifact.kind, + filter: artifact.filter, + creationToken: artifact.creationToken.toString(), + refs: artifact.refs, + prerequisites: artifact.prerequisites, + objectId: artifact.objectId, + bytes: artifact.bytes, + checksum: artifact.checksum, + createdAt: artifact.createdAt, +}); + +const OidString = Schema.String.pipe(Schema.refine(isOid)); +const BundleId = Schema.String.check(Schema.isPattern(/^[a-zA-Z][a-zA-Z0-9-]*$/)); +const TokenString = Schema.String.check(Schema.isPattern(/^[0-9]+$/)); + +const ArtifactWire = Schema.Struct({ + id: BundleId, + kind: Schema.Literals(["full", "incremental"]), + filter: Schema.NullOr(Schema.Literals(["blob:none"])), + creationToken: TokenString, + refs: Schema.Record(Schema.String, OidString), + prerequisites: Schema.Array(OidString), + objectId: Schema.String, + bytes: Schema.Finite, + checksum: Schema.String, + createdAt: Schema.String, +}); + +const FamilyWire = Schema.Struct({ + filter: Schema.NullOr(Schema.Literals(["blob:none"])), + full: Schema.NullOr(ArtifactWire), + incrementals: Schema.Array(ArtifactWire), +}); + +const ManifestWire = Schema.Struct({ + version: Schema.Literals([1]), + families: Schema.Array(FamilyWire), +}); + +const decodeManifestWire = Schema.decodeUnknownResult(ManifestWire); + +const fromWire = (wire: typeof ArtifactWire.Type): BundleArtifact => { + const refs: Record = {}; + for (const [name, value] of Object.entries(wire.refs)) { + if (isOid(value)) refs[name] = value; + } + const prerequisites: Oid[] = []; + for (const value of wire.prerequisites) { + if (isOid(value)) prerequisites.push(value); + } + return { + id: wire.id, + kind: wire.kind, + filter: wire.filter, + creationToken: BigInt(wire.creationToken), + refs, + prerequisites, + objectId: wire.objectId, + bytes: wire.bytes, + checksum: wire.checksum, + createdAt: wire.createdAt, + }; +}; + +export const encodeManifest = (manifest: BundleManifest): string => + JSON.stringify({ + version: 1, + families: manifest.families.map((family) => ({ + filter: family.filter, + full: family.full === null ? null : artifactJson(family.full), + incrementals: family.incrementals.map(artifactJson), + })), + }); + +export const decodeManifest = (text: string): BundleManifest | null => { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + return null; + } + const decoded = decodeManifestWire(parsed); + if (Result.isFailure(decoded)) return null; + return { + version: 1, + families: decoded.success.families.map((family) => ({ + filter: family.filter, + full: family.full === null ? null : fromWire(family.full), + incrementals: family.incrementals.map(fromWire), + })), + }; +}; + +export const emptyManifest = (): BundleManifest => ({ + version: 1, + families: [ + { filter: null, full: null, incrementals: [] }, + { filter: "blob:none", full: null, incrementals: [] }, + ], +}); + +/** + * Advertise `artifact` in `filter`'s family. A new full retires the previous + * full and its incrementals from the advertised lists; they become prune + * candidates. An incremental appends to the current chain. + */ +export const publishArtifact = ( + manifest: BundleManifest | null, + artifact: BundleArtifact, +): BundleManifest => { + const current = manifest ?? emptyManifest(); + const families = current.families.map((family) => { + if (family.filter !== artifact.filter) return family; + if (artifact.kind === "full") { + return { filter: family.filter, full: artifact, incrementals: [] }; + } + return { + filter: family.filter, + full: family.full, + incrementals: [...family.incrementals, artifact], + }; + }); + if (!families.some((family) => family.filter === artifact.filter)) { + families.push( + artifact.kind === "full" + ? { filter: artifact.filter, full: artifact, incrementals: [] } + : { filter: artifact.filter, full: null, incrementals: [artifact] }, + ); + } + return { version: 1, families }; +}; + +/** Every object id the manifest still names. */ +export const advertisedIds = (manifest: BundleManifest | null): ReadonlySet => { + const ids = new Set(); + if (manifest === null) return ids; + for (const family of manifest.families) { + if (family.full !== null) ids.add(family.full.objectId); + for (const incremental of family.incrementals) ids.add(incremental.objectId); + } + return ids; +}; diff --git a/src/server/BundleStore.contract.ts b/src/server/BundleStore.contract.ts new file mode 100644 index 0000000..363e5ae --- /dev/null +++ b/src/server/BundleStore.contract.ts @@ -0,0 +1,111 @@ +/** + * BundleStore contract, run against every backend the way ObjectStore is. + */ +import assert from "node:assert/strict"; + +import { Effect, Stream } from "effect"; + +import { emptyManifest } from "./BundleFormat.ts"; +import { BundleStore } from "./BundleStore.ts"; + +export interface Backend { + readonly run: (effect: Effect.Effect) => Promise; +} + +export interface Runner { + readonly describe: (name: string, body: () => void) => void; + readonly it: (name: string, body: () => Promise | void) => void; +} + +export const bundleStoreContract = (label: string, backend: Backend, runner: Runner): void => { + const { describe, it } = runner; + const { run } = backend; + + describe(`${label}: BundleStore contract`, () => { + it("starts with no manifest", async () => { + const listed = await run(Effect.flatMap(BundleStore, (store) => store.list("repo"))); + assert.equal(listed, null); + }); + + it("writes bytes, stats them, and reads them back", async () => { + const result = await run( + Effect.gen(function* () { + const store = yield* BundleStore; + const payload = new TextEncoder().encode("bundle-bytes"); + const written = yield* store.write("full/1-test.bundle", Stream.fromIterable([payload])); + const stat = yield* store.stat("full/1-test.bundle"); + const chunks = yield* Stream.runCollect(yield* store.read("full/1-test.bundle")); + return { + bytes: written.bytes, + checksum: written.checksum, + stat, + text: new TextDecoder().decode(chunks[0]), + }; + }), + ); + assert.equal(result.bytes, 12); + assert.equal(result.checksum.length, 40); + assert.equal(result.stat?.bytes, 12); + assert.equal(result.text, "bundle-bytes"); + }); + + it("serves a byte range", async () => { + const text = await run( + Effect.gen(function* () { + const store = yield* BundleStore; + yield* store.write( + "full/2-range.bundle", + Stream.fromIterable([new TextEncoder().encode("abcdef")]), + ); + const chunks = yield* Stream.runCollect( + yield* store.read("full/2-range.bundle", { offset: 2, length: 3 }), + ); + return new TextDecoder().decode(chunks[0]); + }), + ); + assert.equal(text, "cde"); + }); + + it("moves an artifact and forgets the old key", async () => { + const state = await run( + Effect.gen(function* () { + const store = yield* BundleStore; + yield* store.write("tmp/x", Stream.fromIterable([new Uint8Array([1, 2, 3])])); + yield* store.move("tmp/x", "full/3-moved.bundle"); + return { + from: yield* store.stat("tmp/x"), + to: yield* store.stat("full/3-moved.bundle"), + }; + }), + ); + assert.equal(state.from, null); + assert.equal(state.to?.bytes, 3); + }); + + it("publishes a manifest atomically relative to unreferenced writes", async () => { + const listed = await run( + Effect.gen(function* () { + const store = yield* BundleStore; + yield* store.write("tmp/unadvertised", Stream.fromIterable([new Uint8Array([9])])); + yield* store.publish("repo", emptyManifest()); + return yield* store.list("repo"); + }), + ); + assert.equal(listed?.version, 1); + assert.equal(listed?.families[0]?.full, null); + }); + + it("lists stored ids including unreferenced ones", async () => { + const ids = await run( + Effect.gen(function* () { + const store = yield* BundleStore; + yield* store.write("tmp/a", Stream.fromIterable([new Uint8Array([1])])); + yield* store.write("full/1-b.bundle", Stream.fromIterable([new Uint8Array([2])])); + return yield* store.listIds("repo"); + }), + ); + assert.ok(ids.includes("tmp/a")); + assert.ok(ids.includes("full/1-b.bundle")); + }); + }); +}; diff --git a/src/server/BundleStore.test.ts b/src/server/BundleStore.test.ts new file mode 100644 index 0000000..cda402e --- /dev/null +++ b/src/server/BundleStore.test.ts @@ -0,0 +1,35 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, it } from "@effect/vitest"; + +import { Effect } from "effect"; + +import { fileLayer } from "./Bundles.node.ts"; +import { bundleStoreContract } from "./BundleStore.contract.ts"; +import { memoryLayer } from "./BundleStore.ts"; + +const runner = { describe, it }; + +bundleStoreContract( + "Memory", + { + run: (effect) => Effect.runPromise(effect.pipe(Effect.provide(memoryLayer))), + }, + runner, +); + +bundleStoreContract( + "Node", + { + run: async (effect) => { + const directory = await mkdtemp(join(tmpdir(), "bundles-node-case-")); + try { + return await Effect.runPromise(effect.pipe(Effect.provide(fileLayer(directory)))); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }, + }, + runner, +); diff --git a/src/server/BundleStore.ts b/src/server/BundleStore.ts new file mode 100644 index 0000000..3b4adfa --- /dev/null +++ b/src/server/BundleStore.ts @@ -0,0 +1,122 @@ +/** + * Derived bundle artifacts, separate from Git object storage. + * + * Git objects and refs are repository truth. Bundles are a cache of that + * truth, addressed by their own keys, and a crash before the manifest is + * published must leave only an unreferenced file — never an advertised + * pointer at incomplete bytes. + */ +import { Context, Effect, Layer, Stream } from "effect"; + +import { StorageFailure } from "../git/Error.ts"; +import { bytesToHex, concatBytes } from "../git/Format.ts"; +import { Sha1 } from "../git/Sha1.ts"; +import type { BundleManifest } from "./BundleFormat.ts"; + +export interface BundleWrite { + readonly objectId: string; + readonly bytes: number; + readonly checksum: string; +} + +export interface BundleStat { + readonly objectId: string; + readonly bytes: number; + readonly checksum: string; +} + +export class BundleStore extends Context.Service< + BundleStore, + { + readonly list: (repo: string) => Effect.Effect; + readonly publish: ( + repo: string, + manifest: BundleManifest, + ) => Effect.Effect; + + readonly stat: (id: string) => Effect.Effect; + readonly read: ( + id: string, + range?: { readonly offset: number; readonly length: number }, + ) => Effect.Effect, StorageFailure>; + readonly write: ( + id: string, + source: Stream.Stream, + ) => Effect.Effect; + readonly move: (from: string, to: string) => Effect.Effect; + readonly delete: (id: string) => Effect.Effect; + /** Every stored object id, advertised or not — what prune walks. */ + readonly listIds: (repo: string) => Effect.Effect, StorageFailure>; + } +>()("server/BundleStore") {} + +const missing = (operation: string, id: string) => + new StorageFailure({ operation, path: id, cause: "no such bundle artifact" }); + +/** In-memory backend, for the contract suite and for tests. */ +export const memory = (): BundleStore["Service"] => { + const artifacts = new Map(); + const checksums = new Map(); + const manifests = new Map(); + + return BundleStore.of({ + list: (repo) => Effect.sync(() => manifests.get(repo) ?? null), + publish: (repo, manifest) => + Effect.sync(() => { + manifests.set(repo, manifest); + }), + stat: (id) => + Effect.sync(() => { + const bytes = artifacts.get(id); + const checksum = checksums.get(id); + return bytes === undefined || checksum === undefined + ? null + : { objectId: id, bytes: bytes.length, checksum }; + }), + read: (id, range) => + Effect.sync(() => { + const bytes = artifacts.get(id); + if (bytes === undefined) { + return Stream.fail(missing("bundles.read", id)); + } + const slice = + range === undefined ? bytes : bytes.subarray(range.offset, range.offset + range.length); + return Stream.fromIterable([new Uint8Array(slice)]); + }), + write: (id, source) => + Effect.gen(function* () { + const hash = new Sha1(); + const chunks: Uint8Array[] = []; + yield* Stream.runForEach(source, (chunk) => + Effect.sync(() => { + hash.update(chunk); + chunks.push(chunk); + }), + ); + const bytes = concatBytes(chunks); + const checksum = bytesToHex(hash.digest()); + artifacts.set(id, bytes); + checksums.set(id, checksum); + return { objectId: id, bytes: bytes.length, checksum }; + }), + move: (from, to) => + Effect.sync(() => { + if (from === to) return; + const bytes = artifacts.get(from); + const checksum = checksums.get(from); + if (bytes === undefined || checksum === undefined) return; + artifacts.set(to, bytes); + checksums.set(to, checksum); + artifacts.delete(from); + checksums.delete(from); + }), + delete: (id) => + Effect.sync(() => { + artifacts.delete(id); + checksums.delete(id); + }), + listIds: () => Effect.sync(() => [...artifacts.keys()]), + }); +}; + +export const memoryLayer = Layer.sync(BundleStore, memory); diff --git a/src/server/Bundles.cloudflare.ts b/src/server/Bundles.cloudflare.ts new file mode 100644 index 0000000..e3dbf44 --- /dev/null +++ b/src/server/Bundles.cloudflare.ts @@ -0,0 +1,187 @@ +/** + * Bundle artifacts in R2. Manifest and maintenance state sit beside them. + * + * /bundles//-.bundle + * /bundles/manifest.json + * /gitplus/state.json + * + * Ports stay `R = never`: the bucket is captured when the layer is built. + */ +import { Effect, Layer, Stream } from "effect"; + +import { StorageFailure } from "../git/Error.ts"; +import { bytesToHex } from "../git/Format.ts"; +import { Sha1 } from "../git/Sha1.ts"; +import { decodeManifest, encodeManifest } from "./BundleFormat.ts"; +import { BundleStore } from "./BundleStore.ts"; +import { MaintenanceMeta, parseRecord } from "./MaintenanceRun.ts"; + +export interface CloudflareBundles { + readonly bucket: R2Bucket; + readonly repo: string; +} + +const failed = (operation: string, path: string) => (cause: unknown) => + new StorageFailure({ operation, path, cause }); + +export const r2Store = (options: CloudflareBundles): BundleStore["Service"] => { + const prefix = `${options.repo}/bundles/`; + const manifestKey = `${prefix}manifest.json`; + const keyOf = (id: string) => `${prefix}${id}`; + + return BundleStore.of({ + list: () => + Effect.tryPromise({ + try: async () => { + const object = await options.bucket.get(manifestKey); + if (object === null) return null; + return decodeManifest(await object.text()); + }, + catch: failed("bundles.list", manifestKey), + }), + + publish: (_repo, manifest) => + Effect.tryPromise({ + try: () => options.bucket.put(manifestKey, encodeManifest(manifest)), + catch: failed("bundles.publish", manifestKey), + }), + + stat: (id) => + Effect.tryPromise({ + try: async () => { + const found = await options.bucket.head(keyOf(id)); + if (found === null) return null; + const checksum = + found.customMetadata?.["checksum"] ?? + id + .split("-") + .at(-1) + ?.replace(/\.bundle$/, "") ?? + ""; + return { objectId: id, bytes: found.size, checksum }; + }, + catch: failed("bundles.stat", keyOf(id)), + }), + + read: (id, range) => + Effect.tryPromise({ + try: () => + range === undefined + ? options.bucket.get(keyOf(id)) + : options.bucket.get(keyOf(id), { + range: { offset: range.offset, length: range.length }, + }), + catch: failed("bundles.read", keyOf(id)), + }).pipe( + Effect.flatMap((object) => + object === null || object.body === null + ? Effect.succeed( + Stream.fail(new StorageFailure({ operation: "bundles.read", path: keyOf(id) })), + ) + : Effect.succeed( + Stream.fromReadableStream({ + evaluate: () => { + // SAFETY: R2 serves an object's bytes; the runtime types + // declare `body` without a chunk type. + return object.body as ReadableStream; + }, + onError: failed("bundles.read", keyOf(id)), + }), + ), + ), + ), + + write: (id, source) => + Effect.gen(function* () { + const hash = new Sha1(); + let bytes = 0; + const hashed = source.pipe( + Stream.tap((chunk) => + Effect.sync(() => { + hash.update(chunk); + bytes += chunk.length; + }), + ), + ); + yield* Effect.tryPromise({ + try: () => options.bucket.put(keyOf(id), Stream.toReadableStream(hashed)), + catch: failed("bundles.write", keyOf(id)), + }); + return { objectId: id, bytes, checksum: bytesToHex(hash.digest()) }; + }), + + move: (from, to) => + Effect.gen(function* () { + if (from === to) return; + const object = yield* Effect.tryPromise({ + try: () => options.bucket.get(keyOf(from)), + catch: failed("bundles.move", keyOf(from)), + }); + if (object === null || object.body === null) return; + yield* Effect.tryPromise({ + try: () => + options.bucket.put(keyOf(to), object.body, { + customMetadata: object.customMetadata, + }), + catch: failed("bundles.move", keyOf(to)), + }); + yield* Effect.tryPromise({ + try: () => options.bucket.delete(keyOf(from)), + catch: failed("bundles.move", keyOf(from)), + }); + }), + + delete: (id) => + Effect.tryPromise({ + try: () => options.bucket.delete(keyOf(id)), + catch: failed("bundles.delete", keyOf(id)), + }), + + listIds: () => + Effect.tryPromise({ + try: async () => { + const ids: string[] = []; + let cursor: string | undefined; + for (;;) { + const page: R2Objects = await options.bucket.list({ + prefix, + cursor, + }); + for (const object of page.objects) { + if (object.key === manifestKey) continue; + ids.push(object.key.slice(prefix.length)); + } + if (!page.truncated) break; + cursor = page.cursor; + } + return ids; + }, + catch: failed("bundles.listIds", prefix), + }), + }); +}; + +export const r2Layer = (options: CloudflareBundles) => + Layer.sync(BundleStore, () => r2Store(options)); + +export const r2Meta = (options: CloudflareBundles): MaintenanceMeta["Service"] => { + const key = `${options.repo}/gitplus/state.json`; + return MaintenanceMeta.of({ + read: Effect.tryPromise({ + try: async () => { + const object = await options.bucket.get(key); + if (object === null) return {}; + return parseRecord(await object.text()); + }, + catch: failed("maintenance.read", key), + }), + write: (record) => + Effect.tryPromise({ + try: () => options.bucket.put(key, JSON.stringify(record)), + catch: failed("maintenance.write", key), + }), + }); +}; + +export const r2MetaLayer = (options: CloudflareBundles) => + Layer.sync(MaintenanceMeta, () => r2Meta(options)); diff --git a/src/server/Bundles.node.ts b/src/server/Bundles.node.ts new file mode 100644 index 0000000..70138b6 --- /dev/null +++ b/src/server/Bundles.node.ts @@ -0,0 +1,249 @@ +/** + * Bundle artifacts on the filesystem, under server-managed metadata. + * + * /gitplus/bundles//-.bundle + * /gitplus/bundles.json + * + * git does not look here. A crash before `publish` leaves an unreferenced + * file that the planner later prunes. + */ +import * as fs from "node:fs/promises"; +import * as path from "node:path"; + +import { Effect, Layer, Predicate, Stream } from "effect"; + +import { StorageFailure } from "../git/Error.ts"; +import { bytesToHex } from "../git/Format.ts"; +import { Sha1 } from "../git/Sha1.ts"; +import { decodeManifest, encodeManifest } from "./BundleFormat.ts"; +import { BundleStore } from "./BundleStore.ts"; +import { MaintenanceMeta, parseRecord } from "./MaintenanceRun.ts"; + +const failed = (operation: string, target: string) => (cause: unknown) => + new StorageFailure({ operation, path: target, cause }); + +const missing = (cause: unknown): boolean => + Predicate.hasProperty(cause, "code") && cause.code === "ENOENT"; + +const safeId = (id: string): string | null => { + if (id.length === 0 || id.includes("\0") || id.includes("..")) return null; + if (path.isAbsolute(id) || id.startsWith("/") || id.startsWith("\\")) return null; + return id; +}; + +const writeFileAtomic = async (target: string, bytes: Uint8Array): Promise => { + await fs.mkdir(path.dirname(target), { recursive: true }); + const temp = `${target}.${process.pid}.${Date.now()}.tmp`; + await fs.writeFile(temp, bytes); + await fs.rename(temp, target); +}; + +export const fileStore = (root: string): BundleStore["Service"] => { + const base = path.join(root, "gitplus"); + const artifactPath = (id: string): string => path.join(base, "bundles", id); + const manifestPath = path.join(base, "bundles.json"); + + return BundleStore.of({ + list: () => + Effect.tryPromise({ + try: async () => { + const text = await fs.readFile(manifestPath, "utf8").catch((cause) => { + if (missing(cause)) return null; + throw cause; + }); + return text === null ? null : decodeManifest(text); + }, + catch: failed("bundles.list", manifestPath), + }), + + publish: (_repo, manifest) => + Effect.tryPromise({ + try: () => + writeFileAtomic(manifestPath, new TextEncoder().encode(encodeManifest(manifest))), + catch: failed("bundles.publish", manifestPath), + }), + + stat: (id) => + Effect.tryPromise({ + try: async () => { + const safe = safeId(id); + if (safe === null) return null; + const target = artifactPath(safe); + const found = await fs.stat(target).catch(() => null); + if (found === null || !found.isFile()) return null; + const checksum = + path + .basename(safe) + .split("-") + .at(-1) + ?.replace(/\.bundle$/, "") ?? ""; + return { objectId: id, bytes: found.size, checksum }; + }, + catch: failed("bundles.stat", id), + }), + + read: (id, range) => + Effect.gen(function* () { + const safe = safeId(id); + if (safe === null) { + return Stream.fail(new StorageFailure({ operation: "bundles.read", path: id })); + } + const target = artifactPath(safe); + const handle = yield* Effect.tryPromise({ + try: () => fs.open(target, "r"), + catch: failed("bundles.read", target), + }); + const start = range?.offset ?? 0; + const length = range?.length; + return Stream.fromAsyncIterable( + (async function* () { + try { + const stat = await handle.stat(); + let remaining = length ?? Math.max(0, stat.size - start); + let offset = start; + const chunk = new Uint8Array(64 * 1024); + while (remaining > 0) { + const take = Math.min(chunk.length, remaining); + const result = await handle.read(chunk, 0, take, offset); + if (result.bytesRead === 0) break; + yield new Uint8Array(chunk.subarray(0, result.bytesRead)); + offset += result.bytesRead; + remaining -= result.bytesRead; + } + } finally { + await handle.close(); + } + })(), + failed("bundles.read", target), + ); + }), + + write: (id, source) => + Effect.gen(function* () { + const safe = safeId(id); + if (safe === null) { + return yield* new StorageFailure({ + operation: "bundles.write", + path: id, + cause: "unsafe artifact id", + }); + } + const target = artifactPath(safe); + yield* Effect.tryPromise({ + try: () => fs.mkdir(path.dirname(target), { recursive: true }), + catch: failed("bundles.write", target), + }); + const temp = `${target}.${process.pid}.writing`; + const handle = yield* Effect.tryPromise({ + try: () => fs.open(temp, "w"), + catch: failed("bundles.write", temp), + }); + const hash = new Sha1(); + let bytes = 0; + yield* Stream.runForEach(source, (chunk) => + Effect.tryPromise({ + try: async () => { + hash.update(chunk); + bytes += chunk.length; + await handle.write(chunk); + }, + catch: failed("bundles.write", temp), + }), + ).pipe( + Effect.ensuring( + Effect.tryPromise({ + try: () => handle.close(), + catch: failed("bundles.write", temp), + }).pipe(Effect.ignore), + ), + ); + yield* Effect.tryPromise({ + try: () => fs.rename(temp, target), + catch: failed("bundles.write", target), + }); + return { objectId: id, bytes, checksum: bytesToHex(hash.digest()) }; + }), + + move: (from, to) => + Effect.gen(function* () { + if (from === to) return; + const source = safeId(from); + const dest = safeId(to); + if (source === null || dest === null) { + return yield* new StorageFailure({ + operation: "bundles.move", + path: to, + cause: "unsafe artifact id", + }); + } + const fromPath = artifactPath(source); + const toPath = artifactPath(dest); + yield* Effect.tryPromise({ + try: async () => { + await fs.mkdir(path.dirname(toPath), { recursive: true }); + await fs.rename(fromPath, toPath); + }, + catch: failed("bundles.move", toPath), + }); + }), + + delete: (id) => + Effect.tryPromise({ + try: async () => { + const safe = safeId(id); + if (safe === null) return; + await fs.unlink(artifactPath(safe)).catch((cause) => { + if (missing(cause)) return; + throw cause; + }); + }, + catch: failed("bundles.delete", id), + }), + + listIds: () => + Effect.tryPromise({ + try: async () => { + const root = path.join(base, "bundles"); + const found: string[] = []; + const walk = async (dir: string, prefix: string) => { + const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []); + for (const entry of entries) { + const relative = prefix === "" ? entry.name : `${prefix}/${entry.name}`; + if (entry.isDirectory()) await walk(path.join(dir, entry.name), relative); + else if (!entry.name.endsWith(".tmp") && !entry.name.endsWith(".writing")) { + found.push(relative); + } + } + }; + await walk(root, ""); + return found; + }, + catch: failed("bundles.listIds", base), + }), + }); +}; + +export const fileLayer = (root: string) => Layer.sync(BundleStore, () => fileStore(root)); + +export const fileMeta = (root: string): MaintenanceMeta["Service"] => { + const target = path.join(root, "gitplus", "state.json"); + return MaintenanceMeta.of({ + read: Effect.tryPromise({ + try: async () => { + const text = await fs.readFile(target, "utf8").catch((cause) => { + if (missing(cause)) return "{}"; + throw cause; + }); + return parseRecord(text); + }, + catch: failed("maintenance.read", target), + }), + write: (record) => + Effect.tryPromise({ + try: () => writeFileAtomic(target, new TextEncoder().encode(JSON.stringify(record))), + catch: failed("maintenance.write", target), + }), + }); +}; + +export const fileMetaLayer = (root: string) => Layer.sync(MaintenanceMeta, () => fileMeta(root)); diff --git a/src/server/Bundles.test.ts b/src/server/Bundles.test.ts new file mode 100644 index 0000000..f2fa485 --- /dev/null +++ b/src/server/Bundles.test.ts @@ -0,0 +1,93 @@ +import assert from "node:assert/strict"; +import { describe, it } from "@effect/vitest"; + +import { Effect, Layer } from "effect"; + +import { type Signature } from "../git/Format.ts"; +import { stores } from "../git/Memory.ts"; +import * as GitRepository from "../git/Repository.ts"; +import { Repository } from "../git/Repository.ts"; +import * as Bundles from "./Bundles.ts"; +import { memoryLayer } from "./BundleStore.ts"; +import { defaultsLayer } from "./Features.ts"; + +const alice: Signature = { + name: "Alice", + email: "alice@example.com", + at: new Date(1_700_000_000_000), + offset: 0, +}; + +const liveRepo = GitRepository.layer.pipe( + Layer.provide(GitRepository.hooksNoop), + Layer.provideMerge(stores), +); +const live = Bundles.layer.pipe( + Layer.provide(memoryLayer), + Layer.provideMerge(liveRepo), + Layer.provideMerge(defaultsLayer), +); + +const scenario = (effect: Effect.Effect) => + Effect.runPromise(effect.pipe(Effect.provide(live))); + +describe("Bundles publication", () => { + it.effect("does not advertise until a verified artifact is published", () => + Effect.promise(async () => { + const { before, built, after, clone } = await scenario( + Effect.gen(function* () { + const bundles = yield* Bundles.Bundles; + const before = yield* bundles.summary; + const repository = yield* Repository; + const blob = yield* repository.writeBlob(new TextEncoder().encode("hi\n")); + const tree = yield* repository.writeTree([{ mode: "100644", name: "a.txt", oid: blob }]); + yield* repository.commit({ branch: "main", tree, message: "hi", author: alice }); + const built = yield* bundles.build({ kind: "full", filter: null }); + const after = yield* bundles.summary; + const clone = yield* bundles.protocolLines("clone", "http://host/repo"); + const catchup = yield* bundles.protocolLines("catchup", "http://host/repo"); + return { before, built, after, clone, catchup }; + }), + ); + + assert.equal( + before.families.some((family) => family.full !== null), + false, + ); + assert.ok(built.bytes > 0); + assert.match(built.id, /^full-/); + assert.equal( + after.families.some((family) => family.full !== null), + true, + ); + assert.ok(clone.some((line) => line.startsWith("bundle.version="))); + assert.ok(clone.some((line) => line.includes(".uri="))); + }), + ); + + it.effect("serves clone lists and range-readable artifacts", () => + Effect.promise(async () => { + const { list, artifact } = await scenario( + Effect.gen(function* () { + const repository = yield* Repository; + const blob = yield* repository.writeBlob(new TextEncoder().encode("hi\n")); + const tree = yield* repository.writeTree([{ mode: "100644", name: "a.txt", oid: blob }]); + yield* repository.commit({ branch: "main", tree, message: "hi", author: alice }); + const bundles = yield* Bundles.Bundles; + const built = yield* bundles.build({ kind: "full", filter: null }); + const list = yield* bundles.handle(new Request("http://host/repo/bundles/clone")); + const artifact = yield* bundles.handle( + new Request(`http://host/repo/bundles/${built.objectId}`, { + headers: { range: "bytes=0-15" }, + }), + ); + return { list, artifact }; + }), + ); + assert.equal(list?.status, 200); + assert.match((await list?.text()) ?? "", /\[bundle\]/); + assert.equal(artifact?.status, 206); + assert.equal(artifact?.headers.get("accept-ranges"), "bytes"); + }), + ); +}); diff --git a/src/server/Bundles.ts b/src/server/Bundles.ts new file mode 100644 index 0000000..11b50b5 --- /dev/null +++ b/src/server/Bundles.ts @@ -0,0 +1,355 @@ +/** + * Bundle list, publication, and HTTP / protocol integration. + * + * Capture refs, generate, verify, store, then publish the manifest — in that + * order, so a client never observes an entry pointing at incomplete bytes. + */ +import { Context, Effect, Layer, Option, Stream } from "effect"; + +import { anonymousReadAllowed } from "./Auth.ts"; +import { project as projectTrust } from "../trust/Projection.ts"; +import { readGenesis } from "../trust/Genesis.ts"; +import { BundleCorrupt, Invalid, type ObjectNotFound, StorageFailure } from "../git/Error.ts"; +import { hiddenFromAdvertisement } from "../git/Refspec.ts"; +import { Repository } from "../git/Repository.ts"; +import { isOid, ObjectStore, type Oid, storageOf } from "../git/Store.ts"; +import * as Artifact from "./Artifact.ts"; +import * as BundleBuilder from "./BundleBuilder.ts"; +import { + advertisedIds, + artifactPath, + type BundleArtifact, + type BundleFilter, + type BundleKind, + type BundleManifest, + type BundleSnapshot, + catchupArtifacts, + cloneArtifacts, + decodeManifest, + encodeConfigList, + encodeProtocolList, + familyState, + artifactId, + latestToken, + listEntries, + nextToken, + objectIdFromPath, + objectIdOf, + publishArtifact, +} from "./BundleFormat.ts"; +import { BundleStore } from "./BundleStore.ts"; +import { defaults, ServerFeatures } from "./Features.ts"; +import type { OperationHandle } from "./Operations.ts"; + +export interface BundleFamilySummary { + readonly filter: BundleFilter; + readonly full: string | null; + readonly incrementals: number; + readonly latestCreationToken: string | null; +} + +export interface BundlesSummary { + readonly enabled: boolean; + readonly families: ReadonlyArray; +} + +export class Bundles extends Context.Service< + Bundles, + { + readonly enabled: Effect.Effect; + readonly snapshot: ( + filter: BundleFilter, + ) => Effect.Effect; + readonly manifest: Effect.Effect; + readonly summary: Effect.Effect; + readonly build: (input: { + readonly kind: BundleKind; + readonly filter: BundleFilter; + readonly operation?: OperationHandle; + }) => Effect.Effect; + readonly protocolLines: ( + kind: "clone" | "catchup", + baseUrl: string, + ) => Effect.Effect, StorageFailure>; + readonly configText: ( + kind: "clone" | "catchup", + baseUrl: string, + ) => Effect.Effect; + readonly handle: (request: Request) => Effect.Effect; + } +>()("server/Bundles") {} + +const featuresOf = Effect.fnUntraced(function* () { + return Option.getOrElse(yield* Effect.serviceOption(ServerFeatures), () => defaults); +}); + +const repoName = Effect.fnUntraced(function* () { + const named = yield* storageOf(); + return named ?? "repository"; +}); + +const advertisedRefs = Effect.fn("Bundles.advertisedRefs")(function* () { + const repository = yield* Repository; + const listed = yield* repository.refs; + const refs: Record = {}; + for (const [name, oid] of listed) { + if (hiddenFromAdvertisement(name) || !isOid(oid)) continue; + refs[name] = oid; + } + return refs; +}); + +const cacheMode = (): Effect.Effect<"public" | "private", never, Repository> => + readGenesis().pipe( + Effect.orElseSucceed(() => null), + Effect.flatMap((stored) => { + if (stored === null) return Effect.succeed<"public">("public"); + return projectTrust(stored.genesis).pipe( + Effect.orElseSucceed(() => null), + Effect.map((projection) => + projection !== null && anonymousReadAllowed(projection) ? "public" : "private", + ), + ); + }), + ); + +const uriOf = (baseUrl: string, artifact: BundleArtifact): string => { + const root = baseUrl.replace(/\/+$/, ""); + return `${root}/bundles/${artifact.objectId}`; +}; + +const entriesFor = ( + manifest: BundleManifest | null, + kind: "clone" | "catchup", + baseUrl: string, +) => { + const artifacts = []; + if (manifest !== null) { + for (const family of manifest.families) { + const listed = kind === "clone" ? cloneArtifacts(family) : catchupArtifacts(family); + for (const artifact of listed) artifacts.push(artifact); + } + } + return listEntries(artifacts, (artifact) => uriOf(baseUrl, artifact)); +}; + +const originOf = (request: Request) => { + const url = new URL(request.url); + const segments = url.pathname.split("/").filter((segment) => segment !== ""); + const first = segments[0]; + const repo = first === undefined || first === "bundles" ? "repository" : first; + return { repo, base: `${url.origin}/${repo}` }; +}; + +export const layer = Layer.effect( + Bundles, + Effect.gen(function* () { + const store = yield* BundleStore; + const context = yield* Effect.context(); + const withRepo = (effect: Effect.Effect) => + effect.pipe(Effect.provide(context)); + + const snapshot = Effect.fn("Bundles.snapshot")(function* ( + filter: BundleFilter, + ): Effect.fn.Return { + return { + createdAt: new Date(), + refs: yield* withRepo(advertisedRefs()), + filter, + } satisfies BundleSnapshot; + }); + + const manifestOf = Effect.fn("Bundles.manifest")(function* () { + return yield* store.list(yield* repoName()); + }); + + const summary = Effect.fn("Bundles.summary")(function* () { + const features = yield* featuresOf(); + const manifest = yield* manifestOf(); + const families = (manifest?.families ?? []).map((family) => { + const latest = + family.incrementals.at(-1)?.creationToken ?? family.full?.creationToken ?? null; + return { + filter: family.filter, + full: family.full?.id ?? null, + incrementals: family.incrementals.length, + latestCreationToken: latest === null ? null : latest.toString(), + } satisfies BundleFamilySummary; + }); + return { enabled: features.bundles.enabled, families } satisfies BundlesSummary; + }); + + const build = Effect.fn("Bundles.build")(function* (input: { + readonly kind: BundleKind; + readonly filter: BundleFilter; + readonly operation?: OperationHandle; + }): Effect.fn.Return< + BundleArtifact, + BundleCorrupt | Invalid | ObjectNotFound | StorageFailure + > { + const repo = yield* repoName(); + const captured = yield* snapshot(input.filter); + if (Object.keys(captured.refs).length === 0) { + return yield* new Invalid({ field: "bundle", reason: "repository has no advertised refs" }); + } + const current = yield* manifestOf(); + const family = familyState(current, input.filter); + const previous = + input.kind === "incremental" ? (family.incrementals.at(-1) ?? family.full) : undefined; + if (input.kind === "incremental" && previous === null) { + return yield* new Invalid({ + field: "bundle", + reason: "cannot build an incremental bundle before a full base exists", + }); + } + yield* input.operation?.info("walking objects") ?? Effect.void; + const request = + previous === undefined || previous === null + ? { snapshot: captured, kind: input.kind } + : { snapshot: captured, kind: input.kind, prerequisiteRefs: previous.refs }; + const built = yield* withRepo(BundleBuilder.build(request)); + yield* ( + input.operation?.progress( + { current: 0, total: built.oids.length, unit: "objects" }, + "writing bundle", + ) ?? Effect.void + ); + + const token = nextToken(latestToken(current), captured.createdAt); + const tempId = `tmp/${token.toString()}-${crypto.randomUUID()}`; + const written = yield* store.write( + tempId, + built.stream.pipe( + Stream.provide(context), + Stream.mapError((error) => + error._tag === "StorageFailure" + ? error + : new StorageFailure({ + operation: "bundle.write", + path: tempId, + cause: error, + }), + ), + ), + ); + const finalId = objectIdOf(input.filter, token, written.checksum); + yield* store.move(tempId, finalId); + + yield* input.operation?.info("verifying bundle") ?? Effect.void; + const bytes = yield* BundleBuilder.collect(yield* store.read(finalId)); + yield* BundleBuilder.verifyBundle(bytes, { + refs: captured.refs, + prerequisites: built.header.prerequisites, + filter: input.filter, + }); + yield* input.operation?.commit ?? Effect.void; + + const artifact: BundleArtifact = { + id: artifactId(input.kind, input.filter, token), + kind: input.kind, + filter: input.filter, + creationToken: token, + refs: captured.refs, + prerequisites: built.header.prerequisites, + objectId: finalId, + bytes: written.bytes, + checksum: written.checksum, + createdAt: captured.createdAt.toISOString(), + }; + yield* store.publish(repo, publishArtifact(current, artifact)); + yield* input.operation?.info("published") ?? Effect.void; + return artifact; + }); + + const protocolLines = Effect.fn("Bundles.protocolLines")(function* ( + kind: "clone" | "catchup", + baseUrl: string, + ) { + return encodeProtocolList(entriesFor(yield* manifestOf(), kind, baseUrl)); + }); + + const configText = Effect.fn("Bundles.configText")(function* ( + kind: "clone" | "catchup", + baseUrl: string, + ) { + return encodeConfigList(entriesFor(yield* manifestOf(), kind, baseUrl)); + }); + + const handle = Effect.fn("Bundles.handle")(function* ( + request: Request, + ): Effect.fn.Return { + if (request.method !== "GET" && request.method !== "HEAD") return null; + const url = new URL(request.url); + const segments = url.pathname.split("/").filter((segment) => segment !== ""); + const at = segments.lastIndexOf("bundles"); + if (at === -1) return null; + const rest = segments.slice(at + 1); + + if (rest.length === 1 && (rest[0] === "clone" || rest[0] === "catchup")) { + const { base } = originOf(request); + const body = yield* configText(rest[0], base); + const bytes = new TextEncoder().encode(body); + return new Response(request.method === "HEAD" ? null : body, { + headers: { + "content-type": "text/plain; charset=utf-8", + "content-length": String(bytes.byteLength), + "cache-control": "no-cache", + }, + }); + } + + const pointer = artifactPath(url.pathname); + if (pointer === null) return null; + const id = objectIdFromPath(pointer); + const stat = yield* store.stat(id); + if (stat === null) return new Response("not found\n", { status: 404 }); + const cache = yield* withRepo(cacheMode()); + return yield* Artifact.respond(request, { + bytes: stat.bytes, + etag: stat.checksum, + contentType: "application/x-git-bundle", + cache, + read: (range) => store.read(id, range), + }); + }); + + return Bundles.of({ + enabled: featuresOf().pipe(Effect.map((features) => features.bundles.enabled)), + snapshot, + manifest: Effect.suspend(() => manifestOf()), + summary: Effect.suspend(() => summary()), + build, + protocolLines, + configText, + handle, + }); + }), +); + +/** Advertise `bundle-uri` when a published full bundle exists. */ +export const advertise = Effect.fn("Bundles.advertise")(function* () { + const service = yield* Effect.serviceOption(Bundles); + if (Option.isNone(service)) return false; + const features = yield* featuresOf(); + if (!features.bundles.enabled) return false; + const summary = yield* service.value.summary; + return summary.families.some((family) => family.full !== null); +}); + +export const tryHandle = Effect.fn("Bundles.tryHandle")(function* (request: Request) { + const service = yield* Effect.serviceOption(Bundles); + if (Option.isNone(service)) return null; + return yield* service.value.handle(request); +}); + +export const protocolList = Effect.fn("Bundles.protocolList")(function* (request: Request) { + const service = yield* Effect.serviceOption(Bundles); + if (Option.isNone(service)) return []; + const url = new URL(request.url); + const segments = url.pathname.split("/").filter((segment) => segment !== ""); + const repo = segments[0] ?? "repository"; + return yield* service.value.protocolLines("clone", `${url.origin}/${repo}`); +}); + +export { advertisedIds, decodeManifest }; +export type { BundleArtifact, BundleManifest, BundleSnapshot }; diff --git a/src/server/Features.ts b/src/server/Features.ts new file mode 100644 index 0000000..c48bf30 --- /dev/null +++ b/src/server/Features.ts @@ -0,0 +1,147 @@ +/** + * Server-side bundle and maintenance configuration. + * + * Read through Effect `Config`, not `process.env`, so a malformed duration + * fails naming the variable. Invalid values fail closed at layer build. + */ +import { Config, Context, Duration, Effect, Layer } from "effect"; + +import { Invalid } from "../git/Error.ts"; +import type { BundleFilter } from "./BundleFormat.ts"; + +export interface BundlePolicy { + readonly enabled: boolean; + readonly fullMaxAge: Duration.Duration; + readonly incrementalMinAge: Duration.Duration; + readonly maxIncrementals: number; + readonly minNewObjects: number; + readonly filters: ReadonlyArray; +} + +export interface MaintenancePolicy { + readonly enabled: boolean; + readonly interval: Duration.Duration; + readonly fsckInterval: Duration.Duration; + readonly gcInterval: Duration.Duration; + readonly repackThreshold: number; +} + +export interface Features { + readonly bundles: BundlePolicy; + readonly maintenance: MaintenancePolicy; +} + +export const defaults: Features = { + bundles: { + enabled: true, + fullMaxAge: Duration.days(7), + incrementalMinAge: Duration.hours(1), + maxIncrementals: 24, + minNewObjects: 1, + filters: [null, "blob:none"], + }, + maintenance: { + enabled: true, + interval: Duration.hours(1), + fsckInterval: Duration.days(7), + gcInterval: Duration.days(1), + repackThreshold: 50, + }, +}; + +const parseFilters = (value: string): Effect.Effect, Invalid> => + Effect.suspend(() => { + const filters: BundleFilter[] = []; + for (const part of value.split(",")) { + const item = part.trim(); + if (item === "" || item === "full" || item === "none") { + if (!filters.includes(null)) filters.push(null); + continue; + } + if (item === "blob:none" || item === "blobnone") { + if (!filters.includes("blob:none")) filters.push("blob:none"); + continue; + } + return Effect.fail( + new Invalid({ + field: "BUNDLES_FILTERS", + reason: `unknown bundle filter '${item}'`, + }), + ); + } + return Effect.succeed(filters.length === 0 ? defaults.bundles.filters : filters); + }); + +const requireCount = (field: string, value: number): Effect.Effect => + Number.isInteger(value) && value >= 0 + ? Effect.succeed(value) + : Effect.fail(new Invalid({ field, reason: "must be a non-negative integer" })); + +export class ServerFeatures extends Context.Service()( + "server/ServerFeatures", +) {} + +export const configuration = Effect.fn("Features.configuration")(function* () { + const enabled = yield* Config.boolean("BUNDLES_ENABLED").pipe( + Config.withDefault(defaults.bundles.enabled), + ); + const fullMaxAge = yield* Config.duration("BUNDLES_FULL_MAX_AGE").pipe( + Config.withDefault(defaults.bundles.fullMaxAge), + ); + const incrementalMinAge = yield* Config.duration("BUNDLES_INCREMENTAL_MIN_AGE").pipe( + Config.withDefault(defaults.bundles.incrementalMinAge), + ); + const maxIncrementals = yield* requireCount( + "BUNDLES_MAX_INCREMENTALS", + yield* Config.number("BUNDLES_MAX_INCREMENTALS").pipe( + Config.withDefault(defaults.bundles.maxIncrementals), + ), + ); + const minNewObjects = yield* requireCount( + "BUNDLES_MIN_NEW_OBJECTS", + yield* Config.number("BUNDLES_MIN_NEW_OBJECTS").pipe( + Config.withDefault(defaults.bundles.minNewObjects), + ), + ); + const filterText = yield* Config.string("BUNDLES_FILTERS").pipe( + Config.withDefault("full,blob:none"), + ); + const filters = yield* parseFilters(filterText); + + const maintenanceEnabled = yield* Config.boolean("MAINTENANCE_ENABLED").pipe( + Config.withDefault(defaults.maintenance.enabled), + ); + const interval = yield* Config.duration("MAINTENANCE_INTERVAL").pipe( + Config.withDefault(defaults.maintenance.interval), + ); + const fsckInterval = yield* Config.duration("MAINTENANCE_FSCK_INTERVAL").pipe( + Config.withDefault(defaults.maintenance.fsckInterval), + ); + const gcInterval = yield* Config.duration("MAINTENANCE_GC_INTERVAL").pipe( + Config.withDefault(defaults.maintenance.gcInterval), + ); + const repackThreshold = yield* requireCount( + "MAINTENANCE_REPACK_THRESHOLD", + yield* Config.number("MAINTENANCE_REPACK_THRESHOLD").pipe( + Config.withDefault(defaults.maintenance.repackThreshold), + ), + ); + + return { + bundles: { enabled, fullMaxAge, incrementalMinAge, maxIncrementals, minNewObjects, filters }, + maintenance: { + enabled: maintenanceEnabled, + interval, + fsckInterval, + gcInterval, + repackThreshold, + }, + } satisfies Features; +}); + +export const layer = Layer.effect( + ServerFeatures, + configuration().pipe(Effect.map((features) => ServerFeatures.of(features))), +); + +export const defaultsLayer = Layer.succeed(ServerFeatures, ServerFeatures.of(defaults)); diff --git a/src/server/MaintenanceRun.test.ts b/src/server/MaintenanceRun.test.ts new file mode 100644 index 0000000..884dc6b --- /dev/null +++ b/src/server/MaintenanceRun.test.ts @@ -0,0 +1,55 @@ +import assert from "node:assert/strict"; +import { describe, it } from "@effect/vitest"; + +import { Effect, Layer } from "effect"; + +import { type Signature } from "../git/Format.ts"; +import { stores } from "../git/Memory.ts"; +import * as GitRepository from "../git/Repository.ts"; +import { Repository } from "../git/Repository.ts"; +import * as Bundles from "./Bundles.ts"; +import { memoryLayer } from "./BundleStore.ts"; +import { defaultsLayer } from "./Features.ts"; +import * as Maintenance from "./MaintenanceRun.ts"; +import { memoryLayer as operationsMemory } from "./Operations.ts"; + +const alice: Signature = { + name: "Alice", + email: "alice@example.com", + at: new Date(1_700_000_000_000), + offset: 0, +}; + +const liveRepo = GitRepository.layer.pipe( + Layer.provide(GitRepository.hooksNoop), + Layer.provideMerge(stores), +); + +const live = Bundles.layer.pipe( + Layer.provide(memoryLayer), + Layer.provideMerge(liveRepo), + Layer.provideMerge(defaultsLayer), + Layer.provideMerge(operationsMemory), + Layer.provideMerge(Maintenance.memoryMetaLayer), +); + +describe("MaintenanceRun", () => { + it.effect("plans a full bundle for a seeded repository and run converges", () => + Effect.promise(async () => { + const { planned, after } = await Effect.runPromise( + Effect.gen(function* () { + const repository = yield* Repository; + const blob = yield* repository.writeBlob(new TextEncoder().encode("x\n")); + const tree = yield* repository.writeTree([{ mode: "100644", name: "x.txt", oid: blob }]); + yield* repository.commit({ branch: "main", tree, message: "x", author: alice }); + const planned = yield* Maintenance.plan(); + yield* Maintenance.runAll(); + const after = yield* Maintenance.plan(); + return { planned, after }; + }).pipe(Effect.provide(live)), + ); + assert.ok(planned.some((unit) => unit.kind === "build-full")); + assert.ok(!after.some((unit) => unit.kind === "build-full")); + }), + ); +}); diff --git a/src/server/MaintenanceRun.ts b/src/server/MaintenanceRun.ts new file mode 100644 index 0000000..6704056 --- /dev/null +++ b/src/server/MaintenanceRun.ts @@ -0,0 +1,270 @@ +/** + * Desired-state maintenance: read a snapshot, plan, execute one unit. + * + * Restart-safe by construction. A crash mid-build leaves no manifest entry, + * so the next tick plans the same unit again. Operations record progress; + * they are not the source of truth for whether the work still needs doing. + */ +import { Context, Effect, Layer, Option, Result, Schema, Stream } from "effect"; + +import { hiddenFromAdvertisement } from "../git/Refspec.ts"; +import { Repository } from "../git/Repository.ts"; +import { PackStore } from "../git/Packed.ts"; +import { isOid, ObjectStore, type Oid, storageOf } from "../git/Store.ts"; +import { type StorageFailure } from "../git/Error.ts"; +import { BundleStore } from "./BundleStore.ts"; +import { Bundles } from "./Bundles.ts"; +import { defaults, type Features, ServerFeatures } from "./Features.ts"; +import { Operations, run as runOperation } from "./Operations.ts"; +import { + type MaintenanceConfig, + type MaintenanceSnapshot, + type MaintenanceUnit, + nextUnit, + planMaintenance, +} from "./Planner.ts"; + +export interface MaintenanceRecord { + readonly lastFsck?: string; + readonly lastGc?: string; + readonly lastRepack?: string; + readonly lastBundleAttempt?: string; +} + +const RecordWire = Schema.Struct({ + lastFsck: Schema.optionalKey(Schema.String), + lastGc: Schema.optionalKey(Schema.String), + lastRepack: Schema.optionalKey(Schema.String), + lastBundleAttempt: Schema.optionalKey(Schema.String), +}); +const decodeRecord = Schema.decodeUnknownResult(RecordWire); + +export const parseRecord = (text: string): MaintenanceRecord => { + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + return {}; + } + const decoded = decodeRecord(parsed); + return Result.isFailure(decoded) ? {} : decoded.success; +}; + +export class MaintenanceMeta extends Context.Service< + MaintenanceMeta, + { + readonly read: Effect.Effect; + readonly write: (record: MaintenanceRecord) => Effect.Effect; + } +>()("server/MaintenanceMeta") {} + +export const memoryMeta = (): MaintenanceMeta["Service"] => { + let record: MaintenanceRecord = {}; + return MaintenanceMeta.of({ + read: Effect.sync(() => record), + write: (next) => + Effect.sync(() => { + record = next; + }), + }); +}; + +export const memoryMetaLayer = Layer.sync(MaintenanceMeta, memoryMeta); + +const featuresOf = Effect.fnUntraced(function* () { + return Option.getOrElse(yield* Effect.serviceOption(ServerFeatures), () => defaults); +}); + +const asConfig = (features: Features): MaintenanceConfig => ({ + bundles: features.bundles, + maintenance: features.maintenance, +}); + +const parseDate = (value: string | undefined): Date | undefined => { + if (value === undefined) return undefined; + const date = new Date(value); + return Number.isNaN(date.getTime()) ? undefined : date; +}; + +export const snapshot = Effect.fn("Maintenance.snapshot")(function* () { + const repository = yield* Repository; + const objects = yield* ObjectStore; + const listed = yield* repository.refs; + const refs: Record = {}; + for (const [name, oid] of listed) { + if (hiddenFromAdvertisement(name) || !isOid(oid)) continue; + refs[name] = oid; + } + + const bundles = yield* Effect.serviceOption(Bundles); + const store = yield* Effect.serviceOption(BundleStore); + const repo = (yield* storageOf()) ?? "repository"; + const manifest = Option.isSome(bundles) + ? yield* bundles.value.manifest + : Option.isSome(store) + ? yield* store.value.list(repo) + : null; + const storedIds = Option.isSome(store) ? yield* store.value.listIds(repo) : []; + + const meta = yield* Effect.serviceOption(MaintenanceMeta); + const record = Option.isSome(meta) ? yield* meta.value.read : {}; + + const packs = yield* Effect.serviceOption(PackStore); + const packCount = Option.isSome(packs) ? (yield* packs.value.list).length : undefined; + + let looseObjectCount: number | undefined; + if (Option.isSome(yield* Effect.serviceOption(ObjectStore))) { + let count = 0; + yield* Stream.runForEach(objects.list, () => + Effect.sync(() => { + count += 1; + }), + ); + looseObjectCount = count; + } + + return { + now: new Date(), + refs, + bundles: manifest, + storedIds, + lastFsck: parseDate(record.lastFsck), + lastGc: parseDate(record.lastGc), + lastRepack: parseDate(record.lastRepack), + lastBundleAttempt: parseDate(record.lastBundleAttempt), + looseObjectCount, + packCount, + } satisfies MaintenanceSnapshot; +}); + +export const plan = Effect.fn("Maintenance.plan")(function* () { + const captured = yield* snapshot(); + const features = yield* featuresOf(); + return planMaintenance(captured, asConfig(features)); +}); + +const stamp = (record: MaintenanceRecord, field: keyof MaintenanceRecord): MaintenanceRecord => ({ + ...record, + [field]: new Date().toISOString(), +}); + +const executeUnit = Effect.fn("Maintenance.execute")(function* (unit: MaintenanceUnit) { + const store = yield* Effect.serviceOption(BundleStore); + const bundles = yield* Effect.serviceOption(Bundles); + const meta = yield* Effect.serviceOption(MaintenanceMeta); + const repository = yield* Repository; + const record = Option.isSome(meta) ? yield* meta.value.read : {}; + + switch (unit.kind) { + case "prune-invalid": + case "prune-retired": { + if (Option.isNone(store)) return { unit, result: { pruned: 0 } }; + yield* Effect.forEach(unit.ids, (id) => store.value.delete(id), { discard: true }); + return { unit, result: { pruned: unit.ids.length } }; + } + case "build-full": + case "build-incremental": { + if (Option.isNone(bundles)) { + return { unit, result: { skipped: "bundles are not configured" } }; + } + const kind = unit.kind === "build-full" ? "full" : "incremental"; + const built = yield* bundles.value.build({ kind, filter: unit.filter }); + if (Option.isSome(meta)) { + yield* meta.value.write(stamp(record, "lastBundleAttempt")); + } + return { unit, result: { id: built.id, bytes: built.bytes, objects: built.objectId } }; + } + case "fsck": { + const report = yield* repository.fsck; + if (Option.isSome(meta)) yield* meta.value.write(stamp(record, "lastFsck")); + return { unit, result: report }; + } + case "gc": { + const report = yield* repository.gc({}); + if (Option.isSome(meta)) yield* meta.value.write(stamp(record, "lastGc")); + return { unit, result: report }; + } + case "repack": { + const report = yield* repository.gc({ repack: true }); + if (Option.isSome(meta)) { + yield* meta.value.write({ + ...stamp(record, "lastGc"), + lastRepack: new Date().toISOString(), + }); + } + return { unit, result: report }; + } + } +}); + +export interface TickResult { + readonly planned: ReadonlyArray; + readonly ran: MaintenanceUnit | null; + readonly result?: unknown; + readonly operationId?: string; +} + +const kindName = (unit: MaintenanceUnit): string => { + switch (unit.kind) { + case "build-full": + return "bundle.build"; + case "build-incremental": + return "bundle.build"; + case "prune-invalid": + case "prune-retired": + return "bundle.prune"; + case "fsck": + return "maintenance.fsck"; + case "gc": + return "maintenance.gc"; + case "repack": + return "maintenance.repack"; + } +}; + +/** Plan and execute the highest-priority unit. */ +export const tick = Effect.fn("Maintenance.tick")(function* () { + const features = yield* featuresOf(); + if (!features.maintenance.enabled && !features.bundles.enabled) { + return { planned: [], ran: null } satisfies TickResult; + } + const captured = yield* snapshot(); + const planned = planMaintenance(captured, asConfig(features)); + const unit = nextUnit(captured, asConfig(features)); + if (unit === null) return { planned, ran: null } satisfies TickResult; + + const operations = yield* Effect.serviceOption(Operations); + const repo = (yield* storageOf()) ?? "repository"; + if (Option.isNone(operations)) { + const executed = yield* executeUnit(unit); + return { planned, ran: unit, result: executed.result } satisfies TickResult; + } + + const ran = yield* runOperation( + { + repo, + kind: kindName(unit), + cancellable: unit.kind === "build-full" || unit.kind === "build-incremental", + }, + (handle) => executeUnit(unit).pipe(Effect.tap(() => handle.info("complete"))), + ); + return { + planned, + ran: unit, + operationId: ran.operation.id, + } satisfies TickResult; +}); + +/** Replan after each unit until nothing remains. */ +export const runAll = Effect.fn("Maintenance.runAll")(function* () { + const ticks: TickResult[] = []; + for (let step = 0; step < 32; step++) { + const result = yield* tick(); + ticks.push(result); + if (result.ran === null) break; + } + return ticks; +}); + +export { planMaintenance }; +export type { MaintenanceUnit }; diff --git a/src/server/Operations.test.ts b/src/server/Operations.test.ts new file mode 100644 index 0000000..31cc59c --- /dev/null +++ b/src/server/Operations.test.ts @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { describe, it } from "@effect/vitest"; + +import { Effect } from "effect"; + +import { memoryLayer, Operations } from "./Operations.ts"; + +const scenario = (effect: Effect.Effect) => + Effect.runPromise(effect.pipe(Effect.provide(memoryLayer))); + +describe("Operations state", () => { + it.effect("transitions queued → running → succeeded", () => + Effect.promise(async () => { + const operation = await scenario( + Effect.gen(function* () { + const operations = yield* Operations; + const created = yield* operations.create({ repo: "demo", kind: "bundle.build" }); + yield* operations.start(created.id); + yield* operations.progress(created.id, { current: 1, total: 2, unit: "objects" }, "walk"); + return yield* operations.succeed(created.id); + }), + ); + assert.equal(operation.state, "succeeded"); + assert.equal(operation.progress?.current, 1); + }), + ); + + it.effect("refuses cancellation after the commit point", () => + Effect.promise(async () => { + const tag = await scenario( + Effect.gen(function* () { + const operations = yield* Operations; + const created = yield* operations.create({ repo: "demo", kind: "bundle.build" }); + yield* operations.start(created.id); + yield* operations.markCommitted(created.id); + return (yield* Effect.flip(operations.cancel(created.id)))._tag; + }), + ); + assert.equal(tag, "Invalid"); + }), + ); + + it.effect("marks a cooperative cancel as cancelled", () => + Effect.promise(async () => { + const state = await scenario( + Effect.gen(function* () { + const operations = yield* Operations; + const created = yield* operations.create({ repo: "demo", kind: "walk" }); + yield* operations.start(created.id); + return (yield* operations.cancel(created.id)).state; + }), + ); + assert.equal(state, "cancelled"); + }), + ); +}); + +describe("Operations.memory layer", () => { + it.effect("lists by state", () => + Effect.promise(async () => { + const running = await scenario( + Effect.gen(function* () { + const operations = yield* Operations; + const first = yield* operations.create({ repo: "a", kind: "maintenance.fsck" }); + yield* operations.start(first.id); + yield* operations.create({ repo: "a", kind: "maintenance.gc" }); + return yield* operations.list({ state: "running" }); + }), + ); + assert.equal(running.length, 1); + assert.equal(running[0]?.kind, "maintenance.fsck"); + }), + ); +}); diff --git a/src/server/Operations.ts b/src/server/Operations.ts new file mode 100644 index 0000000..8748023 --- /dev/null +++ b/src/server/Operations.ts @@ -0,0 +1,470 @@ +/** + * Observable host operations — not hub Tasks. + * + * An Operation wraps work another subsystem already knows how to run. It is + * not a durable queue: if the process disappears, desired-state maintenance + * decides whether the work still needs doing and starts a new operation. + */ +import { Context, Effect, Layer } from "effect"; + +import { Invalid, OperationNotFound } from "../git/Error.ts"; + +export type OperationState = "queued" | "running" | "succeeded" | "failed" | "cancelled"; +export type OperationLevel = "debug" | "info" | "warning" | "error"; + +export interface OperationProgress { + readonly current?: number; + readonly total?: number; + readonly unit?: string; +} + +export interface OperationError { + readonly tag: string; + readonly message: string; +} + +export interface Operation { + readonly id: string; + readonly repo: string; + readonly kind: string; + readonly state: OperationState; + readonly createdAt: string; + readonly startedAt?: string; + readonly finishedAt?: string; + readonly progress?: OperationProgress; + readonly message?: string; + readonly error?: OperationError; + readonly cancellable: boolean; +} + +export interface OperationEvent { + readonly sequence: number; + readonly at: string; + readonly level: OperationLevel; + readonly message: string; + readonly progress?: OperationProgress; +} + +export type OperationSse = + | { readonly type: "progress"; readonly event: OperationEvent } + | { readonly type: "message"; readonly event: OperationEvent } + | { readonly type: "completed"; readonly operation: Operation } + | { readonly type: "failed"; readonly operation: Operation }; + +export interface OperationHandle { + readonly id: string; + readonly progress: (progress: OperationProgress, message?: string) => Effect.Effect; + readonly info: (message: string) => Effect.Effect; + readonly warn: (message: string) => Effect.Effect; + /** Mark the commit point: cancellation may stop follow-up work but not undo. */ + readonly commit: Effect.Effect; +} + +const nowIso = () => new Date().toISOString(); + +const nextId = (): string => crypto.randomUUID(); + +interface Recorded { + operation: Operation; + events: OperationEvent[]; + committed: boolean; + watchers: Set<(event: OperationSse) => void>; +} + +const HISTORY_BOUND = 64; + +export class Operations extends Context.Service< + Operations, + { + readonly create: (input: { + readonly repo: string; + readonly kind: string; + readonly cancellable?: boolean; + }) => Effect.Effect; + readonly get: (id: string) => Effect.Effect; + readonly list: (query?: { + readonly repo?: string; + readonly state?: OperationState; + }) => Effect.Effect>; + readonly history: ( + id: string, + ) => Effect.Effect, OperationNotFound>; + readonly start: (id: string) => Effect.Effect; + readonly progress: ( + id: string, + progress: OperationProgress, + message?: string, + ) => Effect.Effect; + readonly message: ( + id: string, + level: OperationLevel, + message: string, + ) => Effect.Effect; + readonly succeed: (id: string) => Effect.Effect; + readonly fail: ( + id: string, + error: OperationError, + ) => Effect.Effect; + readonly cancel: (id: string) => Effect.Effect; + readonly markCommitted: (id: string) => Effect.Effect; + readonly watch: ( + id: string, + listener: (event: OperationSse) => void, + ) => Effect.Effect<() => void, OperationNotFound>; + } +>()("server/Operations") {} + +const terminal = (state: OperationState): boolean => + state === "succeeded" || state === "failed" || state === "cancelled"; + +export const memory = (): Operations["Service"] => { + const records = new Map(); + + const require = (id: string): Effect.Effect => { + const recorded = records.get(id); + return recorded === undefined + ? Effect.fail(new OperationNotFound({ id })) + : Effect.succeed(recorded); + }; + + const emit = (recorded: Recorded, event: OperationSse) => { + for (const listener of recorded.watchers) listener(event); + }; + + const append = ( + recorded: Recorded, + level: OperationLevel, + message: string, + progress?: OperationProgress, + ): OperationEvent => { + const event: OperationEvent = + progress === undefined + ? { sequence: recorded.events.length + 1, at: nowIso(), level, message } + : { sequence: recorded.events.length + 1, at: nowIso(), level, message, progress }; + recorded.events.push(event); + if (recorded.operation.state !== "queued" && recorded.operation.state !== "running") { + if (recorded.events.length > HISTORY_BOUND) { + recorded.events = recorded.events.slice(-HISTORY_BOUND); + } + } + if (progress !== undefined) { + recorded.operation = { ...recorded.operation, progress, message }; + emit(recorded, { type: "progress", event }); + } else { + recorded.operation = { ...recorded.operation, message }; + emit(recorded, { type: "message", event }); + } + return event; + }; + + return Operations.of({ + create: (input) => + Effect.sync(() => { + const operation: Operation = { + id: nextId(), + repo: input.repo, + kind: input.kind, + state: "queued", + createdAt: nowIso(), + cancellable: input.cancellable !== false, + }; + records.set(operation.id, { + operation, + events: [], + committed: false, + watchers: new Set(), + }); + return operation; + }), + + get: (id) => require(id).pipe(Effect.map((recorded) => recorded.operation)), + + list: (query) => + Effect.sync(() => + [...records.values()] + .map((recorded) => recorded.operation) + .filter((operation) => { + if (query?.repo !== undefined && operation.repo !== query.repo) return false; + if (query?.state !== undefined && operation.state !== query.state) return false; + return true; + }) + .sort((left, right) => right.createdAt.localeCompare(left.createdAt)), + ), + + history: (id) => require(id).pipe(Effect.map((recorded) => recorded.events)), + + start: (id) => + require(id).pipe( + Effect.map((recorded) => { + if (recorded.operation.state !== "queued") return; + recorded.operation = { + ...recorded.operation, + state: "running", + startedAt: nowIso(), + }; + append(recorded, "info", "started"); + }), + ), + + progress: (id, progress, message) => + require(id).pipe( + Effect.map((recorded) => { + append(recorded, "info", message ?? recorded.operation.message ?? "", progress); + }), + ), + + message: (id, level, message) => + require(id).pipe( + Effect.map((recorded) => { + append(recorded, level, message); + }), + ), + + succeed: (id) => + require(id).pipe( + Effect.map((recorded) => { + if (!terminal(recorded.operation.state)) { + recorded.operation = { + ...recorded.operation, + state: "succeeded", + finishedAt: nowIso(), + message: recorded.operation.message ?? "complete", + }; + append(recorded, "info", "complete"); + emit(recorded, { type: "completed", operation: recorded.operation }); + } + return recorded.operation; + }), + ), + + fail: (id, error) => + require(id).pipe( + Effect.map((recorded) => { + if (!terminal(recorded.operation.state)) { + recorded.operation = { + ...recorded.operation, + state: "failed", + finishedAt: nowIso(), + error, + message: error.message, + }; + append(recorded, "error", error.message); + emit(recorded, { type: "failed", operation: recorded.operation }); + } + return recorded.operation; + }), + ), + + cancel: (id) => + require(id).pipe( + Effect.flatMap((recorded) => { + if (terminal(recorded.operation.state)) { + return Effect.succeed(recorded.operation); + } + if (!recorded.operation.cancellable || recorded.committed) { + return Effect.fail( + new Invalid({ + field: "operation", + reason: recorded.committed + ? "this operation has passed its commit point" + : "this operation cannot be cancelled", + }), + ); + } + recorded.operation = { + ...recorded.operation, + state: "cancelled", + finishedAt: nowIso(), + message: "cancelled", + }; + append(recorded, "warning", "cancelled"); + emit(recorded, { type: "failed", operation: recorded.operation }); + return Effect.succeed(recorded.operation); + }), + ), + + markCommitted: (id) => + require(id).pipe( + Effect.map((recorded) => { + recorded.committed = true; + }), + ), + + watch: (id, listener) => + require(id).pipe( + Effect.map((recorded) => { + recorded.watchers.add(listener); + return () => { + recorded.watchers.delete(listener); + }; + }), + ), + }); +}; + +export const memoryLayer = Layer.sync(Operations, memory); + +export const handleOf = (operations: Operations["Service"], id: string): OperationHandle => ({ + id, + progress: (progress, message) => + operations + .progress(id, progress, message) + .pipe(Effect.catchTag("OperationNotFound", () => Effect.void)), + info: (message) => + operations + .message(id, "info", message) + .pipe(Effect.catchTag("OperationNotFound", () => Effect.void)), + warn: (message) => + operations + .message(id, "warning", message) + .pipe(Effect.catchTag("OperationNotFound", () => Effect.void)), + commit: operations.markCommitted(id).pipe(Effect.ignore), +}); + +/** + * Create, start, run, and complete an operation around `work`. + * + * Interruption before the commit point marks the operation cancelled. + * After the commit point the work is allowed to finish reporting success. + */ +export const run = Effect.fn("Operations.run")(function* ( + input: { + readonly repo: string; + readonly kind: string; + readonly cancellable?: boolean; + }, + work: (handle: OperationHandle) => Effect.Effect, +) { + const operations = yield* Operations; + const created = yield* operations.create(input); + yield* operations.start(created.id); + const handle = handleOf(operations, created.id); + + const finished = yield* work(handle).pipe( + Effect.tap(() => operations.succeed(created.id)), + Effect.tapCause(() => operations.fail(created.id, { tag: "Failed", message: "failed" })), + Effect.onInterrupt(() => + operations.cancel(created.id).pipe( + Effect.catchTag(["Invalid", "OperationNotFound"], () => Effect.void), + Effect.asVoid, + ), + ), + ); + + const operation = yield* operations.get(created.id); + return { operation, result: finished }; +}); + +const sseFrame = (event: string, payload: string): string => + `event: ${event}\ndata: ${payload}\n\n`; + +const encodeEvent = (event: OperationEvent): string => + JSON.stringify({ + sequence: event.sequence, + at: event.at, + level: event.level, + message: event.message, + progress: event.progress ?? null, + }); + +const encodeOperation = (operation: Operation): string => + JSON.stringify({ + id: operation.id, + repo: operation.repo, + kind: operation.kind, + state: operation.state, + createdAt: operation.createdAt, + message: operation.message ?? null, + }); + +/** `text/event-stream` body for one operation, replaying history first. */ +export const eventsResponse = Effect.fn("Operations.eventsResponse")(function* (id: string) { + const operations = yield* Operations; + const operation = yield* operations.get(id); + const history = yield* operations.history(id); + const encoder = new TextEncoder(); + let forward: ((event: OperationSse) => void) | undefined; + const stop = yield* operations.watch(id, (event) => { + if (forward !== undefined) forward(event); + }); + + const stream = new ReadableStream({ + start: (controller) => { + const send = (event: OperationSse) => { + if (event.type === "progress") { + controller.enqueue(encoder.encode(sseFrame("progress", encodeEvent(event.event)))); + } else if (event.type === "message") { + controller.enqueue(encoder.encode(sseFrame("message", encodeEvent(event.event)))); + } else if (event.type === "completed") { + controller.enqueue( + encoder.encode(sseFrame("completed", encodeOperation(event.operation))), + ); + stop(); + controller.close(); + } else { + controller.enqueue(encoder.encode(sseFrame("failed", encodeOperation(event.operation)))); + stop(); + controller.close(); + } + }; + forward = send; + + for (const event of history) { + controller.enqueue( + encoder.encode( + sseFrame(event.progress === undefined ? "message" : "progress", encodeEvent(event)), + ), + ); + } + if (terminal(operation.state)) { + controller.enqueue( + encoder.encode( + sseFrame( + operation.state === "succeeded" ? "completed" : "failed", + encodeOperation(operation), + ), + ), + ); + stop(); + controller.close(); + } + }, + cancel: () => { + stop(); + }, + }); + + return new Response(stream, { + headers: { + "content-type": "text/event-stream", + "cache-control": "no-cache", + connection: "keep-alive", + }, + }); +}); + +/** CLI line for one progress event. */ +export const renderLine = (kind: string, event: OperationEvent): string => { + const progress = event.progress; + if (progress?.current !== undefined && progress.total !== undefined) { + const unit = progress.unit === undefined ? "" : ` ${progress.unit}`; + return `* ${kind}: ${event.message} ${progress.current.toLocaleString()} / ${progress.total.toLocaleString()}${unit}`; + } + if (progress?.current !== undefined && progress.total === undefined) { + return `* ${kind}: ${event.message} ${progress.current}%`; + } + return `* ${kind}: ${event.message}`; +}; + +/** + * Route `GET …/operations/:id/events`. `null` when the path is not that. + */ +export const handleEvents = Effect.fn("Operations.handleEvents")(function* (request: Request) { + if (request.method !== "GET") return null; + const segments = new URL(request.url).pathname.split("/").filter((segment) => segment !== ""); + const at = segments.lastIndexOf("operations"); + const id = segments[at + 1]; + if (at === -1 || segments[at + 2] !== "events" || id === undefined) return null; + if (segments.length !== at + 3) return null; + return yield* eventsResponse(id); +}); diff --git a/src/server/Planner.test.ts b/src/server/Planner.test.ts new file mode 100644 index 0000000..461fd0d --- /dev/null +++ b/src/server/Planner.test.ts @@ -0,0 +1,89 @@ +import assert from "node:assert/strict"; +import { describe, it } from "@effect/vitest"; + +import { Duration, Effect } from "effect"; + +import type { Oid } from "../git/Oid.ts"; +import { defaults } from "./Features.ts"; +import { nextUnit, planMaintenance, type MaintenanceSnapshot } from "./Planner.ts"; + +// SAFETY: forty hex characters by construction. +const oid = (char: string): Oid => char.repeat(40) as Oid; + +const now = new Date("2024-06-01T00:00:00.000Z"); + +const empty: MaintenanceSnapshot = { + now, + refs: { "refs/heads/main": oid("a") }, + bundles: null, + storedIds: [], +}; + +describe("Planner priority", () => { + it.effect("cleans incomplete artifacts before building", () => + Effect.sync(() => { + const planned = planMaintenance({ ...empty, storedIds: ["tmp/half"] }, defaults); + assert.equal(planned[0]?.kind, "prune-invalid"); + assert.equal( + nextUnit({ ...empty, storedIds: ["tmp/half"] }, defaults)?.kind, + "prune-invalid", + ); + }), + ); + + it.effect("builds a full bundle before incrementals and fsck", () => + Effect.sync(() => { + const planned = planMaintenance(empty, defaults); + assert.equal(planned[0]?.kind, "build-full"); + assert.ok(planned.some((unit) => unit.kind === "fsck")); + }), + ); + + it.effect("is deterministic for identical state", () => + Effect.sync(() => { + const left = planMaintenance(empty, defaults); + const right = planMaintenance({ ...empty }, defaults); + assert.deepEqual(left, right); + }), + ); + + it.effect("asks for an incremental once a full base exists and is young", () => + Effect.sync(() => { + const snapshot: MaintenanceSnapshot = { + ...empty, + now: new Date(now.getTime() + Duration.toMillis(Duration.hours(2))), + bundles: { + version: 1, + families: [ + { + filter: null, + full: { + id: "full-1", + kind: "full", + filter: null, + creationToken: 1n, + refs: empty.refs, + prerequisites: [], + objectId: "full/1-aa.bundle", + bytes: 1, + checksum: "aa", + createdAt: now.toISOString(), + }, + incrementals: [], + }, + ], + }, + }; + const planned = planMaintenance(snapshot, defaults); + assert.ok(planned.some((unit) => unit.kind === "build-incremental")); + assert.ok(!planned.some((unit) => unit.kind === "build-full" && unit.filter === null)); + }), + ); + + it.effect("does not plan work for a repository with no refs", () => + Effect.sync(() => { + const planned = planMaintenance({ ...empty, refs: {} }, defaults); + assert.ok(!planned.some((unit) => unit.kind.startsWith("build-"))); + }), + ); +}); diff --git a/src/server/Planner.ts b/src/server/Planner.ts new file mode 100644 index 0000000..ee8970b --- /dev/null +++ b/src/server/Planner.ts @@ -0,0 +1,139 @@ +/** + * Pure desired-state maintenance planner. + * + * Given repository state and configuration now, what is the single most + * valuable bounded unit of missing work? The planner does no I/O. A missed + * timer therefore causes delay, not permanently missing state. + */ +import { Duration } from "effect"; + +import type { Oid } from "../git/Oid.ts"; +import { + advertisedIds, + type BundleFilter, + type BundleManifest, + familyState, +} from "./BundleFormat.ts"; +import type { BundlePolicy, MaintenancePolicy } from "./Features.ts"; + +export type MaintenanceUnit = + | { readonly kind: "prune-invalid"; readonly ids: ReadonlyArray } + | { readonly kind: "build-full"; readonly filter: BundleFilter } + | { readonly kind: "build-incremental"; readonly filter: BundleFilter } + | { readonly kind: "prune-retired"; readonly ids: ReadonlyArray } + | { readonly kind: "fsck" } + | { readonly kind: "gc" } + | { readonly kind: "repack" }; + +export interface MaintenanceSnapshot { + readonly now: Date; + readonly refs: Readonly>; + readonly bundles: BundleManifest | null; + readonly storedIds: ReadonlyArray; + readonly lastFsck?: Date; + readonly lastGc?: Date; + readonly lastRepack?: Date; + readonly lastBundleAttempt?: Date; + readonly looseObjectCount?: number; + readonly packCount?: number; +} + +export interface MaintenanceConfig { + readonly bundles: BundlePolicy; + readonly maintenance: MaintenancePolicy; +} + +const olderThan = (last: Date | undefined, now: Date, max: Duration.Duration): boolean => { + if (last === undefined) return true; + return now.getTime() - last.getTime() >= Duration.toMillis(max); +}; + +const newerThan = (last: Date | undefined, now: Date, min: Duration.Duration): boolean => { + if (last === undefined) return false; + return now.getTime() - last.getTime() < Duration.toMillis(min); +}; + +const hasRefs = (refs: Readonly>): boolean => Object.keys(refs).length > 0; + +const latestFamilyActivity = ( + snapshot: MaintenanceSnapshot, + filter: BundleFilter, +): Date | undefined => { + const family = familyState(snapshot.bundles, filter); + const latest = family.incrementals.at(-1) ?? family.full; + if (latest !== undefined && latest !== null) return new Date(latest.createdAt); + return snapshot.lastBundleAttempt; +}; + +const invalidIds = (snapshot: MaintenanceSnapshot): ReadonlyArray => + snapshot.storedIds.filter((id) => id.startsWith("tmp/")); + +const retiredIds = (snapshot: MaintenanceSnapshot): ReadonlyArray => { + const advertised = advertisedIds(snapshot.bundles); + return snapshot.storedIds.filter((id) => !id.startsWith("tmp/") && !advertised.has(id)); +}; + +/** + * Planned units in priority order. The scheduler executes the first expensive + * unit; cheap cleanup may be grouped by the executor when its cost is nothing. + */ +export const planMaintenance = ( + snapshot: MaintenanceSnapshot, + config: MaintenanceConfig, +): ReadonlyArray => { + const units: MaintenanceUnit[] = []; + const advertised = hasRefs(snapshot.refs); + + const invalid = invalidIds(snapshot); + if (invalid.length > 0) units.push({ kind: "prune-invalid", ids: invalid }); + + if (config.bundles.enabled && advertised) { + for (const filter of config.bundles.filters) { + const family = familyState(snapshot.bundles, filter); + const last = latestFamilyActivity(snapshot, filter); + const incrementals = family.incrementals.length; + const fullStale = + family.full === null || + olderThan(new Date(family.full.createdAt), snapshot.now, config.bundles.fullMaxAge); + const overLong = incrementals >= config.bundles.maxIncrementals; + + if (family.full === null || (fullStale && incrementals > 0) || overLong) { + units.push({ kind: "build-full", filter }); + continue; + } + if ( + !newerThan(last, snapshot.now, config.bundles.incrementalMinAge) && + incrementals < config.bundles.maxIncrementals + ) { + units.push({ kind: "build-incremental", filter }); + } + } + } + + const retired = retiredIds(snapshot); + if (retired.length > 0) units.push({ kind: "prune-retired", ids: retired }); + + if (olderThan(snapshot.lastFsck, snapshot.now, config.maintenance.fsckInterval)) { + units.push({ kind: "fsck" }); + } + + const loose = snapshot.looseObjectCount ?? 0; + const packs = snapshot.packCount ?? 0; + if ( + olderThan(snapshot.lastGc, snapshot.now, config.maintenance.gcInterval) || + loose >= config.maintenance.repackThreshold + ) { + units.push({ kind: "gc" }); + } + if (packs >= 2 || loose >= config.maintenance.repackThreshold) { + units.push({ kind: "repack" }); + } + + return units; +}; + +/** Highest-priority unit, or `null` when the repository needs nothing. */ +export const nextUnit = ( + snapshot: MaintenanceSnapshot, + config: MaintenanceConfig, +): MaintenanceUnit | null => planMaintenance(snapshot, config)[0] ?? null; diff --git a/src/server/Protocol.interop.test.ts b/src/server/Protocol.interop.test.ts index 461e208..4894dc7 100644 --- a/src/server/Protocol.interop.test.ts +++ b/src/server/Protocol.interop.test.ts @@ -520,4 +520,75 @@ describe.skipIf(!hasGit)("Protocol interop with git", () => { await git(work, "fsck", "--strict"); }), ); + + const publishBundle = async (repo: string) => { + const { fileLayer } = await import("./Bundles.node.ts"); + const BundlesMod = await import("./Bundles.ts"); + const { defaultsLayer } = await import("./Features.ts"); + const directory = path.join(root, repo); + const bundleStore = fileLayer(directory); + const live = GitRepository.layer.pipe( + Layer.provide(GitRepository.hooksNoop), + Layer.provideMerge(stores(directory)), + ); + return Effect.runPromise( + Effect.gen(function* () { + const bundles = yield* BundlesMod.Bundles; + return yield* bundles.build({ kind: "full", filter: null }); + }).pipe( + Effect.provide( + BundlesMod.layer.pipe( + Layer.provide(bundleStore), + Layer.provideMerge(live), + Layer.provideMerge(defaultsLayer), + ), + ), + ), + ); + }; + + it.effect("stock clone succeeds when bundle infrastructure is unused", () => + Effect.promise(async () => { + await seed("plain-clone"); + const work = path.join(root, "work-plain-clone"); + await git(root, "-c", "protocol.version=1", "clone", "--quiet", `${base}/plain-clone`, work); + await git(work, "fsck", "--strict"); + }), + ); + + it.effect("stock clone succeeds after a bundle is published", () => + Effect.promise(async () => { + await seed("with-bundle"); + const artifact = await publishBundle("with-bundle"); + assert.ok(artifact.bytes > 0); + const work = path.join(root, "work-with-bundle"); + await git(root, "-c", "protocol.version=2", "clone", "--quiet", `${base}/with-bundle`, work); + assert.equal( + await fs.readFile(path.join(work, "hello.txt"), "utf8"), + "hello from the server\n", + ); + await git(work, "fsck", "--strict"); + }), + ); + + it.effect("unavailable bundle artifacts still let a clone finish", () => + Effect.promise(async () => { + await seed("missing-bundle"); + const artifact = await publishBundle("missing-bundle"); + await fs.rm(path.join(root, "missing-bundle", "gitplus", "bundles", artifact.objectId), { + force: true, + }); + const work = path.join(root, "work-missing-bundle"); + await git( + root, + "-c", + "protocol.version=2", + "clone", + "--quiet", + `${base}/missing-bundle`, + work, + ); + await git(work, "fsck", "--strict"); + }), + ); }); diff --git a/src/server/Protocol.test.ts b/src/server/Protocol.test.ts index 1adf148..56bc68b 100644 --- a/src/server/Protocol.test.ts +++ b/src/server/Protocol.test.ts @@ -1127,3 +1127,45 @@ describe("advertisement hiding", () => { }), ); }); + +describe("protocol v2 bundle-uri", () => { + const advertise = () => + new Request("http://host/repo/info/refs?service=git-upload-pack", { + headers: { "git-protocol": "version=2" }, + }); + + const command = () => + new Request("http://host/repo/git-upload-pack", { + method: "POST", + headers: { "git-protocol": "version=2" }, + body: body([pkt("command=bundle-uri"), FLUSH]), + }); + + it.effect("does not advertise bundle-uri when no bundles are published", () => + Effect.promise(async () => { + const text = await scenario( + Effect.gen(function* () { + const response = yield* Protocol.handle(advertise()); + if (response === null) return ""; + return decoder.decode( + new Uint8Array(yield* Effect.promise(() => response.arrayBuffer())), + ); + }), + ); + assert.match(text, /version 2/); + assert.doesNotMatch(text, /bundle-uri/); + }), + ); + + it.effect("answers bundle-uri with an empty list when none are published", () => + Effect.promise(async () => { + const lines = await scenario( + Effect.gen(function* () { + const response = yield* Protocol.uploadPack(command()); + return linesOf(new Uint8Array(yield* Effect.promise(() => response.arrayBuffer()))).lines; + }), + ); + assert.ok(lines.every((line) => !line.includes(".uri="))); + }), + ); +}); diff --git a/src/server/Protocol.ts b/src/server/Protocol.ts index 8cb4b4c..2a54785 100644 --- a/src/server/Protocol.ts +++ b/src/server/Protocol.ts @@ -27,6 +27,7 @@ import { bandChunks, DELIM, FLUSH, pkt, PktReader } from "../git/Pkt.ts"; import { type ReceiveResult, Repository } from "../git/Repository.ts"; import * as Refspec from "../git/Refspec.ts"; import * as Auth from "./Auth.ts"; +import * as Bundles from "./Bundles.ts"; import * as Policy from "./Policy.ts"; import * as Redaction from "../hub/Redaction.ts"; import { checkRefAddress, checkRefName, isOid, type Oid, type RefUpdate } from "../git/Store.ts"; @@ -204,18 +205,30 @@ const isVersionTwo = (request: Request): boolean => * longer sends all of them before the client can say it only wants one. The * refs come back from `ls-refs`, filtered by prefix. */ -const advertiseV2 = (): Response => - new Response( - concat([ - pkt("version 2\n"), - pkt(`${AGENT}\n`), - pkt("ls-refs=unborn\n"), - pkt("fetch=shallow\n"), - pkt("object-format=sha1\n"), - FLUSH, - ]), - { headers: headers("application/x-git-upload-pack-advertisement") }, +const advertiseV2 = Effect.fn("Protocol.advertiseV2")(function* () { + const lines = [ + pkt("version 2\n"), + pkt(`${AGENT}\n`), + pkt("ls-refs=unborn\n"), + pkt("fetch=shallow\n"), + pkt("object-format=sha1\n"), + ]; + if (yield* Bundles.advertise()) lines.push(pkt("bundle-uri\n")); + lines.push(FLUSH); + return new Response(concat(lines), { + headers: headers("application/x-git-upload-pack-advertisement"), + }); +}); + +/** Protocol v2 `bundle-uri`: the clone list as flattened config keys. */ +const bundleUri = Effect.fn("Protocol.bundleUri")(function* (request: Request) { + const lines = yield* Bundles.protocolList(request).pipe( + Effect.orElseSucceed((): ReadonlyArray => []), ); + return new Response(concat([...lines.map((line) => pkt(`${line}\n`)), FLUSH]), { + headers: headers("application/x-git-upload-pack-result"), + }); +}); /** `GET /info/refs?service=…` — refs, capabilities on the first line. */ export const advertise = Effect.fn("Protocol.advertise")(function* ( @@ -329,6 +342,7 @@ export const uploadPack = Effect.fn("Protocol.uploadPack")(function* (request: R const parsed = yield* readV2(new PktReader(body(request))); if (parsed.command === "ls-refs") return yield* lsRefs(parsed); if (parsed.command === "fetch") return yield* fetchV2(parsed); + if (parsed.command === "bundle-uri") return yield* bundleUri(request); return yield* new Invalid({ field: "command", reason: `unknown v2 command '${parsed.command}'`, @@ -1041,7 +1055,7 @@ export const handle = (request: Request): Effect.Effect void) | null = null; + + protected override render(): TemplateResult { + const operation = this.operation; + if (operation === null) return html`${nothing}`; + + const percent = percentOf(operation); + const failed = operation.state === "failed" || operation.state === "cancelled"; + const done = operation.state === "succeeded"; + + return html` +
+
+ ${operation.kind} + ${operation.state} +
+
+ ${operation.message === undefined ? nothing : html`

${operation.message}

`} + ${percent === null ? nothing : html``} + ${ + operation.error === undefined + ? nothing + : html`

+ ${operation.error.tag}: ${operation.error.message} +

` + } + ${ + failed && this.onRetry !== null + ? html`` + : nothing + } + ${done ? html`

complete

` : nothing} +
+
+ `; + } +} + +const percentOf = (operation: OperationProgressView): number | null => { + const progress = operation.progress; + if (progress?.current === undefined) return null; + if (progress.total !== undefined && progress.total > 0) { + return Math.min(100, Math.round((progress.current / progress.total) * 100)); + } + return Math.min(100, Math.max(0, progress.current)); +};