diff --git a/Dockerfile b/Dockerfile index 5f629d1..0a3c598 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ COPY --from=builder --chown=bun:bun /app/server ./server # Only the src modules the server imports at runtime: sync-api needs sync-schema, and # pastes needs code.ts (language detection; its one dependency, highlight.js, is in node_modules), # and billing needs diff.ts (the AI summary is asked about the changed lines). -COPY --from=builder --chown=bun:bun /app/src/sync-schema.ts /app/src/flavors.ts /app/src/emoji.ts /app/src/code.ts /app/src/diff.ts /app/src/sensitive.ts /app/src/sync-crypto.ts ./src/ +COPY --from=builder --chown=bun:bun /app/src/sync-schema.ts /app/src/flavors.ts /app/src/emoji.ts /app/src/code.ts /app/src/diff.ts /app/src/sensitive.ts /app/src/sync-crypto.ts /app/src/blob-store.ts ./src/ COPY --from=deps --chown=bun:bun /app/node_modules ./node_modules RUN mkdir -p /data && chown bun:bun /data diff --git a/README.md b/README.md index 10b6786..3e24634 100644 --- a/README.md +++ b/README.md @@ -371,7 +371,28 @@ work offline. Remote URLs need a connection and a host that permits browser access (CORS); files are fetched directly with no credentials or server proxy. Local files are uploaded only when you explicitly choose Share → Save online. -Imports are limited to 4 MB per file, 20 MB per workspace, and 1,000 files. +#### File limits and storage + +| | Without an account | Free account | Pay as you go | +| --- | --- | --- | --- | +| Largest file | 4 MB | 5 GB | 5 GB | +| Number of files | 1,000 (20 MB in all) | no limit | no limit | +| Synced storage | none | 5 GB | $0.05 per GB a month over 5 GB | + +Signed in, a file over 4 MB is a *stored file*: it opens read-only (a PDF or image is +shown whole, text as its first megabyte) and Download gives it back exactly. On the +device it is kept in browser storage as it is; on the account it is end-to-end +encrypted with its own key in 8 MB chunks, and the key travels only inside your +sealed workspace, so readm3.com holds data it cannot read or name. Small files that +do not fit the synced copy are stored the same way and turn back into ordinary, +editable files when opened. + +Storage over the free 5 GB is charged hourly from the prepaid balance (CoinPay +top-ups, as for AI summaries). When the balance is empty, new uploads past the free +allowance are refused; nothing already stored is deleted. `GET /api/v1/blobs` reports +what an account stores, the allowance and the price. Self-hosted servers set +`READM3_FREE_STORAGE_BYTES`, `READM3_STORAGE_MICROS_PER_GB_MONTH` and +`READM3_BLOB_DIR` (default: `blobs/` beside the database). ### File history and AI summaries @@ -713,8 +734,9 @@ conflicting local edits, including on a device that has never synced. `--force` explicitly overrides a conflict. Replaced local files receive numbered `.bak-NNN` backups. Browser replacements receive backups in IndexedDB, accessible through **Sync → Local backups** for download or restoration. Clearing browser site data -also removes those local backups. Sync supports the browser's workspace limits: -1,000 files, 4 MB per file, and 20 MB of source in total. +also removes those local backups. The CLI syncs a readable workspace of up to 4 MB +per file and 20 MB in total; the browser and desktop app sync any number of files of +up to 5 GB each (see File limits and storage). The authenticated API exposes: diff --git a/desktop/main.cjs b/desktop/main.cjs index 4987cbb..93c4cba 100644 --- a/desktop/main.cjs +++ b/desktop/main.cjs @@ -18,15 +18,17 @@ const { app, BrowserWindow, Menu, ipcMain, net, protocol, session, shell } = require("electron"); const { createHash, randomBytes } = require("node:crypto"); const { createServer } = require("node:http"); -const { readFile, readdir, writeFile, stat } = require("node:fs/promises"); +const { open, readFile, readdir, writeFile, stat } = require("node:fs/promises"); const { basename, join, resolve } = require("node:path"); const { fileURLToPath, pathToFileURL } = require("node:url"); const ORIGIN = (process.env.READM3_URL || "https://readm3.com").replace(/\/$/, ""); const LOCAL = "app://readm3"; -const MAX_FILE_BYTES = 4 * 1024 * 1024; -/** The page's own limits (site/workspace.ts); it applies them again, these only bound the walk. */ -const MAX_FILES = 1000; +/** Larger files go to the page without their bytes; it reads them in ranges (read-range). */ +const INLINE_BYTES = 4 * 1024 * 1024; +/** The largest file a signed-in account may keep (src/blob-store.ts); the page applies its own limits again. */ +const MAX_FILE_BYTES = 5 * 1024 ** 3; +const MAX_RANGE = 16 * 1024 * 1024; const IGNORED = new Set(["node_modules", "dist", "build", "out", "target", "vendor", "coverage", "venv", "__pycache__", ".git"]); const NEVER = /\.(?:zip|gz|tgz|bz2|xz|7z|rar|tar|exe|dll|so|dylib|o|a|class|jar|war|wasm|bin|dat|db|sqlite|sqlite3|mp3|mp4|m4a|mov|avi|mkv|webm|ogg|wav|flac|woff2?|ttf|otf|eot|psd|ai|doc|docx|xls|xlsx|ppt|pptx|iso|dmg|pkg|deb|rpm|apk|lock)$/i; /** The bundled editor: inside the package, or the repo's site build when run with `electron .`. */ @@ -48,13 +50,13 @@ let window; async function load(paths, cwd = process.cwd()) { const files = []; const add = async (path, info, relative) => { - if (info.size > MAX_FILE_BYTES || files.length >= MAX_FILES) return; - files.push({ path, name: basename(path), relative, data: new Uint8Array(await readFile(path)) }); + if (info.size > MAX_FILE_BYTES) return; + const data = info.size > INLINE_BYTES ? undefined : new Uint8Array(await readFile(path)); + files.push({ path, name: basename(path), relative, size: info.size, data }); opened.add(path); }; const walk = async (directory, prefix) => { for (const entry of await readdir(directory, { withFileTypes: true })) { - if (files.length >= MAX_FILES) return; if (entry.name.startsWith(".") || IGNORED.has(entry.name)) continue; const path = join(directory, entry.name); const relative = `${prefix}/${entry.name}`; @@ -241,6 +243,18 @@ else { const info = await stat(path); return { data: new Uint8Array(await readFile(path)), modified: info.mtimeMs }; }); + // A large file reaches the page a range at a time, never whole through one message. + ipcMain.handle("read-range", async (event, path, start, end) => { + if (!trusted(event.senderFrame?.url ?? "")) throw new Error("Not allowed."); + if (typeof path !== "string" || !opened.has(path)) throw new Error("readm3 can only read files it opened."); + if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end - start > MAX_RANGE) throw new Error("Invalid range."); + const handle = await open(path, "r"); + try { + const buffer = Buffer.alloc(end - start); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, start); + return new Uint8Array(buffer.subarray(0, bytesRead)); + } finally { await handle.close(); } + }); ipcMain.handle("write-file", async (event, path, data) => { if (!trusted(event.senderFrame?.url ?? "")) throw new Error("Not allowed."); if (typeof path !== "string" || !opened.has(path)) throw new Error("readm3 can only save files it opened."); diff --git a/desktop/package.json b/desktop/package.json index 3e44184..ddf58a2 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -1,7 +1,7 @@ { "name": "readm3-desktop", "productName": "readm3", - "version": "0.15.0", + "version": "0.16.0", "private": true, "description": "readm3 as a desktop app: a plain, fast text editor for Markdown and code.", "author": "Profullstack, Inc. ", diff --git a/desktop/preload.cjs b/desktop/preload.cjs index 3f79bd8..c6a869b 100644 --- a/desktop/preload.cjs +++ b/desktop/preload.cjs @@ -13,6 +13,7 @@ if (origin === "https://readm3.com" || origin === "app://readm3" || (custom && o signInWithBrowser: () => ipcRenderer.invoke("sign-in-with-browser"), onOpenFiles: (callback) => { ipcRenderer.on("open-files", (_event, files) => callback(files)); }, readFile: (path) => ipcRenderer.invoke("read-file", path), + readRange: (path, start, end) => ipcRenderer.invoke("read-range", path, start, end), writeFile: (path, data) => ipcRenderer.invoke("write-file", path, data), }); } diff --git a/package.json b/package.json index 20a1c8e..f696b20 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/readm3", - "version": "0.15.0", + "version": "0.16.0", "description": "Markdown editing and sharing for people and teams. Web/PWA, CLI, API and MCP with version history, view/edit links, and a terminal reader.", "type": "module", "license": "MIT", diff --git a/server/billing.ts b/server/billing.ts index b8dad3b..82c33fe 100644 --- a/server/billing.ts +++ b/server/billing.ts @@ -112,6 +112,15 @@ export class Billing { return result.changes > 0; } + /** Takes money once per reference (storage charges it by the hour); false when already taken. */ + debit(userId: string, amountMicros: number, ref: string, detail: string): boolean { + const result = this.store.run( + "INSERT OR IGNORE INTO billing_ledger (id,userId,kind,amountMicros,ref,arm,detail,createdAt) VALUES (?,?,?,?,?,NULL,?,?)", + id(), userId, "charge", -amountMicros, ref, detail, now(), + ); + return result.changes > 0; + } + account(user: User) { const arm = this.arm(user); return { diff --git a/server/blobs.ts b/server/blobs.ts new file mode 100644 index 0000000..b676589 --- /dev/null +++ b/server/blobs.ts @@ -0,0 +1,208 @@ +/** + * Stored files (src/blob-store.ts): the encrypted chunks of files too large for the + * synced workspace, kept on disk under `///`, with one row per + * blob so an account's storage is a SUM, not a directory walk. + * + * GET /api/v1/blobs usage, the free allowance, the price, every blob + * PUT /api/v1/blobs//?size= one encrypted chunk (application/octet-stream) + * POST /api/v1/blobs//complete every chunk is there: the blob may be referenced + * GET /api/v1/blobs// one encrypted chunk back + * DELETE /api/v1/blobs/ gone, and no longer counted + * + * The free tier needs an account and nothing else: up to 5 GB per file, any number of + * files, 5 GB stored. Past that it is pay as you go: the prepaid balance (CoinPay + * top-ups, server/payments.ts) is charged every hour for what is stored over the + * allowance, and a new chunk that would go over it needs a positive balance. Nothing + * already stored is deleted when the balance runs out; it is still there to download. + */ +import { mkdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { Store, HttpError, now } from "./store.ts"; +import type { Accounts } from "./accounts.ts"; +import type { Billing } from "./billing.ts"; +import { + chunkCount, chunkStoredBytes, formatStorage, FREE_STORAGE_BYTES, GiB, isBlobId, MAX_BLOB_BYTES, STORAGE_MICROS_PER_GIB_MONTH, +} from "../src/blob-store.ts"; + +export const BLOBS_SCHEMA = ` + CREATE TABLE IF NOT EXISTS blobs ( + userId TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + id TEXT NOT NULL, size INTEGER NOT NULL, chunks INTEGER NOT NULL, + stored INTEGER NOT NULL DEFAULT 0, complete INTEGER NOT NULL DEFAULT 0, + createdAt TEXT NOT NULL, PRIMARY KEY(userId, id) + ); + CREATE TABLE IF NOT EXISTS blob_chunks ( + userId TEXT NOT NULL, blobId TEXT NOT NULL, n INTEGER NOT NULL, bytes INTEGER NOT NULL, + PRIMARY KEY(userId, blobId, n), + FOREIGN KEY(userId, blobId) REFERENCES blobs(userId, id) ON DELETE CASCADE + ); +`; +const HOURS_PER_MONTH = 730; +/** An upload left unfinished this long is abandoned; its chunks stop counting. */ +const ABANDONED_MS = 2 * 24 * 3600 * 1000; + +export interface BlobOptions { + dir: string; + freeBytes?: number; + microsPerGibMonth?: number; + /** Whether top-ups are open, so an over-quota refusal can say how to pay. */ + topups?: () => boolean; +} + +type BlobRow = { id: string; size: number; chunks: number; stored: number; complete: number; createdAt: string }; + +export class Blobs { + readonly dir: string; + readonly freeBytes: number; + readonly microsPerGibMonth: number; + private topups: () => boolean; + constructor(private store: Store, private billing: Billing, options: BlobOptions) { + store.db.exec(BLOBS_SCHEMA); + this.dir = options.dir; + this.freeBytes = options.freeBytes ?? FREE_STORAGE_BYTES; + this.microsPerGibMonth = options.microsPerGibMonth ?? STORAGE_MICROS_PER_GIB_MONTH; + this.topups = options.topups ?? (() => false); + mkdirSync(this.dir, { recursive: true }); + } + + used(userId: string): number { + return this.store.get<{ total: number | null }>("SELECT SUM(stored) AS total FROM blobs WHERE userId=?", userId)?.total ?? 0; + } + + account(userId: string) { + return { + usedBytes: this.used(userId), + freeBytes: this.freeBytes, + maxFileBytes: MAX_BLOB_BYTES, + microsPerGibMonth: this.microsPerGibMonth, + balanceMicros: this.billing.balance(userId), + topups: this.topups(), + blobs: this.store.all("SELECT id,size,chunks,stored,complete,createdAt FROM blobs WHERE userId=? ORDER BY createdAt", userId) + .map((row) => ({ ...row, complete: !!row.complete })), + }; + } + + private path(userId: string, blobId: string, n?: number) { + return n === undefined ? join(this.dir, userId, blobId) : join(this.dir, userId, blobId, String(n)); + } + + /** Stores chunk n of a blob, creating the blob on its first chunk. */ + putChunk(userId: string, blobId: string, n: number, size: number, body: Uint8Array) { + if (!isBlobId(blobId)) throw new HttpError(400, "Invalid stored file id."); + if (!Number.isSafeInteger(size) || size < 0) throw new HttpError(400, "size is the file's length in bytes."); + if (size > MAX_BLOB_BYTES) throw new HttpError(413, `A file may be up to ${formatStorage(MAX_BLOB_BYTES)}.`); + const chunks = chunkCount(size); + if (!Number.isSafeInteger(n) || n < 0 || n >= chunks) throw new HttpError(400, "That chunk is outside the file."); + if (body.byteLength !== chunkStoredBytes(size, n)) throw new HttpError(400, `Chunk ${n} must be ${chunkStoredBytes(size, n)} bytes.`); + const blob = this.store.get("SELECT id,size,chunks,stored,complete,createdAt FROM blobs WHERE userId=? AND id=?", userId, blobId); + if (blob && (blob.size !== size)) throw new HttpError(409, "That stored file has a different size."); + if (blob?.complete) throw new HttpError(409, "That stored file is complete; it cannot change."); + const had = this.store.get<{ bytes: number }>("SELECT bytes FROM blob_chunks WHERE userId=? AND blobId=? AND n=?", userId, blobId, n); + const used = this.used(userId); + const after = used + body.byteLength - (had?.bytes ?? 0); + if (after > this.freeBytes && after > used && this.billing.balance(userId) <= 0) { + throw new HttpError(402, `Your free ${formatStorage(this.freeBytes)} of storage is used. ${this.topups() + ? `Add funds to store more: pay as you go, $${(this.microsPerGibMonth / 1e6).toFixed(2)} per GB a month over ${formatStorage(this.freeBytes)}.` + : "Paid storage is not open yet; delete files you no longer need."}`, { usedBytes: used, freeBytes: this.freeBytes }); + } + mkdirSync(this.path(userId, blobId), { recursive: true }); + const file = this.path(userId, blobId, n); + writeFileSync(`${file}.part`, body); + renameSync(`${file}.part`, file); + this.store.db.transaction(() => { + if (!blob) this.store.run("INSERT OR IGNORE INTO blobs (userId,id,size,chunks,stored,complete,createdAt) VALUES (?,?,?,?,0,0,?)", userId, blobId, size, chunks, now()); + this.store.run("INSERT OR REPLACE INTO blob_chunks (userId,blobId,n,bytes) VALUES (?,?,?,?)", userId, blobId, n, body.byteLength); + this.store.run("UPDATE blobs SET stored=(SELECT COALESCE(SUM(bytes),0) FROM blob_chunks WHERE userId=?1 AND blobId=?2) WHERE userId=?1 AND id=?2", userId, blobId); + }).immediate(); + return { usedBytes: this.used(userId) }; + } + + complete(userId: string, blobId: string) { + const blob = this.store.get("SELECT id,size,chunks,stored,complete,createdAt FROM blobs WHERE userId=? AND id=?", userId, blobId); + if (!blob) throw new HttpError(404, "No such stored file."); + const have = this.store.get<{ count: number }>("SELECT COUNT(*) AS count FROM blob_chunks WHERE userId=? AND blobId=?", userId, blobId)!.count; + if (have !== blob.chunks) throw new HttpError(409, `${blob.chunks - have} chunk(s) are still missing.`); + this.store.run("UPDATE blobs SET complete=1 WHERE userId=? AND id=?", userId, blobId); + return { id: blobId, size: blob.size, complete: true }; + } + + chunk(userId: string, blobId: string, n: number): string { + if (!isBlobId(blobId)) throw new HttpError(400, "Invalid stored file id."); + const row = this.store.get<{ bytes: number }>("SELECT bytes FROM blob_chunks WHERE userId=? AND blobId=? AND n=?", userId, blobId, n); + const file = this.path(userId, blobId, n); + try { if (row && statSync(file).size === row.bytes) return file; } catch { /* Missing on disk: not found. */ } + throw new HttpError(404, "That stored file chunk is not here."); + } + + remove(userId: string, blobId: string) { + if (!isBlobId(blobId)) throw new HttpError(400, "Invalid stored file id."); + this.store.run("DELETE FROM blob_chunks WHERE userId=? AND blobId=?", userId, blobId); + this.store.run("DELETE FROM blobs WHERE userId=? AND id=?", userId, blobId); + rmSync(this.path(userId, blobId), { recursive: true, force: true }); + return { usedBytes: this.used(userId) }; + } + + /** + * Once an hour: charge each account for the hour's storage over the allowance, once + * per hour (the ledger reference is unique), and drop uploads abandoned half way. + */ + tick(at = new Date()) { + const cutoff = new Date(at.getTime() - ABANDONED_MS).toISOString(); + for (const row of this.store.all<{ userId: string; id: string }>("SELECT userId,id FROM blobs WHERE complete=0 AND createdAt("SELECT userId, SUM(stored) AS total FROM blobs GROUP BY userId HAVING total > ?", this.freeBytes)) { + const micros = Math.ceil((total - this.freeBytes) / GiB * this.microsPerGibMonth / HOURS_PER_MONTH); + if (micros > 0 && this.billing.debit(userId, micros, `storage:${userId}:${hour}`, JSON.stringify({ storedBytes: total, freeBytes: this.freeBytes, hour }))) charged++; + } + return { charged }; + } +} + +/** The HTTP surface, signed in by session cookie (same origin) or a personal/OAuth bearer token. */ +export function createBlobApi(blobs: Blobs, accounts: Accounts) { + return async (request: Request): Promise => { + const url = new URL(request.url); + const match = url.pathname.match(/^\/api\/v1\/blobs(?:\/([^/]+)(?:\/([^/]+))?)?$/); + if (!match) return null; + const reply = (body: unknown, status = 200) => Response.json(body, { status, headers: { "cache-control": "no-store", "x-content-type-options": "nosniff" } }); + try { + const origin = request.headers.get("origin"); + if ((origin && origin !== accounts.origin) || request.headers.get("sec-fetch-site") === "cross-site") throw new HttpError(403, "Cross-origin API requests are not allowed."); + const authorization = request.headers.get("authorization"); + const token = authorization ? /^Bearer (\S+)$/i.exec(authorization)?.[1] : accounts.token(request); + const user = token ? accounts.authenticate(token) : null; + if (!user) throw new HttpError(401, "Sign in to store files. The free tier only needs an account."); + if (!authorization && request.method !== "GET" && !origin) throw new HttpError(403, "Browser writes require a same-origin Origin header."); + const [, blobId, part] = match; + if (!blobId) { + if (request.method !== "GET") throw new HttpError(405, "Method not allowed."); + return reply(blobs.account(user.id)); + } + if (!part) { + if (request.method !== "DELETE") throw new HttpError(405, "Method not allowed."); + return reply(blobs.remove(user.id, blobId)); + } + if (part === "complete") { + if (request.method !== "POST") throw new HttpError(405, "Method not allowed."); + return reply(blobs.complete(user.id, blobId)); + } + if (!/^\d{1,6}$/.test(part)) throw new HttpError(404, "Not found."); + const n = Number(part); + if (request.method === "GET") { + return new Response(Bun.file(blobs.chunk(user.id, blobId, n)), { headers: { "content-type": "application/octet-stream", "cache-control": "private, no-store", "x-content-type-options": "nosniff" } }); + } + if (request.method !== "PUT") throw new HttpError(405, "Method not allowed."); + const size = Number(url.searchParams.get("size")); + if (size > MAX_BLOB_BYTES) throw new HttpError(413, `A file may be up to ${formatStorage(MAX_BLOB_BYTES)}.`); + const declared = Number(request.headers.get("content-length")); + if (declared && declared !== chunkStoredBytes(size, n)) throw new HttpError(400, `Chunk ${n} must be ${chunkStoredBytes(size, n)} bytes.`); + const body = new Uint8Array(await request.arrayBuffer()); + return reply(blobs.putChunk(user.id, blobId, n, size, body)); + } catch (error) { + if (error instanceof HttpError) return reply({ ok: false, error: error.message, ...error.details }, error.status); + console.error("readm3 blobs:", error instanceof Error ? error.message : "Unknown error"); + return reply({ ok: false, error: "Unable to store files right now." }, 500); + } + }; +} diff --git a/server/test/blobs.test.ts b/server/test/blobs.test.ts new file mode 100644 index 0000000..8e14854 --- /dev/null +++ b/server/test/blobs.test.ts @@ -0,0 +1,93 @@ +import { afterEach, test, expect } from "bun:test"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Store } from "../store.ts"; +import { Accounts } from "../accounts.ts"; +import { Billing } from "../billing.ts"; +import { Blobs, createBlobApi } from "../blobs.ts"; +import { CHUNK_BYTES, chunkStoredBytes, decryptChunk, encryptChunk, newBlobRef, type BlobRef } from "../../src/blob-store.ts"; +import { verifiedAccount } from "./fixtures.ts"; + +const cleanup: (() => void)[] = []; +afterEach(() => { while (cleanup.length) cleanup.pop()!(); }); + +async function fixture(freeBytes = 64 * 1024 * 1024) { + const root = mkdtempSync(join(tmpdir(), "readm3-blobs-")); + const store = new Store(join(root, "test.sqlite")); + const accounts = new Accounts(store.db, "http://localhost", async () => {}); + const billing = new Billing(store, null); + const blobs = new Blobs(store, billing, { dir: join(root, "blobs"), freeBytes, microsPerGibMonth: 50_000, topups: () => true }); + const api = createBlobApi(blobs, accounts); + const server = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: async (request) => await api(request) || new Response("Missing", { status: 404 }) }); + cleanup.push(() => { server.stop(true); store.close(); rmSync(root, { recursive: true, force: true }); }); + const alice = await verifiedAccount(store, "blob_alice"); + const bob = await verifiedAccount(store, "blob_bob"); + const base = `http://127.0.0.1:${server.port}/api/v1/blobs`; + accounts.origin = new URL(base).origin; + const call = (path: string, token: string, method = "GET", body?: BodyInit) => + fetch(`${base}${path}`, { method, headers: { authorization: `Bearer ${token}` }, body }); + async function put(ref: BlobRef, bytes: Uint8Array, token = alice.token) { + for (let n = 0, start = 0; start < Math.max(1, bytes.length); n++, start += CHUNK_BYTES) { + const response = await call(`/${ref.id}/${n}?size=${ref.size}`, token, "PUT", await encryptChunk(ref, n, bytes.subarray(start, start + CHUNK_BYTES)) as BodyInit); + if (!response.ok) return response; + if (start + CHUNK_BYTES >= bytes.length) break; + } + return call(`/${ref.id}/complete`, token, "POST"); + } + return { root, store, billing, blobs, alice, bob, call, put }; +} + +test("a signed-in account stores a multi-chunk file and reads it back; nobody else can", async () => { + const f = await fixture(); + expect((await f.call("", "nope")).status).toBe(401); + const bytes = new Uint8Array(CHUNK_BYTES + 1234).map((_, i) => i % 251); + const ref = newBlobRef(bytes.length); + expect((await f.put(ref, bytes)).status).toBe(200); + const account = await (await f.call("", f.alice.token)).json(); + expect(account.blobs).toEqual([expect.objectContaining({ id: ref.id, size: bytes.length, complete: true, chunks: 2 })]); + expect(account.usedBytes).toBe(bytes.length + 32); + const back = await Promise.all([0, 1].map(async (n) => decryptChunk(ref, n, new Uint8Array(await (await f.call(`/${ref.id}/${n}`, f.alice.token)).arrayBuffer())))); + expect(Buffer.concat(back)).toEqual(Buffer.from(bytes)); + expect((await f.call(`/${ref.id}/0`, f.bob.token)).status).toBe(404); + expect((await f.call(`/${ref.id}/0?size=${ref.size}`, f.alice.token, "PUT", new Uint8Array(chunkStoredBytes(ref.size, 0)))).status).toBe(409); + expect((await f.call(`/${ref.id}`, f.alice.token, "DELETE")).status).toBe(200); + expect((await (await f.call("", f.alice.token)).json()).usedBytes).toBe(0); + expect(existsSync(join(f.root, "blobs", f.alice.user.id, ref.id))).toBe(false); +}); + +test("a file must arrive whole before it is complete, and chunks must be the declared size", async () => { + const f = await fixture(); + const ref = newBlobRef(CHUNK_BYTES + 1); + expect((await f.call(`/${ref.id}/0?size=${ref.size}`, f.alice.token, "PUT", new Uint8Array(10))).status).toBe(400); + expect((await f.call(`/${ref.id}/2?size=${ref.size}`, f.alice.token, "PUT", new Uint8Array(17))).status).toBe(400); + expect((await f.call(`/${ref.id}/1?size=${ref.size}`, f.alice.token, "PUT", new Uint8Array(17))).status).toBe(200); + expect((await f.call(`/${ref.id}/complete`, f.alice.token, "POST")).status).toBe(409); + expect((await f.call(`/${ref.id}/0?size=${6 * 1024 ** 3}`, f.alice.token, "PUT", new Uint8Array(1))).status).toBe(413); +}); + +test("past the free allowance a new upload needs a balance, and storage is charged once an hour", async () => { + const f = await fixture(1024); + const small = newBlobRef(900); + expect((await f.put(small, new Uint8Array(900))).status).toBe(200); + const over = newBlobRef(900); + const refused = await f.put(over, new Uint8Array(900)); + expect(refused.status).toBe(402); + expect((await refused.json()).error).toContain("pay as you go"); + f.billing.credit(f.alice.user.id, 5_000_000, "test-topup", "test"); + expect((await f.put(over, new Uint8Array(900))).status).toBe(200); + const at = new Date("2026-10-19T12:30:00Z"); + expect(f.blobs.tick(at).charged).toBe(1); + expect(f.blobs.tick(at).charged).toBe(0); + expect(f.billing.balance(f.alice.user.id)).toBeLessThan(5_000_000); + expect(f.billing.balance(f.bob.user.id)).toBe(0); +}); + +test("an upload abandoned half way stops counting after two days", async () => { + const f = await fixture(); + const ref = newBlobRef(CHUNK_BYTES + 1); + expect((await f.call(`/${ref.id}/1?size=${ref.size}`, f.alice.token, "PUT", new Uint8Array(17))).status).toBe(200); + expect(f.blobs.used(f.alice.user.id)).toBe(17); + f.blobs.tick(new Date(Date.now() + 3 * 24 * 3600 * 1000)); + expect(f.blobs.used(f.alice.user.id)).toBe(0); +}); diff --git a/site/assets/viewer.css b/site/assets/viewer.css index 0a5d2ba..92182c0 100644 --- a/site/assets/viewer.css +++ b/site/assets/viewer.css @@ -116,6 +116,7 @@ html, body.reader-app { height: 100%; overflow: hidden; } .media-view img { max-width: 100%; height: auto; border: 1px solid var(--border); border-radius: 6px; background: repeating-conic-gradient(color-mix(in srgb, var(--fg) 6%, transparent) 0 25%, transparent 0 50%) 0 0 / 20px 20px; } .media-note { color: var(--muted); font: 11px var(--mono); } .media-note a { color: var(--primary); } +.large-preview { margin: 12px 0 0; font: 13px/1.5 var(--mono); white-space: pre-wrap; overflow-wrap: anywhere; } .reader-app.dragging::after { content: "Drop files here"; position: fixed; inset: 12px; display: grid; place-items: center; background: color-mix(in srgb, var(--bg) 95%, transparent); border: 2px dashed var(--primary); border-radius: 12px; color: var(--primary); font: 20px var(--mono); pointer-events: none; z-index: 10; } [data-theme="light"] { color-scheme: light; } @media (min-width: 1600px) { #reader { padding-left: 90px; } } diff --git a/site/autosync.ts b/site/autosync.ts index 2cb1bf0..dcb8fa8 100644 --- a/site/autosync.ts +++ b/site/autosync.ts @@ -19,6 +19,8 @@ import { deriveKey, MIN_PASSPHRASE, newKdf, open, seal, SEALED_FILE, WrongPassph import { digestDocument, digests, mergeWorkspaces } from "../src/sync-merge.ts"; import { isSealed, validateSyncFiles, type ReaderSettings, type SyncedWorkspace } from "../src/sync-schema.ts"; import { store, stored } from "./sync.ts"; +import * as blobs from "./blobs.ts"; +import { formatStorage, newBlobRef, SNAPSHOT_INLINE_BYTES, type BlobRef } from "../src/blob-store.ts"; const OFF = "readm3:autosync"; export const autoSyncEnabled = (): boolean => { try { return localStorage.getItem(OFF) !== "off"; } catch { return true; } }; @@ -34,7 +36,7 @@ export interface AutoSyncHost { seedUrl: string; app: string; } -type State = { revision: number; base: Record; settings: string }; +type State = { revision: number; base: Record; settings: string; spills?: Record }; type Latest = { revision: number; snapshot: { version: number; files: Record } }; let host: AutoSyncHost | undefined; @@ -142,6 +144,61 @@ async function starterDigests(): Promise> { return starter; } +/** An upload this recent may belong to another device's save in flight; it is not collected yet. */ +const GC_GRACE_MS = 3600_000; + +/** + * The workspace as it is uploaded: every stored file on the account, and small text past + * the snapshot's share spilled to stored files too (the largest first), so the sealed + * snapshot stays small however many files there are. A spill keeps the text's digest, + * so the merge sees the same document, and is reused while the text is unchanged. + */ +async function forUpload(workspace: SyncedWorkspace, remote: SyncedWorkspace | undefined, spills: Record) { + const account = await blobs.account(); + const complete = new Set(account.blobs.filter((blob) => blob.complete).map((blob) => blob.id)); + const known = new Map(Object.entries(spills)); + for (const doc of remote?.documents ?? []) if (doc.blob?.digest) known.set(doc.blob.digest, doc.blob); + const encoder = new TextEncoder(); + const inline = workspace.documents.filter((doc) => !doc.blob).map((doc) => ({ doc, size: encoder.encode(doc.source).length })).sort((a, b) => a.size - b.size); + const spill = new Set(); + let budget = SNAPSHOT_INLINE_BYTES; + for (const { doc, size } of inline) { if (size <= budget) budget -= size; else spill.add(doc.path); } + const nextSpills: Record = {}; + const documents = []; + for (const doc of workspace.documents) { + if (doc.blob) { + if (!complete.has(doc.blob.id)) { + const kept = await blobs.local(doc.blob); + // A file that is on neither side cannot be sent; its reference travels so it is not lost elsewhere. + if (kept) { host?.status(`Uploading ${doc.path.split("/").at(-1)} (${formatStorage(doc.blob.size)})…`); await blobs.upload(doc.blob, kept); complete.add(doc.blob.id); } + } + documents.push(doc); + continue; + } + if (!spill.has(doc.path)) { documents.push(doc); continue; } + const digest = digestDocument(doc); + let ref = known.get(digest); + if (!ref || !complete.has(ref.id)) { + ref = newBlobRef(encoder.encode(doc.source).length, digest); + await blobs.upload(ref, new Blob([doc.source])); + complete.add(ref.id); + } + nextSpills[digest] = ref; + documents.push({ path: doc.path, source: "", ...(doc.language ? { language: doc.language } : {}), blob: ref }); + } + return { workspace: { ...workspace, documents }, spills: nextSpills, account }; +} + +/** Deletes the account's stored files that the saved workspace no longer names. */ +async function collect(workspace: SyncedWorkspace, account: blobs.StorageAccount) { + const used = new Set(workspace.documents.flatMap((doc) => doc.blob ? [doc.blob.id] : [])); + const cutoff = Date.now() - GC_GRACE_MS; + for (const blob of account.blobs) { + if (used.has(blob.id) || Date.parse(blob.createdAt) > cutoff) continue; + try { await blobs.remove(blob.id); } catch { /* The next save tries again. */ } + } +} + async function latest(userId: string): Promise { try { return await request(`settings?accountId=${encodeURIComponent(userId)}`); } catch (error) { if (error instanceof ApiError && error.status === 404) return undefined; throw error; } @@ -191,8 +248,11 @@ async function pass(user: CloudUser) { const onlyStarter = local.documents.every((doc) => (starter ?? {})[doc.path] === digestDocument(doc)); // A readable copy on the account is replaced by a sealed one at once (the server then deletes the readable revisions). const readableOnServer = !!remote && !isSealed(remote.snapshot.files); + let spills = state?.spills ?? {}; if (!(onlyStarter && !remote) && (merge.upload || (settingsChanged && remote) || readableOnServer || rekey)) { - const plain = { "settings.json": { content: serialize(settings) }, "workspace.json": { content: serialize(merge.workspace) } }; + const outgoing = await forUpload(merge.workspace, remoteWorkspace, spills); + spills = outgoing.spills; + const plain = { "settings.json": { content: serialize(settings) }, "workspace.json": { content: serialize(outgoing.workspace) } }; validateSyncFiles(plain); const sealed = { [SEALED_FILE]: { content: JSON.stringify(await seal(plain, syncKey)) } }; try { @@ -202,6 +262,7 @@ async function pass(user: CloudUser) { rekey = false; if (saved.userId !== user.id) throw new Error("your account changed"); revision = saved.revision; + void collect(outgoing.workspace, outgoing.account); } catch (error) { if (error instanceof ApiError && error.status === 409) continue; // Another device saved first. throw error; @@ -210,7 +271,7 @@ async function pass(user: CloudUser) { // Edits made while this pass ran are not overwritten: the next pass merges them. if (!sameDigests(digests(h.read()), localDigests)) { again = true; return; } if (merge.download || serialize(settings) !== localSettings) await h.apply(merge.workspace, settings); - await store(key, { revision, base: digests(merge.workspace), settings: serialize(settings) } satisfies State); + await store(key, { revision, base: digests(merge.workspace), settings: serialize(settings), spills } satisfies State); known = revision; h.status(merge.conflicts.length ? `Synced · ${merge.conflicts.length} conflict cop${merge.conflicts.length === 1 ? "y" : "ies"} kept` diff --git a/site/blobs.ts b/site/blobs.ts new file mode 100644 index 0000000..5aa12d0 --- /dev/null +++ b/site/blobs.ts @@ -0,0 +1,73 @@ +/** + * Stored files on this device and on the account (src/blob-store.ts, server/blobs.ts). + * + * On the device a stored file is one Blob in IndexedDB under `blob:`; browsers keep a + * large Blob on disk, not in memory, so a 5 GB file is fine where a 5 GB string is not. + * On the account it is encrypted chunks, uploaded and downloaded one at a time. + */ +import { ApiError, request } from "./cloud.ts"; +import { readKey, writeKey } from "./workspace.ts"; +import { CHUNK_BYTES, chunkCount, chunkPlainBytes, decryptChunk, encryptChunk, type BlobRef } from "../src/blob-store.ts"; + +export interface StorageAccount { + usedBytes: number; + freeBytes: number; + maxFileBytes: number; + microsPerGibMonth: number; + balanceMicros: number; + topups: boolean; + blobs: { id: string; size: number; complete: boolean; createdAt: string }[]; +} + +const localKey = (id: string) => `blob:${id}`; +export const local = (ref: Pick) => readKey(localKey(ref.id)).catch(() => undefined); +export const keep = (ref: Pick, blob: Blob) => writeKey(localKey(ref.id), blob); +export const forget = (ref: Pick) => writeKey(localKey(ref.id), undefined).catch(() => undefined); + +export const account = () => request("blobs"); +export const remove = (id: string) => request(`blobs/${id}`, "DELETE"); + +async function failure(response: Response): Promise { + let message = `Storage request failed (${response.status})`; + try { message = (await response.json()).error || message; } catch { /* Not JSON. */ } + return new ApiError(message, response.status); +} + +/** Encrypts and uploads every chunk, then marks the file complete; a failed upload restarts from chunk 0. */ +export async function upload(ref: BlobRef, blob: Blob, onProgress?: (done: number) => void) { + for (let n = 0; n < chunkCount(ref.size); n++) { + const start = n * CHUNK_BYTES; + const plain = new Uint8Array(await blob.slice(start, start + chunkPlainBytes(ref.size, n)).arrayBuffer()); + const sealed = await encryptChunk(ref, n, plain); + const response = await fetch(`/api/v1/blobs/${ref.id}/${n}?size=${ref.size}`, { + method: "PUT", credentials: "same-origin", headers: { "content-type": "application/octet-stream" }, body: sealed as BodyInit, + }); + if (!response.ok) throw await failure(response); + onProgress?.(Math.min(ref.size, start + plain.byteLength)); + } + await request(`blobs/${ref.id}/complete`, "POST", {}); +} + +/** Downloads and decrypts a stored file, chunk by chunk, into one Blob. */ +export async function download(ref: BlobRef, type = "", onProgress?: (done: number) => void): Promise { + const parts: Blob[] = []; + let done = 0; + for (let n = 0; n < chunkCount(ref.size); n++) { + const response = await fetch(`/api/v1/blobs/${ref.id}/${n}`, { credentials: "same-origin", cache: "no-store" }); + if (!response.ok) throw await failure(response); + const plain = await decryptChunk(ref, n, new Uint8Array(await response.arrayBuffer())); + parts.push(new Blob([plain as BlobPart])); + done += plain.byteLength; + onProgress?.(done); + } + return new Blob(parts, { type }); +} + +/** The file's bytes: from this device, or fetched from the account once and kept here. */ +export async function bytes(ref: BlobRef, type = "", onProgress?: (done: number) => void): Promise { + const kept = await local(ref); + if (kept) return type && kept.type !== type ? new Blob([kept], { type }) : kept; + const fetched = await download(ref, type, onProgress); + await keep(ref, fetched).catch(() => { /* Browser storage full: still shown, fetched again next time. */ }); + return fetched; +} diff --git a/site/build.ts b/site/build.ts index 93dbd49..2e2cd11 100644 --- a/site/build.ts +++ b/site/build.ts @@ -255,6 +255,7 @@ function home(): string {

Keep the story of the document.

Every online save creates a version. See who changed it, share a specific version, or restore an earlier draft without losing the history.

A workspace for your team.

Organize people into organizations and teams. Share with a whole group, a few collaborators, or anyone with your link.

A plain text editor.

Line numbers, multiple cursors, find and replace, folding, Ctrl+P to any file and Ctrl+Shift+P for every command. Open a folder from disk and Ctrl+S saves back to it. Open the editor.

+

Files up to 5 GB, as many as you like.

A free account lifts the limits: any number of files, each up to 5 GB, with 5 GB of end-to-end encrypted storage that syncs to every device. Need more? Pay as you go, $0.05 per GB a month, from a CoinPay balance.

Take the editor with you.

Install readm3 from your browser, or get the desktop app: Linux AppImage or Flatpak, macOS for Apple silicon or Intel, Windows. Then readm3 edit . opens a folder from the terminal, like subl .. Local files work offline; shared documents stay connected.

Use the tools you already use.

The web editor, CLI, API, and MCP work with the same documents and permissions. Your terminal and your agents can join the workflow.

diff --git a/site/disk.ts b/site/disk.ts index 57a0b3c..7e85705 100644 --- a/site/disk.ts +++ b/site/disk.ts @@ -1,4 +1,4 @@ -import { accepts, MAX_FILES, readKey, writeKey } from "./workspace.ts"; +import { accepts, readKey, writeKey } from "./workspace.ts"; /** * Files that came from disk and can go back to it, through the File System Access API @@ -38,8 +38,8 @@ export async function pickFolder(): Promise<{ name: string; files: Picked[] } | catch (error) { if (cancelled(error)) return null; throw error; } const files: Picked[] = []; const walk = async (directory: Directory, prefix: string) => { + // No count here: the editor applies the limits, which depend on being signed in. for await (const entry of directory.values()) { - if (files.length >= MAX_FILES) return; const path = `${prefix}/${entry.name}`; if (entry.kind === "directory") { if (accepts(`${path}/x`)) await walk(entry as Directory, path); } else if (accepts(path)) { @@ -65,13 +65,18 @@ export async function launched(handles: readonly FileSystemHandle[]): Promise ({ file: await handle.getFile(), path: handle.name, handle }))); } -/** A file the desktop app opened for the page: its path, its name and its bytes; `relative` when it came in a folder. */ -export interface DesktopFile { path: string; name: string; relative?: string; data: Uint8Array; } +/** + * A file the desktop app opened for the page: its path, its name and its bytes; `relative` + * when it came in a folder. A file over 4 MB comes with its size and no bytes; the page + * reads it in ranges. + */ +export interface DesktopFile { path: string; name: string; relative?: string; size?: number; data?: Uint8Array; } interface DesktopBridge { takeLaunchFiles(): Promise; onOpenFiles(callback: (files: DesktopFile[]) => void): void; /** Absent in desktop builds before 0.13.0, which only had the bytes from launch. */ readFile?(path: string): Promise<{ data: Uint8Array; modified: number }>; + readRange?(path: string, start: number, end: number): Promise; writeFile(path: string, data: string | Uint8Array): Promise; } /** Present only inside the readm3 desktop app (desktop/preload.cjs). */ @@ -82,12 +87,25 @@ export const desktop = typeof window === "undefined" ? undefined : (window as un * picker. It lasts for the session: it cannot be stored, and that is fine, because the * desktop hands the file over again the next time it is opened. */ -export function fromDesktop(files: DesktopFile[]): Picked[] { - return files.map((entry) => { +export async function fromDesktop(files: DesktopFile[]): Promise { + const RANGE = 8 * 1024 * 1024; + /** A large file, read range by range into Blob parts, which the browser keeps on disk. */ + const whole = async (entry: DesktopFile): Promise => { + const parts: Blob[] = []; + for (let start = 0; start < entry.size!; start += RANGE) + parts.push(new Blob([await desktop!.readRange!(entry.path, start, Math.min(entry.size!, start + RANGE)) as BlobPart])); + return new File(parts, entry.name); + }; + const picked: Picked[] = []; + for (const entry of files) { + // A large file from a desktop build without ranges cannot be read; it is left out. + if (!entry.data && !desktop!.readRange) continue; + const file = entry.data ? new File([entry.data.buffer as ArrayBuffer], entry.name) : await whole(entry); const handle = { kind: "file", name: entry.name, // Read from disk each time, so the editor sees a file change under it and a save is not undone. getFile: async () => { + if (!entry.data) return whole(entry); if (!desktop!.readFile) return new File([entry.data.buffer as ArrayBuffer], entry.name); const { data, modified } = await desktop!.readFile(entry.path); return new File([data.buffer as ArrayBuffer], entry.name, { lastModified: modified }); @@ -97,8 +115,9 @@ export function fromDesktop(files: DesktopFile[]): Picked[] { return { write: async (data: BlobPart) => { parts.push(data); }, close: async () => { await desktop!.writeFile(entry.path, new Uint8Array(await new Blob(parts).arrayBuffer())); } }; }, } as unknown as FileSystemFileHandle; - return { file: new File([entry.data.buffer as ArrayBuffer], entry.name), path: entry.relative ?? entry.name, handle }; - }); + picked.push({ file, path: entry.relative ?? entry.name, handle }); + } + return picked; } /** Writes the file, asking for write access first if this session has not been granted it. */ diff --git a/site/server.ts b/site/server.ts index a2d6494..0f85dc9 100644 --- a/site/server.ts +++ b/site/server.ts @@ -7,6 +7,7 @@ import { existsSync, readFileSync, statSync } from "node:fs"; import { footerHtml } from "@profullstack/footer"; import { dirname, join, normalize, sep } from "node:path"; +import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; import { Accounts } from "../server/accounts.ts"; import { accountMailer } from "../server/account-mail.ts"; @@ -17,6 +18,7 @@ import { Payments, paymentOptions, type PaymentOptions } from "../server/payment import { CoinPay } from "../server/coinpay.ts"; import { createOAuth } from "../server/oauth.ts"; import { createSyncApi } from "../server/sync-api.ts"; +import { Blobs, createBlobApi } from "../server/blobs.ts"; const dist = join(dirname(fileURLToPath(import.meta.url)), "dist"); const port = Number(process.env.PORT ?? 3000); @@ -109,10 +111,11 @@ function cacheFor(path: string): string { return "public, max-age=300"; } -export function serveSite(options: { accounts?: Accounts; coinpay?: ConstructorParameters[2]; port?: number; hostname?: string; summarize?: Summarize | null; payments?: PaymentOptions } = {}) { +export function serveSite(options: { accounts?: Accounts; coinpay?: ConstructorParameters[2]; port?: number; hostname?: string; summarize?: Summarize | null; payments?: PaymentOptions; blobDir?: string } = {}) { const listenPort = options.port ?? port; + const database = process.env.READM3_DB || join(dirname(dist), "..", "data", "readm3.sqlite"); const accounts = options.accounts ?? new Accounts( - process.env.READM3_DB || join(dirname(dist), "..", "data", "readm3.sqlite"), + database, process.env.READM3_URL || (process.env.NODE_ENV === "production" ? "https://readm3.com" : `http://127.0.0.1:${listenPort}`), accountMailer(), ); @@ -127,6 +130,16 @@ export function serveSite(options: { accounts?: Accounts; coinpay?: ConstructorP const payments = new Payments(store, billing, options.payments ?? paymentOptions()); const api = createApi(store, accounts.origin, new CoinPay(store, accounts, options.coinpay), { billing, payments }); const syncApi = createSyncApi(store, accounts); + // Stored files (server/blobs.ts) live beside the database on the same volume. The free + // allowance and the pay-as-you-go price can be changed without a release. + const blobs = new Blobs(store, billing, { + dir: options.blobDir ?? process.env.READM3_BLOB_DIR ?? (database === ":memory:" ? join(tmpdir(), `readm3-blobs-${process.pid}`) : join(dirname(database), "blobs")), + freeBytes: Number(process.env.READM3_FREE_STORAGE_BYTES) || undefined, + microsPerGibMonth: Number(process.env.READM3_STORAGE_MICROS_PER_GB_MONTH) || undefined, + topups: () => payments.enabled, + }); + const blobApi = createBlobApi(blobs, accounts); + setInterval(() => { try { blobs.tick(); } catch (error) { console.error("readm3 storage charge:", error instanceof Error ? error.message : error); } }, 3600_000).unref(); // `readm3 login`: OAuth 2.1 for the CLI (server/oauth.ts). const oauth = createOAuth(accounts); return Bun.serve({ @@ -165,6 +178,8 @@ export function serveSite(options: { accounts?: Accounts; coinpay?: ConstructorP if (oauthResponse) return oauthResponse; const syncResponse = await syncApi(request); if (syncResponse) return syncResponse; + const blobResponse = await blobApi(request); + if (blobResponse) return blobResponse; const documentResponse = await api(request, ip); if (documentResponse) return documentResponse; diff --git a/site/test/sync.spec.ts b/site/test/sync.spec.ts index ddb26bd..5e0f9a7 100644 --- a/site/test/sync.spec.ts +++ b/site/test/sync.spec.ts @@ -1,4 +1,5 @@ import { test, expect, type Page, type BrowserContext } from "@playwright/test"; +import { stat } from "node:fs/promises"; import { editorInput } from "./editor.ts"; const origin = process.env.READM3_TEST_URL || `http://127.0.0.1:${process.env.READM3_TEST_PORT || 4318}`; @@ -124,3 +125,32 @@ test("sync requires sign-in and account snapshots are not served from the offlin expect(cached).toBe(false); await context.setOffline(false); }); + +test("a signed-in account opens a file over 4 MB read-only and syncs it as an encrypted stored file", async ({ page, context }) => { + await register(context, username()); + await page.goto("/viewer"); + await setPassphrase(page); + const line = "large log line with a marker\n"; + const big = Buffer.from(line.repeat(Math.ceil(9 * 1024 * 1024 / line.length))); + await page.locator("#files-input").setInputFiles({ name: "server.log", mimeType: "text/plain", buffer: big }); + await expect(page.locator("#document-name")).toHaveText("server.log"); + await expect(page.locator("#code-content")).toContainText("too large to edit here"); + await expect(page.locator("#code-content .large-preview")).toContainText("large log line with a marker"); + await expect(page.locator("#edit-mode")).toBeDisabled(); + await expect(page.locator("#document-info")).toContainText("read-only"); + await expect.poll(async () => { + const account = await (await page.request.get("/api/v1/blobs")).json(); + return account.blobs.filter((blob: { complete: boolean }) => blob.complete).map((blob: { size: number }) => blob.size); + }, { timeout: 20_000 }).toEqual([big.length]); + await expect.poll(async () => JSON.stringify(await sealedSnapshot(page)), { timeout: 20_000 }).toContain("sealed.json"); + expect(JSON.stringify(await sealedSnapshot(page))).not.toContain("server.log"); + const downloading = page.waitForEvent("download"); + await page.locator("#download").click(); + expect((await stat((await (await downloading).path())!)).size).toBe(big.length); +}); + +test("signed out, a file over 4 MB asks for a free account", async ({ page }) => { + await page.goto("/viewer"); + await page.locator("#files-input").setInputFiles({ name: "huge.md", mimeType: "text/markdown", buffer: Buffer.alloc(4 * 1024 * 1024 + 1, "a") }); + await expect(page.locator("#notice")).toContainText("Sign in (free) for files up to 5 GB"); +}); diff --git a/site/viewer.ts b/site/viewer.ts index 3cf7940..97b4d29 100644 --- a/site/viewer.ts +++ b/site/viewer.ts @@ -1,4 +1,6 @@ -import { action, request, type CloudDocument } from "./cloud.ts"; +import { action, request, type CloudDocument, type CloudUser } from "./cloud.ts"; +import * as stored from "./blobs.ts"; +import { formatStorage, INLINE_MAX_BYTES, MAX_BLOB_BYTES, newBlobRef } from "../src/blob-store.ts"; import { pasteDialog, publish, sharing, history as versionHistory } from "./sharing.ts"; import { showSync } from "./sync.ts"; import { askPassphrase, autoSyncEnabled, forgetPassphrase, schedule as scheduleSync, setAutoSync, startAutoSync, syncNow } from "./autosync.ts"; @@ -59,6 +61,15 @@ const kind = () => kindOf(current()); const isCode = () => kind().kind === "code"; const isBinary = () => kind().kind === "binary"; const isMarkdown = () => kind().kind === "markdown"; +/** + * Signed in, the free tier has no file count and files up to 5 GB (src/blob-store.ts); + * signed out, the workspace keeps its small local limits. + */ +let signedIn = false; +const ACCOUNT_LIMITS = "Sign in (free) for files up to 5 GB and no limit on the number of files."; +/** A stored file too large for the workspace (not a spilled small one): shown read-only. */ +const isLarge = (doc = current()) => !!doc.blob && !doc.blob.digest; +const PREVIEW_BYTES = 1024 * 1024; const formatBytes = (n: number) => n < 1024 ? `${n} B` : n < 1024 * 1024 ? `${(n / 1024).toFixed(1)} KB` : `${(n / 1024 / 1024).toFixed(1)} MB`; /** Shows the controls that make sense for the active document's kind. */ @@ -67,15 +78,15 @@ function applyKind() { element("flavor-label").hidden = current.kind !== "markdown"; element("spoilers").hidden = current.kind !== "markdown"; element("code-tools").hidden = current.kind !== "code"; - element("copy").hidden = current.kind === "binary"; + element("copy").hidden = current.kind === "binary" || isLarge(); element("file-mark").textContent = current.kind === "markdown" ? "M↓" : current.kind === "code" ? current.language.extensions[0]!.toUpperCase() : current.binary.label.split(" ")[0]!.toUpperCase(); - element("edit-mode").disabled = current.kind === "binary" || !!pasteDocument || (!!cloudDocument && !cloudDocument.canEdit); - if (!cloudDocument && !pasteDocument) element("history").hidden = current.kind === "binary"; + element("edit-mode").disabled = current.kind === "binary" || isLarge() || !!pasteDocument || (!!cloudDocument && !cloudDocument.canEdit); + if (!cloudDocument && !pasteDocument) element("history").hidden = current.kind === "binary" || isLarge(); } -/** Whether the active file may be edited here: not a paste, not a view-only link, not a PDF or image. */ +/** Whether the active file may be edited here: not a paste, not a view-only link, not a PDF, image or file over 4 MB. */ function canEdit(): boolean { - return !pasteDocument && !isBinary() && (!cloudDocument || cloudDocument.canEdit); + return !pasteDocument && !isBinary() && !isLarge() && (!cloudDocument || cloudDocument.canEdit); } function cloudControls() { @@ -381,12 +392,16 @@ function renderMedia(mime: string, media: "pdf" | "image", label: string) { const key = JSON.stringify(["media", workspace.active, current().source.length]); if (key === previousRender) return; previousRender = key; + showMedia(new Blob([bytesOf(current()).buffer as ArrayBuffer], { type: mime }), media, label); +} + +function showMedia(blob: Blob, media: "pdf" | "image", label: string) { content.hidden = true; codeContent.hidden = true; mediaContent.hidden = false; codeView = null; if (mediaUrl) URL.revokeObjectURL(mediaUrl); - mediaUrl = URL.createObjectURL(new Blob([bytesOf(current()).buffer as ArrayBuffer], { type: mime })); + mediaUrl = URL.createObjectURL(blob); mediaContent.replaceChildren(); if (media === "image") { const image = document.createElement("img"); @@ -395,7 +410,7 @@ function renderMedia(mime: string, media: "pdf" | "image", label: string) { mediaContent.append(image); } else { const embed = document.createElement("embed"); - embed.type = mime; + embed.type = blob.type; embed.src = mediaUrl; embed.title = current().path; mediaContent.append(embed); @@ -411,8 +426,58 @@ function renderMedia(mime: string, media: "pdf" | "image", label: string) { mediaContent.append(note); } +/** + * A stored file (site/blobs.ts), read from this device or fetched from the account. A + * small one spilled from the synced copy turns back into an ordinary file; a large one + * is a PDF or image the browser shows, or text shown as a read-only first megabyte. + */ +function renderStored() { + const doc = current(); + const ref = doc.blob!; + const key = JSON.stringify(["stored", doc.path, ref.id]); + if (key === previousRender) return; + previousRender = key; + const active = kind(); + content.hidden = true; + mediaContent.hidden = true; + codeContent.hidden = false; + codeView = null; + const status = document.createElement("p"); + status.className = "media-note"; + status.textContent = `Opening ${formatStorage(ref.size)}…`; + codeContent.replaceChildren(status); + const still = () => workspace.active === doc.path && current().blob?.id === ref.id; + stored.bytes(ref, active.kind === "binary" ? active.binary.mime : "", (done) => { status.textContent = `Downloading from your account… ${Math.floor(done / Math.max(1, ref.size) * 100)}%`; }) + .then(async (blob) => { + if (!still()) return; + if (ref.digest) { + doc.source = await blob.text(); + delete doc.blob; + previousRender = ""; + applyKind(); + documentInfo(); + renderDocument(); + void save(); + return; + } + if (active.kind === "binary") { showMedia(blob, active.binary.kind, active.binary.label); return; } + const pre = document.createElement("pre"); + pre.className = "large-preview"; + pre.textContent = await blob.slice(0, PREVIEW_BYTES).text(); + if (!still()) return; + status.textContent = `This file is ${formatStorage(ref.size)}, too large to edit here. Showing the first 1 MB, read-only; Download gives the whole file.`; + codeContent.replaceChildren(status, pre); + }) + .catch((error) => { + if (!still()) return; + status.textContent = `${error instanceof Error ? error.message : "This file could not be opened."}${signedIn ? "" : " Sign in to fetch it from your account."}`; + previousRender = ""; + }); +} + function renderDocument() { if (!workspace || editing) return; + if (current().blob) { renderStored(); progress(); return; } const active = kind(); if (active.kind === "code") { const key = JSON.stringify(["code", workspace.active, current().source, active.language.id]); @@ -480,6 +545,7 @@ function setMode(edit: boolean, focus = true, atEnd = false) { if (edit && cloudDocument && !cloudDocument.canEdit) { notice("You have view-only access to this document."); return; } if (edit && pasteDocument) { notice("A private link is read-only. Copy or download the file to edit it."); return; } if (edit && isBinary()) { notice("A PDF or image cannot be edited here. Download it to change it elsewhere."); return; } + if (edit && isLarge()) { notice("A file over 4 MB opens read-only. Download it to change it elsewhere."); return; } editing = edit; editor.hidden = !edit; reader.hidden = edit; @@ -495,6 +561,12 @@ function setMode(edit: boolean, focus = true, atEnd = false) { function documentInfo() { const active = kind(); + const ref = current().blob; + if (ref) { + const label = active.kind === "binary" ? active.binary.label : active.kind === "code" ? active.language.label : "Markdown"; + element("document-info").textContent = `${label} · ${formatStorage(ref.size)}${ref.digest ? "" : " · read-only"}`; + return; + } if (active.kind === "binary") { element("document-info").textContent = `${active.binary.label} · ${formatBytes(bytesOf(current()).length)}`; return; @@ -532,8 +604,8 @@ function addDocuments(incoming: Document[], name?: string): Document[] { const added: Document[] = []; for (const doc of incoming) { const size = encoder.encode(doc.source).length; - if (size > MAX_FILE_BYTES || total + size > MAX_WORKSPACE_BYTES || map.size >= MAX_FILES) { - notice("Workspace limit reached: 4 MB per file, 20 MB total, and 1,000 files. Download files you want to keep."); + if (size > MAX_FILE_BYTES || (!signedIn && (total + size > MAX_WORKSPACE_BYTES || map.size >= MAX_FILES))) { + notice(signedIn ? "A file over 4 MB could not be stored apart. Try opening it again." : `Workspace limit reached: 4 MB per file, 20 MB total, and 1,000 files without an account. ${ACCOUNT_LIMITS}`); break; } // Reopening a file refreshes the copy that matches disk (`readm3 edit .` every morning), @@ -541,7 +613,10 @@ function addDocuments(incoming: Document[], name?: string): Document[] { let path = doc.path; let copy = 2; const existing = map.get(path); - if (existing && (existing.source === doc.source || matchesDisk(existing))) total -= encoder.encode(existing.source).length; + if (existing && (existing.source === doc.source || matchesDisk(existing))) { + total -= encoder.encode(existing.source).length; + if (existing.blob && existing.blob.id !== doc.blob?.id) void stored.forget(existing.blob); + } else while (map.has(path)) { const dot = doc.path.lastIndexOf("."); path = dot >= 0 ? `${doc.path.slice(0, dot)} (${copy++})${doc.path.slice(dot)}` : `${doc.path} (${copy++})`; @@ -569,12 +644,28 @@ async function importFiles(files: disk.Picked[], name = "Your workspace") { const handles: (FileSystemFileHandle | undefined)[] = []; let bytes = 0; let skipped = 0; + let needAccount = 0; let unreadable = 0; for (const { file, path, handle } of files) { if (!accepts(path)) continue; // A PDF or image is kept as base64, a third larger than the file; the limits apply to what is stored. const cost = binaryType(path) ? Math.ceil(file.size * 4 / 3) : file.size; - if (cost > MAX_FILE_BYTES || bytes + cost > MAX_WORKSPACE_BYTES || incoming.length >= MAX_FILES) { skipped++; continue; } + if (cost > MAX_FILE_BYTES) { + // Larger files are stored apart as they are (site/blobs.ts), which needs an account. + if (!signedIn) { needAccount++; continue; } + if (file.size > MAX_BLOB_BYTES) { skipped++; continue; } + if (!binaryType(path) && looksBinary(await file.slice(0, 8192).text())) { unreadable++; continue; } + const doc: Document = { path, source: "", blob: newBlobRef(file.size) }; + if (!binaryType(path) && !languageForName(path)) { + const language = detectLanguage(path, await file.slice(0, 65536).text()); + if (language !== MARKDOWN) doc.language = language; + } + await stored.keep(doc.blob!, file); + incoming.push(doc); + handles.push(handle); + continue; + } + if (!signedIn && (bytes + cost > MAX_WORKSPACE_BYTES || incoming.length >= MAX_FILES)) { needAccount++; continue; } if (binaryType(path)) { incoming.push({ path, source: toBase64(new Uint8Array(await file.arrayBuffer())) }); handles.push(handle); @@ -596,13 +687,18 @@ async function importFiles(files: disk.Picked[], name = "Your workspace") { if (incoming.length) { const added = addDocuments(incoming, name); added.forEach((doc, i) => { const handle = handles[i]; if (handle) diskEntries.set(doc.path, { handle, saved: disk.fingerprint(doc.source) }); }); - for (const doc of added) if (!binaryType(doc.path)) void fileHistory.record(doc.path, doc.source, "opened"); + for (const doc of added) if (!binaryType(doc.path) && !doc.blob) void fileHistory.record(doc.path, doc.source, "opened"); if (handles.some(Boolean)) { void disk.storeEntries(diskEntries); renderFiles(); void save(); } } - else notice("No readable files found. Open text files (Markdown, JSON, code, config), PDFs or images up to 4 MB."); - if (skipped) notice(`${skipped} file(s) skipped because of size or workspace limits.`); + else if (!needAccount) notice(`No readable files found. Open text files (Markdown, JSON, code, config), PDFs or images up to ${signedIn ? "5 GB" : "4 MB"}.`); + if (needAccount) notice(`${needAccount} file(s) skipped: over 4 MB, or past 20 MB and 1,000 files, without an account. ${ACCOUNT_LIMITS}`); + if (skipped) notice(`${skipped} file(s) skipped: larger than 5 GB.`); if (unreadable) notice(`${unreadable} file(s) skipped: not text, PDF or image.`); - } catch { notice("A file could not be read. Try selecting it again."); } + } catch (error) { + notice(error instanceof DOMException && error.name === "QuotaExceededError" + ? "This device's browser storage is full. Free some space, or open fewer large files." + : "A file could not be read. Try selecting it again."); + } finally { importBusy = false; filesInput.value = ""; folderInput.value = ""; } } @@ -629,7 +725,7 @@ const diskSeen = new Map(); * reloaded; with local edits it is left alone and the disk version waits in History. */ async function checkDisk() { - if (!workspace || cloudDocument || pasteDocument || document.hidden || isBinary()) return; + if (!workspace || cloudDocument || pasteDocument || document.hidden || isBinary() || current().blob) return; const doc = current(); const entry = diskEntries.get(doc.path); if (!entry) return; @@ -671,7 +767,7 @@ async function openFromDisk(folder: boolean) { */ async function saveFile(saveAs = false) { if (cloudDocument) { await saveCloud(); return; } - if (pasteDocument || !disk.supported()) { download(); return; } + if (pasteDocument || !disk.supported() || current().blob) { download(); return; } const doc = current(); let handle = saveAs ? undefined : diskEntries.get(doc.path)?.handle; try { @@ -690,16 +786,26 @@ async function saveFile(saveAs = false) { function download() { const active = kind(); - const blob = active.kind === "binary" - ? new Blob([bytesOf(current()).buffer as ArrayBuffer], { type: active.binary.mime }) - : new Blob([current().source], { type: `${active.language.mime};charset=utf-8` }); + const doc = current(); + const name = doc.path.split("/").at(-1)!; + const mime = active.kind === "binary" ? active.binary.mime : `${active.language.mime};charset=utf-8`; + if (doc.blob) { + notice(`Preparing ${name} (${formatStorage(doc.blob.size)})…`); + stored.bytes(doc.blob, mime).then((blob) => saveBlob(blob, name)).catch((error) => notice(error instanceof Error ? error.message : "The file could not be downloaded.")); + return; + } + saveBlob(active.kind === "binary" ? new Blob([bytesOf(doc).buffer as ArrayBuffer], { type: mime }) : new Blob([doc.source], { type: mime }), name); + if (!cloudDocument) void save(); +} + +function saveBlob(blob: Blob, name: string) { const url = URL.createObjectURL(blob); const anchor = document.createElement("a"); anchor.href = url; - anchor.download = current().path.split("/").at(-1)!; + anchor.download = name; anchor.click(); - setTimeout(() => URL.revokeObjectURL(url), 1000); - if (!cloudDocument) void save(); + // A multi-gigabyte file takes a while to hand over; the URL outlives the click. + setTimeout(() => URL.revokeObjectURL(url), 60_000); } function storePreference(key: string, value: string) { try { localStorage.setItem(`readm3:${key}`, value); } catch { /* The reader still works without preferences. */ } } @@ -880,23 +986,24 @@ async function fetchDocument(input: string) { const type = response.headers.get("content-type") ?? ""; const givenName = decodeURIComponent(url.pathname.split("/").at(-1) || "").replace(/[/\\]/g, "-"); if (type.includes("text/html") && !/\.html?$/i.test(givenName)) throw new Error("That URL returns a web page. Use the raw file URL instead."); - if (Number(response.headers.get("content-length")) > MAX_FILE_BYTES) throw new Error("This file exceeds the 4 MB limit."); + const limit = signedIn ? MAX_BLOB_BYTES : MAX_FILE_BYTES; + const tooLarge = () => new Error(signedIn ? "This file is over the 5 GB limit." : `This file is over 4 MB. ${ACCOUNT_LIMITS}`); + if (Number(response.headers.get("content-length")) > limit) throw tooLarge(); const stream = response.body?.getReader(); if (!stream) throw new Error("The host returned an empty response."); - const chunks: Uint8Array[] = []; + // Blob parts, not one array: a browser keeps a large Blob on disk. + const parts: Blob[] = []; let length = 0; try { while (true) { const { done, value } = await stream.read(); if (done) break; length += value.byteLength; - if (length > MAX_FILE_BYTES) { await stream.cancel(); throw new Error("This file exceeds the 4 MB limit."); } - chunks.push(value); + if (length > limit) { await stream.cancel(); throw tooLarge(); } + parts.push(new Blob([value as BlobPart])); } } finally { stream.releaseLock(); } - const bytes = new Uint8Array(length); - let offset = 0; - for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.length; } + const whole = new Blob(parts); let name = givenName || "Document.md"; if (!accepts(name)) name = "Document.md"; // A PDF or image served without an extension in its path still gets one from its type. @@ -904,6 +1011,14 @@ async function fetchDocument(input: string) { if (type.startsWith("application/pdf")) name += ".pdf"; else if (type.startsWith("image/")) name += `.${type.slice(6).split(/[;+]/)[0]}`; } + if (length > MAX_FILE_BYTES || (binaryType(name) && Math.ceil(length * 4 / 3) > MAX_FILE_BYTES)) { + if (!binaryType(name) && looksBinary(await whole.slice(0, 8192).text())) throw new Error("That URL is not a text file, PDF or image."); + const doc: Document = { path: name, source: "", blob: newBlobRef(length) }; + await stored.keep(doc.blob!, whole); + addDocuments([doc]); + return; + } + const bytes = new Uint8Array(await whole.arrayBuffer()); if (binaryType(name)) { addDocuments([{ path: name, source: toBase64(bytes) }]); return; } const text = new TextDecoder().decode(bytes); if (looksBinary(text)) throw new Error("That URL is not a text file, PDF or image."); @@ -929,7 +1044,7 @@ function receiveLaunches() { const open = (files: disk.DesktopFile[]) => { if (!files.length) return; const folder = files.find((file) => file.relative)?.relative!.split("/")[0]; - void importFiles(disk.fromDesktop(files), folder); + void disk.fromDesktop(files).then((picked) => importFiles(picked, folder)).catch(() => notice("A file could not be read from disk.")); }; void disk.desktop.takeLaunchFiles().then(open); disk.desktop.onOpenFiles(open); @@ -1001,6 +1116,11 @@ async function setupPwa() { } catch { notice("Offline setup is unavailable. The reader still works while this page is open."); } } +async function refreshAccount() { + try { signedIn = !!(await request<{ user: CloudUser | null }>("me")).user; } + catch { /* Offline: the last answer stands. */ } +} + async function start() { loadPreferences(); if (pasteToken) { @@ -1053,6 +1173,9 @@ async function start() { openDocument(workspace.active, false); bindEvents(); await save(); + // The account decides the limits before anything is opened from a link or the desktop. + await refreshAccount(); + window.addEventListener("focus", () => void refreshAccount()); const params = new URLSearchParams(location.search); if (params.has("new")) { element("new-file").click(); history.replaceState(null, "", "/viewer"); } const remote = params.get("url")?.trim(); diff --git a/src/blob-store.ts b/src/blob-store.ts new file mode 100644 index 0000000..bd71484 --- /dev/null +++ b/src/blob-store.ts @@ -0,0 +1,99 @@ +/** + * Large files, stored apart from the synced workspace. + * + * A signed-in account may keep files of up to 5 GB, and any number of them. A file that + * large cannot live in the workspace JSON (a string, synced as one request), so it is + * a `blob` document: the workspace keeps a reference, and the bytes live on their own, + * in IndexedDB on the device and in fixed-size chunks on readm3.com. + * + * Every blob has its own random AES-256 key. The key travels only inside the reference, + * which travels only inside the sealed (end-to-end encrypted) workspace, so the server + * stores chunks it cannot read and cannot link to a file name. Chunk n is encrypted + * with AES-256-GCM under an IV that is n itself: unique, since no key is reused across + * blobs, and a chunk moved to another position fails authentication. + * + * Small text files that do not fit the snapshot are stored the same way ("spilled") and + * carry the document's digest, so the merge sees the same document either way. + * + * Browser-safe: Web Crypto only. Shared by the page, the server and the tests. + */ +export const GiB = 1024 ** 3; +/** The largest single file an account may keep. */ +export const MAX_BLOB_BYTES = 5 * GiB; +/** Storage every signed-in account gets free; more is pay as you go from the prepaid balance. */ +export const FREE_STORAGE_BYTES = 5 * GiB; +/** Pay as you go: micro-dollars per GiB stored per month, over the free allowance. */ +export const STORAGE_MICROS_PER_GIB_MONTH = 50_000; +/** Plaintext bytes per chunk; each stored chunk is 16 bytes longer (the GCM tag). */ +export const CHUNK_BYTES = 8 * 1024 * 1024; +export const TAG_BYTES = 16; +/** Text larger than this opens read-only, as a preview; smaller text is edited in the workspace. */ +export const INLINE_MAX_BYTES = 4 * 1024 * 1024; +/** Inline text kept inside one synced snapshot; the rest of the small files spill to blobs. */ +export const SNAPSHOT_INLINE_BYTES = 16 * 1024 * 1024; + +export interface BlobRef { + /** 22 url-safe characters: random, never derived from the content. */ + id: string; + /** Plaintext bytes. */ + size: number; + /** Base64 of the 32-byte AES key. */ + key: string; + /** Set on a spilled text file: the document digest, so it compares equal to the inline copy. */ + digest?: string; +} + +const ID = /^[A-Za-z0-9_-]{22}$/; +const KEY = /^[A-Za-z0-9+/]{43}=$/; +export const isBlobId = (value: unknown): value is string => typeof value === "string" && ID.test(value); + +export function blobRefProblem(value: unknown): string | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return "A stored file reference must be an object."; + const ref = value as Record; + if (Object.keys(ref).some((key) => !["id", "size", "key", "digest"].includes(key))) return "A stored file reference has unknown fields."; + if (!isBlobId(ref.id)) return "A stored file has an invalid id."; + if (!Number.isSafeInteger(ref.size) || (ref.size as number) < 0 || (ref.size as number) > MAX_BLOB_BYTES) return "A stored file is larger than 5 GB."; + if (typeof ref.key !== "string" || !KEY.test(ref.key)) return "A stored file has an invalid key."; + if (ref.digest !== undefined && (typeof ref.digest !== "string" || !/^[a-z0-9]{1,16}:[a-z0-9]{1,16}$/.test(ref.digest))) return "A stored file has an invalid digest."; + return null; +} + +const b64 = (bytes: Uint8Array): string => { + let text = ""; + for (let i = 0; i < bytes.length; i += 0x8000) text += String.fromCharCode(...bytes.subarray(i, i + 0x8000)); + return btoa(text); +}; +const unb64 = (text: string): Uint8Array => Uint8Array.from(atob(text), (c) => c.charCodeAt(0)); + +export function newBlobRef(size: number, digest?: string): BlobRef { + const id = b64(crypto.getRandomValues(new Uint8Array(16))).replace(/=+$/, "").replace(/\+/g, "-").replace(/\//g, "_"); + return { id, size, key: b64(crypto.getRandomValues(new Uint8Array(32))), ...(digest ? { digest } : {}) }; +} + +export const chunkCount = (size: number): number => Math.max(1, Math.ceil(size / CHUNK_BYTES)); +/** Plaintext bytes in chunk n of a blob of `size` bytes. */ +export const chunkPlainBytes = (size: number, n: number): number => Math.max(0, Math.min(CHUNK_BYTES, size - n * CHUNK_BYTES)); +/** What the server stores for chunk n: the ciphertext and its tag. */ +export const chunkStoredBytes = (size: number, n: number): number => chunkPlainBytes(size, n) + TAG_BYTES; +/** Everything the server holds for a blob, which is what the account's storage counts. */ +export const storedBytes = (size: number): number => size + chunkCount(size) * TAG_BYTES; + +const iv = (n: number): Uint8Array => { const bytes = new Uint8Array(12); new DataView(bytes.buffer).setUint32(8, n); return bytes; }; +const keys = new Map>(); +const cryptoKey = (ref: BlobRef) => { + let key = keys.get(ref.key); + if (!key) { key = crypto.subtle.importKey("raw", unb64(ref.key) as BufferSource, "AES-GCM", false, ["encrypt", "decrypt"]); keys.set(ref.key, key); } + return key; +}; + +export async function encryptChunk(ref: BlobRef, n: number, plain: Uint8Array): Promise { + return new Uint8Array(await crypto.subtle.encrypt({ name: "AES-GCM", iv: iv(n) as BufferSource }, await cryptoKey(ref), plain as BufferSource)); +} + +export async function decryptChunk(ref: BlobRef, n: number, stored: Uint8Array): Promise { + try { return new Uint8Array(await crypto.subtle.decrypt({ name: "AES-GCM", iv: iv(n) as BufferSource }, await cryptoKey(ref), stored as BufferSource)); } + catch { throw new Error("A stored file chunk failed to decrypt; it was damaged or replaced."); } +} + +export const formatStorage = (bytes: number): string => + bytes >= GiB ? `${(bytes / GiB).toFixed(bytes >= 10 * GiB ? 0 : 1)} GB` : bytes >= 1024 * 1024 ? `${(bytes / 1024 / 1024).toFixed(1)} MB` : `${Math.ceil(bytes / 1024)} KB`; diff --git a/src/cli.ts b/src/cli.ts index 56b91bf..ccd0434 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -12,7 +12,7 @@ import { run, type ViewerOptions } from "./viewer.ts"; import { isCloudCommand, cloudMain, CLOUD_USAGE } from "./cloud-cli.ts"; import { isDesktopCommand, desktopMain, installedDesktop, prefersDesktop, DESKTOP_USAGE } from "./desktop-cli.ts"; -export const VERSION = "0.15.0"; +export const VERSION = "0.16.0"; /** The help text, exported so a test can hold it to the flags it documents. */ export const USAGE = `readm3 — a plain text editor for Markdown and code diff --git a/src/sync-merge.ts b/src/sync-merge.ts index 32a4f08..d3b5f88 100644 --- a/src/sync-merge.ts +++ b/src/sync-merge.ts @@ -16,8 +16,12 @@ import type { SyncedWorkspace, WorkspaceDocument } from "./sync-schema.ts"; export type Digest = (text: string) => string; -/** FNV-1a over the text and the language: cheap, and only ever compared on one account. */ -export const digestDocument = (doc: Pick): string => { +/** + * FNV-1a over the text and the language: cheap, and only ever compared on one account. + * A spilled text file carries the digest of its text; a large stored file is its blob. + */ +export const digestDocument = (doc: Pick): string => { + if (doc.blob) return doc.blob.digest ?? `blob:${doc.blob.id}`; let hash = 0x811c9dc5; const text = `${doc.language ?? ""}\0${doc.source}`; for (let i = 0; i < text.length; i++) hash = Math.imul(hash ^ text.charCodeAt(i), 0x01000193); diff --git a/src/sync-schema.ts b/src/sync-schema.ts index c7971a1..cad9995 100644 --- a/src/sync-schema.ts +++ b/src/sync-schema.ts @@ -1,13 +1,15 @@ /** Browser-safe account data contract. Only these two files may be synced. */ import { isFlavor, type Flavor } from "./flavors.ts"; import { envelopeProblem, SEALED_FILE } from "./sync-crypto.ts"; +import { blobRefProblem, INLINE_MAX_BYTES, SNAPSHOT_INLINE_BYTES, type BlobRef } from "./blob-store.ts"; /** * A workspace file. `source` is the text, or base64 bytes when the path names a PDF or * an image. `language` is only stored when the name could not decide it (no known - * extension) and the content was sniffed on import. + * extension) and the content was sniffed on import. A file stored apart (src/blob-store.ts: + * larger than 4 MB, or spilled from a full snapshot) has `blob` and an empty `source`. */ -export interface WorkspaceDocument { path: string; source: string; language?: string } +export interface WorkspaceDocument { path: string; source: string; language?: string; blob?: BlobRef } export interface SyncedWorkspace { documents: WorkspaceDocument[]; active: string; name: string } export interface ReaderSettings { theme?: string; flavor?: Flavor } export const SYNC_LIMITS = { maxFileBytes: 32 * 1024 * 1024, maxTotalBytes: 33 * 1024 * 1024, maxFiles: 2 }; @@ -31,7 +33,7 @@ export function validateWorkspace(value: unknown): SyncedWorkspace { if (!value || typeof value !== "object") throw new Error("Workspace must be an object."); const ws = value as SyncedWorkspace; if (Object.keys(ws).some((key) => !["name", "active", "documents"].includes(key))) throw new Error("Workspace can only contain name, active and documents."); - if (typeof ws.name !== "string" || ws.name.length > 200 || !Array.isArray(ws.documents) || !ws.documents.length || ws.documents.length > 1000) throw new Error("Workspace needs a name and 1–1,000 files."); + if (typeof ws.name !== "string" || ws.name.length > 200 || !Array.isArray(ws.documents) || !ws.documents.length) throw new Error("Workspace needs a name and at least one file."); const paths = new Set(); let total = 0; const encoder = new TextEncoder(); @@ -41,10 +43,16 @@ export function validateWorkspace(value: unknown): SyncedWorkspace { if (!doc || typeof doc.path !== "string" || doc.path.length > 1024 || /[\\\x00-\x1f:]/.test(doc.path) || doc.path.split("/").some((p) => !p || p.startsWith(".")) || paths.has(doc.path)) throw new Error("Workspace contains an invalid or duplicate file path."); if (typeof doc.source !== "string") throw new Error("A file's source must be text."); if (doc.language !== undefined && (typeof doc.language !== "string" || !/^[a-z0-9_+-]{1,40}$/.test(doc.language))) throw new Error("A file's language must be a short highlight name."); - if (Object.keys(doc).some((key) => !["path", "source", "language"].includes(key))) throw new Error("Documents can only contain path, source and language."); + if (Object.keys(doc).some((key) => !["path", "source", "language", "blob"].includes(key))) throw new Error("Documents can only contain path, source, language and blob."); + if (doc.blob !== undefined) { + const problem = blobRefProblem(doc.blob); + if (problem) throw new Error(problem); + if (doc.source !== "") throw new Error("A stored file keeps its bytes apart; its source must be empty."); + } const bytes = encoder.encode(doc.source).length; total += bytes; - if (bytes > 4 * 1024 * 1024 || total > 20 * 1024 * 1024) throw new Error("Workspace limit: 4 MB per file and 20 MB total."); + // Larger files, and small ones past the snapshot's share, are stored apart (src/blob-store.ts). + if (bytes > INLINE_MAX_BYTES || total > SNAPSHOT_INLINE_BYTES + INLINE_MAX_BYTES) throw new Error("Workspace limit: 4 MB of text per file inside the synced copy; larger files are stored apart."); paths.add(doc.path); } if (!paths.has(ws.active)) throw new Error("The active document is missing."); diff --git a/test/blob-store.test.ts b/test/blob-store.test.ts new file mode 100644 index 0000000..a6fd120 --- /dev/null +++ b/test/blob-store.test.ts @@ -0,0 +1,51 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + blobRefProblem, CHUNK_BYTES, chunkCount, chunkStoredBytes, decryptChunk, encryptChunk, MAX_BLOB_BYTES, newBlobRef, storedBytes, TAG_BYTES, +} from "../src/blob-store.ts"; +import { validateWorkspace } from "../src/sync-schema.ts"; +import { digestDocument, mergeWorkspaces } from "../src/sync-merge.ts"; + +test("a stored file is encrypted per chunk, and a chunk only opens at its own position", async () => { + const ref = newBlobRef(10); + assert.equal(blobRefProblem(ref), null); + const plain = new TextEncoder().encode("0123456789"); + const sealed = await encryptChunk(ref, 0, plain); + assert.equal(sealed.byteLength, plain.byteLength + TAG_BYTES); + assert.deepEqual(await decryptChunk(ref, 0, sealed), plain); + await assert.rejects(decryptChunk(ref, 1, sealed), /failed to decrypt/); + await assert.rejects(decryptChunk(newBlobRef(10), 0, sealed), /failed to decrypt/); +}); + +test("chunk arithmetic covers a 5 GB file exactly", () => { + assert.equal(chunkCount(0), 1); + assert.equal(chunkCount(CHUNK_BYTES), 1); + assert.equal(chunkCount(CHUNK_BYTES + 1), 2); + assert.equal(chunkCount(MAX_BLOB_BYTES), 640); + assert.equal(chunkStoredBytes(CHUNK_BYTES + 1, 1), 1 + TAG_BYTES); + assert.equal(storedBytes(MAX_BLOB_BYTES), MAX_BLOB_BYTES + 640 * TAG_BYTES); + assert.match(blobRefProblem({ ...newBlobRef(1), size: MAX_BLOB_BYTES + 1 })!, /5 GB/); + assert.match(blobRefProblem({ ...newBlobRef(1), id: "../../etc" })!, /invalid id/); +}); + +test("a synced workspace has no file count limit and carries stored files by reference", () => { + const documents = Array.from({ length: 5000 }, (_, i) => ({ path: `notes/${i}.md`, source: `# ${i}` })); + documents.push({ path: "big/dump.json", source: "", blob: newBlobRef(3 * 1024 ** 3) } as never); + assert.equal(validateWorkspace({ name: "Many", active: "notes/0.md", documents }).documents.length, 5001); + assert.throws(() => validateWorkspace({ name: "x", active: "a.md", documents: [{ path: "a.md", source: "text", blob: newBlobRef(1) }] }), /source must be empty/); +}); + +test("a spilled file compares equal to its inline copy, so moving it never looks like an edit", () => { + const inline = { path: "a.md", source: "# hello" }; + const spilled = { path: "a.md", source: "", blob: newBlobRef(7, digestDocument(inline)) }; + assert.equal(digestDocument(spilled), digestDocument(inline)); + const merge = mergeWorkspaces({ + local: { name: "w", active: "a.md", documents: [inline] }, + remote: { name: "w", active: "a.md", documents: [spilled] }, + base: { "a.md": digestDocument(inline) }, + }); + assert.equal(merge.upload, false); + assert.equal(merge.download, false); + const large = newBlobRef(9 * 1024 * 1024); + assert.notEqual(digestDocument({ source: "", blob: large }), digestDocument({ source: "", blob: newBlobRef(large.size) })); +}); diff --git a/test/sync-schema.test.ts b/test/sync-schema.test.ts index 670b648..4f25d24 100644 --- a/test/sync-schema.test.ts +++ b/test/sync-schema.test.ts @@ -22,7 +22,7 @@ describe("synced workspace files", () => { const base = { name: "x", active: "a.md" }; assert.throws(() => validateWorkspace({ ...base, documents: [{ path: ".env", source: "" }], active: ".env" }), /invalid or duplicate/); assert.throws(() => validateWorkspace({ ...base, documents: [{ path: "a.md", source: "" }, { path: "a.md", source: "" }] }), /invalid or duplicate/); - assert.throws(() => validateWorkspace({ ...base, documents: [{ path: "a.md", source: "", encoding: "base64" }] }), /path, source and language/); + assert.throws(() => validateWorkspace({ ...base, documents: [{ path: "a.md", source: "", encoding: "base64" }] }), /path, source, language and blob/); assert.throws(() => validateWorkspace({ ...base, documents: [{ path: "a.md", source: "", language: "not a language!" }] }), /language must be/); }); });