From 87886b247156489f7b1db3892ba8e43fc2cd777a Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 23 Oct 2026 07:49:16 +0000 Subject: [PATCH] Open big folders fast in the desktop app; don't sync files over 500 MB (v0.22.0) Opening ~/Desktop (22k files, 2.6 GB) sent 644 MB of bytes to the page in one IPC message and copied 1.5 GB of large files into browser storage before the side panel showed anything; signed in, sync then uploaded it all. - The desktop app lists a folder (names and sizes) and the page reads each file when it needs it, with a progress count in the status line. - Large files, PDFs and images the desktop app opened are read from disk on demand (blobs.fromDisk), never copied into IndexedDB. Reopening the folder keeps their stored reference, so sync does not upload them again. - Sync skips files larger than 500 MB by default. The Sync dialog has "Don't sync files larger than [500] MB"; unchecking it syncs everything. Large files on either side stay out of the merge, so a file another device synced with a higher limit is never deleted here. Co-Authored-By: Claude Opus 5.5 --- desktop/main.cjs | 13 +++++----- package.json | 2 +- site/autosync.ts | 56 +++++++++++++++++++++++++++++++++------- site/blobs.ts | 14 ++++++++-- site/disk.ts | 45 ++++++++++++++++++++++---------- site/sync.ts | 31 ++++++++++++++++++++-- site/test/sync.spec.ts | 29 +++++++++++++++++++++ site/test/viewer.spec.ts | 54 ++++++++++++++++++++++++++++++++++++++ site/viewer.ts | 42 ++++++++++++++++++++---------- src/cli.ts | 2 +- 10 files changed, 239 insertions(+), 49 deletions(-) diff --git a/desktop/main.cjs b/desktop/main.cjs index 63aabb6..5ba3e99 100644 --- a/desktop/main.cjs +++ b/desktop/main.cjs @@ -7,8 +7,8 @@ * served as app://readm3 when readm3.com cannot be reached on a first launch. * * Files and folders named on the command line (`readm3 edit .`, like `subl .`), or - * handed over by the desktop's "Open with", are read here and given to the page - * through preload.cjs, which can write back only to files this process gave it. + * handed over by the desktop's "Open with", are listed here and read by the page + * through preload.cjs, which can read and write only files this process listed. * * readm3:// links ("Open in app" on readm3.com) open a readm3.com page in this window, and * "Sign in with your browser" signs the window in through the system browser, where the @@ -24,8 +24,6 @@ const { fileURLToPath, pathToFileURL } = require("node:url"); const ORIGIN = (process.env.READM3_URL || "https://readm3.com").replace(/\/$/, ""); const LOCAL = "app://readm3"; -/** Larger files go to the page without their bytes; it reads them in ranges (read-range). */ -const INLINE_BYTES = 4 * 1024 * 1024; /** The largest file a signed-in account may keep (src/blob-store.ts); the page applies its own limits again. */ const MAX_FILE_BYTES = 5 * 1024 ** 3; const MAX_RANGE = 16 * 1024 * 1024; @@ -46,15 +44,16 @@ let pending = []; let window; /** - * Reads files, and every file under a folder. A folder's files carry `relative`, their + * Lists files, and every file under a folder. A folder's files carry `relative`, their * path from the folder's parent ("notes/todo.md"), so the page opens them as the folder. + * No bytes travel with the list: the page reads each file when it needs it (read-file, + * read-range), so a folder of thousands of files is never one message of gigabytes. */ async function load(paths, cwd = process.cwd()) { const files = []; const add = async (path, info, relative) => { if (info.size > MAX_FILE_BYTES || files.length >= MAX_FILES) return; - const data = info.size > INLINE_BYTES ? undefined : new Uint8Array(await readFile(path)); - files.push({ path, name: basename(path), relative, size: info.size, data }); + files.push({ path, name: basename(path), relative, size: info.size }); opened.add(path); }; const walk = async (directory, prefix) => { diff --git a/package.json b/package.json index 161f3f7..30bfaa9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/readm3", - "version": "0.21.0", + "version": "0.22.0", "description": "Markdown editing and sharing for people and teams. Web/PWA, CLI, API and MCP with version history, view/edit links, and a terminal reader.", "type": "module", "license": "MIT", diff --git a/site/autosync.ts b/site/autosync.ts index dcb8fa8..abb8ef8 100644 --- a/site/autosync.ts +++ b/site/autosync.ts @@ -26,6 +26,28 @@ const OFF = "readm3:autosync"; export const autoSyncEnabled = (): boolean => { try { return localStorage.getItem(OFF) !== "off"; } catch { return true; } }; export function setAutoSync(on: boolean) { try { localStorage.setItem(OFF, on ? "on" : "off"); } catch { /* This device only. */ } if (on) void syncNow(); else host?.status("Sync is off on this device"); } +/** + * Files larger than this stay on this device: a folder of films or a disk image is not + * worth uploading to edit Markdown next to it. 0 syncs every file. Set per device. + */ +const LIMIT = "readm3:sync-limit-mb"; +export const DEFAULT_SYNC_LIMIT_MB = 500; +export function syncLimitMb(): number { + try { + const saved = localStorage.getItem(LIMIT); + const mb = saved === null ? DEFAULT_SYNC_LIMIT_MB : Number(saved); + return Number.isFinite(mb) && mb >= 0 ? mb : DEFAULT_SYNC_LIMIT_MB; + } catch { return DEFAULT_SYNC_LIMIT_MB; } +} +export function setSyncLimitMb(mb: number) { + try { localStorage.setItem(LIMIT, String(Math.max(0, Math.round(mb)))); } catch { /* This device only. */ } + void syncNow(); +} +/** Only stored files can pass the limit: inline text is 4 MB at most. */ +export function keptOnDevice(doc: { blob?: BlobRef }, limitMb = syncLimitMb()): boolean { + return limitMb > 0 && !!doc.blob && doc.blob.size > limitMb * 1024 * 1024; +} + export interface AutoSyncHost { read(): SyncedWorkspace; settings(): ReaderSettings; @@ -233,10 +255,17 @@ async function pass(user: CloudUser) { remoteWorkspace = files["workspace.json"] ? JSON.parse(files["workspace.json"].content) : undefined; remoteSettings = files["settings.json"] ? JSON.parse(files["settings.json"].content) : undefined; } - const local = structuredClone(h.read()); - const localDigests = digests(local); + const all = structuredClone(h.read()); + const localDigests = digests(all); + // Files past the size limit take no part in the merge, on either side: this device never + // uploads its own, and one another device synced (a higher limit there) is left as it is. + const limit = syncLimitMb(); + const deviceOnly = all.documents.filter((doc) => keptOnDevice(doc, limit)); + const accountOnly = remoteWorkspace?.documents.filter((doc) => keptOnDevice(doc, limit)) ?? []; + const local = { ...all, documents: all.documents.filter((doc) => !keptOnDevice(doc, limit)) }; + const remoteSynced = remoteWorkspace && { ...remoteWorkspace, documents: remoteWorkspace.documents.filter((doc) => !keptOnDevice(doc, limit)) }; const merge = mergeWorkspaces({ - local, remote: remoteWorkspace, base: state?.base, starter: await starterDigests(), + local, remote: remoteSynced, base: state?.base, starter: await starterDigests(), conflictLabel: `${h.device} ${new Date().toISOString().slice(0, 10)}`, }); // Settings: this device's choice wins only if it changed since the last sync. @@ -250,9 +279,10 @@ async function pass(user: CloudUser) { const readableOnServer = !!remote && !isSealed(remote.snapshot.files); let spills = state?.spills ?? {}; if (!(onlyStarter && !remote) && (merge.upload || (settingsChanged && remote) || readableOnServer || rekey)) { - const outgoing = await forUpload(merge.workspace, remoteWorkspace, spills); + const outgoing = await forUpload(merge.workspace, remoteSynced, spills); spills = outgoing.spills; - const plain = { "settings.json": { content: serialize(settings) }, "workspace.json": { content: serialize(outgoing.workspace) } }; + const uploaded = { ...outgoing.workspace, documents: [...outgoing.workspace.documents, ...accountOnly].sort((a, b) => a.path.localeCompare(b.path)) }; + const plain = { "settings.json": { content: serialize(settings) }, "workspace.json": { content: serialize(uploaded) } }; validateSyncFiles(plain); const sealed = { [SEALED_FILE]: { content: JSON.stringify(await seal(plain, syncKey)) } }; try { @@ -262,7 +292,7 @@ async function pass(user: CloudUser) { rekey = false; if (saved.userId !== user.id) throw new Error("your account changed"); revision = saved.revision; - void collect(outgoing.workspace, outgoing.account); + void collect(uploaded, outgoing.account); } catch (error) { if (error instanceof ApiError && error.status === 409) continue; // Another device saved first. throw error; @@ -270,12 +300,20 @@ async function pass(user: CloudUser) { } // Edits made while this pass ran are not overwritten: the next pass merges them. if (!sameDigests(digests(h.read()), localDigests)) { again = true; return; } - if (merge.download || serialize(settings) !== localSettings) await h.apply(merge.workspace, settings); + if (merge.download || serialize(settings) !== localSettings) { + // The merge never saw the large files, so they are put back: this device's own first. + const merged = new Set(merge.workspace.documents.map((doc) => doc.path)); + const mine = new Set(deviceOnly.map((doc) => doc.path)); + const documents = [...merge.workspace.documents, ...deviceOnly.filter((doc) => !merged.has(doc.path)), + ...accountOnly.filter((doc) => !merged.has(doc.path) && !mine.has(doc.path))]; + await h.apply({ ...merge.workspace, documents: documents.sort((a, b) => a.path.localeCompare(b.path)) }, settings); + } await store(key, { revision, base: digests(merge.workspace), settings: serialize(settings), spills } satisfies State); known = revision; + const kept = deviceOnly.length ? ` · ${deviceOnly.length} file${deviceOnly.length === 1 ? "" : "s"} over ${limit.toLocaleString()} MB kept on this device` : ""; h.status(merge.conflicts.length - ? `Synced · ${merge.conflicts.length} conflict cop${merge.conflicts.length === 1 ? "y" : "ies"} kept` - : `Synced to your account · ${time()}`); + ? `Synced · ${merge.conflicts.length} conflict cop${merge.conflicts.length === 1 ? "y" : "ies"} kept${kept}` + : `Synced to your account · ${time()}${kept}`); return; } throw new Error("other devices kept saving; trying again shortly"); diff --git a/site/blobs.ts b/site/blobs.ts index 5aa12d0..264857a 100644 --- a/site/blobs.ts +++ b/site/blobs.ts @@ -20,9 +20,19 @@ export interface StorageAccount { } const localKey = (id: string) => `blob:${id}`; -export const local = (ref: Pick) => readKey(localKey(ref.id)).catch(() => undefined); +/** + * Files the desktop app opened from disk: read from there each time, never copied into + * browser storage. A reader lasts for the session; opening the folder again restores it. + */ +const onDisk = new Map Promise>(); +export const fromDisk = (ref: Pick, read: () => Promise) => { onDisk.set(ref.id, read); }; +export async function local(ref: Pick): Promise { + const read = onDisk.get(ref.id); + if (read) { try { return await read(); } catch { /* Moved or deleted on disk: a kept or synced copy may still exist. */ } } + return readKey(localKey(ref.id)).catch(() => undefined); +} export const keep = (ref: Pick, blob: Blob) => writeKey(localKey(ref.id), blob); -export const forget = (ref: Pick) => writeKey(localKey(ref.id), undefined).catch(() => undefined); +export const forget = (ref: Pick) => { onDisk.delete(ref.id); return writeKey(localKey(ref.id), undefined).catch(() => undefined); }; export const account = () => request("blobs"); export const remove = (id: string) => request(`blobs/${id}`, "DELETE"); diff --git a/site/disk.ts b/site/disk.ts index e623c76..b45216e 100644 --- a/site/disk.ts +++ b/site/disk.ts @@ -8,7 +8,13 @@ import { accepts, readKey, writeKey } from "./workspace.ts"; * Browsers without the API keep the old behaviour: files are copied in, Ctrl+S downloads. */ export interface DiskEntry { handle: FileSystemFileHandle; saved: number; } -export interface Picked { file: File; path: string; handle?: FileSystemFileHandle; } +/** + * `onDisk`: opened by the desktop app, so the editor reads it from disk rather than keeping + * a copy. A large one comes as its first 64 KB, enough to tell what it is, with `size` + * saying how large the whole file is. + */ +export interface Picked { file: File; path: string; handle?: FileSystemFileHandle; size?: number; onDisk?: boolean; } +export type Progress = (done: number, total: number) => void; /** The parts of the API that TypeScript's DOM library does not describe yet. */ interface PickerWindow { @@ -32,12 +38,12 @@ export function fingerprint(text: string): number { /** null when the person closes the picker, which is not an error. */ const cancelled = (error: unknown) => error instanceof DOMException && error.name === "AbortError"; -export async function pickFolder(): Promise<{ name: string; files: Picked[] } | null> { - // The desktop app opens folders with the system dialog and reads them itself (desktop/main.cjs). +export async function pickFolder(onProgress?: Progress): Promise<{ name: string; files: Picked[] } | null> { + // The desktop app opens folders with the system dialog and lists them itself (desktop/main.cjs). if (desktop?.pick) { const files = await desktop.pick(true); if (!files) return null; - return { name: files.find((file) => file.relative)?.relative!.split("/")[0] ?? "Your workspace", files: await fromDesktop(files) }; + return { name: files.find((file) => file.relative)?.relative!.split("/")[0] ?? "Your workspace", files: await fromDesktop(files, onProgress) }; } let root: Directory; try { root = await picker().showDirectoryPicker({ id: "readm3", mode: "readwrite" }) as Directory; } @@ -58,10 +64,10 @@ export async function pickFolder(): Promise<{ name: string; files: Picked[] } | return { name: root.name, files }; } -export async function pickFiles(): Promise { +export async function pickFiles(onProgress?: Progress): Promise { if (desktop?.pick) { const files = await desktop.pick(false); - return files && fromDesktop(files); + return files && fromDesktop(files, onProgress); } let handles: FileSystemFileHandle[]; try { handles = await picker().showOpenFilePicker({ id: "readm3", multiple: true }); } @@ -76,9 +82,9 @@ export async function launched(handles: readonly FileSystemHandle[]): Promise { +export async function fromDesktop(files: DesktopFile[], onProgress?: Progress): Promise { const picked: Picked[] = []; - for (const entry of files) { + for (const [i, entry] of files.entries()) { // A large file from a desktop build without ranges cannot be read; it is left out. if (!entry.data && !desktop!.readRange) continue; - const file = entry.data ? new File([entry.data.buffer as ArrayBuffer], entry.name) : await whole(entry); - picked.push({ file, path: entry.relative ?? entry.name, handle: desktopHandle(entry) }); + const handle = desktopHandle(entry); + try { + let file: File; + if (entry.data) file = new File([entry.data.buffer as ArrayBuffer], entry.name); + // Past 4 MB only the start is read now; the rest stays on disk until it is opened. + else if (entry.size! > LARGE) file = new File([await desktop!.readRange!(entry.path, 0, SAMPLE) as BlobPart], entry.name); + else file = await handle.getFile(); + picked.push({ file, path: entry.relative ?? entry.name, handle, size: entry.size ?? file.size, onDisk: true }); + } catch { /* Unreadable now (moved, permissions): the rest still opens. */ } + if (onProgress && (i % 50 === 49 || i === files.length - 1)) onProgress(i + 1, files.length); } return picked; } +const LARGE = 4 * 1024 * 1024; +const SAMPLE = 64 * 1024; const RANGE = 8 * 1024 * 1024; /** A large file, read range by range into Blob parts, which the browser keeps on disk. */ async function whole(entry: DesktopFile): Promise { @@ -126,7 +142,8 @@ function desktopHandle(entry: DesktopFile): FileSystemFileHandle { kind: "file", name: entry.name, // Read from disk each time, so the editor sees a file change under it and a save is not undone. getFile: async () => { - if (!entry.data && entry.size !== undefined) return whole(entry); + // A large file is read a range at a time, never whole through one message. + if (!entry.data && entry.size !== undefined && (entry.size > LARGE || !desktop!.readFile)) return whole(entry); if (!desktop!.readFile) return new File([entry.data!.buffer as ArrayBuffer], entry.name); const { data, modified } = await desktop!.readFile(entry.path); return new File([data.buffer as ArrayBuffer], entry.name, { lastModified: modified }); diff --git a/site/sync.ts b/site/sync.ts index aefe95f..67e774f 100644 --- a/site/sync.ts +++ b/site/sync.ts @@ -14,7 +14,11 @@ function loadMarker(key: string): Marker | undefined { function saveMarker(key: string, marker: Marker) { sessionStorage.setItem(`readm3:${key}`, JSON.stringify(marker)); } export interface SyncWorkspace { /** The automatic sync switch for this device (site/autosync.ts). */ - auto?: { enabled(): boolean; set(on: boolean): void; now(): Promise; changePassphrase(userId: string): void; forgetPassphrase(userId: string): Promise }; + auto?: { + enabled(): boolean; set(on: boolean): void; now(): Promise; changePassphrase(userId: string): void; forgetPassphrase(userId: string): Promise; + /** Files larger than this many MB stay on this device; 0 syncs every file. */ + limit: { get(): number; set(mb: number): void; fallback: number }; + }; read(): SyncedWorkspace; settings(): ReaderSettings; apply(workspace: SyncedWorkspace | undefined, settings: ReaderSettings | undefined): Promise; @@ -88,9 +92,32 @@ export async function showSync(workspace: SyncWorkspace) { auto.checked = workspace.auto.enabled(); auto.onchange = () => workspace.auto!.set(auto.checked); autoLabel.append(auto, " Sync automatically on this device (recommended). Every signed-in device keeps the same files; edits on two devices are merged per file, and a file changed on both keeps a conflict copy."); + // Large files (films, disk images) stay on this device unless the person turns this off. + const { limit } = workspace.auto; + const limitLabel = document.createElement("label"); + const skip = document.createElement("input"); + skip.type = "checkbox"; + skip.id = "sync-limit-on"; + skip.checked = limit.get() > 0; + const size = document.createElement("input"); + size.type = "number"; + size.id = "sync-limit-mb"; + size.min = "1"; + size.step = "1"; + size.style.width = "6em"; + size.value = String(limit.get() || limit.fallback); + size.disabled = !skip.checked; + const apply = () => { + size.disabled = !skip.checked; + const mb = Math.round(Number(size.value)); + limit.set(skip.checked ? (Number.isFinite(mb) && mb >= 1 ? mb : limit.fallback) : 0); + }; + skip.onchange = apply; + size.onchange = apply; + limitLabel.append(skip, " Don't sync files larger than ", size, " MB. They stay on this device."); const e2e = document.createElement("p"); e2e.textContent = "End-to-end encrypted: files are compressed and encrypted on this device with your sync passphrase before upload. readm3.com cannot read them."; - box.body.append(autoLabel, e2e); + box.body.append(autoLabel, limitLabel, e2e); const actions = document.createElement("div"); actions.className = "cloud-actions"; const act = (label: string, run: () => void | Promise) => { diff --git a/site/test/sync.spec.ts b/site/test/sync.spec.ts index 8229eec..a1e1ac9 100644 --- a/site/test/sync.spec.ts +++ b/site/test/sync.spec.ts @@ -151,6 +151,35 @@ test("a signed-in account opens a file over 4 MB read-only and syncs it as an en expect((await stat((await (await downloading).path())!)).size).toBe(big.length); }); +test("files over the sync size limit stay on this device until the limit is turned off", async ({ page, context }) => { + await register(context, username()); + // 500 MB by default; a small limit keeps the test's files small. + await page.addInitScript(() => { if (localStorage.getItem("readm3:sync-limit-mb") === null) localStorage.setItem("readm3:sync-limit-mb", "8"); }); + await page.goto("/viewer"); + await setPassphrase(page); + const line = "a line of a large log\n"; + const big = Buffer.from(line.repeat(Math.ceil(9 * 1024 * 1024 / line.length))); + const medium = Buffer.from(line.repeat(Math.ceil(6 * 1024 * 1024 / line.length))); + await page.locator("#files-input").setInputFiles([ + { name: "big.log", mimeType: "text/plain", buffer: big }, + { name: "medium.log", mimeType: "text/plain", buffer: medium }, + ]); + const uploaded = async () => { + const account = await (await page.request.get("/api/v1/blobs")).json(); + return account.blobs.filter((blob: { complete: boolean }) => blob.complete).map((blob: { size: number }) => blob.size).sort(); + }; + await expect.poll(uploaded, { timeout: 20_000 }).toEqual([medium.length]); + await expect(page.locator("#sync-note")).toContainText("1 file over 8 MB kept on this device"); + await expect(page.locator('#file-list button[title="big.log"]')).toBeVisible(); + await openSync(page); + await expect(page.locator("#sync-limit-on")).toBeChecked(); + await expect(page.locator("#sync-limit-mb")).toHaveValue("8"); + await page.locator("#sync-limit-on").uncheck(); + await expect(page.locator("#sync-limit-mb")).toBeDisabled(); + await expect.poll(uploaded, { timeout: 20_000 }).toEqual([big.length, medium.length].sort()); + expect(await page.evaluate(() => localStorage.getItem("readm3:sync-limit-mb"))).toBe("0"); +}); + test("the header shows usage against the plan's limits, signed out and signed in", async ({ page, context }) => { await page.goto("/viewer"); const usage = page.locator("#usage"); diff --git a/site/test/viewer.spec.ts b/site/test/viewer.spec.ts index cdb4d01..9af5d7b 100644 --- a/site/test/viewer.spec.ts +++ b/site/test/viewer.spec.ts @@ -143,6 +143,60 @@ test("in the desktop app, opens and saves through the system dialogs, never the expect(await page.evaluate(() => (window as unknown as { written: Record }).written["/home/me/Welcome.md"])).toContain("Your docs. Your space."); }); +test("in the desktop app, a folder comes as a list and large files and images are read from disk, not copied", async ({ page }) => { + await page.addInitScript(() => { + const text = new TextEncoder(); + const big = "x".repeat(5 * 1024 * 1024 - 7) + "\nend\n"; + const disk: Record = { + "/d/notes.md": text.encode("# Notes\n"), + "/d/huge.log": text.encode(big), + // A one-pixel PNG. + "/d/dot.png": Uint8Array.from(atob("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="), (c) => c.charCodeAt(0)), + }; + const reads = { file: [] as string[], range: [] as number[] }; + Object.assign(window, { + reads, + readm3Desktop: { + takeLaunchFiles: async () => [], + onOpenFiles: () => {}, + // As main.cjs lists them since 0.22.0: names and sizes, no bytes. + pick: async () => Object.entries(disk).map(([path, bytes]) => ({ path, name: path.split("/").at(-1), relative: `d/${path.split("/").at(-1)}`, size: bytes.length })), + readFile: async (path: string) => { reads.file.push(path); return { data: disk[path], modified: 1 }; }, + readRange: async (path: string, start: number, end: number) => { reads.range.push(end - start); return disk[path].slice(start, end); }, + writeFile: async () => {}, + }, + }); + }); + await page.reload(); + await page.locator("#open-folder").click(); + await expect(page.locator("#workspace-name")).toHaveText("d"); + await expect(page.locator('#file-list button[title="d/huge.log"]')).toBeVisible(); + const reads = () => page.evaluate(() => (window as unknown as { reads: { file: string[]; range: number[] } }).reads); + // Opening read the small files and only the first 64 KB of the large one. + expect((await reads()).range).toEqual([64 * 1024]); + expect((await reads()).file).not.toContain("/d/huge.log"); + // Nothing large was copied into browser storage. + const keptBlobs = () => page.evaluate(() => new Promise((done) => { + const open = indexedDB.open("readm3"); + open.onsuccess = () => { + const db = open.result; + const names = [...db.objectStoreNames]; + if (!names.length) { done(0); return; } + let count = 0; + const tx = db.transaction(names, "readonly"); + for (const name of names) tx.objectStore(name).getAllKeys().onsuccess = (event) => { count += ((event.target as IDBRequest).result as IDBValidKey[]).filter((key) => String(key).startsWith("blob:")).length; }; + tx.oncomplete = () => done(count); + }; + open.onerror = () => done(0); + })); + expect(await keptBlobs()).toBe(0); + await page.locator('#file-list button[title="d/huge.log"]').click(); + await expect(page.locator("#code-content .large-preview")).toContainText("xxxx"); + await page.locator('#file-list button[title="d/dot.png"]').click(); + await expect(page.locator("#media-content img")).toBeVisible(); + expect(await keptBlobs()).toBe(0); +}); + test("keeps a rolling history: opened, saved, changed on disk, with snippets and restore", async ({ page }) => { // A fake disk whose file can change under the editor, as a regenerated output.json does. await page.addInitScript(() => { diff --git a/site/viewer.ts b/site/viewer.ts index e55947d..bb799fd 100644 --- a/site/viewer.ts +++ b/site/viewer.ts @@ -4,7 +4,7 @@ import { refreshUsage, showUsage } from "./usage.ts"; import { formatStorage, INLINE_MAX_BYTES, MAX_ACCOUNT_FILES, MAX_BLOB_BYTES, newBlobRef } from "../src/blob-store.ts"; import { pasteDialog, publish, sharing, history as versionHistory } from "./sharing.ts"; import { showSync } from "./sync.ts"; -import { askPassphrase, autoSyncEnabled, forgetPassphrase, schedule as scheduleSync, setAutoSync, startAutoSync, syncNow } from "./autosync.ts"; +import { askPassphrase, autoSyncEnabled, DEFAULT_SYNC_LIMIT_MB, forgetPassphrase, schedule as scheduleSync, setAutoSync, setSyncLimitMb, startAutoSync, syncLimitMb, syncNow } from "./autosync.ts"; import { renderMarkdown } from "../src/markdown.ts"; import type { Flavor } from "../src/flavors.ts"; import type { ReaderSettings } from "../src/sync-schema.ts"; @@ -502,7 +502,10 @@ function renderStored() { }) .catch((error) => { if (!still()) return; - status.textContent = `${error instanceof Error ? error.message : "This file could not be opened."}${signedIn ? "" : " Sign in to fetch it from your account."}`; + // A file the desktop app reads from disk is there only while the app has it open. + status.textContent = diskSynced.has(doc.path) || diskEntries.has(doc.path) + ? "This file is read from disk. Open its folder again (File: Open folder…) to read it." + : `${error instanceof Error ? error.message : "This file could not be opened."}${signedIn ? "" : " Sign in to fetch it from your account."}`; previousRender = ""; }); } @@ -683,23 +686,28 @@ async function importFiles(files: disk.Picked[], name = "Your workspace") { let needAccount = 0; let overCount = 0; let unreadable = 0; - for (const { file, path, handle } of files) { + for (const { file, path, handle, size = file.size, onDisk } of files) { if (!accepts(path)) continue; // Past the account's file count nothing more is read. if (signedIn && incoming.length >= MAX_ACCOUNT_FILES) { overCount++; continue; } // A PDF or image is kept as base64, a third larger than the file; the limits apply to what is stored. - const cost = binaryType(path) ? Math.ceil(file.size * 4 / 3) : file.size; - if (cost > MAX_FILE_BYTES) { + const cost = binaryType(path) ? Math.ceil(size * 4 / 3) : size; + // A large file, PDF or image the desktop app opened stays on disk and is read from there: no copy. + const readFromDisk = !!onDisk && !!handle && (cost > MAX_FILE_BYTES || !!binaryType(path)); + if (cost > MAX_FILE_BYTES || readFromDisk) { // Larger files are stored apart as they are (site/blobs.ts), which needs an account. - if (!signedIn) { needAccount++; continue; } - if (file.size > MAX_BLOB_BYTES) { skipped++; continue; } + if (!signedIn && (!readFromDisk || incoming.length >= MAX_FILES)) { needAccount++; continue; } + if (size > MAX_BLOB_BYTES) { skipped++; continue; } if (!binaryType(path) && looksBinary(await file.slice(0, 8192).text())) { unreadable++; continue; } - const doc: Document = { path, source: "", blob: newBlobRef(file.size) }; + // Opening the folder again keeps a file's stored reference, so sync does not upload it again. + const known = readFromDisk ? workspace.documents.find((doc) => doc.path === path && doc.blob?.size === size && !doc.blob.digest)?.blob : undefined; + const doc: Document = { path, source: "", blob: known ?? newBlobRef(size) }; if (!binaryType(path) && !languageForName(path)) { const language = detectLanguage(path, await file.slice(0, 65536).text()); if (language !== MARKDOWN) doc.language = language; } - await stored.keep(doc.blob!, file); + if (readFromDisk) stored.fromDisk(doc.blob!, () => handle!.getFile()); + else await stored.keep(doc.blob!, file); incoming.push(doc); handles.push(handle); continue; @@ -807,11 +815,16 @@ async function checkDisk() { /** What an input or a drop hands over: files with their relative paths, no way back to disk. */ const copies = (files: Iterable): disk.Picked[] => [...files].map((file) => ({ file, path: file.webkitRelativePath || file.name })); +/** Progress while the desktop app reads a folder of thousands of files. */ +const reading: disk.Progress = (done, total) => { + if (total >= 200) saveStatus.textContent = done < total ? `Reading files from disk… ${done.toLocaleString()} of ${total.toLocaleString()}` : "Opening…"; +}; + async function openFromDisk(folder: boolean) { if (!disk.supported()) { (folder ? folderInput : filesInput).click(); return; } try { - if (folder) { const picked = await disk.pickFolder(); if (picked) await importFiles(picked.files, picked.name); } - else { const picked = await disk.pickFiles(); if (picked) await importFiles(picked); } + if (folder) { const picked = await disk.pickFolder(reading); if (picked) await importFiles(picked.files, picked.name); } + else { const picked = await disk.pickFiles(reading); if (picked) await importFiles(picked); } } catch (error) { notice(error instanceof Error ? error.message : "Could not open from disk."); } } @@ -876,7 +889,10 @@ function loadPreferences() { function bindEvents() { element("sync-workspace").onclick = () => void showSync({ - auto: { enabled: autoSyncEnabled, set: setAutoSync, now: syncNow, changePassphrase: (userId) => void askPassphrase("create", userId), forgetPassphrase }, + auto: { + enabled: autoSyncEnabled, set: setAutoSync, now: syncNow, changePassphrase: (userId) => void askPassphrase("create", userId), forgetPassphrase, + limit: { get: syncLimitMb, set: setSyncLimitMb, fallback: DEFAULT_SYNC_LIMIT_MB }, + }, read: () => workspace, settings: () => ({ theme: theme.value, flavor: flavor.value as Flavor }), apply: (incoming, settings) => applySynced(incoming as Workspace | undefined, settings), @@ -1100,7 +1116,7 @@ function receiveLaunches() { const open = (files: disk.DesktopFile[]) => { if (!files.length) return; const folder = files.find((file) => file.relative)?.relative!.split("/")[0]; - void disk.fromDesktop(files).then((picked) => importFiles(picked, folder)).catch(() => notice("A file could not be read from disk.")); + void disk.fromDesktop(files, reading).then((picked) => importFiles(picked, folder)).catch(() => notice("A file could not be read from disk.")); }; void disk.desktop.takeLaunchFiles().then(open); disk.desktop.onOpenFiles(open); diff --git a/src/cli.ts b/src/cli.ts index 51844b4..2aea740 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -12,7 +12,7 @@ import { run, type ViewerOptions } from "./viewer.ts"; import { isCloudCommand, cloudMain, CLOUD_USAGE } from "./cloud-cli.ts"; import { isDesktopCommand, desktopMain, installedDesktop, prefersDesktop, DESKTOP_USAGE } from "./desktop-cli.ts"; -export const VERSION = "0.21.0"; +export const VERSION = "0.22.0"; /** The help text, exported so a test can hold it to the flags it documents. */ export const USAGE = `readm3 — a plain text editor for Markdown and code