Open big folders fast in the desktop app; don't sync files over 500 MB (v0.22.0)

desktop/main.cjs

@@ -7,8 +7,8 @@
  * served as app://readm3 when readm3.com cannot be reached on a first launch.
  *
  * Files and folders named on the command line (`readm3 edit .`, like `subl .`), or
- * handed over by the desktop's "Open with", are read here and given to the page
- * through preload.cjs, which can write back only to files this process gave it.
+ * handed over by the desktop's "Open with", are listed here and read by the page
+ * through preload.cjs, which can read and write only files this process listed.
  *
  * readm3:// links ("Open in app" on readm3.com) open a readm3.com page in this window, and
  * "Sign in with your browser" signs the window in through the system browser, where the
@@ -24,8 +24,6 @@ const { fileURLToPath, pathToFileURL } = require("node:url");
 
 const ORIGIN = (process.env.READM3_URL || "https://readm3.com").replace(/\/$/, "");
 const LOCAL = "app://readm3";
-/** Larger files go to the page without their bytes; it reads them in ranges (read-range). */
-const INLINE_BYTES = 4 * 1024 * 1024;
 /** The largest file a signed-in account may keep (src/blob-store.ts); the page applies its own limits again. */
 const MAX_FILE_BYTES = 5 * 1024 ** 3;
 const MAX_RANGE = 16 * 1024 * 1024;
@@ -46,15 +44,16 @@ let pending = [];
 let window;
 
 /**
- * Reads files, and every file under a folder. A folder's files carry `relative`, their
+ * Lists files, and every file under a folder. A folder's files carry `relative`, their
  * path from the folder's parent ("notes/todo.md"), so the page opens them as the folder.
+ * No bytes travel with the list: the page reads each file when it needs it (read-file,
+ * read-range), so a folder of thousands of files is never one message of gigabytes.
  */
 async function load(paths, cwd = process.cwd()) {
   const files = [];
   const add = async (path, info, relative) => {
     if (info.size > MAX_FILE_BYTES || files.length >= MAX_FILES) return;
-    const data = info.size > INLINE_BYTES ? undefined : new Uint8Array(await readFile(path));
-    files.push({ path, name: basename(path), relative, size: info.size, data });
+    files.push({ path, name: basename(path), relative, size: info.size });
     opened.add(path);
   };
   const walk = async (directory, prefix) => {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@profullstack/readm3",
-  "version": "0.21.0",
+  "version": "0.22.0",
   "description": "Markdown editing and sharing for people and teams. Web/PWA, CLI, API and MCP with version history, view/edit links, and a terminal reader.",
   "type": "module",
   "license": "MIT",

site/autosync.ts

@@ -26,6 +26,28 @@ const OFF = "readm3:autosync";
 export const autoSyncEnabled = (): boolean => { try { return localStorage.getItem(OFF) !== "off"; } catch { return true; } };
 export function setAutoSync(on: boolean) { try { localStorage.setItem(OFF, on ? "on" : "off"); } catch { /* This device only. */ } if (on) void syncNow(); else host?.status("Sync is off on this device"); }
 
+/**
+ * Files larger than this stay on this device: a folder of films or a disk image is not
+ * worth uploading to edit Markdown next to it. 0 syncs every file. Set per device.
+ */
+const LIMIT = "readm3:sync-limit-mb";
+export const DEFAULT_SYNC_LIMIT_MB = 500;
+export function syncLimitMb(): number {
+  try {
+    const saved = localStorage.getItem(LIMIT);
+    const mb = saved === null ? DEFAULT_SYNC_LIMIT_MB : Number(saved);
+    return Number.isFinite(mb) && mb >= 0 ? mb : DEFAULT_SYNC_LIMIT_MB;
+  } catch { return DEFAULT_SYNC_LIMIT_MB; }
+}
+export function setSyncLimitMb(mb: number) {
+  try { localStorage.setItem(LIMIT, String(Math.max(0, Math.round(mb)))); } catch { /* This device only. */ }
+  void syncNow();
+}
+/** Only stored files can pass the limit: inline text is 4 MB at most. */
+export function keptOnDevice(doc: { blob?: BlobRef }, limitMb = syncLimitMb()): boolean {
+  return limitMb > 0 && !!doc.blob && doc.blob.size > limitMb * 1024 * 1024;
+}
+
 export interface AutoSyncHost {
   read(): SyncedWorkspace;
   settings(): ReaderSettings;
@@ -233,10 +255,17 @@ async function pass(user: CloudUser) {
       remoteWorkspace = files["workspace.json"] ? JSON.parse(files["workspace.json"].content) : undefined;
       remoteSettings = files["settings.json"] ? JSON.parse(files["settings.json"].content) : undefined;
     }
-    const local = structuredClone(h.read());
-    const localDigests = digests(local);
+    const all = structuredClone(h.read());
+    const localDigests = digests(all);
+    // Files past the size limit take no part in the merge, on either side: this device never
+    // uploads its own, and one another device synced (a higher limit there) is left as it is.
+    const limit = syncLimitMb();
+    const deviceOnly = all.documents.filter((doc) => keptOnDevice(doc, limit));
+    const accountOnly = remoteWorkspace?.documents.filter((doc) => keptOnDevice(doc, limit)) ?? [];
+    const local = { ...all, documents: all.documents.filter((doc) => !keptOnDevice(doc, limit)) };
+    const remoteSynced = remoteWorkspace && { ...remoteWorkspace, documents: remoteWorkspace.documents.filter((doc) => !keptOnDevice(doc, limit)) };
     const merge = mergeWorkspaces({
-      local, remote: remoteWorkspace, base: state?.base, starter: await starterDigests(),
+      local, remote: remoteSynced, base: state?.base, starter: await starterDigests(),
       conflictLabel: `${h.device} ${new Date().toISOString().slice(0, 10)}`,
     });
     // Settings: this device's choice wins only if it changed since the last sync.
@@ -250,9 +279,10 @@ async function pass(user: CloudUser) {
     const readableOnServer = !!remote && !isSealed(remote.snapshot.files);
     let spills = state?.spills ?? {};
     if (!(onlyStarter && !remote) && (merge.upload || (settingsChanged && remote) || readableOnServer || rekey)) {
-      const outgoing = await forUpload(merge.workspace, remoteWorkspace, spills);
+      const outgoing = await forUpload(merge.workspace, remoteSynced, spills);
       spills = outgoing.spills;
-      const plain = { "settings.json": { content: serialize(settings) }, "workspace.json": { content: serialize(outgoing.workspace) } };
+      const uploaded = { ...outgoing.workspace, documents: [...outgoing.workspace.documents, ...accountOnly].sort((a, b) => a.path.localeCompare(b.path)) };
+      const plain = { "settings.json": { content: serialize(settings) }, "workspace.json": { content: serialize(uploaded) } };
       validateSyncFiles(plain);
       const sealed = { [SEALED_FILE]: { content: JSON.stringify(await seal(plain, syncKey)) } };
       try {
@@ -262,20 +292,28 @@ async function pass(user: CloudUser) {
         rekey = false;
         if (saved.userId !== user.id) throw new Error("your account changed");
         revision = saved.revision;
-        void collect(outgoing.workspace, outgoing.account);
+        void collect(uploaded, outgoing.account);
       } catch (error) {
         if (error instanceof ApiError && error.status === 409) continue; // Another device saved first.
         throw error;
       }
     }
     // Edits made while this pass ran are not overwritten: the next pass merges them.
     if (!sameDigests(digests(h.read()), localDigests)) { again = true; return; }
-    if (merge.download || serialize(settings) !== localSettings) await h.apply(merge.workspace, settings);
+    if (merge.download || serialize(settings) !== localSettings) {
+      // The merge never saw the large files, so they are put back: this device's own first.
+      const merged = new Set(merge.workspace.documents.map((doc) => doc.path));
+      const mine = new Set(deviceOnly.map((doc) => doc.path));
+      const documents = [...merge.workspace.documents, ...deviceOnly.filter((doc) => !merged.has(doc.path)),
+        ...accountOnly.filter((doc) => !merged.has(doc.path) && !mine.has(doc.path))];
+      await h.apply({ ...merge.workspace, documents: documents.sort((a, b) => a.path.localeCompare(b.path)) }, settings);
+    }
     await store(key, { revision, base: digests(merge.workspace), settings: serialize(settings), spills } satisfies State);
     known = revision;
+    const kept = deviceOnly.length ? ` · ${deviceOnly.length} file${deviceOnly.length === 1 ? "" : "s"} over ${limit.toLocaleString()} MB kept on this device` : "";
     h.status(merge.conflicts.length
-      ? `Synced · ${merge.conflicts.length} conflict cop${merge.conflicts.length === 1 ? "y" : "ies"} kept`
-      : `Synced to your account · ${time()}`);
+      ? `Synced · ${merge.conflicts.length} conflict cop${merge.conflicts.length === 1 ? "y" : "ies"} kept${kept}`
+      : `Synced to your account · ${time()}${kept}`);
     return;
   }
   throw new Error("other devices kept saving; trying again shortly");

site/blobs.ts

@@ -20,9 +20,19 @@ export interface StorageAccount {
 }
 
 const localKey = (id: string) => `blob:${id}`;
-export const local = (ref: Pick<BlobRef, "id">) => readKey<Blob>(localKey(ref.id)).catch(() => undefined);
+/**
+ * Files the desktop app opened from disk: read from there each time, never copied into
+ * browser storage. A reader lasts for the session; opening the folder again restores it.
+ */
+const onDisk = new Map<string, () => Promise<Blob>>();
+export const fromDisk = (ref: Pick<BlobRef, "id">, read: () => Promise<Blob>) => { onDisk.set(ref.id, read); };
+export async function local(ref: Pick<BlobRef, "id">): Promise<Blob | undefined> {
+  const read = onDisk.get(ref.id);
+  if (read) { try { return await read(); } catch { /* Moved or deleted on disk: a kept or synced copy may still exist. */ } }
+  return readKey<Blob>(localKey(ref.id)).catch(() => undefined);
+}
 export const keep = (ref: Pick<BlobRef, "id">, blob: Blob) => writeKey(localKey(ref.id), blob);
-export const forget = (ref: Pick<BlobRef, "id">) => writeKey(localKey(ref.id), undefined).catch(() => undefined);
+export const forget = (ref: Pick<BlobRef, "id">) => { onDisk.delete(ref.id); return writeKey(localKey(ref.id), undefined).catch(() => undefined); };
 
 export const account = () => request<StorageAccount>("blobs");
 export const remove = (id: string) => request(`blobs/${id}`, "DELETE");

site/disk.ts

@@ -8,7 +8,13 @@ import { accepts, readKey, writeKey } from "./workspace.ts";
  * Browsers without the API keep the old behaviour: files are copied in, Ctrl+S downloads.
  */
 export interface DiskEntry { handle: FileSystemFileHandle; saved: number; }
-export interface Picked { file: File; path: string; handle?: FileSystemFileHandle; }
+/**
+ * `onDisk`: opened by the desktop app, so the editor reads it from disk rather than keeping
+ * a copy. A large one comes as its first 64 KB, enough to tell what it is, with `size`
+ * saying how large the whole file is.
+ */
+export interface Picked { file: File; path: string; handle?: FileSystemFileHandle; size?: number; onDisk?: boolean; }
+export type Progress = (done: number, total: number) => void;
 
 /** The parts of the API that TypeScript's DOM library does not describe yet. */
 interface PickerWindow {
@@ -32,12 +38,12 @@ export function fingerprint(text: string): number {
 /** null when the person closes the picker, which is not an error. */
 const cancelled = (error: unknown) => error instanceof DOMException && error.name === "AbortError";
 
-export async function pickFolder(): Promise<{ name: string; files: Picked[] } | null> {
-  // The desktop app opens folders with the system dialog and reads them itself (desktop/main.cjs).
+export async function pickFolder(onProgress?: Progress): Promise<{ name: string; files: Picked[] } | null> {
+  // The desktop app opens folders with the system dialog and lists them itself (desktop/main.cjs).
   if (desktop?.pick) {
     const files = await desktop.pick(true);
     if (!files) return null;
-    return { name: files.find((file) => file.relative)?.relative!.split("/")[0] ?? "Your workspace", files: await fromDesktop(files) };
+    return { name: files.find((file) => file.relative)?.relative!.split("/")[0] ?? "Your workspace", files: await fromDesktop(files, onProgress) };
   }
   let root: Directory;
   try { root = await picker().showDirectoryPicker({ id: "readm3", mode: "readwrite" }) as Directory; }
@@ -58,10 +64,10 @@ export async function pickFolder(): Promise<{ name: string; files: Picked[] } |
   return { name: root.name, files };
 }
 
-export async function pickFiles(): Promise<Picked[] | null> {
+export async function pickFiles(onProgress?: Progress): Promise<Picked[] | null> {
   if (desktop?.pick) {
     const files = await desktop.pick(false);
-    return files && fromDesktop(files);
+    return files && fromDesktop(files, onProgress);
   }
   let handles: FileSystemFileHandle[];
   try { handles = await picker().showOpenFilePicker({ id: "readm3", multiple: true }); }
@@ -76,9 +82,9 @@ export async function launched(handles: readonly FileSystemHandle[]): Promise<Pi
 }
 
 /**
- * A file the desktop app opened for the page: its path, its name and its bytes; `relative`
- * when it came in a folder. A file over 4 MB comes with its size and no bytes; the page
- * reads it in ranges.
+ * A file the desktop app opened for the page: its path, its name and its size; `relative`
+ * when it came in a folder. The page reads the bytes (read-file, read-range). Builds before
+ * 0.22.0 sent the bytes of files up to 4 MB along (`data`).
  */
 export interface DesktopFile { path: string; name: string; relative?: string; size?: number; data?: Uint8Array; }
 interface DesktopBridge {
@@ -100,17 +106,27 @@ export const desktop = typeof window === "undefined" ? undefined : (window as un
  * picker. It lasts for the session: it cannot be stored, and that is fine, because the
  * desktop hands the file over again the next time it is opened.
  */
-export async function fromDesktop(files: DesktopFile[]): Promise<Picked[]> {
+export async function fromDesktop(files: DesktopFile[], onProgress?: Progress): Promise<Picked[]> {
   const picked: Picked[] = [];
-  for (const entry of files) {
+  for (const [i, entry] of files.entries()) {
     // A large file from a desktop build without ranges cannot be read; it is left out.
     if (!entry.data && !desktop!.readRange) continue;
-    const file = entry.data ? new File([entry.data.buffer as ArrayBuffer], entry.name) : await whole(entry);
-    picked.push({ file, path: entry.relative ?? entry.name, handle: desktopHandle(entry) });
+    const handle = desktopHandle(entry);
+    try {
+      let file: File;
+      if (entry.data) file = new File([entry.data.buffer as ArrayBuffer], entry.name);
+      // Past 4 MB only the start is read now; the rest stays on disk until it is opened.
+      else if (entry.size! > LARGE) file = new File([await desktop!.readRange!(entry.path, 0, SAMPLE) as BlobPart], entry.name);
+      else file = await handle.getFile();
+      picked.push({ file, path: entry.relative ?? entry.name, handle, size: entry.size ?? file.size, onDisk: true });
+    } catch { /* Unreadable now (moved, permissions): the rest still opens. */ }
+    if (onProgress && (i % 50 === 49 || i === files.length - 1)) onProgress(i + 1, files.length);
   }
   return picked;
 }
 
+const LARGE = 4 * 1024 * 1024;
+const SAMPLE = 64 * 1024;
 const RANGE = 8 * 1024 * 1024;
 /** A large file, read range by range into Blob parts, which the browser keeps on disk. */
 async function whole(entry: DesktopFile): Promise<File> {
@@ -126,7 +142,8 @@ function desktopHandle(entry: DesktopFile): FileSystemFileHandle {
     kind: "file", name: entry.name,
     // Read from disk each time, so the editor sees a file change under it and a save is not undone.
     getFile: async () => {
-      if (!entry.data && entry.size !== undefined) return whole(entry);
+      // A large file is read a range at a time, never whole through one message.
+      if (!entry.data && entry.size !== undefined && (entry.size > LARGE || !desktop!.readFile)) return whole(entry);
       if (!desktop!.readFile) return new File([entry.data!.buffer as ArrayBuffer], entry.name);
       const { data, modified } = await desktop!.readFile(entry.path);
       return new File([data.buffer as ArrayBuffer], entry.name, { lastModified: modified });

site/sync.ts

@@ -14,7 +14,11 @@ function loadMarker(key: string): Marker | undefined {
 function saveMarker(key: string, marker: Marker) { sessionStorage.setItem(`readm3:${key}`, JSON.stringify(marker)); }
 export interface SyncWorkspace {
   /** The automatic sync switch for this device (site/autosync.ts). */
-  auto?: { enabled(): boolean; set(on: boolean): void; now(): Promise<void>; changePassphrase(userId: string): void; forgetPassphrase(userId: string): Promise<void> };
+  auto?: {
+    enabled(): boolean; set(on: boolean): void; now(): Promise<void>; changePassphrase(userId: string): void; forgetPassphrase(userId: string): Promise<void>;
+    /** Files larger than this many MB stay on this device; 0 syncs every file. */
+    limit: { get(): number; set(mb: number): void; fallback: number };
+  };
   read(): SyncedWorkspace;
   settings(): ReaderSettings;
   apply(workspace: SyncedWorkspace | undefined, settings: ReaderSettings | undefined): Promise<void>;
@@ -88,9 +92,32 @@ export async function showSync(workspace: SyncWorkspace) {
       auto.checked = workspace.auto.enabled();
       auto.onchange = () => workspace.auto!.set(auto.checked);
       autoLabel.append(auto, " Sync automatically on this device (recommended). Every signed-in device keeps the same files; edits on two devices are merged per file, and a file changed on both keeps a conflict copy.");
+      // Large files (films, disk images) stay on this device unless the person turns this off.
+      const { limit } = workspace.auto;
+      const limitLabel = document.createElement("label");
+      const skip = document.createElement("input");
+      skip.type = "checkbox";
+      skip.id = "sync-limit-on";
+      skip.checked = limit.get() > 0;
+      const size = document.createElement("input");
+      size.type = "number";
+      size.id = "sync-limit-mb";
+      size.min = "1";
+      size.step = "1";
+      size.style.width = "6em";
+      size.value = String(limit.get() || limit.fallback);
+      size.disabled = !skip.checked;
+      const apply = () => {
+        size.disabled = !skip.checked;
+        const mb = Math.round(Number(size.value));
+        limit.set(skip.checked ? (Number.isFinite(mb) && mb >= 1 ? mb : limit.fallback) : 0);
+      };
+      skip.onchange = apply;
+      size.onchange = apply;
+      limitLabel.append(skip, " Don't sync files larger than ", size, " MB. They stay on this device.");
       const e2e = document.createElement("p");
       e2e.textContent = "End-to-end encrypted: files are compressed and encrypted on this device with your sync passphrase before upload. readm3.com cannot read them.";
-      box.body.append(autoLabel, e2e);
+      box.body.append(autoLabel, limitLabel, e2e);
       const actions = document.createElement("div");
       actions.className = "cloud-actions";
       const act = (label: string, run: () => void | Promise<void>) => {