diff --git a/CHANGELOG.md b/CHANGELOG.md index 2895b355..cb36c9c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ Format loosely follows [Keep a Changelog](https://keepachangelog.com). ### Fixed +- BOM-marked UTF-16 text and Markdown files now open, search, and save without corrupting their encoding. [#232](https://github.com/bholmesdev/hubble.md/pull/232) + ## [0.1.25] - 2026-08-06 ### Added diff --git a/apps/desktop/electron/main.ts b/apps/desktop/electron/main.ts index 6418c355..4489434b 100644 --- a/apps/desktop/electron/main.ts +++ b/apps/desktop/electron/main.ts @@ -49,6 +49,7 @@ import type { ThemePreference } from "../src/theme"; import { DeleteUndo } from "./deleteUndo"; import { TelemetryManager } from "./telemetry"; import { setupTerminalIpc } from "./terminal"; +import { readTextFile, writeTextFile } from "./textFile"; import { collectWorkspaceFiles, listSidebarFiles, @@ -1335,7 +1336,7 @@ function registerIpc() { "desktop:read-file-text", async (_event, { path: filePath }) => { const resolved = assertGranted(filePath); - return await fs.readFile(resolved, "utf8"); + return await readTextFile(resolved); }, ); @@ -1371,7 +1372,7 @@ function registerIpc() { const resolved = assertGranted(candidate); const stat = await fs.stat(resolved); if (!stat.isFile() || stat.size > SEARCH_MAX_FILE_BYTES) continue; - const content = await fs.readFile(resolved, "utf8"); + const content = await readTextFile(resolved); const matches = findMatchesInContent(content, needle); if (matches.length > 0) results.push({ path: candidate, matches }); } catch {} @@ -1412,10 +1413,10 @@ function registerIpc() { throw new Error("write-file-text requires encoded bytes"); } await fs.mkdir(path.dirname(resolved), { recursive: true }); - // Text is encoded in preload. Main only writes bytes so it cannot - // accidentally shorten UTF-8 content while crossing string encoders. + // Preload sends UTF-8 bytes so IPC cannot shorten multibyte text. + // The writer restores the existing file's BOM-marked encoding. // See https://github.com/bholmesdev/hubble.md/issues/126 for the repro. - await fs.writeFile(resolved, Uint8Array.from(bytes)); + await writeTextFile(resolved, Uint8Array.from(bytes)); }, ); diff --git a/apps/desktop/electron/textFile.test.ts b/apps/desktop/electron/textFile.test.ts new file mode 100644 index 00000000..bb957cb2 --- /dev/null +++ b/apps/desktop/electron/textFile.test.ts @@ -0,0 +1,56 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { decodeText, readTextFile, writeTextFile } from "./textFile"; + +const text = "Changed files: café 🚀"; +const utf8Bytes = new TextEncoder().encode(`${text} edited`); + +describe("text files", () => { + let root = ""; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), "hubble-text-")); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it.each([ + ["UTF-16LE", Buffer.from([0xff, 0xfe]), Buffer.from(text, "utf16le")], + [ + "UTF-16BE", + Buffer.from([0xfe, 0xff]), + swapBytes(Buffer.from(text, "utf16le")), + ], + ])("decodes and preserves %s files", async (_name, bom, content) => { + const filePath = path.join(root, "note.md"); + await fs.writeFile(filePath, Buffer.concat([bom, content])); + + expect(await readTextFile(filePath)).toBe(text); + await writeTextFile(filePath, utf8Bytes); + + const saved = await fs.readFile(filePath); + expect(saved.subarray(0, 2)).toEqual(bom); + expect(decodeText(saved)).toBe(`${text} edited`); + }); + + it("keeps UTF-8 files as UTF-8", async () => { + const filePath = path.join(root, "note.txt"); + await fs.writeFile(filePath, text, "utf8"); + + await writeTextFile(filePath, utf8Bytes); + + expect(await fs.readFile(filePath)).toEqual(Buffer.from(utf8Bytes)); + }); +}); + +function swapBytes(bytes: Uint8Array): Buffer { + const swapped = Buffer.from(bytes); + for (let index = 0; index < swapped.length; index += 2) { + [swapped[index], swapped[index + 1]] = [swapped[index + 1], swapped[index]]; + } + return swapped; +} diff --git a/apps/desktop/electron/textFile.ts b/apps/desktop/electron/textFile.ts new file mode 100644 index 00000000..88a0f852 --- /dev/null +++ b/apps/desktop/electron/textFile.ts @@ -0,0 +1,78 @@ +import fs from "node:fs/promises"; + +type TextEncoding = "utf8" | "utf8-bom" | "utf16le" | "utf16be"; + +export async function readTextFile(filePath: string): Promise { + return decodeText(await fs.readFile(filePath)); +} + +export async function writeTextFile( + filePath: string, + utf8Bytes: Uint8Array, +): Promise { + let encoding: TextEncoding = "utf8"; + try { + encoding = detectEncoding(await fs.readFile(filePath)); + } catch (error) { + if (!isMissingFile(error)) throw error; + } + const content = Buffer.from(utf8Bytes).toString("utf8"); + await fs.writeFile(filePath, encodeText(content, encoding)); +} + +export function decodeText(bytes: Uint8Array): string { + const encoding = detectEncoding(bytes); + const content = Buffer.from(bytes); + if (encoding === "utf16be") { + return swapBytes(content.subarray(2)).toString("utf16le"); + } + if (encoding === "utf16le") { + return content.subarray(2).toString("utf16le"); + } + if (encoding === "utf8-bom") { + return content.subarray(3).toString("utf8"); + } + return content.toString("utf8"); +} + +function detectEncoding(bytes: Uint8Array): TextEncoding { + if (bytes[0] === 0xff && bytes[1] === 0xfe) return "utf16le"; + if (bytes[0] === 0xfe && bytes[1] === 0xff) return "utf16be"; + if (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) + return "utf8-bom"; + return "utf8"; +} + +function encodeText(content: string, encoding: TextEncoding): Buffer { + const bytes = Buffer.from( + content, + encoding.startsWith("utf16") ? "utf16le" : "utf8", + ); + if (encoding === "utf16be") { + return Buffer.concat([Buffer.from([0xfe, 0xff]), swapBytes(bytes)]); + } + if (encoding === "utf16le") { + return Buffer.concat([Buffer.from([0xff, 0xfe]), bytes]); + } + if (encoding === "utf8-bom") { + return Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), bytes]); + } + return bytes; +} + +function swapBytes(bytes: Uint8Array): Buffer { + const swapped = Buffer.from(bytes); + for (let index = 0; index + 1 < swapped.length; index += 2) { + [swapped[index], swapped[index + 1]] = [swapped[index + 1], swapped[index]]; + } + return swapped; +} + +function isMissingFile(error: unknown): boolean { + return ( + error !== null && + typeof error === "object" && + "code" in error && + error.code === "ENOENT" + ); +}