Decode and preserve BOM-marked UTF-16 text files

CHANGELOG.md

@@ -13,6 +13,8 @@ Format loosely follows [Keep a Changelog](https://keepachangelog.com).
 
 ### Fixed
 
+- BOM-marked UTF-16 text and Markdown files now open, search, and save without corrupting their encoding. [#232](https://github.com/bholmesdev/hubble.md/pull/232)
+
 ## [0.1.25] - 2026-08-06
 
 ### Added

apps/desktop/electron/main.ts

@@ -49,6 +49,7 @@ import type { ThemePreference } from "../src/theme";
 import { DeleteUndo } from "./deleteUndo";
 import { TelemetryManager } from "./telemetry";
 import { setupTerminalIpc } from "./terminal";
+import { readTextFile, writeTextFile } from "./textFile";
 import {
 	collectWorkspaceFiles,
 	listSidebarFiles,
@@ -1335,7 +1336,7 @@ function registerIpc() {
 		"desktop:read-file-text",
 		async (_event, { path: filePath }) => {
 			const resolved = assertGranted(filePath);
-			return await fs.readFile(resolved, "utf8");
+			return await readTextFile(resolved);
 		},
 	);
 
@@ -1371,7 +1372,7 @@ function registerIpc() {
 						const resolved = assertGranted(candidate);
 						const stat = await fs.stat(resolved);
 						if (!stat.isFile() || stat.size > SEARCH_MAX_FILE_BYTES) continue;
-						const content = await fs.readFile(resolved, "utf8");
+						const content = await readTextFile(resolved);
 						const matches = findMatchesInContent(content, needle);
 						if (matches.length > 0) results.push({ path: candidate, matches });
 					} catch {}
@@ -1412,10 +1413,10 @@ function registerIpc() {
 				throw new Error("write-file-text requires encoded bytes");
 			}
 			await fs.mkdir(path.dirname(resolved), { recursive: true });
-			// Text is encoded in preload. Main only writes bytes so it cannot
-			// accidentally shorten UTF-8 content while crossing string encoders.
+			// Preload sends UTF-8 bytes so IPC cannot shorten multibyte text.
+			// The writer restores the existing file's BOM-marked encoding.
 			// See https://github.com/bholmesdev/hubble.md/issues/126 for the repro.
-			await fs.writeFile(resolved, Uint8Array.from(bytes));
+			await writeTextFile(resolved, Uint8Array.from(bytes));
 		},
 	);
 

apps/desktop/electron/textFile.test.ts

@@ -0,0 +1,56 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { decodeText, readTextFile, writeTextFile } from "./textFile";
+
+const text = "Changed files: café 🚀";
+const utf8Bytes = new TextEncoder().encode(`${text} edited`);
+
+describe("text files", () => {
+	let root = "";
+
+	beforeEach(async () => {
+		root = await fs.mkdtemp(path.join(os.tmpdir(), "hubble-text-"));
+	});
+
+	afterEach(async () => {
+		await fs.rm(root, { recursive: true, force: true });
+	});
+
+	it.each([
+		["UTF-16LE", Buffer.from([0xff, 0xfe]), Buffer.from(text, "utf16le")],
+		[
+			"UTF-16BE",
+			Buffer.from([0xfe, 0xff]),
+			swapBytes(Buffer.from(text, "utf16le")),
+		],
+	])("decodes and preserves %s files", async (_name, bom, content) => {
+		const filePath = path.join(root, "note.md");
+		await fs.writeFile(filePath, Buffer.concat([bom, content]));
+
+		expect(await readTextFile(filePath)).toBe(text);
+		await writeTextFile(filePath, utf8Bytes);
+
+		const saved = await fs.readFile(filePath);
+		expect(saved.subarray(0, 2)).toEqual(bom);
+		expect(decodeText(saved)).toBe(`${text} edited`);
+	});
+
+	it("keeps UTF-8 files as UTF-8", async () => {
+		const filePath = path.join(root, "note.txt");
+		await fs.writeFile(filePath, text, "utf8");
+
+		await writeTextFile(filePath, utf8Bytes);
+
+		expect(await fs.readFile(filePath)).toEqual(Buffer.from(utf8Bytes));
+	});
+});
+
+function swapBytes(bytes: Uint8Array): Buffer {
+	const swapped = Buffer.from(bytes);
+	for (let index = 0; index < swapped.length; index += 2) {
+		[swapped[index], swapped[index + 1]] = [swapped[index + 1], swapped[index]];
+	}
+	return swapped;
+}

apps/desktop/electron/textFile.ts

@@ -0,0 +1,78 @@
+import fs from "node:fs/promises";
+
+type TextEncoding = "utf8" | "utf8-bom" | "utf16le" | "utf16be";
+
+export async function readTextFile(filePath: string): Promise<string> {
+	return decodeText(await fs.readFile(filePath));
+}
+
+export async function writeTextFile(
+	filePath: string,
+	utf8Bytes: Uint8Array,
+): Promise<void> {
+	let encoding: TextEncoding = "utf8";
+	try {
+		encoding = detectEncoding(await fs.readFile(filePath));
+	} catch (error) {
+		if (!isMissingFile(error)) throw error;
+	}
+	const content = Buffer.from(utf8Bytes).toString("utf8");
+	await fs.writeFile(filePath, encodeText(content, encoding));
+}
+
+export function decodeText(bytes: Uint8Array): string {
+	const encoding = detectEncoding(bytes);
+	const content = Buffer.from(bytes);
+	if (encoding === "utf16be") {
+		return swapBytes(content.subarray(2)).toString("utf16le");
+	}
+	if (encoding === "utf16le") {
+		return content.subarray(2).toString("utf16le");
+	}
+	if (encoding === "utf8-bom") {
+		return content.subarray(3).toString("utf8");
+	}
+	return content.toString("utf8");
+}
+
+function detectEncoding(bytes: Uint8Array): TextEncoding {
+	if (bytes[0] === 0xff && bytes[1] === 0xfe) return "utf16le";
+	if (bytes[0] === 0xfe && bytes[1] === 0xff) return "utf16be";
+	if (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf)
+		return "utf8-bom";
+	return "utf8";
+}
+
+function encodeText(content: string, encoding: TextEncoding): Buffer {
+	const bytes = Buffer.from(
+		content,
+		encoding.startsWith("utf16") ? "utf16le" : "utf8",
+	);
+	if (encoding === "utf16be") {
+		return Buffer.concat([Buffer.from([0xfe, 0xff]), swapBytes(bytes)]);
+	}
+	if (encoding === "utf16le") {
+		return Buffer.concat([Buffer.from([0xff, 0xfe]), bytes]);
+	}
+	if (encoding === "utf8-bom") {
+		return Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), bytes]);
+	}
+	return bytes;
+}
+
+function swapBytes(bytes: Uint8Array): Buffer {
+	const swapped = Buffer.from(bytes);
+	for (let index = 0; index + 1 < swapped.length; index += 2) {
+		[swapped[index], swapped[index + 1]] = [swapped[index + 1], swapped[index]];
+	}
+	return swapped;
+}
+
+function isMissingFile(error: unknown): boolean {
+	return (
+		error !== null &&
+		typeof error === "object" &&
+		"code" in error &&
+		error.code === "ENOENT"
+	);
+}