diff --git a/README.md b/README.md index a699a6f..08de960 100644 --- a/README.md +++ b/README.md @@ -248,6 +248,50 @@ If a worktree for that reference already exists it is replaced. --- +### `export` + +Export the archive as Markdown files — one `.md` per message — with YAML frontmatter and a human-readable body. + +```sh +backmail export +backmail export --only-folder INBOX --only-folder Sent +backmail export --exclude-folder Spam --verbose +``` + +| Option | Description | +|--------|-------------| +| `--exclude-folder ` | Skip a folder (repeatable) | +| `--only-folder ` | Restrict to a folder (repeatable) | +| `--verbose` | Log one line per folder and per message | + +`--exclude-folder` and `--only-folder` are mutually exclusive. + +Output is written to `export/` next to `archive/`, mirroring the folder structure. Each file contains: + +- YAML frontmatter with `messageId`, `from`, `to`, `subject`, `received`, `folders`, `formats`, and `attachments` +- The subject as a top-level heading +- Plain text body (when present) +- HTML body converted to Markdown, preceded by a labelled separator block (when both parts are present — layout tables are unwrapped to prose, external images stripped, inline attachments preserved) + +Running `export` again is idempotent — existing files are overwritten with identical content. + +Output: + +``` +export: =42 exported +``` + +If any message fails the export continues and the summary line is tagged `[partial]`: + +``` +export [partial]: =38 exported +folder INBOX/Archive failed: read error +``` + +Exit code is non-zero when any folder fails. + +--- + ### `restore` Re-upload messages from a backup to an IMAP server. Useful for migrating to a new provider or recovering deleted mail. @@ -290,6 +334,9 @@ archive/ # git repository worktrees/ # point-in-time checkouts (outside the git repo) 2024-01-15/ abc1234/ +export/ # Markdown export (created by `backmail export`) + INBOX/ + 2024-01-15_subject_aabbccdd.md ``` Messages are content-addressed by Message-ID, so identical emails that appear in multiple IMAP folders (common with Gmail labels) are stored only once. diff --git a/package-lock.json b/package-lock.json index 94f8c84..fd35085 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,8 @@ "mailparser": "^3.9.8", "simple-git": "3.36.0", "slugify": "^1.6.9", + "turndown": "^7.2.4", + "turndown-plugin-gfm": "^1.0.2", "zod": "^4.3.6" }, "bin": { @@ -26,6 +28,7 @@ "devDependencies": { "@types/mailparser": "^3.4.6", "@types/node": "25.6.0", + "@types/turndown": "^5.0.6", "@vitest/coverage-v8": "^4.1.5", "tsx": "4.21.0", "typescript": "6.0.3", @@ -938,6 +941,12 @@ "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==", "license": "MIT" }, + "node_modules/@mixmark-io/domino": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", + "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==", + "license": "BSD-2-Clause" + }, "node_modules/@napi-rs/keyring": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@napi-rs/keyring/-/keyring-1.3.0.tgz", @@ -1562,6 +1571,13 @@ "undici-types": "~7.19.0" } }, + "node_modules/@types/turndown": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/turndown/-/turndown-5.0.6.tgz", + "integrity": "sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==", + "dev": true, + "license": "MIT" + }, "node_modules/@vitest/coverage-v8": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.5.tgz", @@ -3103,6 +3119,25 @@ "fsevents": "~2.3.3" } }, + "node_modules/turndown": { + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.4.tgz", + "integrity": "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==", + "license": "MIT", + "dependencies": { + "@mixmark-io/domino": "^2.2.0" + }, + "engines": { + "node": ">=18", + "npm": ">=9" + } + }, + "node_modules/turndown-plugin-gfm": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz", + "integrity": "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==", + "license": "MIT" + }, "node_modules/typescript": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", diff --git a/package.json b/package.json index 5c3fe26..300c6f6 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "devDependencies": { "@types/mailparser": "^3.4.6", "@types/node": "25.6.0", + "@types/turndown": "^5.0.6", "@vitest/coverage-v8": "^4.1.5", "tsx": "4.21.0", "typescript": "6.0.3", @@ -63,6 +64,8 @@ "mailparser": "^3.9.8", "simple-git": "3.36.0", "slugify": "^1.6.9", + "turndown": "^7.2.4", + "turndown-plugin-gfm": "^1.0.2", "zod": "^4.3.6" } } diff --git a/src/cli/index.ts b/src/cli/index.ts index 4b063a4..64dab6b 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -39,7 +39,7 @@ function getRepoRoot(): string { return repoRoot } -import { syncAccount, getLog, checkoutCommit, listFolders, listMessages, viewMessage, restoreAccount } from '../core/index.js' +import { syncAccount, getLog, checkoutCommit, listFolders, listMessages, viewMessage, restoreAccount, exportAccount } from '../core/index.js' function getErrorMessage(err: unknown): string { if (err instanceof Error) { @@ -373,4 +373,48 @@ program } }) +// ── export ─────────────────────────────────────────────────────────────────── +program + .command('export') + .description('Export archive messages as Markdown files') + .option('--exclude-folder ', 'skip this folder (repeatable)', collectRepeatable, []) + .option('--only-folder ', 'restrict to this folder (repeatable)', collectRepeatable, []) + .option('--verbose', 'log one line per exported file') + .action(async (opts: { excludeFolder: string[]; onlyFolder: string[]; verbose?: boolean }) => { + if (opts.excludeFolder.length > 0 && opts.onlyFolder.length > 0) { + console.error('Error: --exclude-folder and --only-folder are mutually exclusive') + process.exit(1) + } + + try { + const repoRoot = getRepoRoot() + const archivePath = path.join(repoRoot, 'archive') + const exportPath = path.join(repoRoot, 'export') + const verbose = opts.verbose ?? false + + const result = await exportAccount(archivePath, exportPath, { + excludeFolders: opts.excludeFolder, + onlyFolders: opts.onlyFolder, + verbose, + onLog: verbose ? (msg) => console.log(msg) : undefined, + }) + + const partialTag = result.errors > 0 ? ' [partial]' : '' + console.log(`export${partialTag}: =${result.exported} exported`) + + for (const fr of result.folderResults) { + if (fr.error) { + console.error(`folder ${fr.path} failed: ${fr.error.message}`) + } + } + + if (result.errors > 0) { + process.exit(1) + } + } catch (err) { + console.error(getErrorMessage(err)) + process.exit(1) + } + }) + program.parse(process.argv) diff --git a/src/core/export.ts b/src/core/export.ts new file mode 100644 index 0000000..148f1b1 --- /dev/null +++ b/src/core/export.ts @@ -0,0 +1,575 @@ +// src/core/export.ts — no exit calls, no console, no CLI imports. +import fs from 'node:fs/promises' +import path from 'node:path' +import { simpleParser } from 'mailparser' +import type { ParsedMail } from 'mailparser' +import TurndownService from 'turndown' +import { gfm } from 'turndown-plugin-gfm' +import { parseHeaderBlock, extractRawHeader } from './sync.js' +import { listFolders } from './browse.js' + +// ── Types ───────────────────────────────────────────────────────────────────── + +export interface ExportOptions { + excludeFolders?: string[] + onlyFolders?: string[] + verbose?: boolean + onLog?: (msg: string) => void +} + +export interface ExportResult { + exported: number + errors: number + folderResults: FolderExportResult[] +} + +export interface FolderExportResult { + path: string + exported: number + error?: Error +} + +// ── HTML → Markdown ─────────────────────────────────────────────────────────── + +export function preprocessHtml(html: string): string { + // Strip invisible filler characters used as spacers in HTML email + // (soft hyphen U+00AD, zero-width space U+200B, zero-width non-joiner U+200C, + // zero-width joiner U+200D, BOM/zero-width no-break space U+FEFF, + // combining grapheme joiner U+034F \u2014 used in preheader padding sequences) + let out = html.replace(/[\u034f\u00ad\u200b\u200c\u200d\ufeff]/g, '') + + // Strip

Hello

' + expect(preprocessHtml(html)).not.toContain('