diff --git a/README.md b/README.md index 4d740b5..fa70769 100644 --- a/README.md +++ b/README.md @@ -54,11 +54,12 @@ handlers close over. Call it once, before the plugin's hooks can fire. | `file-format-identify` | `graphEntityById` (handed to `matcher.js`'s own `configure(deps)` on each dynamic import, for `getFileHandleAtPath`) | | `ca-data-prep` | `writeFileAtPath` | | `merge` | `readJsonFromFolder`, `graphEntityById` | +| `crate2tables` | `readJsonFromFolder`, `writeFileAtPath`, `getFileHandleAtPath`, `readFileTextFromDirectory` | | `validate-crate` | `loadMasp` | | `ro-crate-json-output` | `crateToJsonString`, `writeFile`, `fileExists` | | `ro-crate-xlsx-output` | `crateToXlsxBytes`, `writeFile`, `fileExists` | | `ro-crate-html-output` | `crateToPreviewHtml`, `crateToMultiPageHtml`, `writeFile`, `writeFileAtPath`, `readJsonFromFolder`, `readFileTextFromDirectory`, `verifyPermission`, `fileExists`, `bustCacheUrl`, `buildGitHubTreeUrl`, `fetchGitHubTextFile`, `listGitHubFolder` | -| `generic-input` (input mode) | `buildFileMetadata`, `buildCrate` | +| `generic-input` (input mode) | `buildFileMetadata`, `buildCrate`, `readJsonFromFolder` (reads the folder's existing crate, if any, to reconcile against rather than replace — chaos2crate SPEC.md §6.1a), `openModal` (confirms which newly-found files to add, via `new-files-confirm.js`) | | `docx-input` (input mode) | `writeFileAtPath` (handed to `docx_crate.js`'s own `configure(deps)` once its dynamic import resolves) | `loadMasp` is a thunk — `() => import("../masp.js")` — rather than the @@ -66,6 +67,20 @@ function itself, so `ro-crate-masp` (a heavy validator library) stays dynamically imported from chaos2crate's own tree instead of becoming a static import anywhere in this package. +`crate2tables` depends on [`roctable`](https://github.com/ptsefton/roctable), +a WIP library not yet on npm — installed as `"roctable": "file:../roctable"` +while both are under active development (swap to a `github:ptsefton/roctable` +git dependency, pinned to a commit, once roctable's own PR lands). It reuses +roctable's own crate-walking functions directly (`ctx.crate` is already an +`ro-crate` `ROCrate` instance, the same shape roctable expects) — including +`load_text`, via a `fileReader` this plugin injects +(`browserFileReader` in `src/crate2tables/index.js`, wrapping +`readFileTextFromDirectory`) rather than roctable's own Node-`fs`-based +default (see roctable's `lib/io.js` and its `SPEC.md` §9.0). Its config +load/save and CSV file writing stay this plugin's own job either way — +roctable's `lib/config.js`/`lib/csv.js` file I/O is Node-`fs`-only and simply +isn't called from here; see `chaos2crate/docs/crate2tables-spec.md`. + ## Writing a new plugin here ```js diff --git a/index.js b/index.js index f09fab0..27f443b 100644 --- a/index.js +++ b/index.js @@ -17,6 +17,7 @@ import { createPlugin as createValidateCrate } from "./src/validate-crate/index. import { createPlugin as createJsonOutput } from "./src/ro-crate-json-output/index.js"; import { createPlugin as createXlsxOutput } from "./src/ro-crate-xlsx-output/index.js"; import { createPlugin as createHtmlOutput } from "./src/ro-crate-html-output/index.js"; +import { createPlugin as createCrate2Tables } from "./src/crate2tables/index.js"; import { createPlugin as createGenericInput } from "./src/generic-input/index.js"; import { createPlugin as createDocxInput } from "./src/docx-input/index.js"; @@ -33,6 +34,7 @@ export const REGISTRY = { "ca-data-prep": createCaDataPrep, "chat-export": createChatExport, "merge": createMerge, + "crate2tables": createCrate2Tables, "validate-crate": createValidateCrate, "ro-crate-json-output": createJsonOutput, "ro-crate-xlsx-output": createXlsxOutput, diff --git a/package-lock.json b/package-lock.json index 3a88b38..6cb479c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,9 +13,24 @@ "mammoth": "^1.12.0", "ro-crate": "^3.7.2", "ro-crate-excel": "^1.2.1", + "roctable": "file:../roctable", "unicode-name": "^1.1.0" } }, + "../roctable": { + "dependencies": { + "commander": "^15.0.0", + "csv-parse": "^7.0.2", + "csv-stringify": "^6.8.3", + "ro-crate": "^3.7.2" + }, + "bin": { + "roctable": "bin/roctable.js" + }, + "devDependencies": { + "vitest": "^3.2.7" + } + }, "node_modules/@fast-csv/format": { "version": "4.3.5", "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", @@ -1524,6 +1539,10 @@ "node": ">=20" } }, + "node_modules/roctable": { + "resolved": "../roctable", + "link": true + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", diff --git a/package.json b/package.json index f2613a4..72e9cb4 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "mammoth": "^1.12.0", "ro-crate": "^3.7.2", "ro-crate-excel": "^1.2.1", + "roctable": "file:../roctable", "unicode-name": "^1.1.0" }, "scripts": { diff --git a/src/crate2tables/index.js b/src/crate2tables/index.js new file mode 100644 index 0000000..b1fbdb9 --- /dev/null +++ b/src/crate2tables/index.js @@ -0,0 +1,136 @@ +// Exports a built RO-Crate as one CSV per configured @type, using roctable +// (https://github.com/ptsefton/roctable) — a WIP, git-installed library that +// flattens an `ro-crate` ROCrate graph into tables according to a JSON +// config. chaos2crate's own crate.js builds its ROCrate with the same +// `ro-crate` package version roctable depends on, so ctx.crate can be handed +// straight to roctable's crate-walking functions with no adapter needed. +// +// roctable's own CLI is a two-step workflow: `roctable inspect` discovers +// every @type/property in a crate and writes/updates a config (new fields +// default to unselected, existing choices are preserved); `roctable csv` +// then extracts tables for whatever the config's "tables" section selects. +// This plugin runs the same two steps on every build instead of requiring a +// separate CLI pass: it always re-discovers against the current crate and +// rewrites crate2tables-config.json (non-destructively — see +// roctable/lib/inspect.js's mergeDiscovered), then extracts+writes CSV for +// whatever the config's "tables" section already selects. A first build +// against a fresh folder therefore selects nothing and only seeds the +// config; a person edits that file (moving a type from "potential_tables" to +// "tables", setting "include"/"expand"/"load_text" on its properties) and +// reruns the build to get output. See docs/crate2tables-spec.md. +// +// load_text (roctable's "read this property's referenced file into the row" +// feature) reads through an injected fileReader (ptsefton/roctable#1) rather +// than roctable's own Node-fs default — browserFileReader below wraps +// chaos2crate's readFileTextFromDirectory, which already returns null for +// "not found", matching what extractTables' loadText expects from a reader. +import { inspectCrate, mergeDiscovered, discoverExpandedProperties } from "roctable/lib/inspect.js"; +import { extractTables } from "roctable/lib/extract.js"; +import { tablesToCsvStrings } from "roctable/lib/csv.js"; +import { defaultConfig } from "roctable/lib/config.js"; + +const CONFIG_FILE = "crate2tables-config.json"; +const OUTPUT_DIR = "crate2tables-output"; + +// Hook names are literal strings and core chaos2crate functions arrive via +// createPlugin(deps) — see this repo's README. +let readJsonFromFolder, writeFileAtPath, getFileHandleAtPath, readFileTextFromDirectory; + +export function createPlugin(deps) { + ({ readJsonFromFolder, writeFileAtPath, getFileHandleAtPath, readFileTextFromDirectory } = deps); + return plugin; +} + +function browserFileReader(dirHandle) { + return { readFile: (relPath) => readFileTextFromDirectory(dirHandle, relPath) }; +} + +async function existsAtPath(dirHandle, relativePath) { + return !!(await getFileHandleAtPath(dirHandle, relativePath)); +} + +const plugin = { + name: "crate2tables", + optionSchema: { + key: "enableCrate2Tables", + label: "Export RO-Crate tables", + default: false, + hint: 'Flattens the crate into one CSV per entity type, using crate2tables-config.json — written to the folder on the first build with every discovered type/property, unselected. Move a type from "potential_tables" to "tables" and set "include": true on the properties you want, then rebuild. See docs/crate2tables-spec.md.', + children: [ + { key: "crate2tablesConfigUpload", type: "file", label: "Table config (JSON)", + accept: "application/json,.json", + hint: "Overrides crate2tables-config.json from the folder, if present." }, + ], + }, + outputPaths: [ + { path: CONFIG_FILE, kind: "file" }, + { path: OUTPUT_DIR, kind: "dir" }, + ], + hooks: { + "crate:built": async (ctx) => { + if (!ctx.options.enableCrate2Tables) return; + const { crate, dirHandle, options, log } = ctx; + + let existingConfig = null; + let configSource = "none — starting fresh"; + if (options.crate2tablesConfigUpload) { + const text = await options.crate2tablesConfigUpload.file.text(); + try { existingConfig = JSON.parse(text); } + catch (e) { throw new Error(`uploaded table config "${options.crate2tablesConfigUpload.name}" is not valid JSON: ${e.message}`); } + configSource = `uploaded (${options.crate2tablesConfigUpload.name})`; + } else { + const folderConfig = await readJsonFromFolder(dirHandle, CONFIG_FILE); + if (folderConfig) { existingConfig = folderConfig; configSource = CONFIG_FILE; } + } + + let config; + try { + config = discoverExpandedProperties(crate, mergeDiscovered(existingConfig || defaultConfig(), inspectCrate(crate))); + } catch (e) { + log(`crate2tables: could not inspect the crate — ${e.message}`, "warn"); + return; + } + + ctx.crate2tables = { config, configSource }; + + const tableNames = Object.keys(config.tables || {}); + if (!tableNames.length) { + log(`crate2tables: no tables selected yet (config source: ${configSource}). Wrote every discovered type to ${CONFIG_FILE} under "potential_tables" — move the ones you want into "tables" and rebuild.`, "warn"); + return; + } + + try { + const data = await extractTables(crate, config, { fileReader: browserFileReader(dirHandle) }); + ctx.crate2tables.csv = tablesToCsvStrings(data); + log(`crate2tables: built ${tableNames.length} table(s) — ${tableNames.join(", ")}.`, "ok"); + } catch (e) { + log(`crate2tables: failed to extract tables — ${e.message}`, "warn"); + } + }, + + "output:write": async (ctx) => { + if (!ctx.options.enableCrate2Tables || !ctx.crate2tables) return; + const { dirHandle, options, log } = ctx; + const { config, csv } = ctx.crate2tables; + + // Non-destructive by construction (mergeDiscovered only ever adds + // newly-seen types/properties, unselected — see roctable/lib/inspect.js), + // so rewriting it every build is the same "keep it fresh" behaviour as + // rerunning `roctable inspect`, not a risk to a hand-edited config. + await writeFileAtPath(dirHandle, CONFIG_FILE, JSON.stringify(config, null, 2) + "\n"); + + if (!csv) return; + let written = 0; + for (const [tableName, text] of Object.entries(csv)) { + const path = `${OUTPUT_DIR}/${tableName}.csv`; + if (options.overwrite || !(await existsAtPath(dirHandle, path))) { + await writeFileAtPath(dirHandle, path, text); + written++; + } else { + log(`${path} exists and overwrite is off — skipped.`, "warn"); + } + } + if (written) log(`crate2tables: wrote ${written} CSV file(s) to ${OUTPUT_DIR}/.`, "ok"); + }, + }, +}; diff --git a/src/generic-input/index.js b/src/generic-input/index.js index 329902b..de3c9a9 100644 --- a/src/generic-input/index.js +++ b/src/generic-input/index.js @@ -11,10 +11,12 @@ // runs per build, dispatched by pipeline.js on ctx.options.inputMode. // Core chaos2crate functions arrive via createPlugin(deps) — see this // repo's README. -let buildFileMetadata, buildCrate; +import { confirmNewFiles } from "./new-files-confirm.js"; + +let buildFileMetadata, buildCrate, readJsonFromFolder, openModal; export function createPlugin(deps) { - ({ buildFileMetadata, buildCrate } = deps); + ({ buildFileMetadata, buildCrate, readJsonFromFolder, openModal } = deps); return plugin; } @@ -31,13 +33,48 @@ const plugin = { ctx.sourceCount = ctx.filesWithMeta.length; }, - buildCrate(ctx) { - ctx.crate = buildCrate(ctx.filesWithMeta, ctx.config, ctx.log, { + // If the folder already has a crate, this build reconciles against it + // (SPEC.md §6.1a) instead of replacing it — buildCrate() (crate.js) only + // needs the parsed JSON to know that; everything else is unchanged. + // "Existing crate" here means the file this same JSON output plugin + // writes, not xlsx-crate-input's additional-ro-crate-metadata.xlsx (a + // deliberately separate, opt-in source — see that plugin's own hooks). + // + // A file the scan found with no matching entity in that existing crate + // isn't added silently — the person building the crate confirms it first + // (new-files-confirm.js's checkbox tree), since reconcileFileEntities' + // fallback for one with no obvious home is to attach it straight to the + // root dataset, and that's exactly the kind of guess a human should sign + // off on rather than discover after the fact in the build log. + async buildCrate(ctx) { + const existingJson = await readJsonFromFolder(ctx.dirHandle, "ro-crate-metadata.json"); + let filesToBuild = ctx.filesWithMeta; + + if (existingJson) { + const existingIds = new Set((existingJson["@graph"] || []).map((e) => e["@id"])); + const newPaths = ctx.filesWithMeta.map((f) => f.id).filter((id) => !existingIds.has(id)); + + if (newPaths.length) { + ctx.log(`${newPaths.length} file(s) not in the existing crate — asking which to add.`, "info"); + const confirmed = await confirmNewFiles({ newPaths, openModal }); + if (confirmed === null) throw new Error("Build cancelled: new files were not confirmed."); + + const confirmedSet = new Set(confirmed); + const skipped = newPaths.filter((id) => !confirmedSet.has(id)); + if (confirmed.length) ctx.log(`Adding ${confirmed.length} confirmed new file(s).`, "ok"); + if (skipped.length) ctx.log(`Skipping ${skipped.length} file(s) this build (not added to the crate): ${skipped.join(", ")}`, "warn"); + + filesToBuild = ctx.filesWithMeta.filter((f) => existingIds.has(f.id) || confirmedSet.has(f.id)); + } + } + + ctx.crate = buildCrate(filesToBuild, ctx.config, ctx.log, { topLevelFolderType: ctx.options.topLevelFolderType, // ctx.xlsxCrate is set at config:prepare, before this runs: a spreadsheet // already describes the entries and what belongs to what, so the folder // scan shouldn't invent a parallel structure alongside it. structureFromMetadata: !!ctx.xlsxCrate, + existingJson, }); }, }; diff --git a/src/generic-input/new-files-confirm.js b/src/generic-input/new-files-confirm.js new file mode 100644 index 0000000..2e81302 --- /dev/null +++ b/src/generic-input/new-files-confirm.js @@ -0,0 +1,128 @@ +// A checkbox file-tree for confirming which newly-discovered files (found by +// a generic-input scan but not yet in an existing crate) should actually be +// added as new File entities — see index.js's buildCrate and chaos2crate +// SPEC.md §6.1a. Built the same "no host markup, no HTML string" way as the +// rest of this app's own plugin UI, using the shared `openModal` helper and +// the host's own `.checkbox`/`.modal .actions` CSS conventions rather than +// inventing new ones. + +// Turns a flat list of relative paths into a nested { name, children, isFile, +// path } tree, splitting on "/" the same way filesWithMeta's own folder +// grouping does. +function buildFileTree(paths) { + const root = { name: "", children: new Map(), isFile: false }; + for (const relPath of paths) { + const parts = relPath.split("/"); + let node = root; + parts.forEach((part, i) => { + const isFile = i === parts.length - 1; + if (!node.children.has(part)) { + node.children.set(part, { name: part, children: new Map(), isFile, path: isFile ? relPath : null }); + } + node = node.children.get(part); + }); + } + return root; +} + +function leafPaths(node) { + if (node.isFile) return [node.path]; + return [...node.children.values()].flatMap(leafPaths); +} + +// Renders `node`'s children as a