From 97050c35dc8c2894bafba6a773bbb30c30b4df9a Mon Sep 17 00:00:00 2001 From: Peter Sefton Date: Wed, 2 Sep 2026 16:10:26 +1000 Subject: [PATCH 1/4] Add crate2tables plugin: export RO-Crate tables via roctable Flattens a built crate into one CSV per entity type using roctable (github:ptsefton/roctable, WIP/not on npm), config-driven the same way its own inspect/csv CLI works: discovers types/properties into crate2tables-config.json on every build (non-destructive to existing choices), extracts+writes CSV for whatever the config already selects into crate2tables-output/. load_text is disabled with a warning rather than crashing, since roctable's implementation reads files via Node's fs, not available in chaos2crate's browser build. See chaos2crate/docs/crate2tables-spec.md for the full design, config schema, and the Phase 2 UI plan for interactive type/field selection. --- README.md | 10 +++ index.js | 2 + package-lock.json | 34 ++++++++ package.json | 1 + src/crate2tables/index.js | 159 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 206 insertions(+) create mode 100644 src/crate2tables/index.js diff --git a/README.md b/README.md index 4d740b5..657356a 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ handlers close over. Call it once, before the plugin's hooks can fire. | `file-format-identify` | `graphEntityById` (handed to `matcher.js`'s own `configure(deps)` on each dynamic import, for `getFileHandleAtPath`) | | `ca-data-prep` | `writeFileAtPath` | | `merge` | `readJsonFromFolder`, `graphEntityById` | +| `crate2tables` | `readJsonFromFolder`, `writeFileAtPath`, `getFileHandleAtPath` | | `validate-crate` | `loadMasp` | | `ro-crate-json-output` | `crateToJsonString`, `writeFile`, `fileExists` | | `ro-crate-xlsx-output` | `crateToXlsxBytes`, `writeFile`, `fileExists` | @@ -66,6 +67,15 @@ function itself, so `ro-crate-masp` (a heavy validator library) stays dynamically imported from chaos2crate's own tree instead of becoming a static import anywhere in this package. +`crate2tables` depends on [`roctable`](https://github.com/ptsefton/roctable), +a WIP library not yet on npm — installed here as a git dependency +(`"roctable": "github:ptsefton/roctable"`). It reuses roctable's own +crate-walking functions directly (`ctx.crate` is already an `ro-crate` +`ROCrate` instance, the same shape roctable expects), but not its config/CSV +file I/O, which is Node-`fs`-only — see `src/crate2tables/index.js` and +`chaos2crate/docs/crate2tables-spec.md` for what's adapted and what's +deliberately unsupported for now (`load_text`). + ## Writing a new plugin here ```js diff --git a/index.js b/index.js index f09fab0..27f443b 100644 --- a/index.js +++ b/index.js @@ -17,6 +17,7 @@ import { createPlugin as createValidateCrate } from "./src/validate-crate/index. import { createPlugin as createJsonOutput } from "./src/ro-crate-json-output/index.js"; import { createPlugin as createXlsxOutput } from "./src/ro-crate-xlsx-output/index.js"; import { createPlugin as createHtmlOutput } from "./src/ro-crate-html-output/index.js"; +import { createPlugin as createCrate2Tables } from "./src/crate2tables/index.js"; import { createPlugin as createGenericInput } from "./src/generic-input/index.js"; import { createPlugin as createDocxInput } from "./src/docx-input/index.js"; @@ -33,6 +34,7 @@ export const REGISTRY = { "ca-data-prep": createCaDataPrep, "chat-export": createChatExport, "merge": createMerge, + "crate2tables": createCrate2Tables, "validate-crate": createValidateCrate, "ro-crate-json-output": createJsonOutput, "ro-crate-xlsx-output": createXlsxOutput, diff --git a/package-lock.json b/package-lock.json index 3a88b38..a05cefc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "mammoth": "^1.12.0", "ro-crate": "^3.7.2", "ro-crate-excel": "^1.2.1", + "roctable": "github:ptsefton/roctable", "unicode-name": "^1.1.0" } }, @@ -439,6 +440,18 @@ "url": "https://github.com/sponsors/fb55" } }, + "node_modules/csv-parse": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-7.0.2.tgz", + "integrity": "sha512-uKZghv9UmPkMVLYy//KZ9HFAIJsl7wkhoEdIL0+rhuSY9pZQlhaeGEDPIe+/w7eh81MOql8Q/9+inAGWG6ZHYA==", + "license": "MIT" + }, + "node_modules/csv-stringify": { + "version": "6.8.3", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.8.3.tgz", + "integrity": "sha512-gIeSCvq5F4VtXV3naV3VAewLhBkiZBz+PPhTOA8H3Y8h/ELa+R1ml0GZck/4/Nzo9ep2lvOluilJ6MJlbZsKMA==", + "license": "MIT" + }, "node_modules/dayjs": { "version": "1.11.23", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.23.tgz", @@ -1524,6 +1537,27 @@ "node": ">=20" } }, + "node_modules/roctable": { + "resolved": "git+ssh://git@github.com/ptsefton/roctable.git#91dd951e1895c789c6889282677e5d3722ac6382", + "dependencies": { + "commander": "^15.0.0", + "csv-parse": "^7.0.2", + "csv-stringify": "^6.8.3", + "ro-crate": "^3.7.2" + }, + "bin": { + "roctable": "bin/roctable.js" + } + }, + "node_modules/roctable/node_modules/commander": { + "version": "15.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-15.0.0.tgz", + "integrity": "sha512-z67u4ZhzCL/Tydu1lJARtEZYWbWaN7oYLHbsuzocr6y4N6WZAagG3RQ4FW61V1/0+jImpj293XfrcYnd1qxtPg==", + "license": "MIT", + "engines": { + "node": ">=22.12.0" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", diff --git a/package.json b/package.json index f2613a4..e2e9bfa 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "mammoth": "^1.12.0", "ro-crate": "^3.7.2", "ro-crate-excel": "^1.2.1", + "roctable": "github:ptsefton/roctable", "unicode-name": "^1.1.0" }, "scripts": { diff --git a/src/crate2tables/index.js b/src/crate2tables/index.js new file mode 100644 index 0000000..3b61f42 --- /dev/null +++ b/src/crate2tables/index.js @@ -0,0 +1,159 @@ +// Exports a built RO-Crate as one CSV per configured @type, using roctable +// (https://github.com/ptsefton/roctable) — a WIP, git-installed library that +// flattens an `ro-crate` ROCrate graph into tables according to a JSON +// config. chaos2crate's own crate.js builds its ROCrate with the same +// `ro-crate` package version roctable depends on, so ctx.crate can be handed +// straight to roctable's crate-walking functions with no adapter needed. +// +// roctable's own CLI is a two-step workflow: `roctable inspect` discovers +// every @type/property in a crate and writes/updates a config (new fields +// default to unselected, existing choices are preserved); `roctable csv` +// then extracts tables for whatever the config's "tables" section selects. +// This plugin runs the same two steps on every build instead of requiring a +// separate CLI pass: it always re-discovers against the current crate and +// rewrites crate2tables-config.json (non-destructively — see +// roctable/lib/inspect.js's mergeDiscovered), then extracts+writes CSV for +// whatever the config's "tables" section already selects. A first build +// against a fresh folder therefore selects nothing and only seeds the +// config; a person edits that file (moving a type from "potential_tables" to +// "tables", setting "include"/"expand"/"load_text" on its properties) and +// reruns the build to get output. See docs/crate2tables-spec.md. +// +// load_text (roctable's "read this property's referenced file into the row" +// feature) is deliberately unsupported here for now: roctable's extractTables +// reads that file with Node's `fs`, which isn't available in chaos2crate's +// browser build. Any property configured with load_text is disabled with a +// warning rather than passed through — see stripUnsupportedFeatures below +// and the spec's "Known limitations" section. +import { inspectCrate, mergeDiscovered, discoverExpandedProperties } from "roctable/lib/inspect.js"; +import { extractTables } from "roctable/lib/extract.js"; +import { tablesToCsvStrings } from "roctable/lib/csv.js"; +import { defaultConfig } from "roctable/lib/config.js"; + +const CONFIG_FILE = "crate2tables-config.json"; +const OUTPUT_DIR = "crate2tables-output"; + +// Hook names are literal strings and core chaos2crate functions arrive via +// createPlugin(deps) — see this repo's README. +let readJsonFromFolder, writeFileAtPath, getFileHandleAtPath; + +export function createPlugin(deps) { + ({ readJsonFromFolder, writeFileAtPath, getFileHandleAtPath } = deps); + return plugin; +} + +// Recursively clears load_text off every property (including expand's own +// nested sub-properties), returning the dotted entityType.property paths it +// touched so the caller can log one clear warning instead of one per field. +function stripUnsupportedFeatures(config) { + const touched = []; + for (const [entityType, tableConfig] of Object.entries(config.tables || {})) { + for (const [prop, propConfig] of Object.entries(tableConfig.properties || {})) { + if (propConfig.load_text) { + propConfig.load_text = false; + touched.push(`${entityType}.${prop}`); + } + for (const [exProp, exPropConfig] of Object.entries(propConfig.properties || {})) { + if (exPropConfig.load_text) { + exPropConfig.load_text = false; + touched.push(`${entityType}.${prop}.${exProp}`); + } + } + } + } + return touched; +} + +async function existsAtPath(dirHandle, relativePath) { + return !!(await getFileHandleAtPath(dirHandle, relativePath)); +} + +const plugin = { + name: "crate2tables", + optionSchema: { + key: "enableCrate2Tables", + label: "Export RO-Crate tables", + default: false, + hint: 'Flattens the crate into one CSV per entity type, using crate2tables-config.json — written to the folder on the first build with every discovered type/property, unselected. Move a type from "potential_tables" to "tables" and set "include": true on the properties you want, then rebuild. See docs/crate2tables-spec.md.', + children: [ + { key: "crate2tablesConfigUpload", type: "file", label: "Table config (JSON)", + accept: "application/json,.json", + hint: "Overrides crate2tables-config.json from the folder, if present." }, + ], + }, + outputPaths: [ + { path: CONFIG_FILE, kind: "file" }, + { path: OUTPUT_DIR, kind: "dir" }, + ], + hooks: { + "crate:built": async (ctx) => { + if (!ctx.options.enableCrate2Tables) return; + const { crate, dirHandle, options, log } = ctx; + + let existingConfig = null; + let configSource = "none — starting fresh"; + if (options.crate2tablesConfigUpload) { + const text = await options.crate2tablesConfigUpload.file.text(); + try { existingConfig = JSON.parse(text); } + catch (e) { throw new Error(`uploaded table config "${options.crate2tablesConfigUpload.name}" is not valid JSON: ${e.message}`); } + configSource = `uploaded (${options.crate2tablesConfigUpload.name})`; + } else { + const folderConfig = await readJsonFromFolder(dirHandle, CONFIG_FILE); + if (folderConfig) { existingConfig = folderConfig; configSource = CONFIG_FILE; } + } + + let config; + try { + config = discoverExpandedProperties(crate, mergeDiscovered(existingConfig || defaultConfig(), inspectCrate(crate))); + } catch (e) { + log(`crate2tables: could not inspect the crate — ${e.message}`, "warn"); + return; + } + + const unsupported = stripUnsupportedFeatures(config); + if (unsupported.length) { + log(`crate2tables: load_text isn't supported yet in the browser build — disabled on ${unsupported.join(", ")}.`, "warn"); + } + + ctx.crate2tables = { config, configSource }; + + const tableNames = Object.keys(config.tables || {}); + if (!tableNames.length) { + log(`crate2tables: no tables selected yet (config source: ${configSource}). Wrote every discovered type to ${CONFIG_FILE} under "potential_tables" — move the ones you want into "tables" and rebuild.`, "warn"); + return; + } + + try { + ctx.crate2tables.csv = tablesToCsvStrings(extractTables(crate, config)); + log(`crate2tables: built ${tableNames.length} table(s) — ${tableNames.join(", ")}.`, "ok"); + } catch (e) { + log(`crate2tables: failed to extract tables — ${e.message}`, "warn"); + } + }, + + "output:write": async (ctx) => { + if (!ctx.options.enableCrate2Tables || !ctx.crate2tables) return; + const { dirHandle, options, log } = ctx; + const { config, csv } = ctx.crate2tables; + + // Non-destructive by construction (mergeDiscovered only ever adds + // newly-seen types/properties, unselected — see roctable/lib/inspect.js), + // so rewriting it every build is the same "keep it fresh" behaviour as + // rerunning `roctable inspect`, not a risk to a hand-edited config. + await writeFileAtPath(dirHandle, CONFIG_FILE, JSON.stringify(config, null, 2) + "\n"); + + if (!csv) return; + let written = 0; + for (const [tableName, text] of Object.entries(csv)) { + const path = `${OUTPUT_DIR}/${tableName}.csv`; + if (options.overwrite || !(await existsAtPath(dirHandle, path))) { + await writeFileAtPath(dirHandle, path, text); + written++; + } else { + log(`${path} exists and overwrite is off — skipped.`, "warn"); + } + } + if (written) log(`crate2tables: wrote ${written} CSV file(s) to ${OUTPUT_DIR}/.`, "ok"); + }, + }, +}; From 435f3b3e4f4eed5ea2240fd2259d043987266c06 Mon Sep 17 00:00:00 2001 From: Peter Sefton Date: Wed, 2 Sep 2026 16:42:17 +1000 Subject: [PATCH 2/4] Wire crate2tables' load_text to roctable's injected fileReader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit roctable's extractTables() now takes an optional fileReader instead of reading load_text files via Node fs (ptsefton/roctable#1). This plugin passes browserFileReader(dirHandle), wrapping readFileTextFromDirectory — dropping the stripUnsupportedFeatures workaround that disabled load_text with a warning. Depends on roctable's file:../roctable sibling checkout while both are under active development — package.json points there for now instead of the github: dependency, to be switched back once ptsefton/roctable#1 merges. Closes #15 --- README.md | 21 ++++++++++------- package-lock.json | 49 ++++++++++++++------------------------- package.json | 2 +- src/crate2tables/index.js | 43 ++++++++-------------------------- 4 files changed, 41 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 657356a..a1ac3cd 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ handlers close over. Call it once, before the plugin's hooks can fire. | `file-format-identify` | `graphEntityById` (handed to `matcher.js`'s own `configure(deps)` on each dynamic import, for `getFileHandleAtPath`) | | `ca-data-prep` | `writeFileAtPath` | | `merge` | `readJsonFromFolder`, `graphEntityById` | -| `crate2tables` | `readJsonFromFolder`, `writeFileAtPath`, `getFileHandleAtPath` | +| `crate2tables` | `readJsonFromFolder`, `writeFileAtPath`, `getFileHandleAtPath`, `readFileTextFromDirectory` | | `validate-crate` | `loadMasp` | | `ro-crate-json-output` | `crateToJsonString`, `writeFile`, `fileExists` | | `ro-crate-xlsx-output` | `crateToXlsxBytes`, `writeFile`, `fileExists` | @@ -68,13 +68,18 @@ dynamically imported from chaos2crate's own tree instead of becoming a static import anywhere in this package. `crate2tables` depends on [`roctable`](https://github.com/ptsefton/roctable), -a WIP library not yet on npm — installed here as a git dependency -(`"roctable": "github:ptsefton/roctable"`). It reuses roctable's own -crate-walking functions directly (`ctx.crate` is already an `ro-crate` -`ROCrate` instance, the same shape roctable expects), but not its config/CSV -file I/O, which is Node-`fs`-only — see `src/crate2tables/index.js` and -`chaos2crate/docs/crate2tables-spec.md` for what's adapted and what's -deliberately unsupported for now (`load_text`). +a WIP library not yet on npm — installed as `"roctable": "file:../roctable"` +while both are under active development (swap to a `github:ptsefton/roctable` +git dependency, pinned to a commit, once roctable's own PR lands). It reuses +roctable's own crate-walking functions directly (`ctx.crate` is already an +`ro-crate` `ROCrate` instance, the same shape roctable expects) — including +`load_text`, via a `fileReader` this plugin injects +(`browserFileReader` in `src/crate2tables/index.js`, wrapping +`readFileTextFromDirectory`) rather than roctable's own Node-`fs`-based +default (see roctable's `lib/io.js` and its `SPEC.md` §9.0). Its config +load/save and CSV file writing stay this plugin's own job either way — +roctable's `lib/config.js`/`lib/csv.js` file I/O is Node-`fs`-only and simply +isn't called from here; see `chaos2crate/docs/crate2tables-spec.md`. ## Writing a new plugin here diff --git a/package-lock.json b/package-lock.json index a05cefc..6cb479c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,10 +13,24 @@ "mammoth": "^1.12.0", "ro-crate": "^3.7.2", "ro-crate-excel": "^1.2.1", - "roctable": "github:ptsefton/roctable", + "roctable": "file:../roctable", "unicode-name": "^1.1.0" } }, + "../roctable": { + "dependencies": { + "commander": "^15.0.0", + "csv-parse": "^7.0.2", + "csv-stringify": "^6.8.3", + "ro-crate": "^3.7.2" + }, + "bin": { + "roctable": "bin/roctable.js" + }, + "devDependencies": { + "vitest": "^3.2.7" + } + }, "node_modules/@fast-csv/format": { "version": "4.3.5", "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", @@ -440,18 +454,6 @@ "url": "https://github.com/sponsors/fb55" } }, - "node_modules/csv-parse": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-7.0.2.tgz", - "integrity": "sha512-uKZghv9UmPkMVLYy//KZ9HFAIJsl7wkhoEdIL0+rhuSY9pZQlhaeGEDPIe+/w7eh81MOql8Q/9+inAGWG6ZHYA==", - "license": "MIT" - }, - "node_modules/csv-stringify": { - "version": "6.8.3", - "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.8.3.tgz", - "integrity": "sha512-gIeSCvq5F4VtXV3naV3VAewLhBkiZBz+PPhTOA8H3Y8h/ELa+R1ml0GZck/4/Nzo9ep2lvOluilJ6MJlbZsKMA==", - "license": "MIT" - }, "node_modules/dayjs": { "version": "1.11.23", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.23.tgz", @@ -1538,25 +1540,8 @@ } }, "node_modules/roctable": { - "resolved": "git+ssh://git@github.com/ptsefton/roctable.git#91dd951e1895c789c6889282677e5d3722ac6382", - "dependencies": { - "commander": "^15.0.0", - "csv-parse": "^7.0.2", - "csv-stringify": "^6.8.3", - "ro-crate": "^3.7.2" - }, - "bin": { - "roctable": "bin/roctable.js" - } - }, - "node_modules/roctable/node_modules/commander": { - "version": "15.0.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-15.0.0.tgz", - "integrity": "sha512-z67u4ZhzCL/Tydu1lJARtEZYWbWaN7oYLHbsuzocr6y4N6WZAagG3RQ4FW61V1/0+jImpj293XfrcYnd1qxtPg==", - "license": "MIT", - "engines": { - "node": ">=22.12.0" - } + "resolved": "../roctable", + "link": true }, "node_modules/safe-buffer": { "version": "5.2.1", diff --git a/package.json b/package.json index e2e9bfa..72e9cb4 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "mammoth": "^1.12.0", "ro-crate": "^3.7.2", "ro-crate-excel": "^1.2.1", - "roctable": "github:ptsefton/roctable", + "roctable": "file:../roctable", "unicode-name": "^1.1.0" }, "scripts": { diff --git a/src/crate2tables/index.js b/src/crate2tables/index.js index 3b61f42..b1fbdb9 100644 --- a/src/crate2tables/index.js +++ b/src/crate2tables/index.js @@ -20,11 +20,10 @@ // reruns the build to get output. See docs/crate2tables-spec.md. // // load_text (roctable's "read this property's referenced file into the row" -// feature) is deliberately unsupported here for now: roctable's extractTables -// reads that file with Node's `fs`, which isn't available in chaos2crate's -// browser build. Any property configured with load_text is disabled with a -// warning rather than passed through — see stripUnsupportedFeatures below -// and the spec's "Known limitations" section. +// feature) reads through an injected fileReader (ptsefton/roctable#1) rather +// than roctable's own Node-fs default — browserFileReader below wraps +// chaos2crate's readFileTextFromDirectory, which already returns null for +// "not found", matching what extractTables' loadText expects from a reader. import { inspectCrate, mergeDiscovered, discoverExpandedProperties } from "roctable/lib/inspect.js"; import { extractTables } from "roctable/lib/extract.js"; import { tablesToCsvStrings } from "roctable/lib/csv.js"; @@ -35,33 +34,15 @@ const OUTPUT_DIR = "crate2tables-output"; // Hook names are literal strings and core chaos2crate functions arrive via // createPlugin(deps) — see this repo's README. -let readJsonFromFolder, writeFileAtPath, getFileHandleAtPath; +let readJsonFromFolder, writeFileAtPath, getFileHandleAtPath, readFileTextFromDirectory; export function createPlugin(deps) { - ({ readJsonFromFolder, writeFileAtPath, getFileHandleAtPath } = deps); + ({ readJsonFromFolder, writeFileAtPath, getFileHandleAtPath, readFileTextFromDirectory } = deps); return plugin; } -// Recursively clears load_text off every property (including expand's own -// nested sub-properties), returning the dotted entityType.property paths it -// touched so the caller can log one clear warning instead of one per field. -function stripUnsupportedFeatures(config) { - const touched = []; - for (const [entityType, tableConfig] of Object.entries(config.tables || {})) { - for (const [prop, propConfig] of Object.entries(tableConfig.properties || {})) { - if (propConfig.load_text) { - propConfig.load_text = false; - touched.push(`${entityType}.${prop}`); - } - for (const [exProp, exPropConfig] of Object.entries(propConfig.properties || {})) { - if (exPropConfig.load_text) { - exPropConfig.load_text = false; - touched.push(`${entityType}.${prop}.${exProp}`); - } - } - } - } - return touched; +function browserFileReader(dirHandle) { + return { readFile: (relPath) => readFileTextFromDirectory(dirHandle, relPath) }; } async function existsAtPath(dirHandle, relativePath) { @@ -110,11 +91,6 @@ const plugin = { return; } - const unsupported = stripUnsupportedFeatures(config); - if (unsupported.length) { - log(`crate2tables: load_text isn't supported yet in the browser build — disabled on ${unsupported.join(", ")}.`, "warn"); - } - ctx.crate2tables = { config, configSource }; const tableNames = Object.keys(config.tables || {}); @@ -124,7 +100,8 @@ const plugin = { } try { - ctx.crate2tables.csv = tablesToCsvStrings(extractTables(crate, config)); + const data = await extractTables(crate, config, { fileReader: browserFileReader(dirHandle) }); + ctx.crate2tables.csv = tablesToCsvStrings(data); log(`crate2tables: built ${tableNames.length} table(s) — ${tableNames.join(", ")}.`, "ok"); } catch (e) { log(`crate2tables: failed to extract tables — ${e.message}`, "warn"); From f6b71970e88d2b1f78695ce4b87136f6075a6667 Mon Sep 17 00:00:00 2001 From: Peter Sefton Date: Wed, 2 Sep 2026 17:33:47 +1000 Subject: [PATCH 3/4] generic-input: reconcile against the folder's existing crate, not replace it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reads ro-crate-metadata.json from the folder (if present) and passes it through to buildCrate() as opts.existingJson, so a build against an already-curated crate updates it instead of throwing it away — see chaos2crate SPEC.md §6.1a and its crate.js change for what reconciling actually does. --- README.md | 2 +- src/generic-input/index.js | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a1ac3cd..6052124 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ handlers close over. Call it once, before the plugin's hooks can fire. | `ro-crate-json-output` | `crateToJsonString`, `writeFile`, `fileExists` | | `ro-crate-xlsx-output` | `crateToXlsxBytes`, `writeFile`, `fileExists` | | `ro-crate-html-output` | `crateToPreviewHtml`, `crateToMultiPageHtml`, `writeFile`, `writeFileAtPath`, `readJsonFromFolder`, `readFileTextFromDirectory`, `verifyPermission`, `fileExists`, `bustCacheUrl`, `buildGitHubTreeUrl`, `fetchGitHubTextFile`, `listGitHubFolder` | -| `generic-input` (input mode) | `buildFileMetadata`, `buildCrate` | +| `generic-input` (input mode) | `buildFileMetadata`, `buildCrate`, `readJsonFromFolder` (reads the folder's existing crate, if any, to reconcile against rather than replace — chaos2crate SPEC.md §6.1a) | | `docx-input` (input mode) | `writeFileAtPath` (handed to `docx_crate.js`'s own `configure(deps)` once its dynamic import resolves) | `loadMasp` is a thunk — `() => import("../masp.js")` — rather than the diff --git a/src/generic-input/index.js b/src/generic-input/index.js index 329902b..551a1a7 100644 --- a/src/generic-input/index.js +++ b/src/generic-input/index.js @@ -11,10 +11,10 @@ // runs per build, dispatched by pipeline.js on ctx.options.inputMode. // Core chaos2crate functions arrive via createPlugin(deps) — see this // repo's README. -let buildFileMetadata, buildCrate; +let buildFileMetadata, buildCrate, readJsonFromFolder; export function createPlugin(deps) { - ({ buildFileMetadata, buildCrate } = deps); + ({ buildFileMetadata, buildCrate, readJsonFromFolder } = deps); return plugin; } @@ -31,13 +31,21 @@ const plugin = { ctx.sourceCount = ctx.filesWithMeta.length; }, - buildCrate(ctx) { + // If the folder already has a crate, this build reconciles against it + // (SPEC.md §6.1a) instead of replacing it — buildCrate() (crate.js) only + // needs the parsed JSON to know that; everything else is unchanged. + // "Existing crate" here means the file this same JSON output plugin + // writes, not xlsx-crate-input's additional-ro-crate-metadata.xlsx (a + // deliberately separate, opt-in source — see that plugin's own hooks). + async buildCrate(ctx) { + const existingJson = await readJsonFromFolder(ctx.dirHandle, "ro-crate-metadata.json"); ctx.crate = buildCrate(ctx.filesWithMeta, ctx.config, ctx.log, { topLevelFolderType: ctx.options.topLevelFolderType, // ctx.xlsxCrate is set at config:prepare, before this runs: a spreadsheet // already describes the entries and what belongs to what, so the folder // scan shouldn't invent a parallel structure alongside it. structureFromMetadata: !!ctx.xlsxCrate, + existingJson, }); }, }; From 1a07b5e717a917f1ac7b42f063e9ee22bc6cfeae Mon Sep 17 00:00:00 2001 From: Peter Sefton Date: Wed, 2 Sep 2026 17:43:52 +1000 Subject: [PATCH 4/4] generic-input: confirm new files with a checkbox tree before adding them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A file the scan found with no matching entity in the existing crate (reconciliation, chaos2crate SPEC.md §6.1a) was previously added automatically, attached to the root dataset with only a warning after the fact — a silent guess about a decision a curator should actually make. buildCrate(ctx) now diffs the scan against the existing crate's @ids first and, if anything's new, blocks on a confirmation modal (new-files-confirm.js): a checkbox file tree, everything checked by default, folders reflecting all/some/none of their contents. Only the confirmed subset reaches buildCrate() at all — an unchecked file simply isn't part of this build, same as one the scan never saw, and gets asked about again next time unless it's removed from the folder. Dismissing the modal outright (as opposed to confirming even an empty selection) cancels the whole build, since that made no explicit choice. Verified the orchestration (fresh build/no-new-files/confirmed/skipped/ cancelled) with a mocked openModal, and the tree itself (build, render, checkbox↔indeterminate propagation on toggle, confirmed-result shape) with a minimal fake DOM, since this repo has no jsdom dependency. --- README.md | 2 +- src/generic-input/index.js | 35 ++++++- src/generic-input/new-files-confirm.js | 128 +++++++++++++++++++++++++ 3 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 src/generic-input/new-files-confirm.js diff --git a/README.md b/README.md index 6052124..fa70769 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ handlers close over. Call it once, before the plugin's hooks can fire. | `ro-crate-json-output` | `crateToJsonString`, `writeFile`, `fileExists` | | `ro-crate-xlsx-output` | `crateToXlsxBytes`, `writeFile`, `fileExists` | | `ro-crate-html-output` | `crateToPreviewHtml`, `crateToMultiPageHtml`, `writeFile`, `writeFileAtPath`, `readJsonFromFolder`, `readFileTextFromDirectory`, `verifyPermission`, `fileExists`, `bustCacheUrl`, `buildGitHubTreeUrl`, `fetchGitHubTextFile`, `listGitHubFolder` | -| `generic-input` (input mode) | `buildFileMetadata`, `buildCrate`, `readJsonFromFolder` (reads the folder's existing crate, if any, to reconcile against rather than replace — chaos2crate SPEC.md §6.1a) | +| `generic-input` (input mode) | `buildFileMetadata`, `buildCrate`, `readJsonFromFolder` (reads the folder's existing crate, if any, to reconcile against rather than replace — chaos2crate SPEC.md §6.1a), `openModal` (confirms which newly-found files to add, via `new-files-confirm.js`) | | `docx-input` (input mode) | `writeFileAtPath` (handed to `docx_crate.js`'s own `configure(deps)` once its dynamic import resolves) | `loadMasp` is a thunk — `() => import("../masp.js")` — rather than the diff --git a/src/generic-input/index.js b/src/generic-input/index.js index 551a1a7..de3c9a9 100644 --- a/src/generic-input/index.js +++ b/src/generic-input/index.js @@ -11,10 +11,12 @@ // runs per build, dispatched by pipeline.js on ctx.options.inputMode. // Core chaos2crate functions arrive via createPlugin(deps) — see this // repo's README. -let buildFileMetadata, buildCrate, readJsonFromFolder; +import { confirmNewFiles } from "./new-files-confirm.js"; + +let buildFileMetadata, buildCrate, readJsonFromFolder, openModal; export function createPlugin(deps) { - ({ buildFileMetadata, buildCrate, readJsonFromFolder } = deps); + ({ buildFileMetadata, buildCrate, readJsonFromFolder, openModal } = deps); return plugin; } @@ -37,9 +39,36 @@ const plugin = { // "Existing crate" here means the file this same JSON output plugin // writes, not xlsx-crate-input's additional-ro-crate-metadata.xlsx (a // deliberately separate, opt-in source — see that plugin's own hooks). + // + // A file the scan found with no matching entity in that existing crate + // isn't added silently — the person building the crate confirms it first + // (new-files-confirm.js's checkbox tree), since reconcileFileEntities' + // fallback for one with no obvious home is to attach it straight to the + // root dataset, and that's exactly the kind of guess a human should sign + // off on rather than discover after the fact in the build log. async buildCrate(ctx) { const existingJson = await readJsonFromFolder(ctx.dirHandle, "ro-crate-metadata.json"); - ctx.crate = buildCrate(ctx.filesWithMeta, ctx.config, ctx.log, { + let filesToBuild = ctx.filesWithMeta; + + if (existingJson) { + const existingIds = new Set((existingJson["@graph"] || []).map((e) => e["@id"])); + const newPaths = ctx.filesWithMeta.map((f) => f.id).filter((id) => !existingIds.has(id)); + + if (newPaths.length) { + ctx.log(`${newPaths.length} file(s) not in the existing crate — asking which to add.`, "info"); + const confirmed = await confirmNewFiles({ newPaths, openModal }); + if (confirmed === null) throw new Error("Build cancelled: new files were not confirmed."); + + const confirmedSet = new Set(confirmed); + const skipped = newPaths.filter((id) => !confirmedSet.has(id)); + if (confirmed.length) ctx.log(`Adding ${confirmed.length} confirmed new file(s).`, "ok"); + if (skipped.length) ctx.log(`Skipping ${skipped.length} file(s) this build (not added to the crate): ${skipped.join(", ")}`, "warn"); + + filesToBuild = ctx.filesWithMeta.filter((f) => existingIds.has(f.id) || confirmedSet.has(f.id)); + } + } + + ctx.crate = buildCrate(filesToBuild, ctx.config, ctx.log, { topLevelFolderType: ctx.options.topLevelFolderType, // ctx.xlsxCrate is set at config:prepare, before this runs: a spreadsheet // already describes the entries and what belongs to what, so the folder diff --git a/src/generic-input/new-files-confirm.js b/src/generic-input/new-files-confirm.js new file mode 100644 index 0000000..2e81302 --- /dev/null +++ b/src/generic-input/new-files-confirm.js @@ -0,0 +1,128 @@ +// A checkbox file-tree for confirming which newly-discovered files (found by +// a generic-input scan but not yet in an existing crate) should actually be +// added as new File entities — see index.js's buildCrate and chaos2crate +// SPEC.md §6.1a. Built the same "no host markup, no HTML string" way as the +// rest of this app's own plugin UI, using the shared `openModal` helper and +// the host's own `.checkbox`/`.modal .actions` CSS conventions rather than +// inventing new ones. + +// Turns a flat list of relative paths into a nested { name, children, isFile, +// path } tree, splitting on "/" the same way filesWithMeta's own folder +// grouping does. +function buildFileTree(paths) { + const root = { name: "", children: new Map(), isFile: false }; + for (const relPath of paths) { + const parts = relPath.split("/"); + let node = root; + parts.forEach((part, i) => { + const isFile = i === parts.length - 1; + if (!node.children.has(part)) { + node.children.set(part, { name: part, children: new Map(), isFile, path: isFile ? relPath : null }); + } + node = node.children.get(part); + }); + } + return root; +} + +function leafPaths(node) { + if (node.isFile) return [node.path]; + return [...node.children.values()].flatMap(leafPaths); +} + +// Renders `node`'s children as a
    . Returns { el, refresh } — refresh() +// recomputes every checkbox's checked/indeterminate state from `checkedSet` +// (a folder is checked only when every leaf beneath it is, indeterminate +// when some but not all are) and must be called after any change to it, from +// anywhere in the tree, since a folder's own state depends only on its +// leaves, not on which specific one changed. +function renderNode(node, checkedSet, onAnyChange, depth = 0) { + const ul = document.createElement("ul"); + ul.style.cssText = `list-style:none; margin:0; padding:${depth === 0 ? "0" : "0 0 0 20px"};`; + const refreshers = []; + + const children = [...node.children.values()].sort((a, b) => ( + a.isFile !== b.isFile ? (a.isFile ? 1 : -1) : a.name.localeCompare(b.name) + )); + + for (const child of children) { + const li = document.createElement("li"); + const row = document.createElement("label"); + row.className = "checkbox"; + row.style.margin = "3px 0"; + const checkbox = document.createElement("input"); + checkbox.type = "checkbox"; + const text = document.createElement("span"); + text.textContent = child.isFile ? child.name : `${child.name}/`; + row.append(checkbox, text); + li.appendChild(row); + + const paths = leafPaths(child); + refreshers.push(() => { + const checkedCount = paths.filter((p) => checkedSet.has(p)).length; + checkbox.checked = checkedCount === paths.length; + checkbox.indeterminate = checkedCount > 0 && checkedCount < paths.length; + }); + + checkbox.addEventListener("change", () => { + for (const p of paths) { if (checkbox.checked) checkedSet.add(p); else checkedSet.delete(p); } + onAnyChange(); + }); + + if (!child.isFile) { + const sub = renderNode(child, checkedSet, onAnyChange, depth + 1); + li.appendChild(sub.el); + refreshers.push(sub.refresh); + } + ul.appendChild(li); + } + + return { el: ul, refresh: () => refreshers.forEach((r) => r()) }; +} + +// Opens the confirmation modal and resolves to the array of confirmed paths +// (possibly empty — the user can uncheck everything), or null if dismissed — +// the caller treats null as "cancel the whole build", not "add nothing", +// since a dismissal made no explicit choice at all. +export async function confirmNewFiles({ newPaths, openModal }) { + const checkedSet = new Set(newPaths); // default: everything checked + const tree = buildFileTree(newPaths); + + return openModal({ + title: `${newPaths.length} new file${newPaths.length === 1 ? "" : "s"} found`, + onDismiss: () => null, + render(body, close) { + const intro = document.createElement("p"); + intro.textContent = + "These files aren't in the existing crate yet. Choose which to add — " + + "anything left unchecked is skipped this build (you'll be asked again next time, unless it's removed from the folder)."; + body.appendChild(intro); + + const countLabel = document.createElement("p"); + countLabel.style.fontWeight = "600"; + const updateCount = () => { countLabel.textContent = `${checkedSet.size} of ${newPaths.length} selected`; }; + + const { el: treeEl, refresh } = renderNode(tree, checkedSet, () => { updateCount(); refresh(); }); + refresh(); + updateCount(); + + const scrollWrap = document.createElement("div"); + scrollWrap.style.cssText = "max-height:260px; overflow-y:auto; border:1px solid var(--border); border-radius:8px; padding:8px 12px; margin-bottom:16px;"; + scrollWrap.appendChild(treeEl); + body.append(countLabel, scrollWrap); + + const actions = document.createElement("div"); + actions.className = "actions"; + const skipBtn = document.createElement("button"); + skipBtn.type = "button"; skipBtn.className = "secondary"; + skipBtn.textContent = "Add none"; + skipBtn.addEventListener("click", () => close([])); + const confirmBtn = document.createElement("button"); + confirmBtn.type = "button"; + confirmBtn.textContent = "Add selected"; + confirmBtn.addEventListener("click", () => close([...checkedSet])); + actions.append(skipBtn, confirmBtn); + body.appendChild(actions); + }, + }); +}