Add crate2tables plugin: export RO-Crate tables via roctable

README.md

@@ -54,18 +54,33 @@ handlers close over. Call it once, before the plugin's hooks can fire.
 | `file-format-identify` | `graphEntityById` (handed to `matcher.js`'s own `configure(deps)` on each dynamic import, for `getFileHandleAtPath`) |
 | `ca-data-prep` | `writeFileAtPath` |
 | `merge` | `readJsonFromFolder`, `graphEntityById` |
+| `crate2tables` | `readJsonFromFolder`, `writeFileAtPath`, `getFileHandleAtPath`, `readFileTextFromDirectory` |
 | `validate-crate` | `loadMasp` |
 | `ro-crate-json-output` | `crateToJsonString`, `writeFile`, `fileExists` |
 | `ro-crate-xlsx-output` | `crateToXlsxBytes`, `writeFile`, `fileExists` |
 | `ro-crate-html-output` | `crateToPreviewHtml`, `crateToMultiPageHtml`, `writeFile`, `writeFileAtPath`, `readJsonFromFolder`, `readFileTextFromDirectory`, `verifyPermission`, `fileExists`, `bustCacheUrl`, `buildGitHubTreeUrl`, `fetchGitHubTextFile`, `listGitHubFolder` |
-| `generic-input` (input mode) | `buildFileMetadata`, `buildCrate` |
+| `generic-input` (input mode) | `buildFileMetadata`, `buildCrate`, `readJsonFromFolder` (reads the folder's existing crate, if any, to reconcile against rather than replace — chaos2crate SPEC.md §6.1a), `openModal` (confirms which newly-found files to add, via `new-files-confirm.js`) |
 | `docx-input` (input mode) | `writeFileAtPath` (handed to `docx_crate.js`'s own `configure(deps)` once its dynamic import resolves) |
 
 `loadMasp` is a thunk — `() => import("../masp.js")` — rather than the
 function itself, so `ro-crate-masp` (a heavy validator library) stays
 dynamically imported from chaos2crate's own tree instead of becoming a
 static import anywhere in this package.
 
+`crate2tables` depends on [`roctable`](https://github.com/ptsefton/roctable),
+a WIP library not yet on npm — installed as `"roctable": "file:../roctable"`
+while both are under active development (swap to a `github:ptsefton/roctable`
+git dependency, pinned to a commit, once roctable's own PR lands). It reuses
+roctable's own crate-walking functions directly (`ctx.crate` is already an
+`ro-crate` `ROCrate` instance, the same shape roctable expects) — including
+`load_text`, via a `fileReader` this plugin injects
+(`browserFileReader` in `src/crate2tables/index.js`, wrapping
+`readFileTextFromDirectory`) rather than roctable's own Node-`fs`-based
+default (see roctable's `lib/io.js` and its `SPEC.md` §9.0). Its config
+load/save and CSV file writing stay this plugin's own job either way —
+roctable's `lib/config.js`/`lib/csv.js` file I/O is Node-`fs`-only and simply
+isn't called from here; see `chaos2crate/docs/crate2tables-spec.md`.
+
 ## Writing a new plugin here
 
 ```js

index.js

@@ -17,6 +17,7 @@ import { createPlugin as createValidateCrate } from "./src/validate-crate/index.
 import { createPlugin as createJsonOutput } from "./src/ro-crate-json-output/index.js";
 import { createPlugin as createXlsxOutput } from "./src/ro-crate-xlsx-output/index.js";
 import { createPlugin as createHtmlOutput } from "./src/ro-crate-html-output/index.js";
+import { createPlugin as createCrate2Tables } from "./src/crate2tables/index.js";
 import { createPlugin as createGenericInput } from "./src/generic-input/index.js";
 import { createPlugin as createDocxInput } from "./src/docx-input/index.js";
 
@@ -33,6 +34,7 @@ export const REGISTRY = {
   "ca-data-prep": createCaDataPrep,
   "chat-export": createChatExport,
   "merge": createMerge,
+  "crate2tables": createCrate2Tables,
   "validate-crate": createValidateCrate,
   "ro-crate-json-output": createJsonOutput,
   "ro-crate-xlsx-output": createXlsxOutput,

package.json

@@ -15,6 +15,7 @@
     "mammoth": "^1.12.0",
     "ro-crate": "^3.7.2",
     "ro-crate-excel": "^1.2.1",
+    "roctable": "file:../roctable",
     "unicode-name": "^1.1.0"
   },
   "scripts": {

src/crate2tables/index.js

@@ -0,0 +1,136 @@
+// Exports a built RO-Crate as one CSV per configured @type, using roctable
+// (https://github.com/ptsefton/roctable) — a WIP, git-installed library that
+// flattens an `ro-crate` ROCrate graph into tables according to a JSON
+// config. chaos2crate's own crate.js builds its ROCrate with the same
+// `ro-crate` package version roctable depends on, so ctx.crate can be handed
+// straight to roctable's crate-walking functions with no adapter needed.
+//
+// roctable's own CLI is a two-step workflow: `roctable inspect` discovers
+// every @type/property in a crate and writes/updates a config (new fields
+// default to unselected, existing choices are preserved); `roctable csv`
+// then extracts tables for whatever the config's "tables" section selects.
+// This plugin runs the same two steps on every build instead of requiring a
+// separate CLI pass: it always re-discovers against the current crate and
+// rewrites crate2tables-config.json (non-destructively — see
+// roctable/lib/inspect.js's mergeDiscovered), then extracts+writes CSV for
+// whatever the config's "tables" section already selects. A first build
+// against a fresh folder therefore selects nothing and only seeds the
+// config; a person edits that file (moving a type from "potential_tables" to
+// "tables", setting "include"/"expand"/"load_text" on its properties) and
+// reruns the build to get output. See docs/crate2tables-spec.md.
+//
+// load_text (roctable's "read this property's referenced file into the row"
+// feature) reads through an injected fileReader (ptsefton/roctable#1) rather
+// than roctable's own Node-fs default — browserFileReader below wraps
+// chaos2crate's readFileTextFromDirectory, which already returns null for
+// "not found", matching what extractTables' loadText expects from a reader.
+import { inspectCrate, mergeDiscovered, discoverExpandedProperties } from "roctable/lib/inspect.js";
+import { extractTables } from "roctable/lib/extract.js";
+import { tablesToCsvStrings } from "roctable/lib/csv.js";
+import { defaultConfig } from "roctable/lib/config.js";
+
+const CONFIG_FILE = "crate2tables-config.json";
+const OUTPUT_DIR = "crate2tables-output";
+
+// Hook names are literal strings and core chaos2crate functions arrive via
+// createPlugin(deps) — see this repo's README.
+let readJsonFromFolder, writeFileAtPath, getFileHandleAtPath, readFileTextFromDirectory;
+
+export function createPlugin(deps) {
+  ({ readJsonFromFolder, writeFileAtPath, getFileHandleAtPath, readFileTextFromDirectory } = deps);
+  return plugin;
+}
+
+function browserFileReader(dirHandle) {
+  return { readFile: (relPath) => readFileTextFromDirectory(dirHandle, relPath) };
+}
+
+async function existsAtPath(dirHandle, relativePath) {
+  return !!(await getFileHandleAtPath(dirHandle, relativePath));
+}
+
+const plugin = {
+  name: "crate2tables",
+  optionSchema: {
+    key: "enableCrate2Tables",
+    label: "Export RO-Crate tables",
+    default: false,
+    hint: 'Flattens the crate into one CSV per entity type, using crate2tables-config.json — written to the folder on the first build with every discovered type/property, unselected. Move a type from "potential_tables" to "tables" and set "include": true on the properties you want, then rebuild. See docs/crate2tables-spec.md.',
+    children: [
+      { key: "crate2tablesConfigUpload", type: "file", label: "Table config (JSON)",
+        accept: "application/json,.json",
+        hint: "Overrides crate2tables-config.json from the folder, if present." },
+    ],
+  },
+  outputPaths: [
+    { path: CONFIG_FILE, kind: "file" },
+    { path: OUTPUT_DIR, kind: "dir" },
+  ],
+  hooks: {
+    "crate:built": async (ctx) => {
+      if (!ctx.options.enableCrate2Tables) return;
+      const { crate, dirHandle, options, log } = ctx;
+
+      let existingConfig = null;
+      let configSource = "none — starting fresh";
+      if (options.crate2tablesConfigUpload) {
+        const text = await options.crate2tablesConfigUpload.file.text();
+        try { existingConfig = JSON.parse(text); }
+        catch (e) { throw new Error(`uploaded table config "${options.crate2tablesConfigUpload.name}" is not valid JSON: ${e.message}`); }
+        configSource = `uploaded (${options.crate2tablesConfigUpload.name})`;
+      } else {
+        const folderConfig = await readJsonFromFolder(dirHandle, CONFIG_FILE);
+        if (folderConfig) { existingConfig = folderConfig; configSource = CONFIG_FILE; }
+      }
+
+      let config;
+      try {
+        config = discoverExpandedProperties(crate, mergeDiscovered(existingConfig || defaultConfig(), inspectCrate(crate)));
+      } catch (e) {
+        log(`crate2tables: could not inspect the crate — ${e.message}`, "warn");
+        return;
+      }
+
+      ctx.crate2tables = { config, configSource };
+
+      const tableNames = Object.keys(config.tables || {});
+      if (!tableNames.length) {
+        log(`crate2tables: no tables selected yet (config source: ${configSource}). Wrote every discovered type to ${CONFIG_FILE} under "potential_tables" — move the ones you want into "tables" and rebuild.`, "warn");
+        return;
+      }
+
+      try {
+        const data = await extractTables(crate, config, { fileReader: browserFileReader(dirHandle) });
+        ctx.crate2tables.csv = tablesToCsvStrings(data);
+        log(`crate2tables: built ${tableNames.length} table(s) — ${tableNames.join(", ")}.`, "ok");
+      } catch (e) {
+        log(`crate2tables: failed to extract tables — ${e.message}`, "warn");
+      }
+    },
+
+    "output:write": async (ctx) => {
+      if (!ctx.options.enableCrate2Tables || !ctx.crate2tables) return;
+      const { dirHandle, options, log } = ctx;
+      const { config, csv } = ctx.crate2tables;
+
+      // Non-destructive by construction (mergeDiscovered only ever adds
+      // newly-seen types/properties, unselected — see roctable/lib/inspect.js),
+      // so rewriting it every build is the same "keep it fresh" behaviour as
+      // rerunning `roctable inspect`, not a risk to a hand-edited config.
+      await writeFileAtPath(dirHandle, CONFIG_FILE, JSON.stringify(config, null, 2) + "\n");
+
+      if (!csv) return;
+      let written = 0;
+      for (const [tableName, text] of Object.entries(csv)) {
+        const path = `${OUTPUT_DIR}/${tableName}.csv`;
+        if (options.overwrite || !(await existsAtPath(dirHandle, path))) {
+          await writeFileAtPath(dirHandle, path, text);
+          written++;
+        } else {
+          log(`${path} exists and overwrite is off — skipped.`, "warn");
+        }
+      }
+      if (written) log(`crate2tables: wrote ${written} CSV file(s) to ${OUTPUT_DIR}/.`, "ok");
+    },
+  },
+};

src/generic-input/index.js

@@ -11,10 +11,12 @@
 // runs per build, dispatched by pipeline.js on ctx.options.inputMode.
 // Core chaos2crate functions arrive via createPlugin(deps) — see this
 // repo's README.
-let buildFileMetadata, buildCrate;
+import { confirmNewFiles } from "./new-files-confirm.js";
+
+let buildFileMetadata, buildCrate, readJsonFromFolder, openModal;
 
 export function createPlugin(deps) {
-  ({ buildFileMetadata, buildCrate } = deps);
+  ({ buildFileMetadata, buildCrate, readJsonFromFolder, openModal } = deps);
   return plugin;
 }
 
@@ -31,13 +33,48 @@ const plugin = {
     ctx.sourceCount = ctx.filesWithMeta.length;
   },
 
-  buildCrate(ctx) {
-    ctx.crate = buildCrate(ctx.filesWithMeta, ctx.config, ctx.log, {
+  // If the folder already has a crate, this build reconciles against it
+  // (SPEC.md §6.1a) instead of replacing it — buildCrate() (crate.js) only
+  // needs the parsed JSON to know that; everything else is unchanged.
+  // "Existing crate" here means the file this same JSON output plugin
+  // writes, not xlsx-crate-input's additional-ro-crate-metadata.xlsx (a
+  // deliberately separate, opt-in source — see that plugin's own hooks).
+  //
+  // A file the scan found with no matching entity in that existing crate
+  // isn't added silently — the person building the crate confirms it first
+  // (new-files-confirm.js's checkbox tree), since reconcileFileEntities'
+  // fallback for one with no obvious home is to attach it straight to the
+  // root dataset, and that's exactly the kind of guess a human should sign
+  // off on rather than discover after the fact in the build log.
+  async buildCrate(ctx) {
+    const existingJson = await readJsonFromFolder(ctx.dirHandle, "ro-crate-metadata.json");
+    let filesToBuild = ctx.filesWithMeta;
+
+    if (existingJson) {
+      const existingIds = new Set((existingJson["@graph"] || []).map((e) => e["@id"]));
+      const newPaths = ctx.filesWithMeta.map((f) => f.id).filter((id) => !existingIds.has(id));
+
+      if (newPaths.length) {
+        ctx.log(`${newPaths.length} file(s) not in the existing crate — asking which to add.`, "info");
+        const confirmed = await confirmNewFiles({ newPaths, openModal });
+        if (confirmed === null) throw new Error("Build cancelled: new files were not confirmed.");
+
+        const confirmedSet = new Set(confirmed);
+        const skipped = newPaths.filter((id) => !confirmedSet.has(id));
+        if (confirmed.length) ctx.log(`Adding ${confirmed.length} confirmed new file(s).`, "ok");
+        if (skipped.length) ctx.log(`Skipping ${skipped.length} file(s) this build (not added to the crate): ${skipped.join(", ")}`, "warn");
+
+        filesToBuild = ctx.filesWithMeta.filter((f) => existingIds.has(f.id) || confirmedSet.has(f.id));
+      }
+    }
+
+    ctx.crate = buildCrate(filesToBuild, ctx.config, ctx.log, {
       topLevelFolderType: ctx.options.topLevelFolderType,
       // ctx.xlsxCrate is set at config:prepare, before this runs: a spreadsheet
       // already describes the entries and what belongs to what, so the folder
       // scan shouldn't invent a parallel structure alongside it.
       structureFromMetadata: !!ctx.xlsxCrate,
+      existingJson,
     });
   },
 };