diff --git a/README.md b/README.md index 5f209c0e..e2c4cf6c 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ deployment, the StableHLO path for native and edge targets. ### Edge AI: Minerva Secure MCU Export - Export supported static MLP graphs to Minerva project bundles for secure MCU inference -- Emits compiler NPZ input, libminerva weights, a manifest, host harness, firmware example, and host verification results +- Emits compiler NPZ input, libminerva weights, a fingerprinted manifest, host harness, firmware example, and host verification results - Start with the [Minerva export guide](docs/modules/ROOT/pages/how-to/minerva-export.adoc) ### Compiler: MLIR / StableHLO diff --git a/docs/export/minerva.md b/docs/export/minerva.md index 6563ed05..0ce865c0 100644 --- a/docs/export/minerva.md +++ b/docs/export/minerva.md @@ -34,7 +34,7 @@ Do not commit real device keys. `include/secrets.example.h` contains placeholder | Shapes | Fully known rank-2 tensor shapes | | Pattern | `Input -> MatMul -> Add? -> activation?`, repeated in sequence | | Activations | `Relu`, `Sigmoid`, `Tanh` after a dense layer | -| Out of scope | CNNs, attention, recurrent models, dynamic shapes, branching graphs, transformers, arbitrary ONNX operators | +| Out of scope | CNNs, attention, recurrent models, dynamic shapes, branching graphs, transformers, and arbitrary imported operator sets | ## Export API @@ -77,6 +77,12 @@ build/minerva/TinySecureMlp/ main.c ``` +## Manifest Provenance + +`manifest.json` records the export target, quantization, libminerva root, compiler command summary, NPZ schema version, layer count, reference fixture paths, generated files, and a `generatedFileSha256` map. The hashes cover generated artifacts such as `generated/model.npz`, `generated/weights.c`, `include/weights.h`, host fixtures, host harness sources, and firmware examples. + +Use the manifest as the handoff record between the source model, Kotlin export, libminerva compilation, and host verification. It redacts compiler key-file arguments and does not copy real device key material; keep real keys outside the generated bundle. + ## Host Verification and CI Host verification checks package structure, generated weight files, `model.npz` integrity, placeholder secret hygiene, and SKaiNET reference fixture generation. The packager writes deterministic `host/reference-input.txt` and `host/reference-output.txt` files and records them in `manifest.json`. A real host run can write comma- or whitespace-separated float outputs to `host/observed-output.txt` for zero-config parity comparison. @@ -124,9 +130,11 @@ CI recipe: `minervaHostVerification` is skipped by default. When enabled, it runs `jvmTest` and `runMinervaTinyMlpSample` with CMake and CTest host verification enabled unless `-Pminerva.hostVerification.runCmakeBuild=false` or `-Pminerva.hostVerification.runCTest=false` is set. -## ONNX to Minerva +## Model Sources + +Minerva export starts from a supported SKaiNET `ComputeGraph`. That graph can come from the Kotlin DSL, a traced forward pass, a hand-built graph, an imported model, or any other path that preserves the phase-one static MLP contract. -Use the existing ONNX loader to inspect a model and reject unsupported operators before constructing a compatible SKaiNET `ComputeGraph`: +For ONNX inputs, use the existing ONNX loader as an inspection step before constructing a compatible SKaiNET graph: ```kotlin val loaded = OnnxLoader.fromModelSource { @@ -137,7 +145,7 @@ val ops = graph.node.map { it.opType }.toSet() require(ops.all { it in setOf("MatMul", "Gemm", "Add", "Relu", "Sigmoid", "Tanh") }) ``` -The first phase does not include a general ONNX-to-Minerva importer. +The first phase does not include a general ONNX-to-Minerva importer. Once any source graph is exported, keep `manifest.json` with the source-model artifact or training metadata so reviewers can compare provenance, generated `model.npz`, and host verification fixtures. ## Firmware Integration diff --git a/docs/modules/ROOT/pages/how-to/minerva-export.adoc b/docs/modules/ROOT/pages/how-to/minerva-export.adoc index 619b469b..c29c66d0 100644 --- a/docs/modules/ROOT/pages/how-to/minerva-export.adoc +++ b/docs/modules/ROOT/pages/how-to/minerva-export.adoc @@ -87,7 +87,7 @@ export MINERVA_HOST_LIBRARIES=minerva | `Relu`, `Sigmoid`, and `Tanh` after a dense layer. | Out of scope -| CNNs, attention blocks, recurrent models, dynamic shapes, branching graphs, transformers, and arbitrary ONNX operators. +| CNNs, attention blocks, recurrent models, dynamic shapes, branching graphs, transformers, and arbitrary imported operator sets. |=== == Export API @@ -140,7 +140,15 @@ build/minerva/TinySecureMlp/ main.c ---- -The manifest records the target, quantization, libminerva root, compiler command summary, NPZ schema version, layer count, reference fixture paths, and generated files. `secrets.example.h` contains placeholder values only. +The manifest records the target, quantization, libminerva root, compiler command summary, NPZ schema version, layer count, reference fixture paths, generated files, and SHA-256 fingerprints for generated artifacts. `secrets.example.h` contains placeholder values only. + +== Manifest Provenance + +`manifest.json` is the release handoff record for Minerva export. It includes a `generatedFileSha256` object keyed by generated project path, covering artifacts such as `generated/model.npz`, `generated/weights.c`, `include/weights.h`, host fixtures, host harness sources, and firmware examples. + +Use those fingerprints to audit that the source model, Kotlin export, libminerva compiler output, host verification fixtures, and firmware handoff all refer to the same generated bundle. Compiler key-file arguments are redacted from the command summary, and real device key material must stay outside the generated bundle. + +For imported or trained models, keep the original source artifact or training metadata next to the manifest in your release evidence. The Minerva manifest fingerprints the generated Minerva bundle; it is not a replacement for source-model provenance. == Host Verification @@ -205,9 +213,11 @@ Before flashing firmware: * Re-run host verification after any compiler, runtime, calibration, or key change. * Keep `manifest.json` with release artifacts so the compiler command and schema version remain auditable. -== ONNX to Minerva Workflow +== Model Source Workflow + +Minerva export starts from a supported SKaiNET `ComputeGraph`. That graph can come from the Kotlin DSL, a traced forward pass, a hand-built graph, an imported model, or any other path that preserves the phase-one static MLP contract. -Use the existing ONNX loader path to inspect the model and extract a supported static MLP graph before calling Minerva export. The first phase does not include a general ONNX-to-Minerva importer; unsupported ONNX operators should be rejected before export. +For ONNX inputs, use the existing ONNX loader path as an inspection step before constructing a compatible SKaiNET graph. The first phase does not include a general ONNX-to-Minerva importer; unsupported imported operators should be rejected before export. After export, keep `manifest.json` with the source-model artifact or training metadata so reviewers can compare provenance with the generated Minerva bundle fingerprints. [source,kotlin] ---- diff --git a/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackager.kt b/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackager.kt index 47834638..a18c66d9 100644 --- a/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackager.kt +++ b/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackager.kt @@ -7,6 +7,7 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths import java.nio.file.StandardCopyOption +import java.security.MessageDigest import sk.ainet.compile.export.GraphExportArtifact import sk.ainet.compile.export.GraphExportArtifactRole import sk.ainet.compile.export.GraphExportContext @@ -409,11 +410,15 @@ public class JvmMinervaProjectPackager @kotlin.jvm.JvmOverloads constructor( val manifestPath = projectDir.resolve("manifest.json") val generatedRelative = generatedPaths.map { relativePath(projectDir, it) } + val generatedFileSha256 = generatedPaths.map { path -> + relativePath(projectDir, path) to fileSha256(path) + } Files.writeString( manifestPath, manifestJson( request = request, generatedFiles = generatedRelative, + generatedFileSha256 = generatedFileSha256, manifestPath = relativePath(projectDir, manifestPath) ) ) @@ -507,6 +512,7 @@ public class JvmMinervaProjectPackager @kotlin.jvm.JvmOverloads constructor( private fun manifestJson( request: MinervaProjectPackageRequest, generatedFiles: List, + generatedFileSha256: List>, manifestPath: String ): String { val options = request.options @@ -525,7 +531,8 @@ public class JvmMinervaProjectPackager @kotlin.jvm.JvmOverloads constructor( "referenceInputPath" to jsonString("host/$HOST_REFERENCE_INPUT_FILE"), "referenceOutputPath" to jsonString("host/$HOST_REFERENCE_OUTPUT_FILE"), "manifestPath" to jsonString(manifestPath), - "generatedFiles" to generatedFiles.joinToString(prefix = "[", postfix = "]") { jsonString(it) } + "generatedFiles" to generatedFiles.joinToString(prefix = "[", postfix = "]") { jsonString(it) }, + "generatedFileSha256" to jsonObject(generatedFileSha256) ) return values.entries.joinToString(prefix = "{\n", postfix = "\n}\n", separator = ",\n") { (key, value) -> " ${jsonString(key)}: $value" @@ -725,6 +732,27 @@ public class JvmMinervaProjectPackager @kotlin.jvm.JvmOverloads constructor( .replace("\"", "\\\"") } + private fun fileSha256(path: Path): String { + val digest = MessageDigest.getInstance("SHA-256") + Files.newInputStream(path).use { input -> + val buffer = ByteArray(8192) + while (true) { + val read = input.read(buffer) + if (read < 0) break + digest.update(buffer, 0, read) + } + } + return digest.digest().joinToString(separator = "") { byte -> + (byte.toInt() and 0xff).toString(16).padStart(2, '0') + } + } + + private fun jsonObject(values: List>): String { + return values.joinToString(prefix = "{", postfix = "}") { (key, value) -> + "${jsonString(key)}: ${jsonString(value)}" + } + } + private fun jsonString(value: String): String { val escaped = buildString { value.forEach { char -> diff --git a/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackagerTest.kt b/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackagerTest.kt index 3c47ad13..5c8cfee5 100644 --- a/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackagerTest.kt +++ b/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackagerTest.kt @@ -144,6 +144,11 @@ class MinervaJvmCompilerAndPackagerTest { assertTrue(manifest.contains("\"compilerCommand\": \"fake-minerva --key-file \"")) assertTrue(manifest.contains("\"referenceInputPath\": \"host/reference-input.txt\"")) assertTrue(manifest.contains("\"referenceOutputPath\": \"host/reference-output.txt\"")) + assertTrue(manifest.contains("\"generatedFileSha256\": {")) + assertTrue(Regex("\"generated/model\\.npz\": \"[0-9a-f]{64}\"").containsMatchIn(manifest)) + assertTrue(Regex("\"generated/weights\\.c\": \"[0-9a-f]{64}\"").containsMatchIn(manifest)) + assertTrue(Regex("\"include/weights\\.h\": \"[0-9a-f]{64}\"").containsMatchIn(manifest)) + assertFalse(manifest.contains("REAL_SECRET_KEY_MATERIAL")) assertEquals("manifest.json", bundle.manifestPath) assertTrue(bundle.generatedFiles.contains("generated/weights_debug.npz")) assertTrue(bundle.generatedFiles.contains("include/secrets.example.h"))