From b1cc940c60fee7bec8a6989666f1b9dad98c5f47 Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Sun, 7 Jun 2026 20:50:16 +0200 Subject: [PATCH 1/3] feat(minerva): add compiler packager flow Closes #694 --- .../api/skainet-compile-minerva.api | 158 ++++- .../compile/minerva/MinervaCompilerModels.kt | 124 ++++ .../compile/minerva/MinervaExportFacade.kt | 187 +++++- .../compile/minerva/MinervaExportModels.kt | 18 +- .../sk/ainet/compile/minerva/package.kt | 6 +- .../minerva/MinervaExportFacadeTest.kt | 128 +++- .../minerva/MinervaJvmCompilerAndPackager.kt | 610 ++++++++++++++++++ .../MinervaJvmCompilerAndPackagerTest.kt | 171 +++++ 8 files changed, 1359 insertions(+), 43 deletions(-) create mode 100644 skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaCompilerModels.kt create mode 100644 skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackager.kt create mode 100644 skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackagerTest.kt diff --git a/skainet-compile/skainet-compile-minerva/api/skainet-compile-minerva.api b/skainet-compile/skainet-compile-minerva/api/skainet-compile-minerva.api index 2cc6e05d..16389c64 100644 --- a/skainet-compile/skainet-compile-minerva/api/skainet-compile-minerva.api +++ b/skainet-compile/skainet-compile-minerva/api/skainet-compile-minerva.api @@ -1,3 +1,11 @@ +public final class sk/ainet/compile/minerva/JvmMinervaProjectPackager : sk/ainet/compile/minerva/MinervaProjectPackager { + public fun ()V + public fun (Ljava/lang/String;)V + public synthetic fun (Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun getBackendName ()Ljava/lang/String; + public fun packageProject (Lsk/ainet/compile/minerva/MinervaProjectPackageRequest;Lsk/ainet/compile/export/GraphExportContext;)Lsk/ainet/compile/minerva/MinervaExportBundle; +} + public final class sk/ainet/compile/minerva/MinervaActivation : java/lang/Enum { public static final field RELU Lsk/ainet/compile/minerva/MinervaActivation; public static final field SIGMOID Lsk/ainet/compile/minerva/MinervaActivation; @@ -91,6 +99,69 @@ public final class sk/ainet/compile/minerva/MinervaCompatibilityValidator$Compan public final fun getLayerOperations ()Ljava/util/Set; } +public abstract interface class sk/ainet/compile/minerva/MinervaCompilerAdapter { + public abstract fun compile (Lsk/ainet/compile/minerva/MinervaCompilerRequest;Lsk/ainet/compile/export/GraphExportContext;)Lsk/ainet/compile/minerva/MinervaCompilerOutput; + public abstract fun getBackendName ()Ljava/lang/String; +} + +public final class sk/ainet/compile/minerva/MinervaCompilerException : java/lang/IllegalStateException { + public fun (Ljava/lang/String;Ljava/lang/String;ZLjava/lang/String;Ljava/lang/String;Ljava/lang/Integer;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;)V + public synthetic fun (Ljava/lang/String;Ljava/lang/String;ZLjava/lang/String;Ljava/lang/String;Ljava/lang/Integer;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getCode ()Ljava/lang/String; + public final fun getCommandSummary ()Ljava/lang/String; + public final fun getDetails ()Ljava/util/Map; + public final fun getExitCode ()Ljava/lang/Integer; + public final fun getPrerequisite ()Z + public final fun getRemediation ()Ljava/lang/String; + public final fun getStderr ()Ljava/lang/String; + public final fun getStdout ()Ljava/lang/String; +} + +public final class sk/ainet/compile/minerva/MinervaCompilerOutput { + public fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILjava/util/List;Ljava/util/Map;)V + public synthetic fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILjava/util/List;Ljava/util/Map;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun component1 ()Ljava/lang/String; + public final fun component10 ()Ljava/util/Map; + public final fun component2 ()Ljava/lang/String; + public final fun component3 ()Ljava/lang/String; + public final fun component4 ()Ljava/lang/String; + public final fun component5 ()Ljava/lang/String; + public final fun component6 ()Ljava/lang/String; + public final fun component7 ()Ljava/lang/String; + public final fun component8 ()I + public final fun component9 ()Ljava/util/List; + public final fun copy (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILjava/util/List;Ljava/util/Map;)Lsk/ainet/compile/minerva/MinervaCompilerOutput; + public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaCompilerOutput;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ILjava/util/List;Ljava/util/Map;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaCompilerOutput; + public fun equals (Ljava/lang/Object;)Z + public final fun getCommandSummary ()Ljava/lang/String; + public final fun getDebugWeightsPath ()Ljava/lang/String; + public final fun getExitCode ()I + public final fun getGeneratedFiles ()Ljava/util/List; + public final fun getMetadata ()Ljava/util/Map; + public final fun getOutputDir ()Ljava/lang/String; + public final fun getStderr ()Ljava/lang/String; + public final fun getStdout ()Ljava/lang/String; + public final fun getWeightsCPath ()Ljava/lang/String; + public final fun getWeightsHPath ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + +public final class sk/ainet/compile/minerva/MinervaCompilerRequest { + public fun (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;)V + public final fun component1 ()Lsk/ainet/compile/minerva/MinervaExportOptions; + public final fun component2 ()Lsk/ainet/compile/minerva/MinervaIntermediate; + public final fun component3 ()Lsk/ainet/compile/minerva/MinervaNpzModel; + public final fun copy (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;)Lsk/ainet/compile/minerva/MinervaCompilerRequest; + public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaCompilerRequest;Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaCompilerRequest; + public fun equals (Ljava/lang/Object;)Z + public final fun getIntermediate ()Lsk/ainet/compile/minerva/MinervaIntermediate; + public final fun getNpzModel ()Lsk/ainet/compile/minerva/MinervaNpzModel; + public final fun getOptions ()Lsk/ainet/compile/minerva/MinervaExportOptions; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class sk/ainet/compile/minerva/MinervaExportBackend { public static final field INSTANCE Lsk/ainet/compile/minerva/MinervaExportBackend; public static final field backendName Ljava/lang/String; @@ -98,17 +169,19 @@ public final class sk/ainet/compile/minerva/MinervaExportBackend { } public final class sk/ainet/compile/minerva/MinervaExportBundle { - public fun (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/util/List;Ljava/lang/String;)V - public synthetic fun (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/util/List;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/util/List;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompilerOutput;)V + public synthetic fun (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/util/List;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompilerOutput;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1 ()Ljava/lang/String; public final fun component2 ()Ljava/lang/String; public final fun component3 ()Lsk/ainet/compile/minerva/MinervaTarget; public final fun component4 ()Lsk/ainet/compile/minerva/MinervaQuantization; public final fun component5 ()Ljava/util/List; public final fun component6 ()Ljava/lang/String; - public final fun copy (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/util/List;Ljava/lang/String;)Lsk/ainet/compile/minerva/MinervaExportBundle; - public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaExportBundle;Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/util/List;Ljava/lang/String;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaExportBundle; + public final fun component7 ()Lsk/ainet/compile/minerva/MinervaCompilerOutput; + public final fun copy (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/util/List;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompilerOutput;)Lsk/ainet/compile/minerva/MinervaExportBundle; + public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaExportBundle;Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/util/List;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompilerOutput;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaExportBundle; public fun equals (Ljava/lang/Object;)Z + public final fun getCompilerOutput ()Lsk/ainet/compile/minerva/MinervaCompilerOutput; public final fun getGeneratedFiles ()Ljava/util/List; public final fun getManifestPath ()Ljava/lang/String; public final fun getOutputDir ()Ljava/lang/String; @@ -125,14 +198,18 @@ public final class sk/ainet/compile/minerva/MinervaExportFacade { public fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompatibilityValidator;)V public fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompatibilityValidator;Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer;)V public fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompatibilityValidator;Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer;Lsk/ainet/compile/minerva/MinervaNpzModelWriter;)V - public synthetic fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompatibilityValidator;Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer;Lsk/ainet/compile/minerva/MinervaNpzModelWriter;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompatibilityValidator;Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer;Lsk/ainet/compile/minerva/MinervaNpzModelWriter;Lsk/ainet/compile/minerva/MinervaCompilerAdapter;)V + public fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompatibilityValidator;Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer;Lsk/ainet/compile/minerva/MinervaNpzModelWriter;Lsk/ainet/compile/minerva/MinervaCompilerAdapter;Lsk/ainet/compile/minerva/MinervaProjectPackager;)V + public synthetic fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompatibilityValidator;Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer;Lsk/ainet/compile/minerva/MinervaNpzModelWriter;Lsk/ainet/compile/minerva/MinervaCompilerAdapter;Lsk/ainet/compile/minerva/MinervaProjectPackager;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun exportGraph (Lsk/ainet/lang/graph/ComputeGraph;Lsk/ainet/compile/minerva/MinervaExportOptions;)Lsk/ainet/compile/minerva/MinervaExportResult; public final fun exportModel (Ljava/lang/Object;Lkotlin/jvm/functions/Function1;Lsk/ainet/compile/minerva/MinervaExportOptions;)Lsk/ainet/compile/minerva/MinervaExportResult; public final fun exportModel (Ljava/lang/Object;Lsk/ainet/compile/minerva/MinervaExportOptions;)Lsk/ainet/compile/minerva/MinervaExportResult; public final fun getBackendName ()Ljava/lang/String; public final fun getCompatibilityValidator ()Lsk/ainet/compile/minerva/MinervaCompatibilityValidator; + public final fun getCompilerAdapter ()Lsk/ainet/compile/minerva/MinervaCompilerAdapter; public final fun getGraphCanonicalizer ()Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer; public final fun getNpzWriter ()Lsk/ainet/compile/minerva/MinervaNpzModelWriter; + public final fun getProjectPackager ()Lsk/ainet/compile/minerva/MinervaProjectPackager; } public final class sk/ainet/compile/minerva/MinervaExportFailure { @@ -157,10 +234,13 @@ public final class sk/ainet/compile/minerva/MinervaExportFailure { public final class sk/ainet/compile/minerva/MinervaExportFailureKind : java/lang/Enum { public static final field COMPATIBILITY_VALIDATION_FAILED Lsk/ainet/compile/minerva/MinervaExportFailureKind; + public static final field COMPILER_FAILED Lsk/ainet/compile/minerva/MinervaExportFailureKind; + public static final field COMPILER_PREREQUISITE_FAILED Lsk/ainet/compile/minerva/MinervaExportFailureKind; public static final field GRAPH_VALIDATION_FAILED Lsk/ainet/compile/minerva/MinervaExportFailureKind; public static final field LOWERING_FAILED Lsk/ainet/compile/minerva/MinervaExportFailureKind; public static final field NOT_IMPLEMENTED Lsk/ainet/compile/minerva/MinervaExportFailureKind; public static final field NPZ_SCHEMA_FAILED Lsk/ainet/compile/minerva/MinervaExportFailureKind; + public static final field PACKAGING_FAILED Lsk/ainet/compile/minerva/MinervaExportFailureKind; public static final field RECORDING_FAILED Lsk/ainet/compile/minerva/MinervaExportFailureKind; public static final field UNSUPPORTED_MODEL_TYPE Lsk/ainet/compile/minerva/MinervaExportFailureKind; public static fun getEntries ()Lkotlin/enums/EnumEntries; @@ -169,13 +249,14 @@ public final class sk/ainet/compile/minerva/MinervaExportFailureKind : java/lang } public final class sk/ainet/compile/minerva/MinervaExportOptions { - public fun (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZLjava/util/Map;)V - public synthetic fun (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZLjava/util/Map;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZLjava/util/Map;)V + public synthetic fun (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZLjava/util/Map;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1 ()Ljava/lang/String; public final fun component10 ()Z public final fun component11 ()Z public final fun component12 ()Z - public final fun component13 ()Ljava/util/Map; + public final fun component13 ()Z + public final fun component14 ()Ljava/util/Map; public final fun component2 ()Ljava/lang/String; public final fun component3 ()Lsk/ainet/compile/minerva/MinervaTarget; public final fun component4 ()Lsk/ainet/compile/minerva/MinervaQuantization; @@ -183,9 +264,9 @@ public final class sk/ainet/compile/minerva/MinervaExportOptions { public final fun component6 ()Ljava/lang/String; public final fun component7 ()Ljava/lang/String; public final fun component8 ()Ljava/lang/String; - public final fun component9 ()Z - public final fun copy (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZLjava/util/Map;)Lsk/ainet/compile/minerva/MinervaExportOptions; - public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaExportOptions;Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZLjava/util/Map;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaExportOptions; + public final fun component9 ()Ljava/lang/String; + public final fun copy (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZLjava/util/Map;)Lsk/ainet/compile/minerva/MinervaExportOptions; + public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaExportOptions;Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZLjava/util/Map;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaExportOptions; public fun equals (Ljava/lang/Object;)Z public final fun getCalibrationNpz ()Ljava/lang/String; public final fun getCompilerScript ()Ljava/lang/String; @@ -196,6 +277,7 @@ public final class sk/ainet/compile/minerva/MinervaExportOptions { public final fun getMetadata ()Ljava/util/Map; public final fun getOutputDir ()Ljava/lang/String; public final fun getProjectName ()Ljava/lang/String; + public final fun getPythonExecutable ()Ljava/lang/String; public final fun getQuantization ()Lsk/ainet/compile/minerva/MinervaQuantization; public final fun getRunHostVerification ()Z public final fun getRuntimeRoot ()Ljava/lang/String; @@ -206,10 +288,11 @@ public final class sk/ainet/compile/minerva/MinervaExportOptions { } public final class sk/ainet/compile/minerva/MinervaExportResult { - public fun (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;)V - public synthetic fun (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;)V + public synthetic fun (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1 ()Lsk/ainet/compile/minerva/MinervaExportOptions; public final fun component10 ()Lsk/ainet/compile/minerva/MinervaNpzModel; + public final fun component11 ()Lsk/ainet/compile/minerva/MinervaCompilerOutput; public final fun component2 ()Lsk/ainet/compile/export/GraphExportStatus; public final fun component3 ()Lsk/ainet/compile/minerva/MinervaExportBundle; public final fun component4 ()Lsk/ainet/compile/export/GraphExportDiagnosticReport; @@ -218,12 +301,13 @@ public final class sk/ainet/compile/minerva/MinervaExportResult { public final fun component7 ()Ljava/util/Map; public final fun component8 ()Lsk/ainet/compile/minerva/MinervaCompatibilityReport; public final fun component9 ()Lsk/ainet/compile/minerva/MinervaIntermediate; - public final fun copy (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;)Lsk/ainet/compile/minerva/MinervaExportResult; - public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaExportResult;Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaExportResult; + public final fun copy (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;)Lsk/ainet/compile/minerva/MinervaExportResult; + public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaExportResult;Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaExportResult; public fun equals (Ljava/lang/Object;)Z public final fun getArtifacts ()Ljava/util/List; public final fun getBundle ()Lsk/ainet/compile/minerva/MinervaExportBundle; public final fun getCompatibilityReport ()Lsk/ainet/compile/minerva/MinervaCompatibilityReport; + public final fun getCompilerOutput ()Lsk/ainet/compile/minerva/MinervaCompilerOutput; public final fun getDiagnostics ()Lsk/ainet/compile/export/GraphExportDiagnosticReport; public final fun getFailed ()Z public final fun getFailure ()Lsk/ainet/compile/minerva/MinervaExportFailure; @@ -410,6 +494,42 @@ public final class sk/ainet/compile/minerva/MinervaNpzSchemaException : java/lan public final fun getLayerId ()Ljava/lang/String; } +public final class sk/ainet/compile/minerva/MinervaPackagingException : java/lang/IllegalStateException { + public fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;)V + public synthetic fun (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getCode ()Ljava/lang/String; + public final fun getDetails ()Ljava/util/Map; + public final fun getRemediation ()Ljava/lang/String; +} + +public final class sk/ainet/compile/minerva/MinervaPlatformExportDefaults { + public static final field INSTANCE Lsk/ainet/compile/minerva/MinervaPlatformExportDefaults; + public final fun compilerAdapter ()Lsk/ainet/compile/minerva/MinervaCompilerAdapter; + public final fun projectPackager ()Lsk/ainet/compile/minerva/MinervaProjectPackager; +} + +public final class sk/ainet/compile/minerva/MinervaProjectPackageRequest { + public fun (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;)V + public final fun component1 ()Lsk/ainet/compile/minerva/MinervaExportOptions; + public final fun component2 ()Lsk/ainet/compile/minerva/MinervaIntermediate; + public final fun component3 ()Lsk/ainet/compile/minerva/MinervaNpzModel; + public final fun component4 ()Lsk/ainet/compile/minerva/MinervaCompilerOutput; + public final fun copy (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;)Lsk/ainet/compile/minerva/MinervaProjectPackageRequest; + public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaProjectPackageRequest;Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaProjectPackageRequest; + public fun equals (Ljava/lang/Object;)Z + public final fun getCompilerOutput ()Lsk/ainet/compile/minerva/MinervaCompilerOutput; + public final fun getIntermediate ()Lsk/ainet/compile/minerva/MinervaIntermediate; + public final fun getNpzModel ()Lsk/ainet/compile/minerva/MinervaNpzModel; + public final fun getOptions ()Lsk/ainet/compile/minerva/MinervaExportOptions; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + +public abstract interface class sk/ainet/compile/minerva/MinervaProjectPackager { + public abstract fun getBackendName ()Ljava/lang/String; + public abstract fun packageProject (Lsk/ainet/compile/minerva/MinervaProjectPackageRequest;Lsk/ainet/compile/export/GraphExportContext;)Lsk/ainet/compile/minerva/MinervaExportBundle; +} + public final class sk/ainet/compile/minerva/MinervaQuantization : java/lang/Enum { public static final field Q8 Lsk/ainet/compile/minerva/MinervaQuantization; public final fun getCompilerId ()Ljava/lang/String; @@ -467,3 +587,11 @@ public final class sk/ainet/compile/minerva/MinervaTensorRole : java/lang/Enum { public static fun values ()[Lsk/ainet/compile/minerva/MinervaTensorRole; } +public final class sk/ainet/compile/minerva/PythonMinervaCompilerAdapter : sk/ainet/compile/minerva/MinervaCompilerAdapter { + public fun ()V + public fun (Ljava/lang/String;)V + public synthetic fun (Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun compile (Lsk/ainet/compile/minerva/MinervaCompilerRequest;Lsk/ainet/compile/export/GraphExportContext;)Lsk/ainet/compile/minerva/MinervaCompilerOutput; + public fun getBackendName ()Ljava/lang/String; +} + diff --git a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaCompilerModels.kt b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaCompilerModels.kt new file mode 100644 index 00000000..c58d9086 --- /dev/null +++ b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaCompilerModels.kt @@ -0,0 +1,124 @@ +package sk.ainet.compile.minerva + +import sk.ainet.compile.export.GraphExportContext + +/** + * Platform defaults for compiler execution and project packaging. + */ +public expect object MinervaPlatformExportDefaults { + public fun compilerAdapter(): MinervaCompilerAdapter + public fun projectPackager(): MinervaProjectPackager +} + +/** + * Compiler invocation input after lowering and NPZ emission. + */ +public data class MinervaCompilerRequest( + public val options: MinervaExportOptions, + public val intermediate: MinervaIntermediate, + public val npzModel: MinervaNpzModel +) { + init { + require(options.projectName == intermediate.projectName) { + "compiler request options and intermediate project names must match" + } + } +} + +/** + * Paths and diagnostics returned by a successful Minerva compiler invocation. + */ +public data class MinervaCompilerOutput( + public val outputDir: String, + public val weightsCPath: String, + public val weightsHPath: String, + public val debugWeightsPath: String? = null, + public val commandSummary: String, + public val stdout: String = "", + public val stderr: String = "", + public val exitCode: Int = 0, + public val generatedFiles: List = listOf(weightsCPath, weightsHPath) + + listOfNotNull(debugWeightsPath), + public val metadata: Map = emptyMap() +) { + init { + require(outputDir.isNotBlank()) { "compiler outputDir cannot be blank" } + require(weightsCPath.isNotBlank()) { "weightsCPath cannot be blank" } + require(weightsHPath.isNotBlank()) { "weightsHPath cannot be blank" } + require(debugWeightsPath == null || debugWeightsPath.isNotBlank()) { + "debugWeightsPath cannot be blank when provided" + } + require(commandSummary.isNotBlank()) { "commandSummary cannot be blank" } + require(exitCode == 0) { "successful compiler output must have exitCode 0" } + require(generatedFiles.all { it.isNotBlank() }) { "generatedFiles cannot contain blank paths" } + } +} + +/** + * Typed compiler error that can be mapped into [MinervaExportFailure]. + */ +public class MinervaCompilerException( + message: String, + public val code: String, + public val prerequisite: Boolean = false, + public val stdout: String = "", + public val stderr: String = "", + public val exitCode: Int? = null, + public val commandSummary: String? = null, + public val remediation: String, + public val details: Map = emptyMap() +) : IllegalStateException(message) { + init { + require(code.isNotBlank()) { "compiler exception code cannot be blank" } + require(remediation.isNotBlank()) { "compiler exception remediation cannot be blank" } + } +} + +/** + * Invokes libminerva compiler tooling for a lowered model. + */ +public interface MinervaCompilerAdapter { + public val backendName: String + + public fun compile( + request: MinervaCompilerRequest, + context: GraphExportContext + ): MinervaCompilerOutput +} + +/** + * Packaging input produced after successful compiler invocation. + */ +public data class MinervaProjectPackageRequest( + public val options: MinervaExportOptions, + public val intermediate: MinervaIntermediate, + public val npzModel: MinervaNpzModel, + public val compilerOutput: MinervaCompilerOutput +) + +/** + * Typed packaging error that can be mapped into [MinervaExportFailure]. + */ +public class MinervaPackagingException( + message: String, + public val code: String, + public val remediation: String, + public val details: Map = emptyMap() +) : IllegalStateException(message) { + init { + require(code.isNotBlank()) { "packaging exception code cannot be blank" } + require(remediation.isNotBlank()) { "packaging exception remediation cannot be blank" } + } +} + +/** + * Packages compiler outputs into a Minerva project directory. + */ +public interface MinervaProjectPackager { + public val backendName: String + + public fun packageProject( + request: MinervaProjectPackageRequest, + context: GraphExportContext + ): MinervaExportBundle +} diff --git a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportFacade.kt b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportFacade.kt index 2009ccb1..7a8fee4f 100644 --- a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportFacade.kt +++ b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportFacade.kt @@ -14,15 +14,16 @@ import sk.ainet.tape.Execution * Public Minerva export facade. * * This scaffold accepts direct [ComputeGraph] inputs and exposes the same - * traced-forward-pass shape used by other SKaiNET export facades. Real - * compatibility validation and Minerva compiler invocation start in later - * implementation issues. + * traced-forward-pass shape used by other SKaiNET export facades. Host + * verification remains a follow-up stage after compiler packaging. */ public class MinervaExportFacade @kotlin.jvm.JvmOverloads constructor( public val backendName: String = MinervaExportBackend.backendName, public val compatibilityValidator: MinervaCompatibilityValidator = MinervaCompatibilityValidator(), public val graphCanonicalizer: MinervaGraphCanonicalizer = MinervaGraphCanonicalizer(), - public val npzWriter: MinervaNpzModelWriter = MinervaNpzModelWriter() + public val npzWriter: MinervaNpzModelWriter = MinervaNpzModelWriter(), + public val compilerAdapter: MinervaCompilerAdapter = MinervaPlatformExportDefaults.compilerAdapter(), + public val projectPackager: MinervaProjectPackager = MinervaPlatformExportDefaults.projectPackager() ) { /** @@ -111,19 +112,87 @@ public class MinervaExportFacade @kotlin.jvm.JvmOverloads constructor( ) } + val compilerOutput = try { + compilerAdapter.compile( + MinervaCompilerRequest( + options = options, + intermediate = intermediate, + npzModel = npzModel + ), + context + ) + } catch (exception: MinervaCompilerException) { + return compilerFailedResult( + options = options, + context = context, + compatibilityReport = compatibilityReport, + intermediate = intermediate, + npzModel = npzModel, + exception = exception + ) + } + + val bundle = try { + projectPackager.packageProject( + MinervaProjectPackageRequest( + options = options, + intermediate = intermediate, + npzModel = npzModel, + compilerOutput = compilerOutput + ), + context + ) + } catch (exception: MinervaPackagingException) { + return packagingFailedResult( + options = options, + context = context, + compatibilityReport = compatibilityReport, + intermediate = intermediate, + npzModel = npzModel, + compilerOutput = compilerOutput, + exception = exception + ) + } + + if (!options.runHostVerification) { + context.info( + stage = GraphExportStage.PACKAGING, + code = "minerva.export.completed_without_verification", + message = "Minerva export packaged project outputs; host verification was disabled.", + details = mapOf( + "projectDir" to bundle.outputDir, + "generatedFiles" to bundle.generatedFiles.size.toString() + ) + ) + return MinervaExportResult( + options = options, + status = GraphExportStatus.SUCCESS, + bundle = bundle, + diagnostics = context.diagnosticReport(), + artifacts = context.artifacts, + metadata = context.metadata, + compatibilityReport = compatibilityReport, + intermediate = intermediate, + npzModel = npzModel, + compilerOutput = compilerOutput + ) + } + val failure = MinervaExportFailure( kind = MinervaExportFailureKind.NOT_IMPLEMENTED, - stage = GraphExportStage.PACKAGING, + stage = GraphExportStage.VERIFICATION, code = "minerva.export.not_implemented", - message = "Minerva export lowered the graph and emitted the NPZ compiler input; compiler invocation, packaging, and verification are implemented in follow-up issues.", + message = "Minerva export packaged the project; host verification and parity checks are implemented in a follow-up issue.", details = mapOf( - "nextStep" to "Invoke libminerva compiler and package generated outputs.", - "issue" to "#694", + "nextStep" to "Build the packaged host harness and compare Minerva output with SKaiNET output.", + "issue" to "#695", "layers" to intermediate.layerCount.toString(), "input" to intermediate.input.id, "output" to intermediate.output.id, "npzPath" to npzModel.logicalPath, - "npzBytes" to npzModel.bytes.size.toString() + "npzBytes" to npzModel.bytes.size.toString(), + "projectDir" to bundle.outputDir, + "generatedFiles" to bundle.generatedFiles.size.toString() ) ) context.error( @@ -138,7 +207,9 @@ public class MinervaExportFacade @kotlin.jvm.JvmOverloads constructor( failure = failure, compatibilityReport = compatibilityReport, intermediate = intermediate, - npzModel = npzModel + npzModel = npzModel, + compilerOutput = compilerOutput, + bundle = bundle ) } @@ -275,27 +346,119 @@ public class MinervaExportFacade @kotlin.jvm.JvmOverloads constructor( ) } + private fun compilerFailedResult( + options: MinervaExportOptions, + context: GraphExportContext, + compatibilityReport: MinervaCompatibilityReport, + intermediate: MinervaIntermediate, + npzModel: MinervaNpzModel, + exception: MinervaCompilerException + ): MinervaExportResult { + val details = mutableMapOf( + "code" to exception.code, + "issue" to "#694", + "remediation" to exception.remediation + ) + exception.exitCode?.let { details["exitCode"] = it.toString() } + exception.commandSummary?.let { details["command"] = it } + if (exception.stdout.isNotBlank()) details["stdout"] = diagnosticExcerpt(exception.stdout) + if (exception.stderr.isNotBlank()) details["stderr"] = diagnosticExcerpt(exception.stderr) + details += exception.details + val failure = MinervaExportFailure( + kind = if (exception.prerequisite) { + MinervaExportFailureKind.COMPILER_PREREQUISITE_FAILED + } else { + MinervaExportFailureKind.COMPILER_FAILED + }, + stage = GraphExportStage.PACKAGING, + code = exception.code, + message = exception.message ?: "Minerva compiler invocation failed.", + details = details + ) + context.error( + stage = failure.stage, + code = failure.code, + message = failure.message, + details = failure.details + ) + return failedResult( + options = options, + context = context, + failure = failure, + compatibilityReport = compatibilityReport, + intermediate = intermediate, + npzModel = npzModel + ) + } + + private fun packagingFailedResult( + options: MinervaExportOptions, + context: GraphExportContext, + compatibilityReport: MinervaCompatibilityReport, + intermediate: MinervaIntermediate, + npzModel: MinervaNpzModel, + compilerOutput: MinervaCompilerOutput, + exception: MinervaPackagingException + ): MinervaExportResult { + val details = mutableMapOf( + "code" to exception.code, + "issue" to "#694", + "remediation" to exception.remediation + ) + details += exception.details + val failure = MinervaExportFailure( + kind = MinervaExportFailureKind.PACKAGING_FAILED, + stage = GraphExportStage.PACKAGING, + code = exception.code, + message = exception.message ?: "Minerva project packaging failed.", + details = details + ) + context.error( + stage = failure.stage, + code = failure.code, + message = failure.message, + details = failure.details + ) + return failedResult( + options = options, + context = context, + failure = failure, + compatibilityReport = compatibilityReport, + intermediate = intermediate, + npzModel = npzModel, + compilerOutput = compilerOutput + ) + } + private fun failedResult( options: MinervaExportOptions, context: GraphExportContext, failure: MinervaExportFailure, compatibilityReport: MinervaCompatibilityReport? = null, intermediate: MinervaIntermediate? = null, - npzModel: MinervaNpzModel? = null + npzModel: MinervaNpzModel? = null, + compilerOutput: MinervaCompilerOutput? = null, + bundle: MinervaExportBundle? = null ): MinervaExportResult { return MinervaExportResult( options = options, status = GraphExportStatus.FAILED, + bundle = bundle, diagnostics = context.diagnosticReport(), artifacts = context.artifacts, failure = failure, metadata = context.metadata, compatibilityReport = compatibilityReport, intermediate = intermediate, - npzModel = npzModel + npzModel = npzModel, + compilerOutput = compilerOutput ) } + private fun diagnosticExcerpt(value: String, limit: Int = 4000): String { + return if (value.length <= limit) value else value.take(limit) + "..." + } + private fun exportContext(options: MinervaExportOptions): GraphExportContext { return GraphExportContext( backendName = backendName, diff --git a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportModels.kt b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportModels.kt index 8eb723bd..41047990 100644 --- a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportModels.kt +++ b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportModels.kt @@ -34,15 +34,16 @@ public enum class MinervaTarget( /** * Export options for the Minerva backend. * - * Path values are strings so the API stays usable from common code. The phase - * one scaffold validates shape and intent but does not require a libminerva - * checkout until compiler integration lands. + * Path values are strings so the API stays usable from common code. The JVM + * compiler adapter validates configured paths immediately before process + * execution. */ public data class MinervaExportOptions( public val outputDir: String, public val projectName: String, public val target: MinervaTarget = MinervaTarget.ATMEGA328P, public val quantization: MinervaQuantization = MinervaQuantization.Q8, + public val pythonExecutable: String = "python3", public val runtimeRoot: String? = null, public val compilerScript: String? = null, public val keyFile: String? = null, @@ -56,6 +57,7 @@ public data class MinervaExportOptions( init { require(outputDir.isNotBlank()) { "outputDir cannot be blank" } require(projectName.isNotBlank()) { "projectName cannot be blank" } + require(pythonExecutable.isNotBlank()) { "pythonExecutable cannot be blank" } require(projectName.none { it == '/' || it == '\\' }) { "projectName must be a simple project directory name" } @@ -70,6 +72,7 @@ public data class MinervaExportOptions( return metadata + mapOf( "target" to target.compilerId, "quantization" to quantization.compilerId, + "pythonExecutable" to pythonExecutable, "phaseOneScope" to MinervaExportBackend.phaseOneScope, "generateHostHarness" to generateHostHarness.toString(), "generateFirmwareExample" to generateFirmwareExample.toString(), @@ -93,6 +96,9 @@ public enum class MinervaExportFailureKind { COMPATIBILITY_VALIDATION_FAILED, LOWERING_FAILED, NPZ_SCHEMA_FAILED, + COMPILER_PREREQUISITE_FAILED, + COMPILER_FAILED, + PACKAGING_FAILED, NOT_IMPLEMENTED } @@ -121,7 +127,8 @@ public data class MinervaExportBundle( public val target: MinervaTarget, public val quantization: MinervaQuantization, public val generatedFiles: List = emptyList(), - public val manifestPath: String? = null + public val manifestPath: String? = null, + public val compilerOutput: MinervaCompilerOutput? = null ) { init { require(projectName.isNotBlank()) { "projectName cannot be blank" } @@ -205,7 +212,8 @@ public data class MinervaExportResult( public val metadata: Map = emptyMap(), public val compatibilityReport: MinervaCompatibilityReport? = null, public val intermediate: MinervaIntermediate? = null, - public val npzModel: MinervaNpzModel? = null + public val npzModel: MinervaNpzModel? = null, + public val compilerOutput: MinervaCompilerOutput? = null ) { init { require(status != GraphExportStatus.SUCCESS || bundle != null) { diff --git a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/package.kt b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/package.kt index 8959946a..b8656290 100644 --- a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/package.kt +++ b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/package.kt @@ -3,9 +3,9 @@ package sk.ainet.compile.minerva /** * Minerva graph export support for secure MCU inference. * - * The first implementation slice is intentionally JVM-first and API-only: it - * defines the SKaiNET-facing export surface and result model before the - * validator, lowering, compiler adapter, packager, and host verifier are added. + * The phase-one implementation is JVM-first and targets static sequential MLP + * graphs with Q8 libminerva compilation. Host verification is intentionally a + * separate stage so compiler packaging can be tested without MCU hardware. */ public object MinervaExportBackend { public const val backendName: String = "minerva" diff --git a/skainet-compile/skainet-compile-minerva/src/commonTest/kotlin/sk/ainet/compile/minerva/MinervaExportFacadeTest.kt b/skainet-compile/skainet-compile-minerva/src/commonTest/kotlin/sk/ainet/compile/minerva/MinervaExportFacadeTest.kt index c040ddb9..22b0431c 100644 --- a/skainet-compile/skainet-compile-minerva/src/commonTest/kotlin/sk/ainet/compile/minerva/MinervaExportFacadeTest.kt +++ b/skainet-compile/skainet-compile-minerva/src/commonTest/kotlin/sk/ainet/compile/minerva/MinervaExportFacadeTest.kt @@ -7,6 +7,8 @@ import kotlin.test.assertFalse import kotlin.test.assertNotNull import kotlin.test.assertTrue import sk.ainet.compile.export.GraphExportArtifactRole +import sk.ainet.compile.export.GraphExportContext +import sk.ainet.compile.export.GraphExportStage import sk.ainet.compile.export.GraphExportStatus import sk.ainet.lang.graph.DefaultComputeGraph @@ -20,8 +22,11 @@ class MinervaExportFacadeTest { assertEquals(MinervaExportBackend.backendName, facade.backendName) assertEquals(MinervaExportBackend.backendName, facade.graphCanonicalizer.backendName) assertEquals(MinervaExportBackend.backendName, facade.npzWriter.backendName) + assertEquals(MinervaExportBackend.backendName, facade.compilerAdapter.backendName) + assertEquals(MinervaExportBackend.backendName, facade.projectPackager.backendName) assertEquals(MinervaTarget.ATMEGA328P, options.target) assertEquals(MinervaQuantization.Q8, options.quantization) + assertEquals("python3", options.pythonExecutable) assertEquals("jvm-sequential-mlp-q8", options.toMetadata()["phaseOneScope"]) } @@ -36,6 +41,11 @@ class MinervaExportFacadeTest { minervaTestOptions(projectName = "nested/project") } assertTrue(projectError.message?.contains("simple project directory name") == true) + + val pythonError = assertFailsWith { + MinervaExportOptions(outputDir = "build/minerva", projectName = "TinyMlp", pythonExecutable = "") + } + assertTrue(pythonError.message?.contains("pythonExecutable cannot be blank") == true) } @Test @@ -51,13 +61,14 @@ class MinervaExportFacadeTest { } @Test - fun exportGraphReturnsNotImplementedForValidatedGraph() { + fun exportGraphFailsCompilerPrerequisiteWhenCompilerScriptMissing() { val result = MinervaExportFacade().exportGraph(validMinervaMlpGraph(), minervaTestOptions()) assertEquals(GraphExportStatus.FAILED, result.status) - assertEquals(MinervaExportFailureKind.NOT_IMPLEMENTED, result.failure?.kind) - assertEquals("minerva.export.not_implemented", result.failure?.code) + assertEquals(MinervaExportFailureKind.COMPILER_PREREQUISITE_FAILED, result.failure?.kind) + assertEquals("minerva.compiler.script_missing", result.failure?.code) assertEquals("#694", result.failure?.details?.get("issue")) + assertTrue(result.failure?.details?.get("remediation")?.contains("compilerScript") == true) assertTrue(result.diagnostics.infos.any { it.code == "minerva.graph.validation.passed" }) assertTrue(result.diagnostics.infos.any { it.code == "minerva.lowering.completed" }) assertTrue(result.diagnostics.infos.any { it.code == "minerva.npz.completed" }) @@ -76,7 +87,7 @@ class MinervaExportFacadeTest { val graph = validMinervaMlpGraph() val result = MinervaExportFacade().exportModel(graph, minervaTestOptions()) - assertEquals(MinervaExportFailureKind.NOT_IMPLEMENTED, result.failure?.kind) + assertEquals(MinervaExportFailureKind.COMPILER_PREREQUISITE_FAILED, result.failure?.kind) assertTrue(result.compatibilityReport?.compatible == true) assertEquals(MinervaActivation.RELU, result.intermediate?.layers?.single()?.activation) assertEquals(listOf("layer_0_w", "layer_0_b", "layer_0_act"), result.npzModel?.arrayNames?.filter { it.startsWith("layer_0") }?.take(3)) @@ -129,7 +140,7 @@ class MinervaExportFacadeTest { } @Test - fun exportGraphCarriesLoweredIntermediateBeforeCompilerStage() { + fun exportGraphCarriesLoweredIntermediateBeforeCompilerFailure() { val result = MinervaExportFacade().exportGraph( graph = validMinervaMlpGraph(), options = minervaTestOptions(projectName = "LoweredMlp") @@ -137,14 +148,115 @@ class MinervaExportFacadeTest { val intermediate = assertNotNull(result.intermediate) assertEquals(GraphExportStatus.FAILED, result.status) - assertEquals(MinervaExportFailureKind.NOT_IMPLEMENTED, result.failure?.kind) + assertEquals(MinervaExportFailureKind.COMPILER_PREREQUISITE_FAILED, result.failure?.kind) assertEquals("LoweredMlp", intermediate.projectName) assertEquals(MinervaTensorRole.INPUT, intermediate.input.role) assertEquals(MinervaTensorRole.OUTPUT, intermediate.output.role) assertEquals("matmul", intermediate.layers.single().id) - assertEquals("1", result.failure?.details?.get("layers")) assertEquals("#694", result.failure?.details?.get("issue")) - assertEquals("model.npz", result.failure?.details?.get("npzPath")) assertTrue(assertNotNull(result.npzModel).bytes.isNotEmpty()) } + + @Test + fun exportGraphPackagesProjectWhenVerificationDisabled() { + val result = packagingFacade().exportGraph( + graph = validMinervaMlpGraph(), + options = minervaTestOptions(projectName = "PackagedMlp").copy(runHostVerification = false) + ) + + assertEquals(GraphExportStatus.SUCCESS, result.status) + assertTrue(result.succeeded) + val bundle = result.requireSuccess() + assertEquals("PackagedMlp", bundle.projectName) + assertEquals("build/minerva/PackagedMlp", bundle.outputDir) + assertEquals("manifest.json", bundle.manifestPath) + assertTrue(bundle.generatedFiles.contains("generated/weights.c")) + assertTrue(bundle.generatedFiles.contains("include/secrets.example.h")) + assertEquals("build/minerva/PackagedMlp/generated/weights.c", result.compilerOutput?.weightsCPath) + assertTrue(result.diagnostics.infos.any { it.code == "minerva.compiler.completed" }) + assertTrue(result.diagnostics.infos.any { it.code == "minerva.packaging.completed" }) + assertTrue(result.diagnostics.infos.any { it.code == "minerva.export.completed_without_verification" }) + } + + @Test + fun exportGraphStopsAtVerificationPlaceholderAfterPackagingByDefault() { + val result = packagingFacade().exportGraph( + graph = validMinervaMlpGraph(), + options = minervaTestOptions(projectName = "NeedsVerification") + ) + + assertEquals(GraphExportStatus.FAILED, result.status) + assertEquals(MinervaExportFailureKind.NOT_IMPLEMENTED, result.failure?.kind) + assertEquals(GraphExportStage.VERIFICATION, result.failure?.stage) + assertEquals("#695", result.failure?.details?.get("issue")) + assertEquals("build/minerva/NeedsVerification", result.bundle?.outputDir) + assertNotNull(result.compilerOutput) + assertNotNull(result.npzModel) + } + + private fun packagingFacade(): MinervaExportFacade { + return MinervaExportFacade( + compilerAdapter = FakeCompilerAdapter(), + projectPackager = FakeProjectPackager() + ) + } + + private class FakeCompilerAdapter : MinervaCompilerAdapter { + override val backendName: String = MinervaExportBackend.backendName + + override fun compile( + request: MinervaCompilerRequest, + context: GraphExportContext + ): MinervaCompilerOutput { + val generatedDir = "${request.options.outputDir}/${request.options.projectName}/generated" + context.info( + stage = GraphExportStage.PACKAGING, + code = "minerva.compiler.completed", + message = "Fake Minerva compiler completed.", + details = mapOf("outputDir" to generatedDir) + ) + return MinervaCompilerOutput( + outputDir = generatedDir, + weightsCPath = "$generatedDir/weights.c", + weightsHPath = "$generatedDir/weights.h", + commandSummary = "fake-minerva-compiler --model model.npz", + stdout = "ok" + ) + } + } + + private class FakeProjectPackager : MinervaProjectPackager { + override val backendName: String = MinervaExportBackend.backendName + + override fun packageProject( + request: MinervaProjectPackageRequest, + context: GraphExportContext + ): MinervaExportBundle { + val projectDir = "${request.options.outputDir}/${request.options.projectName}" + val generatedFiles = listOf( + "generated/model.npz", + "generated/weights.c", + "include/weights.h", + "include/secrets.example.h", + "host/main.c", + "firmware/main.c", + "manifest.json" + ) + context.info( + stage = GraphExportStage.PACKAGING, + code = "minerva.packaging.completed", + message = "Fake Minerva project packaged.", + details = mapOf("projectDir" to projectDir) + ) + return MinervaExportBundle( + projectName = request.options.projectName, + outputDir = projectDir, + target = request.options.target, + quantization = request.options.quantization, + generatedFiles = generatedFiles, + manifestPath = "manifest.json", + compilerOutput = request.compilerOutput + ) + } + } } diff --git a/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackager.kt b/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackager.kt new file mode 100644 index 00000000..5aa3c4a4 --- /dev/null +++ b/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackager.kt @@ -0,0 +1,610 @@ +package sk.ainet.compile.minerva + +import java.io.ByteArrayOutputStream +import java.io.IOException +import java.io.InputStream +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.StandardCopyOption +import sk.ainet.compile.export.GraphExportArtifact +import sk.ainet.compile.export.GraphExportArtifactRole +import sk.ainet.compile.export.GraphExportContext +import sk.ainet.compile.export.GraphExportStage + +public actual object MinervaPlatformExportDefaults { + public actual fun compilerAdapter(): MinervaCompilerAdapter = PythonMinervaCompilerAdapter() + + public actual fun projectPackager(): MinervaProjectPackager = JvmMinervaProjectPackager() +} + +/** + * JVM adapter that invokes the Python libminerva compiler entry point. + */ +public class PythonMinervaCompilerAdapter @kotlin.jvm.JvmOverloads constructor( + override val backendName: String = MinervaExportBackend.backendName +) : MinervaCompilerAdapter { + + override fun compile( + request: MinervaCompilerRequest, + context: GraphExportContext + ): MinervaCompilerOutput { + val options = request.options + val compilerScript = options.compilerScript + ?: prerequisiteFailure( + code = "minerva.compiler.script_missing", + message = "Minerva compiler script path is required before compiler invocation.", + remediation = "Set MinervaExportOptions.compilerScript to the libminerva compiler Python entry point." + ) + val scriptPath = requireRegularFile( + field = "compilerScript", + value = compilerScript, + code = "minerva.compiler.script_not_found", + remediation = "Point compilerScript at an existing libminerva compiler Python file." + ) + options.runtimeRoot?.let { + requireDirectory( + field = "runtimeRoot", + value = it, + code = "minerva.compiler.runtime_root_not_found", + remediation = "Point runtimeRoot at an existing libminerva checkout or install directory." + ) + } + options.keyFile?.let { + requireRegularFile( + field = "keyFile", + value = it, + code = "minerva.compiler.key_file_not_found", + remediation = "Point keyFile at an existing key file, or omit it for non-secure local compiler tests." + ) + } + options.calibrationNpz?.let { + requireRegularFile( + field = "calibrationNpz", + value = it, + code = "minerva.compiler.calibration_not_found", + remediation = "Point calibrationNpz at an existing calibration archive, or omit it when not required." + ) + } + + val projectDir = Paths.get(options.outputDir).resolve(options.projectName).normalize() + val generatedDir = projectDir.resolve("generated") + val modelPath = generatedDir.resolve(request.npzModel.logicalPath.substringAfterLast('/')) + try { + Files.createDirectories(generatedDir) + Files.write(modelPath, request.npzModel.bytes) + } catch (exception: IOException) { + throw MinervaCompilerException( + message = "Failed to prepare Minerva compiler input: ${exception.message ?: exception.toString()}", + code = "minerva.compiler.input_write_failed", + prerequisite = true, + remediation = "Ensure outputDir is writable and has enough space.", + details = mapOf("modelPath" to modelPath.toString()) + ) + } + + val command = buildCommand(options, scriptPath, modelPath, generatedDir) + val commandSummary = summarizeCommand(command) + context.info( + stage = GraphExportStage.PACKAGING, + code = "minerva.compiler.started", + message = "Invoking libminerva compiler.", + details = mapOf( + "command" to commandSummary, + "model" to modelPath.toString(), + "outputDir" to generatedDir.toString() + ) + ) + + val processResult = runProcess(command, projectDir, commandSummary) + if (processResult.exitCode != 0) { + throw MinervaCompilerException( + message = "Minerva compiler failed with exit code ${processResult.exitCode}.", + code = "minerva.compiler.process_failed", + stdout = processResult.stdout, + stderr = processResult.stderr, + exitCode = processResult.exitCode, + commandSummary = commandSummary, + remediation = "Inspect compiler stdout/stderr and verify target, quantization, calibration, and key configuration.", + details = mapOf("outputDir" to generatedDir.toString()) + ) + } + + val weightsC = generatedDir.resolve("weights.c") + val weightsH = generatedDir.resolve("weights.h") + requireCompilerOutput(weightsC, "weights.c", commandSummary, processResult) + requireCompilerOutput(weightsH, "weights.h", commandSummary, processResult) + val debugWeights = generatedDir.resolve("weights_debug.npz").takeIf { Files.exists(it) } + val output = MinervaCompilerOutput( + outputDir = generatedDir.toString(), + weightsCPath = weightsC.toString(), + weightsHPath = weightsH.toString(), + debugWeightsPath = debugWeights?.toString(), + commandSummary = commandSummary, + stdout = processResult.stdout, + stderr = processResult.stderr, + exitCode = processResult.exitCode, + metadata = mapOf( + "target" to options.target.compilerId, + "quantization" to options.quantization.compilerId, + "model" to modelPath.toString() + ) + ) + context.addArtifact( + GraphExportArtifact( + path = output.weightsCPath, + role = GraphExportArtifactRole.SOURCE, + description = "Minerva compiler weights source" + ) + ) + context.addArtifact( + GraphExportArtifact( + path = output.weightsHPath, + role = GraphExportArtifactRole.HEADER, + description = "Minerva compiler weights header" + ) + ) + context.info( + stage = GraphExportStage.PACKAGING, + code = "minerva.compiler.completed", + message = "libminerva compiler completed successfully.", + details = mapOf( + "weightsC" to output.weightsCPath, + "weightsH" to output.weightsHPath, + "exitCode" to output.exitCode.toString() + ) + ) + return output + } + + private fun buildCommand( + options: MinervaExportOptions, + scriptPath: Path, + modelPath: Path, + generatedDir: Path + ): List { + val command = mutableListOf( + options.pythonExecutable, + scriptPath.toAbsolutePath().normalize().toString(), + "--model", + modelPath.toAbsolutePath().normalize().toString(), + "--out-dir", + generatedDir.toAbsolutePath().normalize().toString(), + "--target", + options.target.compilerId, + "--quantization", + options.quantization.compilerId + ) + options.runtimeRoot?.let { + command += listOf("--runtime-root", Paths.get(it).toAbsolutePath().normalize().toString()) + } + options.keyFile?.let { + command += listOf("--key-file", Paths.get(it).toAbsolutePath().normalize().toString()) + } + options.calibrationNpz?.let { + command += listOf("--calibration", Paths.get(it).toAbsolutePath().normalize().toString()) + } + if (options.dumpWeights) command += "--dump-weights" + return command + } + + private fun runProcess( + command: List, + workingDir: Path, + commandSummary: String + ): ProcessResult { + try { + Files.createDirectories(workingDir) + val process = ProcessBuilder(command) + .directory(workingDir.toFile()) + .start() + val stdout = StreamCollector(process.inputStream).also { it.start() } + val stderr = StreamCollector(process.errorStream).also { it.start() } + val exitCode = process.waitFor() + stdout.join() + stderr.join() + return ProcessResult( + exitCode = exitCode, + stdout = stdout.text(), + stderr = stderr.text() + ) + } catch (exception: IOException) { + throw MinervaCompilerException( + message = "Failed to start Minerva compiler process: ${exception.message ?: exception.toString()}", + code = "minerva.compiler.process_start_failed", + prerequisite = true, + commandSummary = commandSummary, + remediation = "Verify pythonExecutable and compilerScript are executable in this environment.", + details = mapOf("workingDir" to workingDir.toString()) + ) + } catch (exception: InterruptedException) { + Thread.currentThread().interrupt() + throw MinervaCompilerException( + message = "Minerva compiler process was interrupted.", + code = "minerva.compiler.process_interrupted", + commandSummary = commandSummary, + remediation = "Retry the export when the build process can run to completion.", + details = mapOf("workingDir" to workingDir.toString()) + ) + } + } + + private fun requireCompilerOutput( + path: Path, + name: String, + commandSummary: String, + processResult: ProcessResult + ) { + if (!Files.isRegularFile(path)) { + throw MinervaCompilerException( + message = "Minerva compiler did not produce required output '$name'.", + code = "minerva.compiler.output_missing", + stdout = processResult.stdout, + stderr = processResult.stderr, + exitCode = processResult.exitCode, + commandSummary = commandSummary, + remediation = "Verify the libminerva compiler version and output directory contract.", + details = mapOf("missingPath" to path.toString()) + ) + } + } + + private fun requireRegularFile( + field: String, + value: String, + code: String, + remediation: String + ): Path { + val path = Paths.get(value) + if (!Files.isRegularFile(path)) { + prerequisiteFailure( + code = code, + message = "Minerva compiler prerequisite '$field' does not exist or is not a file: $value", + remediation = remediation, + details = mapOf(field to value) + ) + } + return path + } + + private fun requireDirectory( + field: String, + value: String, + code: String, + remediation: String + ): Path { + val path = Paths.get(value) + if (!Files.isDirectory(path)) { + prerequisiteFailure( + code = code, + message = "Minerva compiler prerequisite '$field' does not exist or is not a directory: $value", + remediation = remediation, + details = mapOf(field to value) + ) + } + return path + } + + private fun prerequisiteFailure( + code: String, + message: String, + remediation: String, + details: Map = emptyMap() + ): Nothing { + throw MinervaCompilerException( + message = message, + code = code, + prerequisite = true, + remediation = remediation, + details = details + ) + } + + private fun summarizeCommand(command: List): String { + return command.mapIndexed { index, value -> + val previous = command.getOrNull(index - 1) + when (previous) { + "--key-file" -> "" + else -> value + } + }.joinToString(" ") + } +} + +/** + * JVM packager that writes a host/firmware-oriented Minerva project layout. + */ +public class JvmMinervaProjectPackager @kotlin.jvm.JvmOverloads constructor( + override val backendName: String = MinervaExportBackend.backendName +) : MinervaProjectPackager { + + override fun packageProject( + request: MinervaProjectPackageRequest, + context: GraphExportContext + ): MinervaExportBundle { + val options = request.options + val projectDir = Paths.get(options.outputDir).resolve(options.projectName).normalize() + val generatedDir = projectDir.resolve("generated") + val includeDir = projectDir.resolve("include") + val hostDir = projectDir.resolve("host") + val firmwareDir = projectDir.resolve("firmware") + context.info( + stage = GraphExportStage.PACKAGING, + code = "minerva.packaging.started", + message = "Packaging Minerva project outputs.", + details = mapOf("projectDir" to projectDir.toString()) + ) + + try { + listOf(projectDir, generatedDir, includeDir, hostDir, firmwareDir).forEach(Files::createDirectories) + val modelPath = generatedDir.resolve(request.npzModel.logicalPath.substringAfterLast('/')) + Files.write(modelPath, request.npzModel.bytes) + val weightsC = copyCompilerOutput( + source = Paths.get(request.compilerOutput.weightsCPath), + target = generatedDir.resolve("weights.c"), + logicalName = "weights.c" + ) + val weightsH = copyCompilerOutput( + source = Paths.get(request.compilerOutput.weightsHPath), + target = includeDir.resolve("weights.h"), + logicalName = "weights.h" + ) + val debugWeights = request.compilerOutput.debugWeightsPath?.let { debugWeightsPath -> + val source = Paths.get(debugWeightsPath) + val fileName = source.fileName?.toString() ?: "weights_debug.npz" + copyCompilerOutput( + source = source, + target = generatedDir.resolve(fileName), + logicalName = fileName + ) + } + val secretsExample = includeDir.resolve("secrets.example.h") + Files.writeString(secretsExample, secretsExampleHeader(options)) + val generatedPaths = mutableListOf(modelPath, weightsC, weightsH, secretsExample) + debugWeights?.let(generatedPaths::add) + if (options.generateHostHarness) { + val hostCmake = hostDir.resolve("CMakeLists.txt") + val hostMain = hostDir.resolve("main.c") + Files.writeString(hostCmake, hostCmake(options)) + Files.writeString(hostMain, hostMain(request)) + generatedPaths.add(hostCmake) + generatedPaths.add(hostMain) + } + if (options.generateFirmwareExample) { + val firmwareMain = firmwareDir.resolve("main.c") + Files.writeString(firmwareMain, firmwareMain(request)) + generatedPaths.add(firmwareMain) + } + + val manifestPath = projectDir.resolve("manifest.json") + val generatedRelative = generatedPaths.map { relativePath(projectDir, it) } + Files.writeString( + manifestPath, + manifestJson( + request = request, + generatedFiles = generatedRelative, + manifestPath = relativePath(projectDir, manifestPath) + ) + ) + val allRelative = generatedRelative + relativePath(projectDir, manifestPath) + recordArtifacts(context, projectDir, manifestPath, generatedPaths) + context.info( + stage = GraphExportStage.PACKAGING, + code = "minerva.packaging.completed", + message = "Packaged Minerva project outputs.", + details = mapOf( + "projectDir" to projectDir.toString(), + "files" to allRelative.size.toString(), + "manifest" to relativePath(projectDir, manifestPath) + ) + ) + return MinervaExportBundle( + projectName = options.projectName, + outputDir = projectDir.toString(), + target = options.target, + quantization = options.quantization, + generatedFiles = allRelative, + manifestPath = relativePath(projectDir, manifestPath), + compilerOutput = request.compilerOutput + ) + } catch (exception: MinervaPackagingException) { + throw exception + } catch (exception: IOException) { + throw MinervaPackagingException( + message = "Failed to package Minerva project: ${exception.message ?: exception.toString()}", + code = "minerva.packaging.io_failed", + remediation = "Ensure outputDir is writable and compiler outputs are readable.", + details = mapOf("projectDir" to projectDir.toString()) + ) + } + } + + private fun copyCompilerOutput(source: Path, target: Path, logicalName: String): Path { + if (!Files.isRegularFile(source)) { + throw MinervaPackagingException( + message = "Cannot package missing compiler output '$logicalName'.", + code = "minerva.packaging.compiler_output_missing", + remediation = "Run the compiler adapter successfully before packaging.", + details = mapOf("missingPath" to source.toString()) + ) + } + Files.createDirectories(target.parent) + if (source.toAbsolutePath().normalize() != target.toAbsolutePath().normalize()) { + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING) + } + return target + } + + private fun recordArtifacts( + context: GraphExportContext, + projectDir: Path, + manifestPath: Path, + generatedPaths: List + ) { + context.addArtifact( + GraphExportArtifact( + path = projectDir.toString(), + role = GraphExportArtifactRole.PROJECT_DIRECTORY, + description = "Minerva packaged project directory" + ) + ) + context.addArtifact( + GraphExportArtifact( + path = manifestPath.toString(), + role = GraphExportArtifactRole.MANIFEST, + description = "Minerva export manifest" + ) + ) + generatedPaths.forEach { path -> + val role = when { + path.fileName.toString().endsWith(".h") -> GraphExportArtifactRole.HEADER + path.fileName.toString().endsWith(".npz") -> GraphExportArtifactRole.INTERMEDIATE + else -> GraphExportArtifactRole.SOURCE + } + context.addArtifact( + GraphExportArtifact( + path = path.toString(), + role = role, + description = "Minerva packaged file ${relativePath(projectDir, path)}", + sensitive = false + ) + ) + } + } + + private fun manifestJson( + request: MinervaProjectPackageRequest, + generatedFiles: List, + manifestPath: String + ): String { + val options = request.options + val values = mapOf( + "projectName" to jsonString(options.projectName), + "skainetVersion" to jsonString(options.metadata["skainetVersion"] ?: "unknown"), + "libminerva" to jsonString(options.runtimeRoot ?: "unspecified"), + "target" to jsonString(options.target.compilerId), + "quantization" to jsonString(options.quantization.compilerId), + "compilerCommand" to jsonString(request.compilerOutput.commandSummary), + "compilerExitCode" to request.compilerOutput.exitCode.toString(), + "npzSchemaVersion" to request.npzModel.schemaVersion.toString(), + "layers" to request.intermediate.layerCount.toString(), + "hostHarness" to options.generateHostHarness.toString(), + "firmwareExample" to options.generateFirmwareExample.toString(), + "manifestPath" to jsonString(manifestPath), + "generatedFiles" to generatedFiles.joinToString(prefix = "[", postfix = "]") { jsonString(it) } + ) + return values.entries.joinToString(prefix = "{\n", postfix = "\n}\n", separator = ",\n") { (key, value) -> + " ${jsonString(key)}: $value" + } + } + + private fun secretsExampleHeader(options: MinervaExportOptions): String { + return """ + |#pragma once + | + |/* + | * Example-only Minerva secret configuration for ${options.projectName}. + | * Replace these placeholders in a private, untracked file. + | */ + |#define MINERVA_DEVICE_KEY_HEX "replace-with-device-key" + |#define MINERVA_KEY_ID "replace-with-key-id" + | + """.trimMargin() + } + + private fun hostCmake(options: MinervaExportOptions): String { + return """ + |cmake_minimum_required(VERSION 3.20) + |project(${options.projectName}_host C) + | + |add_executable(${options.projectName}_host main.c ../generated/weights.c) + |target_include_directories(${options.projectName}_host PRIVATE ../include) + | + """.trimMargin() + } + + private fun hostMain(request: MinervaProjectPackageRequest): String { + val inputCount = request.intermediate.input.elementCount + val outputCount = request.intermediate.output.elementCount + return """ + |#include + |#include + |#include "weights.h" + | + |int main(void) { + | float input[$inputCount] = {0}; + | float output[$outputCount] = {0}; + | + | /* Link this harness with libminerva and call the runtime inference entry point here. */ + | (void)input; + | (void)output; + | puts("Minerva host harness packaged successfully."); + | return 0; + |} + | + """.trimMargin() + } + + private fun firmwareMain(request: MinervaProjectPackageRequest): String { + val inputCount = request.intermediate.input.elementCount + val outputCount = request.intermediate.output.elementCount + return """ + |#include + |#include "weights.h" + |#include "secrets.example.h" + | + |void setup(void) { + | /* Initialize Minerva runtime and seed the PRNG before inference. */ + |} + | + |void loop(void) { + | float input[$inputCount] = {0}; + | float output[$outputCount] = {0}; + | + | /* Call the libminerva inference function with input and output buffers here. */ + | (void)input; + | (void)output; + |} + | + """.trimMargin() + } + + private fun relativePath(root: Path, path: Path): String { + return root.toAbsolutePath().normalize() + .relativize(path.toAbsolutePath().normalize()) + .toString() + .replace('\\', '/') + } + + private fun jsonString(value: String): String { + val escaped = buildString { + value.forEach { char -> + when (char) { + '\\' -> append("\\\\") + '"' -> append("\\\"") + '\n' -> append("\\n") + '\r' -> append("\\r") + '\t' -> append("\\t") + else -> append(char) + } + } + } + return "\"$escaped\"" + } +} + +private data class ProcessResult( + val exitCode: Int, + val stdout: String, + val stderr: String +) + +private class StreamCollector(private val stream: InputStream) : Thread("minerva-process-stream") { + private val output = ByteArrayOutputStream() + + override fun run() { + stream.use { input -> input.copyTo(output) } + } + + fun text(): String = output.toString(Charsets.UTF_8.name()) +} diff --git a/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackagerTest.kt b/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackagerTest.kt new file mode 100644 index 00000000..0c29ab72 --- /dev/null +++ b/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackagerTest.kt @@ -0,0 +1,171 @@ +package sk.ainet.compile.minerva + +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue +import sk.ainet.compile.export.GraphExportArtifactRole +import sk.ainet.compile.export.GraphExportContext + +class MinervaJvmCompilerAndPackagerTest { + + @Test + fun pythonAdapterFailsBeforeProcessWhenCompilerScriptMissing() { + val (options, intermediate, npzModel) = artifacts( + outputDir = tempDir("missing-script").toString(), + projectName = "MissingScript" + ) + val exception = assertFailsWith { + PythonMinervaCompilerAdapter().compile( + MinervaCompilerRequest(options, intermediate, npzModel), + minervaContext(options) + ) + } + + assertEquals("minerva.compiler.script_missing", exception.code) + assertTrue(exception.prerequisite) + assertTrue(exception.remediation.contains("compilerScript")) + } + + @Test + fun pythonAdapterInvokesConfiguredExecutableAndReturnsWeights() { + val root = tempDir("fake-compiler") + val fakeExecutable = root.resolve("fake-python") + val compilerScript = root.resolve("compile.py") + Files.writeString(compilerScript, "# fake script marker\n") + Files.writeString( + fakeExecutable, + """ + |#!/bin/sh + |out="" + |while [ "${'$'}#" -gt 0 ]; do + | case "${'$'}1" in + | --out-dir) + | shift + | out="${'$'}1" + | ;; + | esac + | shift + |done + |mkdir -p "${'$'}out" + |printf '%s\n' 'int minerva_weights = 1;' > "${'$'}out/weights.c" + |printf '%s\n' '#pragma once' > "${'$'}out/weights.h" + |printf '%s\n' 'compiler ok' + | + """.trimMargin() + ) + assertTrue(fakeExecutable.toFile().setExecutable(true)) + + val (options, intermediate, npzModel) = artifacts( + outputDir = root.resolve("out").toString(), + projectName = "AdapterMlp", + compilerScript = compilerScript.toString(), + pythonExecutable = fakeExecutable.toString() + ) + val context = minervaContext(options) + + val output = PythonMinervaCompilerAdapter().compile( + MinervaCompilerRequest(options, intermediate, npzModel), + context + ) + + assertEquals(0, output.exitCode) + assertTrue(Files.isRegularFile(Path.of(output.weightsCPath))) + assertTrue(Files.isRegularFile(Path.of(output.weightsHPath))) + assertTrue(output.stdout.contains("compiler ok")) + assertTrue(output.commandSummary.contains("--model")) + assertTrue(context.diagnostics.any { it.code == "minerva.compiler.completed" }) + assertTrue(context.artifacts.any { it.role == GraphExportArtifactRole.SOURCE && it.path.endsWith("weights.c") }) + } + + @Test + fun projectPackagerWritesManifestSamplesAndSecretTemplate() { + val root = tempDir("packager") + val compilerDir = root.resolve("compiler") + Files.createDirectories(compilerDir) + val weightsC = compilerDir.resolve("weights.c") + val weightsH = compilerDir.resolve("weights.h") + val debugWeights = compilerDir.resolve("weights_debug.npz") + Files.writeString(weightsC, "int minerva_weights = 1;\n") + Files.writeString(weightsH, "#pragma once\n") + Files.write(debugWeights, byteArrayOf(1, 2, 3, 4)) + val keyFile = root.resolve("device.key") + Files.writeString(keyFile, "REAL_SECRET_KEY_MATERIAL") + val (options, intermediate, npzModel) = artifacts( + outputDir = root.resolve("package").toString(), + projectName = "PackagedJvmMlp", + compilerScript = root.resolve("compile.py").toString(), + keyFile = keyFile.toString() + ) + val compilerOutput = MinervaCompilerOutput( + outputDir = compilerDir.toString(), + weightsCPath = weightsC.toString(), + weightsHPath = weightsH.toString(), + debugWeightsPath = debugWeights.toString(), + commandSummary = "fake-minerva --key-file " + ) + val context = minervaContext(options) + + val bundle = JvmMinervaProjectPackager().packageProject( + MinervaProjectPackageRequest(options, intermediate, npzModel, compilerOutput), + context + ) + + val projectDir = Path.of(bundle.outputDir) + assertTrue(Files.isRegularFile(projectDir.resolve("manifest.json"))) + assertTrue(Files.isRegularFile(projectDir.resolve("generated/model.npz"))) + assertTrue(Files.isRegularFile(projectDir.resolve("generated/weights.c"))) + assertTrue(Files.isRegularFile(projectDir.resolve("generated/weights_debug.npz"))) + assertTrue(Files.isRegularFile(projectDir.resolve("include/weights.h"))) + assertTrue(Files.isRegularFile(projectDir.resolve("host/main.c"))) + assertTrue(Files.isRegularFile(projectDir.resolve("firmware/main.c"))) + val secretsExample = Files.readString(projectDir.resolve("include/secrets.example.h")) + assertTrue(secretsExample.contains("replace-with-device-key")) + assertFalse(secretsExample.contains("REAL_SECRET_KEY_MATERIAL")) + val manifest = Files.readString(projectDir.resolve("manifest.json")) + assertTrue(manifest.contains("\"target\": \"atmega328p\"")) + assertTrue(manifest.contains("\"compilerCommand\": \"fake-minerva --key-file \"")) + assertEquals("manifest.json", bundle.manifestPath) + assertTrue(bundle.generatedFiles.contains("generated/weights_debug.npz")) + assertTrue(bundle.generatedFiles.contains("include/secrets.example.h")) + assertTrue(context.artifacts.any { it.role == GraphExportArtifactRole.PROJECT_DIRECTORY }) + assertTrue(context.diagnostics.any { it.code == "minerva.packaging.completed" }) + } + + private fun artifacts( + outputDir: String, + projectName: String, + compilerScript: String? = null, + pythonExecutable: String = "python3", + keyFile: String? = null + ): Triple { + val options = minervaTestOptions( + outputDir = outputDir, + projectName = projectName + ).copy( + compilerScript = compilerScript, + pythonExecutable = pythonExecutable, + keyFile = keyFile, + runHostVerification = false + ) + val context = minervaContext(options) + val intermediate = MinervaGraphCanonicalizer().convert(validMinervaMlpGraph(), context) + val npzModel = MinervaNpzModelWriter().write(intermediate, context) + return Triple(options, intermediate, npzModel) + } + + private fun minervaContext(options: MinervaExportOptions): GraphExportContext { + return GraphExportContext( + backendName = MinervaExportBackend.backendName, + targetName = options.projectName, + metadata = options.toMetadata() + ) + } + + private fun tempDir(prefix: String): Path { + return Files.createTempDirectory("skainet-minerva-$prefix-") + } +} From 70969f60551d229e8cd76acb8133496e1b1ca0ae Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Sun, 7 Jun 2026 21:28:24 +0200 Subject: [PATCH 2/3] feat(minerva): add host verification flow Closes #695 --- .../api/skainet-compile-minerva.api | 115 ++++- .../skainet-compile-minerva/build.gradle.kts | 19 + .../compile/minerva/MinervaCompilerModels.kt | 1 + .../compile/minerva/MinervaExportFacade.kt | 134 ++++-- .../compile/minerva/MinervaExportModels.kt | 9 +- .../minerva/MinervaHostVerificationModels.kt | 158 ++++++ .../sk/ainet/compile/minerva/package.kt | 5 +- .../minerva/MinervaExportFacadeTest.kt | 79 ++- .../minerva/MinervaJvmCompilerAndPackager.kt | 452 ++++++++++++++++++ .../MinervaJvmCompilerAndPackagerTest.kt | 110 +++++ 10 files changed, 1027 insertions(+), 55 deletions(-) create mode 100644 skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaHostVerificationModels.kt diff --git a/skainet-compile/skainet-compile-minerva/api/skainet-compile-minerva.api b/skainet-compile/skainet-compile-minerva/api/skainet-compile-minerva.api index 16389c64..5a3cb7b2 100644 --- a/skainet-compile/skainet-compile-minerva/api/skainet-compile-minerva.api +++ b/skainet-compile/skainet-compile-minerva/api/skainet-compile-minerva.api @@ -1,3 +1,11 @@ +public final class sk/ainet/compile/minerva/JvmMinervaHostVerifier : sk/ainet/compile/minerva/MinervaHostVerifier { + public fun ()V + public fun (Ljava/lang/String;)V + public synthetic fun (Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun getBackendName ()Ljava/lang/String; + public fun verify (Lsk/ainet/compile/minerva/MinervaHostVerificationRequest;Lsk/ainet/compile/export/GraphExportContext;)Lsk/ainet/compile/minerva/MinervaHostVerification; +} + public final class sk/ainet/compile/minerva/JvmMinervaProjectPackager : sk/ainet/compile/minerva/MinervaProjectPackager { public fun ()V public fun (Ljava/lang/String;)V @@ -200,7 +208,8 @@ public final class sk/ainet/compile/minerva/MinervaExportFacade { public fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompatibilityValidator;Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer;Lsk/ainet/compile/minerva/MinervaNpzModelWriter;)V public fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompatibilityValidator;Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer;Lsk/ainet/compile/minerva/MinervaNpzModelWriter;Lsk/ainet/compile/minerva/MinervaCompilerAdapter;)V public fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompatibilityValidator;Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer;Lsk/ainet/compile/minerva/MinervaNpzModelWriter;Lsk/ainet/compile/minerva/MinervaCompilerAdapter;Lsk/ainet/compile/minerva/MinervaProjectPackager;)V - public synthetic fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompatibilityValidator;Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer;Lsk/ainet/compile/minerva/MinervaNpzModelWriter;Lsk/ainet/compile/minerva/MinervaCompilerAdapter;Lsk/ainet/compile/minerva/MinervaProjectPackager;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompatibilityValidator;Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer;Lsk/ainet/compile/minerva/MinervaNpzModelWriter;Lsk/ainet/compile/minerva/MinervaCompilerAdapter;Lsk/ainet/compile/minerva/MinervaProjectPackager;Lsk/ainet/compile/minerva/MinervaHostVerifier;)V + public synthetic fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaCompatibilityValidator;Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer;Lsk/ainet/compile/minerva/MinervaNpzModelWriter;Lsk/ainet/compile/minerva/MinervaCompilerAdapter;Lsk/ainet/compile/minerva/MinervaProjectPackager;Lsk/ainet/compile/minerva/MinervaHostVerifier;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun exportGraph (Lsk/ainet/lang/graph/ComputeGraph;Lsk/ainet/compile/minerva/MinervaExportOptions;)Lsk/ainet/compile/minerva/MinervaExportResult; public final fun exportModel (Ljava/lang/Object;Lkotlin/jvm/functions/Function1;Lsk/ainet/compile/minerva/MinervaExportOptions;)Lsk/ainet/compile/minerva/MinervaExportResult; public final fun exportModel (Ljava/lang/Object;Lsk/ainet/compile/minerva/MinervaExportOptions;)Lsk/ainet/compile/minerva/MinervaExportResult; @@ -208,6 +217,7 @@ public final class sk/ainet/compile/minerva/MinervaExportFacade { public final fun getCompatibilityValidator ()Lsk/ainet/compile/minerva/MinervaCompatibilityValidator; public final fun getCompilerAdapter ()Lsk/ainet/compile/minerva/MinervaCompilerAdapter; public final fun getGraphCanonicalizer ()Lsk/ainet/compile/minerva/MinervaGraphCanonicalizer; + public final fun getHostVerifier ()Lsk/ainet/compile/minerva/MinervaHostVerifier; public final fun getNpzWriter ()Lsk/ainet/compile/minerva/MinervaNpzModelWriter; public final fun getProjectPackager ()Lsk/ainet/compile/minerva/MinervaProjectPackager; } @@ -243,20 +253,22 @@ public final class sk/ainet/compile/minerva/MinervaExportFailureKind : java/lang public static final field PACKAGING_FAILED Lsk/ainet/compile/minerva/MinervaExportFailureKind; public static final field RECORDING_FAILED Lsk/ainet/compile/minerva/MinervaExportFailureKind; public static final field UNSUPPORTED_MODEL_TYPE Lsk/ainet/compile/minerva/MinervaExportFailureKind; + public static final field VERIFICATION_FAILED Lsk/ainet/compile/minerva/MinervaExportFailureKind; public static fun getEntries ()Lkotlin/enums/EnumEntries; public static fun valueOf (Ljava/lang/String;)Lsk/ainet/compile/minerva/MinervaExportFailureKind; public static fun values ()[Lsk/ainet/compile/minerva/MinervaExportFailureKind; } public final class sk/ainet/compile/minerva/MinervaExportOptions { - public fun (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZLjava/util/Map;)V - public synthetic fun (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZLjava/util/Map;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZFLjava/util/Map;)V + public synthetic fun (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZFLjava/util/Map;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1 ()Ljava/lang/String; public final fun component10 ()Z public final fun component11 ()Z public final fun component12 ()Z public final fun component13 ()Z - public final fun component14 ()Ljava/util/Map; + public final fun component14 ()F + public final fun component15 ()Ljava/util/Map; public final fun component2 ()Ljava/lang/String; public final fun component3 ()Lsk/ainet/compile/minerva/MinervaTarget; public final fun component4 ()Lsk/ainet/compile/minerva/MinervaQuantization; @@ -265,14 +277,15 @@ public final class sk/ainet/compile/minerva/MinervaExportOptions { public final fun component7 ()Ljava/lang/String; public final fun component8 ()Ljava/lang/String; public final fun component9 ()Ljava/lang/String; - public final fun copy (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZLjava/util/Map;)Lsk/ainet/compile/minerva/MinervaExportOptions; - public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaExportOptions;Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZLjava/util/Map;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaExportOptions; + public final fun copy (Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZFLjava/util/Map;)Lsk/ainet/compile/minerva/MinervaExportOptions; + public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaExportOptions;Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZZZZFLjava/util/Map;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaExportOptions; public fun equals (Ljava/lang/Object;)Z public final fun getCalibrationNpz ()Ljava/lang/String; public final fun getCompilerScript ()Ljava/lang/String; public final fun getDumpWeights ()Z public final fun getGenerateFirmwareExample ()Z public final fun getGenerateHostHarness ()Z + public final fun getHostVerificationTolerance ()F public final fun getKeyFile ()Ljava/lang/String; public final fun getMetadata ()Ljava/util/Map; public final fun getOutputDir ()Ljava/lang/String; @@ -288,11 +301,12 @@ public final class sk/ainet/compile/minerva/MinervaExportOptions { } public final class sk/ainet/compile/minerva/MinervaExportResult { - public fun (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;)V - public synthetic fun (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;Lsk/ainet/compile/minerva/MinervaHostVerification;)V + public synthetic fun (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;Lsk/ainet/compile/minerva/MinervaHostVerification;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1 ()Lsk/ainet/compile/minerva/MinervaExportOptions; public final fun component10 ()Lsk/ainet/compile/minerva/MinervaNpzModel; public final fun component11 ()Lsk/ainet/compile/minerva/MinervaCompilerOutput; + public final fun component12 ()Lsk/ainet/compile/minerva/MinervaHostVerification; public final fun component2 ()Lsk/ainet/compile/export/GraphExportStatus; public final fun component3 ()Lsk/ainet/compile/minerva/MinervaExportBundle; public final fun component4 ()Lsk/ainet/compile/export/GraphExportDiagnosticReport; @@ -301,8 +315,8 @@ public final class sk/ainet/compile/minerva/MinervaExportResult { public final fun component7 ()Ljava/util/Map; public final fun component8 ()Lsk/ainet/compile/minerva/MinervaCompatibilityReport; public final fun component9 ()Lsk/ainet/compile/minerva/MinervaIntermediate; - public final fun copy (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;)Lsk/ainet/compile/minerva/MinervaExportResult; - public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaExportResult;Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaExportResult; + public final fun copy (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;Lsk/ainet/compile/minerva/MinervaHostVerification;)Lsk/ainet/compile/minerva/MinervaExportResult; + public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaExportResult;Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/export/GraphExportStatus;Lsk/ainet/compile/minerva/MinervaExportBundle;Lsk/ainet/compile/export/GraphExportDiagnosticReport;Ljava/util/List;Lsk/ainet/compile/minerva/MinervaExportFailure;Ljava/util/Map;Lsk/ainet/compile/minerva/MinervaCompatibilityReport;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;Lsk/ainet/compile/minerva/MinervaHostVerification;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaExportResult; public fun equals (Ljava/lang/Object;)Z public final fun getArtifacts ()Ljava/util/List; public final fun getBundle ()Lsk/ainet/compile/minerva/MinervaExportBundle; @@ -311,6 +325,7 @@ public final class sk/ainet/compile/minerva/MinervaExportResult { public final fun getDiagnostics ()Lsk/ainet/compile/export/GraphExportDiagnosticReport; public final fun getFailed ()Z public final fun getFailure ()Lsk/ainet/compile/minerva/MinervaExportFailure; + public final fun getHostVerification ()Lsk/ainet/compile/minerva/MinervaHostVerification; public final fun getIntermediate ()Lsk/ainet/compile/minerva/MinervaIntermediate; public final fun getMetadata ()Ljava/util/Map; public final fun getNpzModel ()Lsk/ainet/compile/minerva/MinervaNpzModel; @@ -333,6 +348,85 @@ public final class sk/ainet/compile/minerva/MinervaGraphCanonicalizer : sk/ainet public final fun getPatternRegistry ()Lsk/ainet/compile/minerva/MinervaLayerPatternRegistry; } +public final class sk/ainet/compile/minerva/MinervaHostVerification { + public fun (Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;FLjava/lang/Float;Ljava/util/List;Ljava/util/List;Ljava/lang/String;Ljava/util/Map;)V + public synthetic fun (Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;FLjava/lang/Float;Ljava/util/List;Ljava/util/List;Ljava/lang/String;Ljava/util/Map;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun component1 ()Lsk/ainet/compile/minerva/MinervaHostVerificationStatus; + public final fun component10 ()Ljava/util/List; + public final fun component11 ()Ljava/lang/String; + public final fun component12 ()Ljava/util/Map; + public final fun component2 ()Ljava/lang/String; + public final fun component3 ()Ljava/lang/String; + public final fun component4 ()Lsk/ainet/compile/minerva/MinervaHostVerificationStatus; + public final fun component5 ()Lsk/ainet/compile/minerva/MinervaHostVerificationStatus; + public final fun component6 ()Lsk/ainet/compile/minerva/MinervaHostVerificationStatus; + public final fun component7 ()F + public final fun component8 ()Ljava/lang/Float; + public final fun component9 ()Ljava/util/List; + public final fun copy (Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;FLjava/lang/Float;Ljava/util/List;Ljava/util/List;Ljava/lang/String;Ljava/util/Map;)Lsk/ainet/compile/minerva/MinervaHostVerification; + public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaHostVerification;Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;Ljava/lang/String;Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;Lsk/ainet/compile/minerva/MinervaHostVerificationStatus;FLjava/lang/Float;Ljava/util/List;Ljava/util/List;Ljava/lang/String;Ljava/util/Map;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaHostVerification; + public fun equals (Ljava/lang/Object;)Z + public final fun getCode ()Ljava/lang/String; + public final fun getDetails ()Ljava/util/Map; + public final fun getExpectedOutput ()Ljava/util/List; + public final fun getFailed ()Z + public final fun getHostBuildStatus ()Lsk/ainet/compile/minerva/MinervaHostVerificationStatus; + public final fun getHostRunStatus ()Lsk/ainet/compile/minerva/MinervaHostVerificationStatus; + public final fun getMaxAbsoluteError ()Ljava/lang/Float; + public final fun getMessage ()Ljava/lang/String; + public final fun getObservedOutput ()Ljava/util/List; + public final fun getParityStatus ()Lsk/ainet/compile/minerva/MinervaHostVerificationStatus; + public final fun getPassed ()Z + public final fun getRemediation ()Ljava/lang/String; + public final fun getSkipped ()Z + public final fun getStatus ()Lsk/ainet/compile/minerva/MinervaHostVerificationStatus; + public final fun getTolerance ()F + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + +public final class sk/ainet/compile/minerva/MinervaHostVerificationMetadata { + public static final field CMAKE_EXECUTABLE Ljava/lang/String; + public static final field CTEST_EXECUTABLE Ljava/lang/String; + public static final field HOST_OUTPUT_PATH Ljava/lang/String; + public static final field INSTANCE Lsk/ainet/compile/minerva/MinervaHostVerificationMetadata; + public static final field RUN_CMAKE_BUILD Ljava/lang/String; + public static final field RUN_CTEST Ljava/lang/String; +} + +public final class sk/ainet/compile/minerva/MinervaHostVerificationRequest { + public fun (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;Lsk/ainet/compile/minerva/MinervaExportBundle;)V + public final fun component1 ()Lsk/ainet/compile/minerva/MinervaExportOptions; + public final fun component2 ()Lsk/ainet/compile/minerva/MinervaIntermediate; + public final fun component3 ()Lsk/ainet/compile/minerva/MinervaNpzModel; + public final fun component4 ()Lsk/ainet/compile/minerva/MinervaCompilerOutput; + public final fun component5 ()Lsk/ainet/compile/minerva/MinervaExportBundle; + public final fun copy (Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;Lsk/ainet/compile/minerva/MinervaExportBundle;)Lsk/ainet/compile/minerva/MinervaHostVerificationRequest; + public static synthetic fun copy$default (Lsk/ainet/compile/minerva/MinervaHostVerificationRequest;Lsk/ainet/compile/minerva/MinervaExportOptions;Lsk/ainet/compile/minerva/MinervaIntermediate;Lsk/ainet/compile/minerva/MinervaNpzModel;Lsk/ainet/compile/minerva/MinervaCompilerOutput;Lsk/ainet/compile/minerva/MinervaExportBundle;ILjava/lang/Object;)Lsk/ainet/compile/minerva/MinervaHostVerificationRequest; + public fun equals (Ljava/lang/Object;)Z + public final fun getBundle ()Lsk/ainet/compile/minerva/MinervaExportBundle; + public final fun getCompilerOutput ()Lsk/ainet/compile/minerva/MinervaCompilerOutput; + public final fun getIntermediate ()Lsk/ainet/compile/minerva/MinervaIntermediate; + public final fun getNpzModel ()Lsk/ainet/compile/minerva/MinervaNpzModel; + public final fun getOptions ()Lsk/ainet/compile/minerva/MinervaExportOptions; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + +public final class sk/ainet/compile/minerva/MinervaHostVerificationStatus : java/lang/Enum { + public static final field FAILED Lsk/ainet/compile/minerva/MinervaHostVerificationStatus; + public static final field PASSED Lsk/ainet/compile/minerva/MinervaHostVerificationStatus; + public static final field SKIPPED Lsk/ainet/compile/minerva/MinervaHostVerificationStatus; + public static fun getEntries ()Lkotlin/enums/EnumEntries; + public static fun valueOf (Ljava/lang/String;)Lsk/ainet/compile/minerva/MinervaHostVerificationStatus; + public static fun values ()[Lsk/ainet/compile/minerva/MinervaHostVerificationStatus; +} + +public abstract interface class sk/ainet/compile/minerva/MinervaHostVerifier { + public abstract fun getBackendName ()Ljava/lang/String; + public abstract fun verify (Lsk/ainet/compile/minerva/MinervaHostVerificationRequest;Lsk/ainet/compile/export/GraphExportContext;)Lsk/ainet/compile/minerva/MinervaHostVerification; +} + public final class sk/ainet/compile/minerva/MinervaIntermediate { public fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Lsk/ainet/compile/minerva/MinervaTensorRef;Lsk/ainet/compile/minerva/MinervaTensorRef;Ljava/util/List;Ljava/util/List;Ljava/util/Map;)V public synthetic fun (Ljava/lang/String;Lsk/ainet/compile/minerva/MinervaTarget;Lsk/ainet/compile/minerva/MinervaQuantization;Lsk/ainet/compile/minerva/MinervaTensorRef;Lsk/ainet/compile/minerva/MinervaTensorRef;Ljava/util/List;Ljava/util/List;Ljava/util/Map;ILkotlin/jvm/internal/DefaultConstructorMarker;)V @@ -505,6 +599,7 @@ public final class sk/ainet/compile/minerva/MinervaPackagingException : java/lan public final class sk/ainet/compile/minerva/MinervaPlatformExportDefaults { public static final field INSTANCE Lsk/ainet/compile/minerva/MinervaPlatformExportDefaults; public final fun compilerAdapter ()Lsk/ainet/compile/minerva/MinervaCompilerAdapter; + public final fun hostVerifier ()Lsk/ainet/compile/minerva/MinervaHostVerifier; public final fun projectPackager ()Lsk/ainet/compile/minerva/MinervaProjectPackager; } diff --git a/skainet-compile/skainet-compile-minerva/build.gradle.kts b/skainet-compile/skainet-compile-minerva/build.gradle.kts index 72464821..bf41b2fe 100644 --- a/skainet-compile/skainet-compile-minerva/build.gradle.kts +++ b/skainet-compile/skainet-compile-minerva/build.gradle.kts @@ -28,3 +28,22 @@ kotlin { } } } + +val minervaHostVerificationEnabled = providers.gradleProperty("minerva.hostVerification.enabled") + .map { it.toBoolean() } + .orElse(false) +val minervaRuntimeRoot = providers.gradleProperty("minerva.runtimeRoot") +val minervaCompilerScript = providers.gradleProperty("minerva.compilerScript") + +tasks.register("minervaHostVerification") { + group = "verification" + description = "Gated lifecycle hook for external Minerva host verification in CI." + enabled = minervaHostVerificationEnabled.get() && + minervaRuntimeRoot.isPresent && + minervaCompilerScript.isPresent + if (enabled) { + dependsOn("jvmTest") + } + inputs.property("minerva.runtimeRoot", minervaRuntimeRoot.orElse("")) + inputs.property("minerva.compilerScript", minervaCompilerScript.orElse("")) +} diff --git a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaCompilerModels.kt b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaCompilerModels.kt index c58d9086..965e85bc 100644 --- a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaCompilerModels.kt +++ b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaCompilerModels.kt @@ -8,6 +8,7 @@ import sk.ainet.compile.export.GraphExportContext public expect object MinervaPlatformExportDefaults { public fun compilerAdapter(): MinervaCompilerAdapter public fun projectPackager(): MinervaProjectPackager + public fun hostVerifier(): MinervaHostVerifier } /** diff --git a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportFacade.kt b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportFacade.kt index 7a8fee4f..685ea531 100644 --- a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportFacade.kt +++ b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportFacade.kt @@ -14,8 +14,7 @@ import sk.ainet.tape.Execution * Public Minerva export facade. * * This scaffold accepts direct [ComputeGraph] inputs and exposes the same - * traced-forward-pass shape used by other SKaiNET export facades. Host - * verification remains a follow-up stage after compiler packaging. + * traced-forward-pass shape used by other SKaiNET export facades. */ public class MinervaExportFacade @kotlin.jvm.JvmOverloads constructor( public val backendName: String = MinervaExportBackend.backendName, @@ -23,7 +22,8 @@ public class MinervaExportFacade @kotlin.jvm.JvmOverloads constructor( public val graphCanonicalizer: MinervaGraphCanonicalizer = MinervaGraphCanonicalizer(), public val npzWriter: MinervaNpzModelWriter = MinervaNpzModelWriter(), public val compilerAdapter: MinervaCompilerAdapter = MinervaPlatformExportDefaults.compilerAdapter(), - public val projectPackager: MinervaProjectPackager = MinervaPlatformExportDefaults.projectPackager() + public val projectPackager: MinervaProjectPackager = MinervaPlatformExportDefaults.projectPackager(), + public val hostVerifier: MinervaHostVerifier = MinervaPlatformExportDefaults.hostVerifier() ) { /** @@ -155,13 +155,15 @@ public class MinervaExportFacade @kotlin.jvm.JvmOverloads constructor( } if (!options.runHostVerification) { + val hostVerification = skippedHostVerification(options) context.info( - stage = GraphExportStage.PACKAGING, + stage = GraphExportStage.VERIFICATION, code = "minerva.export.completed_without_verification", message = "Minerva export packaged project outputs; host verification was disabled.", details = mapOf( "projectDir" to bundle.outputDir, - "generatedFiles" to bundle.generatedFiles.size.toString() + "generatedFiles" to bundle.generatedFiles.size.toString(), + "verificationStatus" to hostVerification.status.name ) ) return MinervaExportResult( @@ -174,42 +176,46 @@ public class MinervaExportFacade @kotlin.jvm.JvmOverloads constructor( compatibilityReport = compatibilityReport, intermediate = intermediate, npzModel = npzModel, - compilerOutput = compilerOutput + compilerOutput = compilerOutput, + hostVerification = hostVerification ) } - val failure = MinervaExportFailure( - kind = MinervaExportFailureKind.NOT_IMPLEMENTED, - stage = GraphExportStage.VERIFICATION, - code = "minerva.export.not_implemented", - message = "Minerva export packaged the project; host verification and parity checks are implemented in a follow-up issue.", - details = mapOf( - "nextStep" to "Build the packaged host harness and compare Minerva output with SKaiNET output.", - "issue" to "#695", - "layers" to intermediate.layerCount.toString(), - "input" to intermediate.input.id, - "output" to intermediate.output.id, - "npzPath" to npzModel.logicalPath, - "npzBytes" to npzModel.bytes.size.toString(), - "projectDir" to bundle.outputDir, - "generatedFiles" to bundle.generatedFiles.size.toString() - ) - ) - context.error( - stage = failure.stage, - code = failure.code, - message = failure.message, - details = failure.details + val hostVerification = hostVerifier.verify( + MinervaHostVerificationRequest( + options = options, + intermediate = intermediate, + npzModel = npzModel, + compilerOutput = compilerOutput, + bundle = bundle + ), + context ) - return failedResult( + if (hostVerification.failed) { + return verificationFailedResult( + options = options, + context = context, + compatibilityReport = compatibilityReport, + intermediate = intermediate, + npzModel = npzModel, + compilerOutput = compilerOutput, + bundle = bundle, + hostVerification = hostVerification + ) + } + + return MinervaExportResult( options = options, - context = context, - failure = failure, + status = GraphExportStatus.SUCCESS, + bundle = bundle, + diagnostics = context.diagnosticReport(), + artifacts = context.artifacts, + metadata = context.metadata, compatibilityReport = compatibilityReport, intermediate = intermediate, npzModel = npzModel, compilerOutput = compilerOutput, - bundle = bundle + hostVerification = hostVerification ) } @@ -430,6 +436,54 @@ public class MinervaExportFacade @kotlin.jvm.JvmOverloads constructor( ) } + private fun verificationFailedResult( + options: MinervaExportOptions, + context: GraphExportContext, + compatibilityReport: MinervaCompatibilityReport, + intermediate: MinervaIntermediate, + npzModel: MinervaNpzModel, + compilerOutput: MinervaCompilerOutput, + bundle: MinervaExportBundle, + hostVerification: MinervaHostVerification + ): MinervaExportResult { + val details = mutableMapOf( + "code" to hostVerification.code, + "issue" to "#695", + "status" to hostVerification.status.name, + "hostBuildStatus" to hostVerification.hostBuildStatus.name, + "hostRunStatus" to hostVerification.hostRunStatus.name, + "parityStatus" to hostVerification.parityStatus.name, + "tolerance" to hostVerification.tolerance.toString(), + "remediation" to hostVerification.remediation + ) + hostVerification.maxAbsoluteError?.let { details["maxAbsoluteError"] = it.toString() } + details += hostVerification.details + val failure = MinervaExportFailure( + kind = MinervaExportFailureKind.VERIFICATION_FAILED, + stage = GraphExportStage.VERIFICATION, + code = hostVerification.code, + message = hostVerification.message, + details = details + ) + context.error( + stage = failure.stage, + code = failure.code, + message = failure.message, + details = failure.details + ) + return failedResult( + options = options, + context = context, + failure = failure, + compatibilityReport = compatibilityReport, + intermediate = intermediate, + npzModel = npzModel, + compilerOutput = compilerOutput, + bundle = bundle, + hostVerification = hostVerification + ) + } + private fun failedResult( options: MinervaExportOptions, context: GraphExportContext, @@ -438,7 +492,8 @@ public class MinervaExportFacade @kotlin.jvm.JvmOverloads constructor( intermediate: MinervaIntermediate? = null, npzModel: MinervaNpzModel? = null, compilerOutput: MinervaCompilerOutput? = null, - bundle: MinervaExportBundle? = null + bundle: MinervaExportBundle? = null, + hostVerification: MinervaHostVerification? = null ): MinervaExportResult { return MinervaExportResult( options = options, @@ -451,7 +506,18 @@ public class MinervaExportFacade @kotlin.jvm.JvmOverloads constructor( compatibilityReport = compatibilityReport, intermediate = intermediate, npzModel = npzModel, - compilerOutput = compilerOutput + compilerOutput = compilerOutput, + hostVerification = hostVerification + ) + } + + private fun skippedHostVerification(options: MinervaExportOptions): MinervaHostVerification { + return MinervaHostVerification( + status = MinervaHostVerificationStatus.SKIPPED, + code = "minerva.host_verification.disabled", + message = "Minerva host verification was disabled by export options.", + tolerance = options.hostVerificationTolerance, + remediation = "Set runHostVerification=true before using generated outputs as verified artifacts." ) } diff --git a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportModels.kt b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportModels.kt index 41047990..ecc457c5 100644 --- a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportModels.kt +++ b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaExportModels.kt @@ -52,6 +52,7 @@ public data class MinervaExportOptions( public val generateHostHarness: Boolean = true, public val generateFirmwareExample: Boolean = true, public val runHostVerification: Boolean = true, + public val hostVerificationTolerance: Float = 1.0e-3f, public val metadata: Map = emptyMap() ) { init { @@ -65,6 +66,9 @@ public data class MinervaExportOptions( requireOptionalPath("compilerScript", compilerScript) requireOptionalPath("keyFile", keyFile) requireOptionalPath("calibrationNpz", calibrationNpz) + require(hostVerificationTolerance.isFinite() && hostVerificationTolerance > 0.0f) { + "hostVerificationTolerance must be positive and finite" + } require(metadata.keys.all { it.isNotBlank() }) { "metadata keys cannot be blank" } } @@ -77,6 +81,7 @@ public data class MinervaExportOptions( "generateHostHarness" to generateHostHarness.toString(), "generateFirmwareExample" to generateFirmwareExample.toString(), "runHostVerification" to runHostVerification.toString(), + "hostVerificationTolerance" to hostVerificationTolerance.toString(), "dumpWeights" to dumpWeights.toString() ) } @@ -99,6 +104,7 @@ public enum class MinervaExportFailureKind { COMPILER_PREREQUISITE_FAILED, COMPILER_FAILED, PACKAGING_FAILED, + VERIFICATION_FAILED, NOT_IMPLEMENTED } @@ -213,7 +219,8 @@ public data class MinervaExportResult( public val compatibilityReport: MinervaCompatibilityReport? = null, public val intermediate: MinervaIntermediate? = null, public val npzModel: MinervaNpzModel? = null, - public val compilerOutput: MinervaCompilerOutput? = null + public val compilerOutput: MinervaCompilerOutput? = null, + public val hostVerification: MinervaHostVerification? = null ) { init { require(status != GraphExportStatus.SUCCESS || bundle != null) { diff --git a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaHostVerificationModels.kt b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaHostVerificationModels.kt new file mode 100644 index 00000000..09c38318 --- /dev/null +++ b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/MinervaHostVerificationModels.kt @@ -0,0 +1,158 @@ +package sk.ainet.compile.minerva + +import kotlin.math.abs +import kotlin.math.exp +import kotlin.math.tanh +import sk.ainet.compile.export.GraphExportContext + +/** + * Metadata keys understood by the JVM host verifier. + */ +public object MinervaHostVerificationMetadata { + public const val RUN_CMAKE_BUILD: String = "minerva.hostVerification.runCmakeBuild" + public const val RUN_CTEST: String = "minerva.hostVerification.runCTest" + public const val CMAKE_EXECUTABLE: String = "minerva.hostVerification.cmakeExecutable" + public const val CTEST_EXECUTABLE: String = "minerva.hostVerification.ctestExecutable" + public const val HOST_OUTPUT_PATH: String = "minerva.hostVerification.hostOutputPath" +} + +/** + * Status for the overall host verification step and individual substeps. + */ +public enum class MinervaHostVerificationStatus { + PASSED, + FAILED, + SKIPPED +} + +/** + * Input passed to a Minerva host verifier after project packaging. + */ +public data class MinervaHostVerificationRequest( + public val options: MinervaExportOptions, + public val intermediate: MinervaIntermediate, + public val npzModel: MinervaNpzModel, + public val compilerOutput: MinervaCompilerOutput, + public val bundle: MinervaExportBundle +) { + init { + require(options.projectName == intermediate.projectName) { + "verification request options and intermediate project names must match" + } + require(options.projectName == bundle.projectName) { + "verification request options and bundle project names must match" + } + } +} + +/** + * Host verification result exposed on [MinervaExportResult]. + */ +public data class MinervaHostVerification( + public val status: MinervaHostVerificationStatus, + public val code: String, + public val message: String, + public val hostBuildStatus: MinervaHostVerificationStatus = MinervaHostVerificationStatus.SKIPPED, + public val hostRunStatus: MinervaHostVerificationStatus = MinervaHostVerificationStatus.SKIPPED, + public val parityStatus: MinervaHostVerificationStatus = MinervaHostVerificationStatus.SKIPPED, + public val tolerance: Float = 1.0e-3f, + public val maxAbsoluteError: Float? = null, + public val expectedOutput: List = emptyList(), + public val observedOutput: List = emptyList(), + public val remediation: String = "", + public val details: Map = emptyMap() +) { + init { + require(code.isNotBlank()) { "verification code cannot be blank" } + require(message.isNotBlank()) { "verification message cannot be blank" } + require(tolerance.isFinite() && tolerance > 0.0f) { + "verification tolerance must be positive and finite" + } + require(maxAbsoluteError == null || maxAbsoluteError.isFinite()) { + "maxAbsoluteError must be finite when provided" + } + require(expectedOutput.all { it.isFinite() }) { "expectedOutput values must be finite" } + require(observedOutput.all { it.isFinite() }) { "observedOutput values must be finite" } + } + + public val passed: Boolean + get() = status == MinervaHostVerificationStatus.PASSED + + public val failed: Boolean + get() = status == MinervaHostVerificationStatus.FAILED + + public val skipped: Boolean + get() = status == MinervaHostVerificationStatus.SKIPPED +} + +/** + * Verifies a packaged Minerva project on the host. + */ +public interface MinervaHostVerifier { + public val backendName: String + + public fun verify( + request: MinervaHostVerificationRequest, + context: GraphExportContext + ): MinervaHostVerification +} + +internal object MinervaReferenceEvaluator { + fun referenceInput(input: MinervaTensorRef): List { + val count = input.elementCount + return List(count) { index -> (index + 1).toFloat() / count.toFloat() } + } + + fun evaluate(intermediate: MinervaIntermediate, input: List = referenceInput(intermediate.input)): List { + require(input.size == intermediate.input.elementCount) { + "reference input size must match Minerva input tensor size" + } + return intermediate.layers.fold(input) { values, layer -> + evaluateLayer(values, layer) + } + } + + fun maxAbsoluteError(expected: List, observed: List): Float { + require(expected.size == observed.size) { "expected and observed outputs must have the same size" } + return expected.zip(observed).maxOfOrNull { (left, right) -> abs(left - right) } ?: 0.0f + } + + private fun evaluateLayer(input: List, layer: MinervaLayer): List { + val weightShape = layer.weights.shape + require(weightShape.size == 2) { + "Minerva dense layer weights must be rank-2" + } + val inputWidth = weightShape[0] + val outputWidth = weightShape[1] + require(input.size % inputWidth == 0) { + "layer input size must be divisible by the weight input width" + } + val weights = requireValues(layer.weights, layer.id) + val bias = layer.bias?.let { requireValues(it, layer.id) } + val batchSize = input.size / inputWidth + val output = MutableList(batchSize * outputWidth) { 0.0f } + for (batch in 0 until batchSize) { + for (out in 0 until outputWidth) { + var sum = bias?.get(out % bias.size) ?: 0.0f + for (inside in 0 until inputWidth) { + sum += input[(batch * inputWidth) + inside] * weights[(inside * outputWidth) + out] + } + output[(batch * outputWidth) + out] = activate(sum, layer.activation) + } + } + return output + } + + private fun requireValues(tensor: MinervaTensorRef, layerId: String): List { + return tensor.values ?: error("Tensor '${tensor.id}' on layer '$layerId' does not have numeric values.") + } + + private fun activate(value: Float, activation: MinervaActivation?): Float { + return when (activation) { + null -> value + MinervaActivation.RELU -> if (value > 0.0f) value else 0.0f + MinervaActivation.SIGMOID -> (1.0 / (1.0 + exp(-value.toDouble()))).toFloat() + MinervaActivation.TANH -> tanh(value.toDouble()).toFloat() + } + } +} diff --git a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/package.kt b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/package.kt index b8656290..f13d05da 100644 --- a/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/package.kt +++ b/skainet-compile/skainet-compile-minerva/src/commonMain/kotlin/sk/ainet/compile/minerva/package.kt @@ -4,8 +4,9 @@ package sk.ainet.compile.minerva * Minerva graph export support for secure MCU inference. * * The phase-one implementation is JVM-first and targets static sequential MLP - * graphs with Q8 libminerva compilation. Host verification is intentionally a - * separate stage so compiler packaging can be tested without MCU hardware. + * graphs with Q8 libminerva compilation. Host verification runs after compiler + * packaging and can stay lightweight by default or opt into external CMake + * checks when a libminerva environment is configured. */ public object MinervaExportBackend { public const val backendName: String = "minerva" diff --git a/skainet-compile/skainet-compile-minerva/src/commonTest/kotlin/sk/ainet/compile/minerva/MinervaExportFacadeTest.kt b/skainet-compile/skainet-compile-minerva/src/commonTest/kotlin/sk/ainet/compile/minerva/MinervaExportFacadeTest.kt index 22b0431c..f1877177 100644 --- a/skainet-compile/skainet-compile-minerva/src/commonTest/kotlin/sk/ainet/compile/minerva/MinervaExportFacadeTest.kt +++ b/skainet-compile/skainet-compile-minerva/src/commonTest/kotlin/sk/ainet/compile/minerva/MinervaExportFacadeTest.kt @@ -24,9 +24,11 @@ class MinervaExportFacadeTest { assertEquals(MinervaExportBackend.backendName, facade.npzWriter.backendName) assertEquals(MinervaExportBackend.backendName, facade.compilerAdapter.backendName) assertEquals(MinervaExportBackend.backendName, facade.projectPackager.backendName) + assertEquals(MinervaExportBackend.backendName, facade.hostVerifier.backendName) assertEquals(MinervaTarget.ATMEGA328P, options.target) assertEquals(MinervaQuantization.Q8, options.quantization) assertEquals("python3", options.pythonExecutable) + assertEquals(1.0e-3f, options.hostVerificationTolerance) assertEquals("jvm-sequential-mlp-q8", options.toMetadata()["phaseOneScope"]) } @@ -46,6 +48,11 @@ class MinervaExportFacadeTest { MinervaExportOptions(outputDir = "build/minerva", projectName = "TinyMlp", pythonExecutable = "") } assertTrue(pythonError.message?.contains("pythonExecutable cannot be blank") == true) + + val toleranceError = assertFailsWith { + MinervaExportOptions(outputDir = "build/minerva", projectName = "TinyMlp", hostVerificationTolerance = 0.0f) + } + assertTrue(toleranceError.message?.contains("hostVerificationTolerance") == true) } @Test @@ -173,31 +180,50 @@ class MinervaExportFacadeTest { assertTrue(bundle.generatedFiles.contains("generated/weights.c")) assertTrue(bundle.generatedFiles.contains("include/secrets.example.h")) assertEquals("build/minerva/PackagedMlp/generated/weights.c", result.compilerOutput?.weightsCPath) + assertEquals(MinervaHostVerificationStatus.SKIPPED, result.hostVerification?.status) assertTrue(result.diagnostics.infos.any { it.code == "minerva.compiler.completed" }) assertTrue(result.diagnostics.infos.any { it.code == "minerva.packaging.completed" }) assertTrue(result.diagnostics.infos.any { it.code == "minerva.export.completed_without_verification" }) } @Test - fun exportGraphStopsAtVerificationPlaceholderAfterPackagingByDefault() { + fun exportGraphRunsHostVerificationBeforeSuccessByDefault() { val result = packagingFacade().exportGraph( graph = validMinervaMlpGraph(), - options = minervaTestOptions(projectName = "NeedsVerification") + options = minervaTestOptions(projectName = "VerifiedMlp") + ) + + assertEquals(GraphExportStatus.SUCCESS, result.status) + assertEquals("build/minerva/VerifiedMlp", result.bundle?.outputDir) + assertEquals(MinervaHostVerificationStatus.PASSED, result.hostVerification?.status) + assertEquals(MinervaHostVerificationStatus.PASSED, result.hostVerification?.parityStatus) + assertNotNull(result.compilerOutput) + assertNotNull(result.npzModel) + assertTrue(result.diagnostics.infos.any { it.code == "minerva.host_verification.passed" }) + } + + @Test + fun exportGraphReturnsTypedVerificationFailure() { + val result = packagingFacade(hostVerifier = FakeHostVerifier(MinervaHostVerificationStatus.FAILED)).exportGraph( + graph = validMinervaMlpGraph(), + options = minervaTestOptions(projectName = "BadVerification") ) assertEquals(GraphExportStatus.FAILED, result.status) - assertEquals(MinervaExportFailureKind.NOT_IMPLEMENTED, result.failure?.kind) + assertEquals(MinervaExportFailureKind.VERIFICATION_FAILED, result.failure?.kind) assertEquals(GraphExportStage.VERIFICATION, result.failure?.stage) assertEquals("#695", result.failure?.details?.get("issue")) - assertEquals("build/minerva/NeedsVerification", result.bundle?.outputDir) - assertNotNull(result.compilerOutput) - assertNotNull(result.npzModel) + assertEquals(MinervaHostVerificationStatus.FAILED, result.hostVerification?.status) + assertEquals("minerva.host_verification.fake_failed", result.failure?.code) } - private fun packagingFacade(): MinervaExportFacade { + private fun packagingFacade( + hostVerifier: MinervaHostVerifier = FakeHostVerifier(MinervaHostVerificationStatus.PASSED) + ): MinervaExportFacade { return MinervaExportFacade( compilerAdapter = FakeCompilerAdapter(), - projectPackager = FakeProjectPackager() + projectPackager = FakeProjectPackager(), + hostVerifier = hostVerifier ) } @@ -259,4 +285,41 @@ class MinervaExportFacadeTest { ) } } + + private class FakeHostVerifier( + private val status: MinervaHostVerificationStatus + ) : MinervaHostVerifier { + override val backendName: String = MinervaExportBackend.backendName + + override fun verify( + request: MinervaHostVerificationRequest, + context: GraphExportContext + ): MinervaHostVerification { + val code = if (status == MinervaHostVerificationStatus.FAILED) { + "minerva.host_verification.fake_failed" + } else { + "minerva.host_verification.passed" + } + context.info( + stage = GraphExportStage.VERIFICATION, + code = code, + message = "Fake Minerva host verification completed.", + details = mapOf("projectDir" to request.bundle.outputDir) + ) + return MinervaHostVerification( + status = status, + code = code, + message = "Fake Minerva host verification completed.", + hostBuildStatus = MinervaHostVerificationStatus.PASSED, + hostRunStatus = status, + parityStatus = status, + tolerance = request.options.hostVerificationTolerance, + expectedOutput = listOf(1.0f), + observedOutput = if (status == MinervaHostVerificationStatus.PASSED) listOf(1.0f) else listOf(2.0f), + maxAbsoluteError = if (status == MinervaHostVerificationStatus.PASSED) 0.0f else 1.0f, + remediation = "Use a real host verifier.", + details = mapOf("fake" to "true") + ) + } + } } diff --git a/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackager.kt b/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackager.kt index 5aa3c4a4..3ab4100d 100644 --- a/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackager.kt +++ b/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackager.kt @@ -16,6 +16,8 @@ public actual object MinervaPlatformExportDefaults { public actual fun compilerAdapter(): MinervaCompilerAdapter = PythonMinervaCompilerAdapter() public actual fun projectPackager(): MinervaProjectPackager = JvmMinervaProjectPackager() + + public actual fun hostVerifier(): MinervaHostVerifier = JvmMinervaHostVerifier() } /** @@ -520,6 +522,11 @@ public class JvmMinervaProjectPackager @kotlin.jvm.JvmOverloads constructor( |add_executable(${options.projectName}_host main.c ../generated/weights.c) |target_include_directories(${options.projectName}_host PRIVATE ../include) | + |include(CTest) + |if(BUILD_TESTING) + | add_test(NAME minerva_host_smoke COMMAND ${options.projectName}_host) + |endif() + | """.trimMargin() } @@ -593,6 +600,451 @@ public class JvmMinervaProjectPackager @kotlin.jvm.JvmOverloads constructor( } } +/** + * JVM host verifier for packaged Minerva projects. + */ +public class JvmMinervaHostVerifier @kotlin.jvm.JvmOverloads constructor( + override val backendName: String = MinervaExportBackend.backendName +) : MinervaHostVerifier { + + override fun verify( + request: MinervaHostVerificationRequest, + context: GraphExportContext + ): MinervaHostVerification { + val options = request.options + val projectDir = Paths.get(request.bundle.outputDir).normalize() + val tolerance = options.hostVerificationTolerance + context.info( + stage = GraphExportStage.VERIFICATION, + code = "minerva.host_verification.started", + message = "Verifying packaged Minerva host project.", + details = mapOf( + "projectDir" to projectDir.toString(), + "tolerance" to tolerance.toString() + ) + ) + + try { + structuralFailure(request, projectDir)?.let { return it } + } catch (exception: IOException) { + return failed( + code = "minerva.host_verification.package_read_failed", + message = "Unable to read packaged Minerva project files during host verification.", + tolerance = tolerance, + remediation = "Ensure the packaged project directory is readable and was not modified during verification.", + details = mapOf("reason" to (exception.message ?: exception.toString())) + ) + } + val expectedOutput = try { + MinervaReferenceEvaluator.evaluate(request.intermediate) + } catch (exception: RuntimeException) { + return failed( + code = "minerva.host_verification.reference_unavailable", + message = "Unable to compute SKaiNET reference output for Minerva parity verification.", + tolerance = tolerance, + remediation = "Ensure lowered Minerva weights and biases contain numeric initializer values.", + details = mapOf("reason" to (exception.message ?: exception.toString())) + ) + } + + var hostBuildStatus = MinervaHostVerificationStatus.SKIPPED + var hostRunStatus = MinervaHostVerificationStatus.SKIPPED + if (metadataFlag(options, MinervaHostVerificationMetadata.RUN_CMAKE_BUILD)) { + val buildFailure = runCmakeBuild(projectDir, options, context, tolerance, expectedOutput) + if (buildFailure != null) return buildFailure + hostBuildStatus = MinervaHostVerificationStatus.PASSED + if (metadataFlag(options, MinervaHostVerificationMetadata.RUN_CTEST)) { + val testFailure = runCTest(projectDir, options, context, tolerance, expectedOutput) + if (testFailure != null) return testFailure + hostRunStatus = MinervaHostVerificationStatus.PASSED + } + } + + val hostOutputPath = options.metadata[MinervaHostVerificationMetadata.HOST_OUTPUT_PATH] + val observedOutput = if (hostOutputPath != null) { + val outputPath = resolveProjectPath(projectDir, hostOutputPath) + try { + readFloatOutput(outputPath) + } catch (exception: IllegalArgumentException) { + return failed( + code = "minerva.host_verification.host_output_invalid", + message = "Configured Minerva host output could not be parsed.", + tolerance = tolerance, + expectedOutput = expectedOutput, + remediation = "Write host output as whitespace- or comma-separated finite float values.", + details = mapOf( + "hostOutputPath" to outputPath.toString(), + "reason" to (exception.message ?: exception.toString()) + ) + ) + } + } else { + emptyList() + } + + val parityStatus: MinervaHostVerificationStatus + val maxAbsoluteError: Float? + if (observedOutput.isNotEmpty()) { + if (expectedOutput.size != observedOutput.size) { + return failed( + code = "minerva.host_verification.parity_shape_mismatch", + message = "Minerva host output length does not match the SKaiNET reference output length.", + tolerance = tolerance, + expectedOutput = expectedOutput, + observedOutput = observedOutput, + remediation = "Regenerate the host output from the same packaged project and reference input.", + details = mapOf( + "expected" to expectedOutput.size.toString(), + "observed" to observedOutput.size.toString() + ) + ) + } + maxAbsoluteError = MinervaReferenceEvaluator.maxAbsoluteError(expectedOutput, observedOutput) + if (maxAbsoluteError > tolerance) { + return failed( + code = "minerva.host_verification.parity_failed", + message = "Minerva host output differs from the SKaiNET reference output.", + tolerance = tolerance, + maxAbsoluteError = maxAbsoluteError, + expectedOutput = expectedOutput, + observedOutput = observedOutput, + remediation = "Inspect compiler inputs, generated weights, quantization settings, and host runtime configuration.", + details = mapOf("maxAbsoluteError" to maxAbsoluteError.toString()) + ) + } + parityStatus = MinervaHostVerificationStatus.PASSED + hostRunStatus = MinervaHostVerificationStatus.PASSED + } else { + parityStatus = MinervaHostVerificationStatus.SKIPPED + maxAbsoluteError = null + } + + val verification = MinervaHostVerification( + status = MinervaHostVerificationStatus.PASSED, + code = "minerva.host_verification.passed", + message = "Minerva host verification completed.", + hostBuildStatus = hostBuildStatus, + hostRunStatus = hostRunStatus, + parityStatus = parityStatus, + tolerance = tolerance, + maxAbsoluteError = maxAbsoluteError, + expectedOutput = expectedOutput, + observedOutput = observedOutput, + details = mapOf( + "projectDir" to projectDir.toString(), + "referenceOutputValues" to expectedOutput.size.toString() + ) + ) + context.info( + stage = GraphExportStage.VERIFICATION, + code = verification.code, + message = verification.message, + details = mapOf( + "hostBuildStatus" to verification.hostBuildStatus.name, + "hostRunStatus" to verification.hostRunStatus.name, + "parityStatus" to verification.parityStatus.name, + "maxAbsoluteError" to (verification.maxAbsoluteError?.toString() ?: "n/a") + ) + ) + return verification + } + + private fun structuralFailure( + request: MinervaHostVerificationRequest, + projectDir: Path + ): MinervaHostVerification? { + val tolerance = request.options.hostVerificationTolerance + val requiredFiles = buildList { + add(projectDir.resolve("manifest.json")) + add(projectDir.resolve("generated").resolve(request.npzModel.logicalPath.substringAfterLast('/'))) + add(projectDir.resolve("generated/weights.c")) + add(projectDir.resolve("include/weights.h")) + add(projectDir.resolve("include/secrets.example.h")) + if (request.options.generateHostHarness) { + add(projectDir.resolve("host/CMakeLists.txt")) + add(projectDir.resolve("host/main.c")) + } + if (request.options.generateFirmwareExample) { + add(projectDir.resolve("firmware/main.c")) + } + } + requiredFiles.firstOrNull { !Files.isRegularFile(it) }?.let { missing -> + return failed( + code = "minerva.host_verification.required_file_missing", + message = "Packaged Minerva project is missing a required generated file.", + tolerance = tolerance, + remediation = "Re-run Minerva packaging and verify the generated project layout.", + details = mapOf("missingPath" to missing.toString()) + ) + } + listOf(projectDir.resolve("generated/weights.c"), projectDir.resolve("include/weights.h")) + .firstOrNull { Files.size(it) == 0L } + ?.let { empty -> + return failed( + code = "minerva.host_verification.empty_generated_file", + message = "Packaged Minerva project contains an empty compiler output.", + tolerance = tolerance, + remediation = "Inspect the libminerva compiler invocation and generated weights.", + details = mapOf("emptyPath" to empty.toString()) + ) + } + val packagedModel = projectDir.resolve("generated").resolve(request.npzModel.logicalPath.substringAfterLast('/')) + if (!Files.readAllBytes(packagedModel).contentEquals(request.npzModel.bytes)) { + return failed( + code = "minerva.host_verification.model_tampered", + message = "Packaged Minerva model bytes differ from the NPZ compiler input produced by SKaiNET.", + tolerance = tolerance, + remediation = "Recreate the package from the original export result before running host verification.", + details = mapOf("modelPath" to packagedModel.toString()) + ) + } + secretLeakFailure(request, projectDir)?.let { return it } + return null + } + + private fun secretLeakFailure( + request: MinervaHostVerificationRequest, + projectDir: Path + ): MinervaHostVerification? { + val keyPath = request.options.keyFile?.let(Paths::get) ?: return null + if (!Files.isRegularFile(keyPath) || Files.size(keyPath) == 0L || Files.size(keyPath) > 4096L) return null + val keyMaterial = Files.readString(keyPath).trim() + if (keyMaterial.isBlank()) return null + val secretsExample = projectDir.resolve("include/secrets.example.h") + val template = Files.readString(secretsExample) + if (!template.contains(keyMaterial)) return null + return failed( + code = "minerva.host_verification.secret_leak", + message = "Generated Minerva secret template contains real key material.", + tolerance = request.options.hostVerificationTolerance, + remediation = "Remove real secrets from generated artifacts and regenerate secrets.example.h with placeholders.", + details = mapOf("secretsExample" to secretsExample.toString()) + ) + } + + private fun runCmakeBuild( + projectDir: Path, + options: MinervaExportOptions, + context: GraphExportContext, + tolerance: Float, + expectedOutput: List + ): MinervaHostVerification? { + val hostDir = projectDir.resolve("host") + val buildDir = hostDir.resolve("build") + Files.createDirectories(buildDir) + val cmake = options.metadata[MinervaHostVerificationMetadata.CMAKE_EXECUTABLE] ?: "cmake" + val configure = runExternalCommand( + command = listOf(cmake, "-S", hostDir.toString(), "-B", buildDir.toString(), "-DBUILD_TESTING=ON"), + workingDir = projectDir, + logPath = buildDir.resolve("cmake-configure.log"), + context = context + ) + if (configure.exitCode != 0) { + return failedExternalStep( + code = "minerva.host_verification.cmake_configure_failed", + message = "CMake configuration failed for the packaged Minerva host project.", + tolerance = tolerance, + expectedOutput = expectedOutput, + result = configure, + logPath = buildDir.resolve("cmake-configure.log") + ) + } + val build = runExternalCommand( + command = listOf(cmake, "--build", buildDir.toString()), + workingDir = projectDir, + logPath = buildDir.resolve("cmake-build.log"), + context = context + ) + if (build.exitCode != 0) { + return failedExternalStep( + code = "minerva.host_verification.cmake_build_failed", + message = "CMake build failed for the packaged Minerva host project.", + tolerance = tolerance, + expectedOutput = expectedOutput, + result = build, + logPath = buildDir.resolve("cmake-build.log") + ) + } + return null + } + + private fun runCTest( + projectDir: Path, + options: MinervaExportOptions, + context: GraphExportContext, + tolerance: Float, + expectedOutput: List + ): MinervaHostVerification? { + val buildDir = projectDir.resolve("host/build") + val ctest = options.metadata[MinervaHostVerificationMetadata.CTEST_EXECUTABLE] ?: "ctest" + val result = runExternalCommand( + command = listOf(ctest, "--test-dir", buildDir.toString(), "--output-on-failure"), + workingDir = projectDir, + logPath = buildDir.resolve("ctest.log"), + context = context + ) + if (result.exitCode != 0) { + return failedExternalStep( + code = "minerva.host_verification.ctest_failed", + message = "CTest failed for the packaged Minerva host project.", + tolerance = tolerance, + expectedOutput = expectedOutput, + hostBuildStatus = MinervaHostVerificationStatus.PASSED, + hostRunStatus = MinervaHostVerificationStatus.FAILED, + result = result, + logPath = buildDir.resolve("ctest.log") + ) + } + return null + } + + private fun failedExternalStep( + code: String, + message: String, + tolerance: Float, + expectedOutput: List, + hostBuildStatus: MinervaHostVerificationStatus = MinervaHostVerificationStatus.FAILED, + hostRunStatus: MinervaHostVerificationStatus = MinervaHostVerificationStatus.SKIPPED, + result: ProcessResult, + logPath: Path + ): MinervaHostVerification { + return failed( + code = code, + message = message, + tolerance = tolerance, + expectedOutput = expectedOutput, + hostBuildStatus = hostBuildStatus, + hostRunStatus = hostRunStatus, + parityStatus = MinervaHostVerificationStatus.SKIPPED, + remediation = "Inspect the host verification log and ensure CMake, compiler, and libminerva paths are configured.", + details = mapOf( + "exitCode" to result.exitCode.toString(), + "logPath" to logPath.toString(), + "stdout" to excerpt(result.stdout), + "stderr" to excerpt(result.stderr) + ) + ) + } + + private fun runExternalCommand( + command: List, + workingDir: Path, + logPath: Path, + context: GraphExportContext + ): ProcessResult { + return try { + Files.createDirectories(logPath.parent) + val result = runProcess(command, workingDir) + Files.writeString( + logPath, + buildString { + appendLine("command: ${command.joinToString(" ")}") + appendLine("exitCode: ${result.exitCode}") + appendLine() + appendLine("stdout:") + appendLine(result.stdout) + appendLine("stderr:") + appendLine(result.stderr) + } + ) + context.addArtifact( + GraphExportArtifact( + path = logPath.toString(), + role = GraphExportArtifactRole.LOG, + description = "Minerva host verification log" + ) + ) + result + } catch (exception: IOException) { + ProcessResult( + exitCode = -1, + stdout = "", + stderr = exception.message ?: exception.toString() + ) + } catch (exception: InterruptedException) { + Thread.currentThread().interrupt() + ProcessResult( + exitCode = -1, + stdout = "", + stderr = exception.message ?: exception.toString() + ) + } + } + + private fun runProcess(command: List, workingDir: Path): ProcessResult { + val process = ProcessBuilder(command) + .directory(workingDir.toFile()) + .start() + val stdout = StreamCollector(process.inputStream).also { it.start() } + val stderr = StreamCollector(process.errorStream).also { it.start() } + val exitCode = process.waitFor() + stdout.join() + stderr.join() + return ProcessResult( + exitCode = exitCode, + stdout = stdout.text(), + stderr = stderr.text() + ) + } + + private fun readFloatOutput(path: Path): List { + if (!Files.isRegularFile(path)) { + throw IllegalArgumentException("host output file does not exist: $path") + } + val tokens = Files.readString(path) + .trim() + .split(Regex("[,\\s]+")) + .filter { it.isNotBlank() } + require(tokens.isNotEmpty()) { "host output file does not contain numeric values" } + return tokens.map { token -> + token.toFloatOrNull()?.takeIf { it.isFinite() } + ?: throw IllegalArgumentException("host output token is not a finite float: $token") + } + } + + private fun resolveProjectPath(projectDir: Path, value: String): Path { + val path = Paths.get(value) + return if (path.isAbsolute) path.normalize() else projectDir.resolve(path).normalize() + } + + private fun metadataFlag(options: MinervaExportOptions, key: String): Boolean { + return options.metadata[key]?.equals("true", ignoreCase = true) == true + } + + private fun failed( + code: String, + message: String, + tolerance: Float, + hostBuildStatus: MinervaHostVerificationStatus = MinervaHostVerificationStatus.SKIPPED, + hostRunStatus: MinervaHostVerificationStatus = MinervaHostVerificationStatus.SKIPPED, + parityStatus: MinervaHostVerificationStatus = MinervaHostVerificationStatus.FAILED, + maxAbsoluteError: Float? = null, + expectedOutput: List = emptyList(), + observedOutput: List = emptyList(), + remediation: String, + details: Map = emptyMap() + ): MinervaHostVerification { + return MinervaHostVerification( + status = MinervaHostVerificationStatus.FAILED, + code = code, + message = message, + hostBuildStatus = hostBuildStatus, + hostRunStatus = hostRunStatus, + parityStatus = parityStatus, + tolerance = tolerance, + maxAbsoluteError = maxAbsoluteError, + expectedOutput = expectedOutput, + observedOutput = observedOutput, + remediation = remediation, + details = details + ) + } + + private fun excerpt(value: String, limit: Int = 2000): String { + return if (value.length <= limit) value else value.take(limit) + "..." + } +} + private data class ProcessResult( val exitCode: Int, val stdout: String, diff --git a/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackagerTest.kt b/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackagerTest.kt index 0c29ab72..99164fb4 100644 --- a/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackagerTest.kt +++ b/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/MinervaJvmCompilerAndPackagerTest.kt @@ -122,6 +122,7 @@ class MinervaJvmCompilerAndPackagerTest { assertTrue(Files.isRegularFile(projectDir.resolve("include/weights.h"))) assertTrue(Files.isRegularFile(projectDir.resolve("host/main.c"))) assertTrue(Files.isRegularFile(projectDir.resolve("firmware/main.c"))) + assertTrue(Files.readString(projectDir.resolve("host/CMakeLists.txt")).contains("add_test")) val secretsExample = Files.readString(projectDir.resolve("include/secrets.example.h")) assertTrue(secretsExample.contains("replace-with-device-key")) assertFalse(secretsExample.contains("REAL_SECRET_KEY_MATERIAL")) @@ -135,6 +136,72 @@ class MinervaJvmCompilerAndPackagerTest { assertTrue(context.diagnostics.any { it.code == "minerva.packaging.completed" }) } + @Test + fun hostVerifierPassesStructuralAndReferenceChecksWithoutExternalRuntime() { + val fixture = packagedProject(tempDir("host-verify-lightweight"), "LightweightVerify") + + val verification = JvmMinervaHostVerifier().verify(fixture.request, fixture.context) + + assertEquals(MinervaHostVerificationStatus.PASSED, verification.status) + assertEquals(MinervaHostVerificationStatus.SKIPPED, verification.hostBuildStatus) + assertEquals(MinervaHostVerificationStatus.SKIPPED, verification.hostRunStatus) + assertEquals(MinervaHostVerificationStatus.SKIPPED, verification.parityStatus) + assertTrue(verification.expectedOutput.isNotEmpty()) + assertTrue(fixture.context.diagnostics.any { it.code == "minerva.host_verification.passed" }) + } + + @Test + fun hostVerifierComparesConfiguredHostOutput() { + val fixture = packagedProject(tempDir("host-verify-output"), "OutputVerify") + val baseline = JvmMinervaHostVerifier().verify(fixture.request, fixture.context) + val hostOutputPath = fixture.projectDir.resolve("host-output.txt") + Files.writeString(hostOutputPath, baseline.expectedOutput.joinToString(separator = "\n")) + val request = fixture.request.copy( + options = fixture.request.options.copy( + metadata = fixture.request.options.metadata + + (MinervaHostVerificationMetadata.HOST_OUTPUT_PATH to "host-output.txt") + ) + ) + + val verification = JvmMinervaHostVerifier().verify(request, fixture.context) + + assertEquals(MinervaHostVerificationStatus.PASSED, verification.status) + assertEquals(MinervaHostVerificationStatus.PASSED, verification.hostRunStatus) + assertEquals(MinervaHostVerificationStatus.PASSED, verification.parityStatus) + assertEquals(0.0f, verification.maxAbsoluteError) + assertEquals(baseline.expectedOutput, verification.observedOutput) + } + + @Test + fun hostVerifierFailsWhenHostOutputExceedsTolerance() { + val fixture = packagedProject(tempDir("host-verify-mismatch"), "MismatchVerify") + Files.writeString(fixture.projectDir.resolve("host-output.txt"), "999 999 999") + val request = fixture.request.copy( + options = fixture.request.options.copy( + metadata = fixture.request.options.metadata + + (MinervaHostVerificationMetadata.HOST_OUTPUT_PATH to "host-output.txt") + ) + ) + + val verification = JvmMinervaHostVerifier().verify(request, fixture.context) + + assertEquals(MinervaHostVerificationStatus.FAILED, verification.status) + assertEquals("minerva.host_verification.parity_failed", verification.code) + assertEquals(MinervaHostVerificationStatus.FAILED, verification.parityStatus) + assertTrue((verification.maxAbsoluteError ?: 0.0f) > verification.tolerance) + } + + @Test + fun hostVerifierFailsWhenPackagedModelWasTampered() { + val fixture = packagedProject(tempDir("host-verify-tampered"), "TamperedVerify") + Files.write(fixture.projectDir.resolve("generated/model.npz"), byteArrayOf(0, 1, 2, 3)) + + val verification = JvmMinervaHostVerifier().verify(fixture.request, fixture.context) + + assertEquals(MinervaHostVerificationStatus.FAILED, verification.status) + assertEquals("minerva.host_verification.model_tampered", verification.code) + } + private fun artifacts( outputDir: String, projectName: String, @@ -165,7 +232,50 @@ class MinervaJvmCompilerAndPackagerTest { ) } + private fun packagedProject(root: Path, projectName: String): PackagedProjectFixture { + val compilerDir = root.resolve("compiler") + Files.createDirectories(compilerDir) + val weightsC = compilerDir.resolve("weights.c") + val weightsH = compilerDir.resolve("weights.h") + Files.writeString(weightsC, "int minerva_weights = 1;\n") + Files.writeString(weightsH, "#pragma once\nextern int minerva_weights;\n") + val (options, intermediate, npzModel) = artifacts( + outputDir = root.resolve("package").toString(), + projectName = projectName, + compilerScript = root.resolve("compile.py").toString() + ) + val compilerOutput = MinervaCompilerOutput( + outputDir = compilerDir.toString(), + weightsCPath = weightsC.toString(), + weightsHPath = weightsH.toString(), + commandSummary = "fake-minerva --model model.npz" + ) + val context = minervaContext(options) + val bundle = JvmMinervaProjectPackager().packageProject( + MinervaProjectPackageRequest(options, intermediate, npzModel, compilerOutput), + context + ) + val request = MinervaHostVerificationRequest( + options = options, + intermediate = intermediate, + npzModel = npzModel, + compilerOutput = compilerOutput, + bundle = bundle + ) + return PackagedProjectFixture( + request = request, + context = context, + projectDir = Path.of(bundle.outputDir) + ) + } + private fun tempDir(prefix: String): Path { return Files.createTempDirectory("skainet-minerva-$prefix-") } + + private data class PackagedProjectFixture( + val request: MinervaHostVerificationRequest, + val context: GraphExportContext, + val projectDir: Path + ) } From 93507c401bfb31f69c6c7c03eeba095998a06f6b Mon Sep 17 00:00:00 2001 From: Michal Harakal Date: Sun, 7 Jun 2026 22:19:09 +0200 Subject: [PATCH 3/3] feat(minerva): document export workflow Closes #696 --- README.md | 12 + docs/export/minerva.md | 123 +++++++++ docs/modules/ROOT/nav.adoc | 1 + .../ROOT/pages/how-to/minerva-export.adoc | 239 ++++++++++++++++++ .../examples/MinervaTinyMlpExportSample.kt | 231 +++++++++++++++++ .../MinervaTinyMlpExportSampleTest.kt | 57 +++++ 6 files changed, 663 insertions(+) create mode 100644 docs/export/minerva.md create mode 100644 docs/modules/ROOT/pages/how-to/minerva-export.adoc create mode 100644 skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/examples/MinervaTinyMlpExportSample.kt create mode 100644 skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/examples/MinervaTinyMlpExportSampleTest.kt diff --git a/README.md b/README.md index 5a8120c7..1b1ddc08 100644 --- a/README.md +++ b/README.md @@ -185,12 +185,24 @@ deployment, the StableHLO path for native and edge targets. - Export trained models to standalone, optimized C99 with static memory allocation - Ready-to-use Arduino library output +### Edge AI: Minerva Secure MCU Export + +- Export supported static MLP graphs to Minerva project bundles for secure MCU inference +- Emits compiler NPZ input, libminerva weights, a manifest, host harness, firmware example, and host verification results +- Start with the [Minerva export guide](docs/modules/ROOT/pages/how-to/minerva-export.adoc) + ### Compiler: MLIR / StableHLO - Lower Kotlin DSL to MLIR StableHLO dialect - Optimization passes: constant folding, operation fusion, dead code elimination - Valid IREE-compilable output with streaming API and public `HloGenerator` +### Choosing an Export Path + +- Use **StableHLO** when you want portable MLIR/IREE-compatible graphs for native, accelerator, or ecosystem compiler flows. +- Use **Arduino / C99 export** when you want standalone generated C with static memory allocation and no external secure runtime. +- Use **Minerva export** when you need a secure MCU project bundle that goes through libminerva packaging and host verification. + --- ## What's New in 0.28.1 diff --git a/docs/export/minerva.md b/docs/export/minerva.md new file mode 100644 index 00000000..ca467d74 --- /dev/null +++ b/docs/export/minerva.md @@ -0,0 +1,123 @@ +# Minerva Secure MCU Export + +Minerva export packages a supported SKaiNET compute graph for secure MCU inference through libminerva. The maintained docs-site version is [`docs/modules/ROOT/pages/how-to/minerva-export.adoc`](../modules/ROOT/pages/how-to/minerva-export.adoc); this Markdown entrypoint keeps the repository path requested by the planning issue and is friendly to GitHub browsing. + +## Setup + +Inside this repository, use `project(":skainet-compile:skainet-compile-minerva")`. Published applications should import the SKaiNET BOM and add `sk.ainet.core:skainet-compile-minerva`. + +Configure libminerva through `MinervaExportOptions` or the JVM sample environment: + +```bash +export MINERVA_COMPILER_SCRIPT=/opt/libminerva/tools/compile_model.py +export MINERVA_RUNTIME_ROOT=/opt/libminerva +export MINERVA_CALIBRATION_NPZ=/secure/project/calibration.npz +export MINERVA_KEY_FILE=/secure/project/device.key +export MINERVA_RUN_CMAKE=true +export MINERVA_RUN_CTEST=true +``` + +Do not commit real device keys. `include/secrets.example.h` contains placeholders only. + +## Compatibility + +| Area | Phase-one support | +|---|---| +| Host platform | JVM export path | +| Target | `MinervaTarget.ATMEGA328P` | +| Quantization | `MinervaQuantization.Q8` | +| Graphs | Static, single-path, sequential MLPs | +| Shapes | Fully known rank-2 tensor shapes | +| Pattern | `Input -> MatMul -> Add? -> activation?`, repeated in sequence | +| Activations | `Relu`, `Sigmoid`, `Tanh` after a dense layer | +| Out of scope | CNNs, attention, recurrent models, dynamic shapes, branching graphs, transformers, arbitrary ONNX operators | + +## Export API + +```kotlin +val options = MinervaExportOptions( + outputDir = "build/minerva", + projectName = "TinySecureMlp", + compilerScript = "/opt/libminerva/tools/compile_model.py", + runtimeRoot = "/opt/libminerva", + calibrationNpz = "/secure/project/calibration.npz", + keyFile = "/secure/project/device.key" +) + +val result = MinervaExportFacade().exportGraph(graph, options) +val bundle = result.requireSuccess() +println(bundle.outputDir) +``` + +If the compiler script is missing, export still runs compatibility validation, lowering, and NPZ generation before returning a typed compiler prerequisite failure. + +## Generated Layout + +```text +build/minerva/TinySecureMlp/ + manifest.json + generated/ + model.npz + weights.c + include/ + weights.h + secrets.example.h + host/ + CMakeLists.txt + main.c + firmware/ + main.c +``` + +## Host Verification and CI + +Host verification checks package structure, generated weight files, `model.npz` integrity, placeholder secret hygiene, and SKaiNET reference output generation. Add these metadata keys to opt into CMake, CTest, and parity comparison with a host output file: + +```kotlin +metadata = mapOf( + MinervaHostVerificationMetadata.RUN_CMAKE_BUILD to "true", + MinervaHostVerificationMetadata.RUN_CTEST to "true", + MinervaHostVerificationMetadata.HOST_OUTPUT_PATH to "host-output.txt" +) +``` + +CI recipe: + +```bash +./gradlew :skainet-compile:skainet-compile-minerva:jvmTest +./gradlew :skainet-compile:skainet-compile-minerva:minervaHostVerification \ + -Pminerva.hostVerification.enabled=true \ + -Pminerva.runtimeRoot="$MINERVA_RUNTIME_ROOT" \ + -Pminerva.compilerScript="$MINERVA_COMPILER_SCRIPT" +cmake -S build/minerva/TinySecureMlp/host -B build/minerva/TinySecureMlp/host/build +ctest --test-dir build/minerva/TinySecureMlp/host/build --output-on-failure +``` + +## ONNX to Minerva + +Use the existing ONNX loader to inspect a model and reject unsupported operators before constructing a compatible SKaiNET `ComputeGraph`: + +```kotlin +val loaded = OnnxLoader.fromModelSource { + File(path).inputStream().asSource() +}.load() +val graph = loaded.proto.graph ?: error("ONNX model has no graph") +val ops = graph.node.map { it.opType }.toSet() +require(ops.all { it in setOf("MatMul", "Gemm", "Add", "Relu", "Sigmoid", "Tanh") }) +``` + +The first phase does not include a general ONNX-to-Minerva importer. + +## Firmware Integration + +The generated firmware example intentionally contains placeholders. Confirm the libminerva inference entry point and output-authentication API names against the pinned libminerva version used by your product build before flashing firmware. + +## Maintained JVM Sample + +`sk.ainet.compile.minerva.examples.MinervaTinyMlpExportSample` builds a tiny two-layer MLP, reads Minerva paths from environment variables, invokes the export facade, and prints bundle and verification status. `MinervaTinyMlpExportSampleTest` validates the sample graph and NPZ generation without real device keys. + +## Export Path Choice + +- Use StableHLO for portable MLIR/IREE-compatible compiler flows. +- Use Arduino / C99 export for standalone generated C with static memory allocation. +- Use Minerva export for secure MCU bundles compiled by libminerva and checked by host verification. diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 4b167c87..416f9db4 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -12,6 +12,7 @@ ** xref:how-to/io-readers.adoc[Load models (GGUF, SafeTensors, ONNX)] ** xref:how-to/java-model-training.adoc[Train a model from Java] ** xref:how-to/arduino-c-codegen.adoc[Generate C for Arduino] +** xref:how-to/minerva-export.adoc[Export secure MCU bundles with Minerva] * Reference ** xref:reference/architecture.adoc[Architecture] ** xref:reference/graph-export-architecture.adoc[Graph export architecture] diff --git a/docs/modules/ROOT/pages/how-to/minerva-export.adoc b/docs/modules/ROOT/pages/how-to/minerva-export.adoc new file mode 100644 index 00000000..31455636 --- /dev/null +++ b/docs/modules/ROOT/pages/how-to/minerva-export.adoc @@ -0,0 +1,239 @@ += Minerva Secure MCU Export + +Minerva export packages a supported SKaiNET compute graph for secure MCU inference through libminerva. Phase one is JVM-first and intentionally narrow: static sequential MLP graphs, Q8 quantization, and the ATmega328P target. + +Use this backend when the output must be a Minerva project bundle with compiler input, generated weights, firmware and host harnesses, a manifest, and host verification metadata. + +== When to Use Each Export Path + +[cols="1,2",options="header"] +|=== +| Export path | Use it when + +| StableHLO +| You need a portable MLIR/IREE-compatible graph for native, accelerator, or ecosystem compiler flows. + +| Arduino / C99 +| You need standalone generated C with static memory allocation and no external secure runtime. + +| Minerva +| You need a secure MCU project bundle that is compiled by libminerva and checked by host verification. +|=== + +== Setup + +Inside this repository, use the Minerva module directly: + +[source,kotlin] +---- +dependencies { + implementation(project(":skainet-compile:skainet-compile-minerva")) +} +---- + +For a published application, use the SKaiNET BOM and the Minerva artifact: + +[source,kotlin] +---- +dependencies { + implementation(platform("sk.ainet:skainet-bom:0.28.1")) + implementation("sk.ainet.core:skainet-compile-minerva") +} +---- + +Configure libminerva through `MinervaExportOptions` or environment variables used by the maintained JVM sample: + +[source,bash] +---- +export MINERVA_COMPILER_SCRIPT=/opt/libminerva/tools/compile_model.py +export MINERVA_RUNTIME_ROOT=/opt/libminerva +export MINERVA_CALIBRATION_NPZ=/secure/project/calibration.npz +export MINERVA_KEY_FILE=/secure/project/device.key +export MINERVA_RUN_CMAKE=true +export MINERVA_RUN_CTEST=true +---- + +`MINERVA_KEY_FILE` and the generated `include/secrets.example.h` are placeholders for integration. Do not commit real device keys or derived secrets. + +== Compatibility Matrix + +[cols="1,2",options="header"] +|=== +| Area | Phase-one support + +| Host platform +| JVM export path. + +| Target +| `MinervaTarget.ATMEGA328P`. + +| Quantization +| `MinervaQuantization.Q8`. + +| Graph shape +| Static, single-path, sequential MLPs. + +| Tensor shapes +| Fully known rank-2 shapes. + +| Layer pattern +| `Input -> MatMul -> Add? -> activation?`, repeated in sequence. + +| Activations +| `Relu`, `Sigmoid`, and `Tanh` after a dense layer. + +| Out of scope +| CNNs, attention blocks, recurrent models, dynamic shapes, branching graphs, transformers, and arbitrary ONNX operators. +|=== + +== Export API + +The public entry point is `MinervaExportFacade`. The facade accepts an already-built `ComputeGraph`, or it can record a representative forward pass for compatible SKaiNET models. + +[source,kotlin] +---- +import sk.ainet.compile.minerva.MinervaExportFacade +import sk.ainet.compile.minerva.MinervaExportOptions + +val options = MinervaExportOptions( + outputDir = "build/minerva", + projectName = "TinySecureMlp", + compilerScript = "/opt/libminerva/tools/compile_model.py", + runtimeRoot = "/opt/libminerva", + calibrationNpz = "/secure/project/calibration.npz", + keyFile = "/secure/project/device.key" +) + +val result = MinervaExportFacade().exportGraph(graph, options) +val bundle = result.requireSuccess() +println(bundle.outputDir) +---- + +If `compilerScript` is missing, export still performs compatibility checks, Minerva lowering, and NPZ schema creation before returning a typed compiler prerequisite failure. That makes local validation possible before libminerva is installed. + +== Generated Project Layout + +A successful export writes a project directory under `outputDir/projectName`: + +[source,text] +---- +build/minerva/TinySecureMlp/ + manifest.json + generated/ + model.npz + weights.c + include/ + weights.h + secrets.example.h + host/ + CMakeLists.txt + main.c + firmware/ + main.c +---- + +The manifest records the target, quantization, libminerva root, compiler command summary, NPZ schema version, layer count, and generated files. `secrets.example.h` contains placeholder values only. + +== Host Verification + +Host verification always checks the package structure, generated weight files, `model.npz` integrity, and placeholder secret hygiene. It also computes the SKaiNET reference output for a deterministic reference input. + +Use these metadata keys to opt into external host checks: + +[source,kotlin] +---- +metadata = mapOf( + MinervaHostVerificationMetadata.RUN_CMAKE_BUILD to "true", + MinervaHostVerificationMetadata.RUN_CTEST to "true", + MinervaHostVerificationMetadata.HOST_OUTPUT_PATH to "host-output.txt" +) +---- + +`RUN_CMAKE_BUILD` configures and builds `host/CMakeLists.txt`. `RUN_CTEST` runs the packaged CTest smoke test. `HOST_OUTPUT_PATH` lets a real host run write comma- or whitespace-separated float outputs that are compared with the SKaiNET reference output using `hostVerificationTolerance`. + +Local CI recipe: + +[source,bash] +---- +./gradlew :skainet-compile:skainet-compile-minerva:jvmTest +./gradlew :skainet-compile:skainet-compile-minerva:minervaHostVerification \ + -Pminerva.hostVerification.enabled=true \ + -Pminerva.runtimeRoot="$MINERVA_RUNTIME_ROOT" \ + -Pminerva.compilerScript="$MINERVA_COMPILER_SCRIPT" +cmake -S build/minerva/TinySecureMlp/host -B build/minerva/TinySecureMlp/host/build +ctest --test-dir build/minerva/TinySecureMlp/host/build --output-on-failure +---- + +The Gradle `minervaHostVerification` task is gated. It only runs when the `minerva.hostVerification.enabled`, `minerva.runtimeRoot`, and `minerva.compilerScript` properties are present. + +== Firmware Integration + +The generated firmware example intentionally contains integration placeholders. Wire it to the runtime API exposed by your pinned libminerva checkout and keep the real secret configuration in a private header that is excluded from source control. + +Before flashing firmware: + +* Replace `secrets.example.h` with a private header outside the repository. +* Confirm the libminerva inference entry point and output-authentication API names against the libminerva version used by your product build. +* Re-run host verification after any compiler, runtime, calibration, or key change. +* Keep `manifest.json` with release artifacts so the compiler command and schema version remain auditable. + +== ONNX to Minerva Workflow + +Use the existing ONNX loader path to inspect the model and extract a supported static MLP graph before calling Minerva export. The first phase does not include a general ONNX-to-Minerva importer; unsupported ONNX operators should be rejected before export. + +[source,kotlin] +---- +import kotlinx.io.asSource +import sk.ainet.io.onnx.OnnxLoader +import java.io.File + +suspend fun loadOnnxForMinerva(path: String) { + val loaded = OnnxLoader.fromModelSource { + File(path).inputStream().asSource() + }.load() + val graph = loaded.proto.graph ?: error("ONNX model has no graph") + + val ops = graph.node.map { it.opType }.toSet() + require(ops.all { it in setOf("MatMul", "Gemm", "Add", "Relu", "Sigmoid", "Tanh") }) { + "ONNX graph contains operators outside the Minerva phase-one scope: $ops" + } + + // Convert the inspected static MLP to a SKaiNET ComputeGraph, then call MinervaExportFacade. +} +---- + +== Maintained JVM Sample + +The maintained sample is `sk.ainet.compile.minerva.examples.MinervaTinyMlpExportSample` in the Minerva module. It builds a tiny two-layer MLP, reads Minerva paths from environment variables, invokes the export facade, and prints bundle and verification status. + +Run the sample after configuring libminerva: + +[source,bash] +---- +./gradlew :skainet-compile:skainet-compile-minerva:jvmTest +./gradlew :skainet-compile:skainet-compile-minerva:jvmJar +---- + +The sample graph is covered by `MinervaTinyMlpExportSampleTest`, which validates compatibility, lowering, and NPZ generation without requiring real device keys. + +== Troubleshooting + +[cols="1,2",options="header"] +|=== +| Symptom | Fix + +| `minerva.compiler.script_missing` +| Set `MinervaExportOptions.compilerScript` or `MINERVA_COMPILER_SCRIPT`. + +| `minerva.compiler.runtime_root_not_found` +| Point `runtimeRoot` or `MINERVA_RUNTIME_ROOT` at the libminerva checkout or install directory. + +| Compatibility fails for an unsupported operation +| Reduce the graph to the phase-one MLP pattern, or use StableHLO for a general compiler flow. + +| CMake or CTest verification fails +| Inspect `minerva-host-verification.log`, confirm CMake is installed, and confirm the generated host harness is linked against the pinned libminerva runtime. + +| Secret leak check fails +| Remove real secrets from generated files and regenerate the bundle. Only placeholders belong in source control. +|=== diff --git a/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/examples/MinervaTinyMlpExportSample.kt b/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/examples/MinervaTinyMlpExportSample.kt new file mode 100644 index 00000000..40adf456 --- /dev/null +++ b/skainet-compile/skainet-compile-minerva/src/jvmMain/kotlin/sk/ainet/compile/minerva/examples/MinervaTinyMlpExportSample.kt @@ -0,0 +1,231 @@ +package sk.ainet.compile.minerva.examples + +import sk.ainet.compile.minerva.MinervaExportFacade +import sk.ainet.compile.minerva.MinervaExportOptions +import sk.ainet.compile.minerva.MinervaHostVerificationMetadata +import sk.ainet.lang.graph.DefaultComputeGraph +import sk.ainet.lang.graph.GraphEdge +import sk.ainet.lang.graph.GraphNode +import sk.ainet.lang.tensor.ops.AddOperation +import sk.ainet.lang.tensor.ops.InputOperation +import sk.ainet.lang.tensor.ops.MatmulOperation +import sk.ainet.lang.tensor.ops.ReluOperation +import sk.ainet.lang.tensor.ops.SigmoidOperation +import sk.ainet.lang.tensor.ops.TensorSpec +import sk.ainet.lang.types.DType + +/** + * Maintained JVM sample for the Minerva secure MCU export path. + */ +internal object MinervaTinyMlpExportSample { + + @JvmStatic + internal fun main(args: Array): Unit { + val env = System.getenv() + val compilerScript = envPath(env, "MINERVA_COMPILER_SCRIPT") + if (compilerScript == null) { + println("Set MINERVA_COMPILER_SCRIPT to run the Minerva tiny MLP export sample.") + return + } + + val options = exportOptions( + compilerScript = compilerScript, + runtimeRoot = envPath(env, "MINERVA_RUNTIME_ROOT"), + keyFile = envPath(env, "MINERVA_KEY_FILE"), + calibrationNpz = envPath(env, "MINERVA_CALIBRATION_NPZ"), + runCmakeBuild = envFlag(env, "MINERVA_RUN_CMAKE"), + runCTest = envFlag(env, "MINERVA_RUN_CTEST"), + hostOutputPath = envPath(env, "MINERVA_HOST_OUTPUT_PATH") + ) + val result = MinervaExportFacade().exportGraph(tinyMlpGraph(), options) + + println("Minerva export status: ${result.status}") + result.bundle?.let { bundle -> + println("Project bundle: ${bundle.outputDir}") + println("Manifest: ${bundle.manifestPath}") + } + result.hostVerification?.let { verification -> + println("Host verification: ${verification.status}") + } + if (result.failed) { + error(result.failure?.message ?: "Minerva export failed.") + } + } + + internal fun tinyMlpGraph(): DefaultComputeGraph { + val inputSpec = spec("x", 1, 4) + val hiddenWeightsSpec = spec("w0", 4, 3, values = linearValues(12, start = 0.10f)) + val hiddenMatmulSpec = spec("matmul0", 1, 3) + val hiddenBiasSpec = spec("b0", 1, 3, values = linearValues(3, start = 0.01f)) + val hiddenAddSpec = spec("hidden_add", 1, 3) + val hiddenSpec = spec("hidden", 1, 3) + val outputWeightsSpec = spec("w1", 3, 2, values = linearValues(6, start = -0.20f)) + val outputMatmulSpec = spec("matmul1", 1, 2) + val outputBiasSpec = spec("b1", 1, 2, values = linearValues(2, start = -0.03f)) + val outputAddSpec = spec("output_add", 1, 2) + val outputSpec = spec("y", 1, 2) + + val input = inputNode("input", inputSpec) + val hiddenWeights = inputNode("hidden_weights", hiddenWeightsSpec) + val hiddenMatmul = matmulNode("matmul0", inputSpec, hiddenWeightsSpec, hiddenMatmulSpec) + val hiddenBias = inputNode("hidden_bias", hiddenBiasSpec) + val hiddenAdd = addNode("hidden_bias_add", hiddenMatmulSpec, hiddenBiasSpec, hiddenAddSpec) + val hiddenRelu = reluNode("hidden_relu", hiddenAddSpec, hiddenSpec) + val outputWeights = inputNode("output_weights", outputWeightsSpec) + val outputMatmul = matmulNode("matmul1", hiddenSpec, outputWeightsSpec, outputMatmulSpec) + val outputBias = inputNode("output_bias", outputBiasSpec) + val outputAdd = addNode("output_bias_add", outputMatmulSpec, outputBiasSpec, outputAddSpec) + val sigmoid = GraphNode( + id = "output_sigmoid", + operation = SigmoidOperation(), + inputs = listOf(outputAddSpec), + outputs = listOf(outputSpec) + ) + + return graphOf( + nodes = listOf( + input, + hiddenWeights, + hiddenMatmul, + hiddenBias, + hiddenAdd, + hiddenRelu, + outputWeights, + outputMatmul, + outputBias, + outputAdd, + sigmoid + ), + edges = listOf( + edge("x_to_matmul0", input, hiddenMatmul, inputSpec, destinationInputIndex = 0), + edge("w0_to_matmul0", hiddenWeights, hiddenMatmul, hiddenWeightsSpec, destinationInputIndex = 1), + edge("matmul0_to_add", hiddenMatmul, hiddenAdd, hiddenMatmulSpec, destinationInputIndex = 0), + edge("b0_to_add", hiddenBias, hiddenAdd, hiddenBiasSpec, destinationInputIndex = 1), + edge("hidden_add_to_relu", hiddenAdd, hiddenRelu, hiddenAddSpec), + edge("hidden_to_matmul1", hiddenRelu, outputMatmul, hiddenSpec, destinationInputIndex = 0), + edge("w1_to_matmul1", outputWeights, outputMatmul, outputWeightsSpec, destinationInputIndex = 1), + edge("matmul1_to_add", outputMatmul, outputAdd, outputMatmulSpec, destinationInputIndex = 0), + edge("b1_to_add", outputBias, outputAdd, outputBiasSpec, destinationInputIndex = 1), + edge("output_add_to_sigmoid", outputAdd, sigmoid, outputAddSpec) + ) + ) + } + + internal fun exportOptions( + outputDir: String = "build/minerva", + projectName: String = "TinySecureMlp", + compilerScript: String, + runtimeRoot: String? = null, + keyFile: String? = null, + calibrationNpz: String? = null, + runCmakeBuild: Boolean = false, + runCTest: Boolean = false, + hostOutputPath: String? = null + ): MinervaExportOptions { + val metadata = mutableMapOf("sample" to "minerva-tiny-mlp") + if (runCmakeBuild) { + metadata[MinervaHostVerificationMetadata.RUN_CMAKE_BUILD] = "true" + } + if (runCTest) { + metadata[MinervaHostVerificationMetadata.RUN_CTEST] = "true" + } + hostOutputPath?.let { + metadata[MinervaHostVerificationMetadata.HOST_OUTPUT_PATH] = it + } + return MinervaExportOptions( + outputDir = outputDir, + projectName = projectName, + runtimeRoot = runtimeRoot, + compilerScript = compilerScript, + keyFile = keyFile, + calibrationNpz = calibrationNpz, + metadata = metadata + ) + } + + private fun inputNode(id: String, output: TensorSpec): GraphNode { + return GraphNode( + id = id, + operation = InputOperation(), + inputs = emptyList(), + outputs = listOf(output) + ) + } + + private fun matmulNode( + id: String, + left: TensorSpec, + right: TensorSpec, + output: TensorSpec + ): GraphNode { + return GraphNode( + id = id, + operation = MatmulOperation(), + inputs = listOf(left, right), + outputs = listOf(output) + ) + } + + private fun addNode( + id: String, + left: TensorSpec, + right: TensorSpec, + output: TensorSpec + ): GraphNode { + return GraphNode( + id = id, + operation = AddOperation(), + inputs = listOf(left, right), + outputs = listOf(output) + ) + } + + private fun reluNode(id: String, input: TensorSpec, output: TensorSpec): GraphNode { + return GraphNode( + id = id, + operation = ReluOperation(), + inputs = listOf(input), + outputs = listOf(output) + ) + } + + private fun spec(name: String, vararg shape: Int, values: List? = null): TensorSpec { + val metadata: Map = values?.let { mapOf("values" to it.toFloatArray()) } ?: emptyMap() + return TensorSpec(name, shape.toList(), "Float32", metadata = metadata) + } + + private fun linearValues(count: Int, start: Float): List { + return List(count) { index -> start + (index * 0.05f) } + } + + private fun graphOf(nodes: List, edges: List): DefaultComputeGraph { + val graph = DefaultComputeGraph() + nodes.forEach { graph.addNode(it) } + edges.forEach { graph.addEdge(it) } + return graph + } + + private fun edge( + id: String, + source: GraphNode, + destination: GraphNode, + spec: TensorSpec, + destinationInputIndex: Int = 0 + ): GraphEdge { + return GraphEdge( + id = id, + source = source, + destination = destination, + destinationInputIndex = destinationInputIndex, + tensorSpec = spec + ) + } + + private fun envPath(env: Map, name: String): String? { + return env[name]?.trim()?.takeIf { it.isNotEmpty() } + } + + private fun envFlag(env: Map, name: String): Boolean { + return env[name]?.equals("true", ignoreCase = true) == true + } +} diff --git a/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/examples/MinervaTinyMlpExportSampleTest.kt b/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/examples/MinervaTinyMlpExportSampleTest.kt new file mode 100644 index 00000000..cdeafaea --- /dev/null +++ b/skainet-compile/skainet-compile-minerva/src/jvmTest/kotlin/sk/ainet/compile/minerva/examples/MinervaTinyMlpExportSampleTest.kt @@ -0,0 +1,57 @@ +package sk.ainet.compile.minerva.examples + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue +import sk.ainet.compile.export.GraphExportArtifactRole +import sk.ainet.compile.export.GraphExportStatus +import sk.ainet.compile.minerva.MinervaCompatibilityValidator +import sk.ainet.compile.minerva.MinervaExportFacade +import sk.ainet.compile.minerva.MinervaExportFailureKind +import sk.ainet.compile.minerva.MinervaHostVerificationMetadata + +class MinervaTinyMlpExportSampleTest { + + @Test + fun sampleGraphIsCompatibleAndLowersToNpz() { + val graph = MinervaTinyMlpExportSample.tinyMlpGraph() + val options = MinervaTinyMlpExportSample.exportOptions( + compilerScript = "/opt/libminerva/tools/compile_model.py", + runtimeRoot = "/opt/libminerva", + keyFile = "/secure/project/device.key", + calibrationNpz = "/secure/project/calibration.npz" + ) + val report = MinervaCompatibilityValidator().validate(graph, options) + + assertTrue(report.compatible, report.issues.joinToString { it.message }) + assertEquals(2, report.layerCount) + + val result = MinervaExportFacade().exportGraph(graph, options.copy(compilerScript = null)) + + assertEquals(GraphExportStatus.FAILED, result.status) + assertEquals(MinervaExportFailureKind.COMPILER_PREREQUISITE_FAILED, result.failure?.kind) + assertTrue(assertNotNull(result.npzModel).bytes.isNotEmpty()) + assertEquals(2, result.intermediate?.layerCount) + assertTrue( + result.artifacts.any { + it.role == GraphExportArtifactRole.INTERMEDIATE && it.path == "model.npz" + } + ) + } + + @Test + fun sampleOptionsCarryHostVerificationMetadata() { + val options = MinervaTinyMlpExportSample.exportOptions( + compilerScript = "/opt/libminerva/tools/compile_model.py", + runCmakeBuild = true, + runCTest = true, + hostOutputPath = "host-output.txt" + ) + + assertEquals("minerva-tiny-mlp", options.metadata["sample"]) + assertEquals("true", options.metadata[MinervaHostVerificationMetadata.RUN_CMAKE_BUILD]) + assertEquals("true", options.metadata[MinervaHostVerificationMetadata.RUN_CTEST]) + assertEquals("host-output.txt", options.metadata[MinervaHostVerificationMetadata.HOST_OUTPUT_PATH]) + } +}