diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cad005b..2bf37d3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,6 +17,6 @@ repos: - id: ruff-format - repo: https://github.com/RobertCraigie/pyright-python - rev: v1.1.409 + rev: v1.1.414 hooks: - id: pyright diff --git a/AGENTS.md b/AGENTS.md index 1f030bf..bb89332 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -208,9 +208,9 @@ before publication. XGBoost supports exact XGBClassifier/XGBRegressor tree wrappers with closed probability/regression objectives and one target. Original class labels supplied through output-major `class_labels=` are bound to a native copy through an mlbundle-owned booster attribute and verified on load. Omitted -labels use encoded indices; sklearn/CatBoost reject `class_labels=` because they own classes_. Local `training_data=` inference uses mlschema with `__`, excludes a named target, -and records training dtypes. Only explicit `sample_rows=N` includes real rows. -Schema/report and sample validation is shared by export, inspect and load. +labels use encoded indices; sklearn/CatBoost reject `class_labels=` because they own classes_. Every export requires the schema extra: mlschema infers from `training_data=` or one float-1.0 row, recording schema v2 `source` and configurable `onehot_separator` (default `__`). Real frames exclude every declared target and record training dtypes; default dtypes stay null. Only explicit `sample_rows=N` includes real rows. +Model-origin mappings use names; other origins use positions and generated names start at `feature_0`. `target_names` declares one unique bounded name per output; real frames exclude all target columns. Reports map by output position with unique string labels, while manifest labels retain their types. +Schema/report and sample validation is shared by export, inspect and load without schema libraries; missing schema and old schema versions require re-export. ONNX preserves path/bytes unchanged (ModelProto is serialized once), validates bytes once per export/load with bounded scalar scans, and rejects external tensors, custom domains, functions and training/runtime configuration before a CPU session. It verifies graph signatures, metadata and environment @@ -220,14 +220,14 @@ labels use count-checked declarations. Structure is null. See the normative ONNX section linked from `docs/format.md` for exact layouts and resource bounds. CatBoost supports exact numeric single-target symmetric classifiers/regressors, with native state commitments and in-memory loading. Exact outer sklearn Pipelines ending in XGBoost/CatBoost use separately checked skops/native components; other native placements remain rejected. The closed member list is: required `manifest.json`, one -native member, or declared `model.skops` + `model.ubj`/`model.cbm` pair, optional `schema.json`, and +native member, or declared `model.skops` + `model.ubj`/`model.cbm` pair, required `schema.json`, and `structure.json` when the supported model has extractable structure. Optional `samples.json` requires explicit user intent. The ceiling is six members; no others are allowed. The manifest must record format name/version, bundle id, UTC creation time and writer version; model library, estimator class, classifier/regressor kind, hyperparameters and feature/class/output counts; feature names, order, dtypes -and origin; original class labels and target name; training Python, OS, +and origin; original class labels and output-major target names; training Python, OS, architecture and package versions; member SHA-256 hashes and sizes; and free-form user metadata. Sample rows are allowed only on explicit request. All of this is untrusted input, and model-derived fields must be verified. @@ -263,14 +263,14 @@ its kind/count inference remains ONNX-owned. Keep native behavior in adapters; f - `src/mlbundle/canonical.py` and `_canonical_work.py` — deterministic JSON and bounded decoding/expansion. - `src/mlbundle/model_io.py`, `_native_io.py`, `_export_manifest.py` — public I/O, native round trips and manifest construction. - `src/mlbundle/_composite/`, `_pipeline_contract.py` — strict recipe, separate I/O, closed reconstruction/claims/prediction and input/inner boundaries. Read `docs/format-pipeline.md` when changing composites. -- `src/mlbundle/training.py` — lazy local DataFrame alignment and schema inference. +- `src/mlbundle/training.py` — lazy local/default inference and feature origin/dtype resolution. - `src/mlbundle/schema.py` and `_schema_fields.py` — closed schema/report declarations and reader cross-checks without mlschema. - `src/mlbundle/samples.py` — explicit feature-row selection and bounded cell conversion. - `src/mlbundle/_sklearn/` — owned trust/skops/state/parameters/structure, transformer route and closed fitted-width/name proofs. Composite commitments/native structure bind the rebuilt Pipeline; all-sklearn structure stays null. - `src/mlbundle/_xgboost/` — closed wrappers/objectives, native UBJSON preflight/loading, label binding, recoverable config, complete tree structure and validated prediction. - `src/mlbundle/_catboost/` — closed numeric symmetric wrappers/losses, CBM preflight/in-memory loading, complete native state/structure and validated prediction. - `src/mlbundle/_onnx/` — bounded protobuf/graph validation, unchanged bytes, native signatures/labels, CPU sessions and tabular prediction. -- `src/mlbundle/declarations.py` — export-time class labels and resolution for native models without labels. +- `src/mlbundle/declarations.py` — class-label resolution and shared output-major target-name validation. - `src/mlbundle/claims.py` — shared extracted claims and exact claim/structure comparison. - `src/mlbundle/manifest.py` — manifest sections, cross-field validation and JSON boundaries. - `src/mlbundle/integrity.py` — member SHA-256/size records and verification. diff --git a/README.md b/README.md index e9f98aa..cc335c2 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ mlbundle replaces three gaps in separate model/DataFrame uploads: - **Training data leaving the machine:** schema inference runs locally; training rows are included only when explicitly requested as samples. -**Pre-alpha v0.1.0a1, published to reserve the PyPI name.** Format version 1 is +**Pre-alpha v0.1.0a2, not yet published.** Version 0.1.0a1 reserved the PyPI name. Format version 1 is not frozen; bundles may need re-export. The security design has not had an independent review: do not accept bundles from untrusted parties. Requires **Python >=3.14**; tested on Windows/Linux. Schema requires Python 3.14. @@ -20,11 +20,9 @@ Requires **Python >=3.14**; tested on Windows/Linux. Schema requires Python 3.14 ## Install and try scikit-learn Install the pre-release with -`python -m pip install --pre "mlbundle[sklearn]"`. -Replace `[sklearn]` with `[xgboost]`, `[catboost]`, `[onnx]`, `[schema]` or `[all]` for the -corresponding dependencies. Combine extras as `[sklearn,schema]` for local -DataFrame inference. Omit the brackets for a base install that can inspect -every bundle without model libraries. Load/verify needs the matching model extra. +`python -m pip install "mlbundle[sklearn,schema]==0.1.0a2"` once published. +Replace `sklearn` with `xgboost`, `catboost` or `onnx`, or use `[all]` for all dependencies. +Export needs the `schema` extra; reading does not. Omit the brackets for metadata-only inspection; load/verify needs the matching model extra. ```python from sklearn.tree import DecisionTreeClassifier @@ -47,8 +45,9 @@ regressors use `bundle.predict`. Results are plain lists: output-major classifie labels/probabilities, row-major regression values. `native_model` supports native inference/explanations; mutating it invalidates the metadata verification. -Native names win; otherwise origin is `declared`, `dataframe`, or `generated` (`feature_1` through `feature_N`). -`target_name=` and `metadata=` remain author declarations. +Native names win; otherwise origin is `declared`, `dataframe`, or `generated` (`feature_0` through `feature_{N-1}`). +For models without native names, pass a 2-D NumPy array or nested numeric rows in manifest feature order; declared names do not add name-based prediction. +`target_names=` and `metadata=` remain author declarations. ## XGBoost @@ -163,8 +162,8 @@ no visualization structure. See [the exact ONNX contract](https://github.com/Ull ## Local schema and explicit samples Install a model extra plus `schema` on Python 3.14. Supply a nonempty pandas -DataFrame with string feature names in model input order. A named target must -exist and is excluded; without `target_name`, the frame contains features only. +DataFrame with string feature names in model input order. `target_names` has one +unique name per output; every target column must exist and is excluded. Without it the frame contains features only. ```python import pandas as pd @@ -177,7 +176,7 @@ training = pd.DataFrame( ) features = training.drop(columns="target") model = DecisionTreeClassifier(random_state=7).fit(features, training["target"]) -export(model, "schema.mlbundle", training_data=training, target_name="target") +export(model, "schema.mlbundle", training_data=training, target_names=["target"]) view = inspect("schema.mlbundle") assert view.manifest.features.dtypes == ["float64", "float64"] assert view.schema["reports"][0]["labels"] == ["cold", "warm"] @@ -188,12 +187,16 @@ export(model, "sampled.mlbundle", training_data=features, sample_rows=2) assert inspect("sampled.mlbundle").samples == features.head(2).to_dict("records") ``` -`schema.json` contains a schema version, locally inferred fields and one report -per target output. `color__red`/`color__blue` binary columns can become one -category field using mlschema's `__` convention. Frame names must match native -names in order; `feature_names=` cannot rename/reorder a supplied frame. +`schema.json` is required and uses schema version `2`, with `source` equal to +`training_data` or `default`. Without a frame, mlschema infers fields from one +synthetic row of float `1.0` values; training dtypes remain null. `onehot_separator=` +defaults to `__` and is stored in the schema (1–16 characters). `color__red`/`color__blue` +binary columns can become one category field. Native-name mappings are strings; all other origins map by zero-based feature position. Reports map by output position, +including `0` for one output, with string display labels; manifest class labels retain +their original types. Frame names must match native names in order; `feature_names=` cannot rename/reorder a supplied frame. Training dtypes are declarations, separate from verified model dtypes. Inspect/load validate schema mappings, reports and sample keys without pandas or mlschema. +Bundles written by 0.1.0a1 without schema, or with schema version `1`, must be re-exported. **Data leaving the machine:** names/dtypes, field kinds/nullability, dtype-based steps, categories (including unused), one-hot suffixes, pair-axis metadata and @@ -261,12 +264,12 @@ bounds do not cap process memory, prediction rows or CPU. ## Bundle contents and security -Every bundle contains `manifest.json` and one native member, or the declared -`model.skops` + `model.ubj`/`model.cbm` composite pair. Single-step composites use only the leaf. Optional members are `schema.json`, `structure.json` +Every bundle contains `manifest.json`, `schema.json` and one native member, or the declared +`model.skops` + `model.ubj`/`model.cbm` composite pair. Single-step composites use only the leaf. Optional members are `structure.json` (required when the adapter extracts structure) and explicitly requested `samples.json`. No training DataFrame or arbitrary files are included. The manifest records identity/time/writer, model parameters/counts/state, -ordered features/dtypes/origin, target labels/name, environment, member +ordered features/dtypes/origin, target labels/names, environment, member SHA-256/sizes, references and free-form user metadata. The normative [format specification](https://github.com/UlloaSP/mlbundle/blob/main/docs/format.md) owns these records and verification rules. diff --git a/SECURITY.md b/SECURITY.md index 1073721..0838459 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,6 +1,6 @@ # Security policy -The published v0.1.0a1 pre-alpha has not had an independent security review; +The v0.1.0a2 development pre-alpha has not had an independent security review; do not accept bundles from untrusted parties. Format version 1 is not frozen. It implements strict manifest validation, secure container reading/writing, metadata-only `inspect`, export/load, sklearn, XGBoost, CatBoost and ONNX adapters, local schema inference, @@ -256,8 +256,10 @@ Omitting samples does not prevent model memorization or model-side disclosure. Only explicit `sample_rows=N` includes real feature rows: exactly the first N, without target or index. The schema/sample members have 16 MiB/depth-32 limits. -Inspect and load validate closed shapes, feature mappings/row keys and report -kind/labels without mlschema. Load also verifies native model claims; label-free +Schema v2 is required, inferred locally from training data or a synthetic default; +source/separator remain declarations. Older bundles require re-export. Inspect/load +validate mappings by feature origin and string reports by output position without +mlschema/pandas; native label verification retains original types. Load also verifies native model claims; label-free ONNX original labels remain declarations. Consumers must treat schema and sample values as untrusted display/input data, never executable instructions. See [the exact data-derived contract](docs/format-schema.md) and review what is shared. @@ -276,5 +278,5 @@ behavior, and the impact. Share malicious bundles or sensitive details through that private report rather than a public issue or pull request. Remove personal or confidential training data from any reproducer you submit. -There are no published versions yet. Report findings against the repository's -current code and identify the commit you tested. +Version 0.1.0a1 is published; 0.1.0a2 is not yet published. Report the affected +version and commit you tested. diff --git a/docs/format-adapters.md b/docs/format-adapters.md index 986a3be..ecbac3f 100644 --- a/docs/format-adapters.md +++ b/docs/format-adapters.md @@ -70,12 +70,12 @@ optional dependency reports `pip install "mlbundle[EXTRA]"`. An unrecognized object raises `UnsupportedModelError`. `export(model, destination, *, training_data=None, feature_names=None, class_labels=None, -target_name=None, sample_rows=None, metadata=None, overwrite=False, limits=None)` +target_names=None, sample_rows=None, onehot_separator="__", metadata=None, overwrite=False, limits=None)` returns a `MetadataView` of the written bundle. It serializes, extracts, reloads and compares both claims and structure before writing. Model-exposed names take precedence over supplied names. It captures distribution metadata, including `claims.onnx` in the environment for -the ONNX adapter. [Local schema export](format-schema.md) owns frame alignment, +the ONNX adapter. [Required schema export](format-schema.md) owns local/default inference, frame alignment, training dtypes, schema/report declarations and explicit sample selection. `load(source, *, limits=None)` validates the container, resolves its closed enum, deserializes, recomputes claims/structure and compares canonical bytes before @@ -210,9 +210,9 @@ bounded `MLBundleError` subclasses without cell contents. Shape errors additiona retain `ValueError` compatibility through `PredictionInputError`. `MetadataView` carries `manifest`, a tuple of `MemberName` values including -the manifest, and optional `schema` and `samples` JSON values; it has no model. +the manifest, required `schema` and optional `samples` JSON values; it has no model. `LoadedBundle` additionally carries the exact -native object, a uniform predictor, optional schema/structure/sample values, and +native object, a uniform predictor, required schema and optional structure/samples, and properties for kind, library, class name, ordered feature names and origin, and output-major original class labels. Calling the prediction method for the wrong kind raises a bounded `ValueError`. Result records and typed models prevent field diff --git a/docs/format-container.md b/docs/format-container.md index afdbb54..fa0f1a9 100644 --- a/docs/format-container.md +++ b/docs/format-container.md @@ -97,13 +97,13 @@ deserializes a native model or resolves an adapter. Writing accepts a `Manifest` or JSON-ready manifest parts, plus non-manifest member bytes. The writer computes and replaces all integrity records; callers -must already set the model pairing and optional references. JSON-ready parts +must already set the model pairing, required schema reference and optional references. JSON-ready parts may omit integrity. Optional JSON members pass the bounded canonical decoder; their original bytes are preserved. The container validates JSON syntax and bounds, while later model/schema APIs own semantic structure and schema checks. Metadata inspection does not infer whether a model requires structure. -Internally, optional JSON members retain their validated values within one +Internally, schema and optional JSON members retain their validated values within one operation. Inspect/load reuse that decode for schema/sample checks, and load compares freshly encoded native extraction with stored structure bytes. Equality proves canonical identity; other wire spellings encode the retained value once. @@ -148,6 +148,6 @@ Container/layout/CRC inconsistencies raise `InvalidContainerError`; resource ceilings raise `LimitExceededError`. Invalid manifest JSON or typed fields raise `InvalidManifestError`; unknown version strings under the mlbundle identity raise `UnsupportedFormatVersionError` before validating version-specific fields. -Invalid optional JSON raises `InvalidJSONError`. Member hash or integrity-size +Invalid member JSON raises `InvalidJSONError`. Member hash or integrity-size disagreement raises `IntegrityMismatchError` naming the allowed member. Messages do not include untrusted names, JSON blobs or ZIP exception text. diff --git a/docs/format-onnx.md b/docs/format-onnx.md index d6a8074..3571c62 100644 --- a/docs/format-onnx.md +++ b/docs/format-onnx.md @@ -95,7 +95,7 @@ hyperparameter ceiling applies; no training parameters are inferred from it. by original domain), IR version and producer name/version. These fields, graph signatures, counts, kind, names/dtypes and native labels are all recomputed with the writer's extraction code and compared by exact canonical bytes. Historical -environment, training dtypes, supplied matrix feature names, target name and +environment, training dtypes, supplied matrix feature names, target names and label-free mappings remain declarations. Producer name/version each allow 0–65,536 characters, including multiline build diff --git a/docs/format-pipeline.md b/docs/format-pipeline.md index 9fcf174..74e20b8 100644 --- a/docs/format-pipeline.md +++ b/docs/format-pipeline.md @@ -70,10 +70,10 @@ A single-step composite has only its native member and null `transformers`. Passthrough/None preceding steps still require the prefix member to preserve the recipe. A pair without its declaration, a missing declared member, ONNX pairs, two boosters or three model members fail. Manifest integrity covers -exactly the declared model members and referenced optional members. +exactly the declared model members, required schema and referenced optional members. -Composite `structure.json` is required. Schema and explicit samples are optional; -six members permit both with the two model members and manifest. Every other +Composite `structure.json` and schema are required; explicit samples are optional. +Six members permit all with the two model members and manifest. Every other archive ceiling is unchanged. Callers may lower member count and other limits. Export gates scope/options before serialization. The leaf adapter receives diff --git a/docs/format-schema.md b/docs/format-schema.md index 77957ba..a938c28 100644 --- a/docs/format-schema.md +++ b/docs/format-schema.md @@ -1,43 +1,56 @@ -# Version 1 schema and samples +# Format 1 schema v2 and samples Normative section of [the format contract](format.md). Local inference uses -the mlschema 0.2 vocabulary, tested with 0.2.1. Schema and samples come from data the -reader never sees. Integrity does not authenticate that data or its history. - -## Export inputs - -`export(training_data=frame)` accepts a nonempty pandas DataFrame. Without -`target_name`, all columns are features. With `target_name`, that column must -exist and is excluded before inference, dtype capture and sample selection. -Without a frame, `target_name` remains a metadata declaration as before. -Index values are never included. The input frame is not modified. - -Column names must be unique, nonempty strings of at most 1,024 characters. -Their count must equal the native feature count. When native names exist, -columns must match those names **in order**, after target exclusion. Supplying -`feature_names` with a frame requires exactly those feature columns in order; -it cannot rename or reorder them. Frame errors raise `InvalidSchemaError` before -publication. Names remain model-origin when the model exposes them; otherwise -the frame yields dataframe-origin names. Training dtypes are `str(dtype)` in +mlschema >=0.2.1,<0.3, tested with 0.2.1. Fields, source and separator are untrusted +declarations; integrity cannot authenticate the training data or its history. + +## Export inputs and defaults + +Every export writes `schema.json`. `export(training_data=frame)` accepts a +nonempty pandas DataFrame. `target_names` is an output-major sequence: one unique +nonempty bounded name per verified output, each at most 1,024 characters. A bare +string is rejected with a bounded error requiring a list. Without `target_names`, +all columns are features; otherwise every target column must exist and is excluded +from inference, dtype capture and samples. Without a frame they are declarations. +Index values never leave through inference/samples; the frame is not modified. + +Training columns must be unique, nonempty strings of at most 1,024 characters, +with count equal to the native feature count. Native names must match columns +in order after target exclusion. `feature_names=` with a frame must equal its +feature columns in order; it cannot rename or reorder them. Errors raise +`InvalidSchemaError` before publication. Native names retain model origin; +otherwise a frame gives dataframe origin. Training dtypes are `str(dtype)` in feature order, separate from verified `model_dtypes`. -Export calls exactly `mlschema.infer_schema(feature_frame, onehot_separator="__")`, -without builders, overrides or custom kinds. It uses the same separator as -MLSuite's `schema_service.py`. Unlike MLSuite's older positional fallback, named -frame columns are preserved even when a model has no names; inferred mappings -therefore always refer to manifest names, never integer positions. Fields retain -the direct mlschema output. MLSuite's MLForm integration still converts a `series` -field's `field1`/`field2` into `columns` with those IDs and removes nested -`mappedTo`; this is presentation mapping and needs no DataFrame. Consumers may -format typed class labels for display and select reports by output index. - -`mlbundle[schema]` optionally installs mlschema >=0.2.1,<0.3 from PyPI, transitively -pandas; it is included in `all`. The dependency is marked for Python 3.14 because -that mlschema version requires `>=3.14,<3.15`. mlbundle's own `>=3.14` range and -sole core dependency (Pydantic) are unchanged. Training export with missing -pandas/mlschema raises `MissingOptionalDependencyError` naming `schema`. -mlbundle import and inspection do not import either library. Loading does not -require them, although native sklearn imports an installed pandas opportunistically. +Without `training_data`, inference uses a synthetic one-row pandas DataFrame of +float `1.0` values, one column per feature. Native or declared names are its +columns; generated origin uses columns `range(n_features)`. mlschema supplies +all field shapes, including numeric `step: 0.1` and one-hot grouping for named +binary columns. Synthetic dtypes are not training evidence: manifest training +dtypes remain null. Generated names/labels are `feature_0` through `feature_{N-1}`. + +`export(..., onehot_separator="__")` accepts a strict nonempty string of at most +16 characters. Inference calls `mlschema.infer_schema(frame, onehot_separator=...)` +without builders, overrides or custom kinds. The separator is stored and read +from the schema; consumers must not assume the default. + +Model-origin fields retain name mappings. Dataframe/declared inference preserves +named columns for grouping, then rewrites every `mappedTo`, including series +subfields and one-hot options, to its zero-based feature position. Generated +origin is inferred on positional columns directly. Labels remain feature names. +For models without native names, prediction accepts 2-D NumPy arrays or nested +numeric rows in manifest feature order. Adapter-supported sparse arrays and +DataFrames retain native restrictions; supplied names do not add name-based +selection or reordering. Positional schema mappings do not change predictor inputs. + +Export requires `mlbundle[schema]` even without training data. This separate extra +installs mlschema and transitively pandas on Python 3.14; it is included in `all`, +not the core or model extras. Missing pandas/mlschema raises +`MissingOptionalDependencyError` naming `schema`, before publication. +Import, inspect and schema validation on load do not import these libraries. +Load needs only the matching model extra; native libraries can opportunistically +import an installed pandas. mlschema requires Python >=3.14,<3.15; mlbundle's own +Python range and sole core dependency, Pydantic, are unchanged. ## `schema.json` @@ -45,64 +58,79 @@ The exact top-level object is: ```json { - "schema_version": "1", + "schema_version": "2", + "source": "training_data", + "onehot_separator": "__", "fields": [ {"kind":"number","label":"age","mappedTo":"age","required":true,"step":1.0} ], "reports": [ - {"kind":"classifier","label":"Predicted class","mappedTo":"classifier", - "output":0,"labels":["cold","warm"],"showClassProbabilities":true} + {"kind":"classifier","label":"Predicted class","mappedTo":0, + "labels":["0","1"],"showClassProbabilities":true} ] } ``` -All keys shown are required. Unknown keys and versions are rejected. Arrays are -ordered; `fields` has 1–100,000 entries and `reports` 1–1,024. Typed records use -strict types, finite numbers and no arbitrary extensibility. This envelope -supports only the default inferred vocabulary, not every mlschema override. - -Every ordinary field has exactly `kind`, `label`, `mappedTo`, `required`, plus -the kind-specific keys below. Labels are nonempty strings of at most 100 -characters; `mappedTo` is a manifest feature name; `required` is a boolean. -At the top level, an ordinary field's label equals its feature name. +All five keys are required and unknown keys are forbidden. `source` is exactly +`training_data` or `default`; `onehot_separator` is bounded as above. Version `1` +and unknown versions raise bounded `InvalidSchemaError` requiring re-export. +A missing schema member raises bounded `InvalidContainerError` saying the bundle +predates the required schema and must be re-exported, through shared member rules. +Thus 0.1.0a1 bundles without schema or with schema v1 are incompatible with 0.1.0a2. +Format version remains `1`, whose pre-release wire contract is not frozen. + +Arrays preserve order. `fields` has 1–100,000 entries and `reports` 1–1,024. +Typed records use strict types, finite numbers and no arbitrary extensibility. +The vocabulary supports default inference, not every mlschema override. + +Ordinary fields have exactly `kind`, `label`, `mappedTo`, `required`, plus the +kind-specific keys below. Labels are nonempty strings of at most 100 characters; +`required` is boolean. For model origin, `mappedTo` must be a manifest feature +name (nonempty, at most 1,024 characters). For every other origin, it must be a +strict integer position in `[0, n_features)`, never a bool or string. The other +mapping type is rejected. Resolve to an index before coverage/order checks; +a top-level ordinary label must equal the feature name at that index. | Kind | Additional required keys | | --- | --- | | `text`, `boolean`, `date` | None. | -| `number` | `step`: finite positive number; default inference emits `1.0` for integers or `0.1` for floats. | +| `number` | `step`: finite positive number; integers infer `1.0`, floats `0.1`. | | `category` | `options`: 1–100,000 unique strings, retaining categorical order and unused categories. | -| `series` | `field1`, `field2`: ordinary scalar fields from the same feature column; no nested series or one-hot groups. Subfield labels describe pair axes; each `mappedTo` equals its parent mapping. | +| `series` | `field1`, `field2`: scalar fields from the same feature column; no nested series/one-hot groups. Subfield labels describe pair axes; mappings resolve to the parent's index. | | `onehot-category` | Special shape below. | -A one-hot group has exactly `kind`, `label`, `required`, `options`; no parent -`mappedTo`. Its label is the shared prefix before the first `__`. At least two -binary columns with that prefix form a group. There are 2–100,000 options, each -exactly `{"label":SUFFIX,"value":SUFFIX,"mappedTo":COLUMN}`. These three strings -are nonempty and at most 1,024 characters. `COLUMN = PREFIX + "__" + SUFFIX`; -the option label and value agree. Other columns stay ordinary fields, even if -their names contain `__`; the reader cannot verify whether their values were -binary. Group options preserve feature order, and fields are emitted in order -of their first mapped feature. Non-adjacent group columns are supported. - -Field/option mappings must cover every manifest feature exactly once, with no -duplicates, missing features, declared target or additional columns. The field count may be -smaller than the feature count because of one-hot groups. The reader verifies -these mappings and ordering without mlschema, not historical inference outcomes. - -There is exactly one report per model output, in output order. `output` is a -zero-based integer index; every report has the model's kind and `mappedTo` equal -to that kind. A classifier report has exactly the classifier keys in the example: -its label is `Predicted class`, `showClassProbabilities` is exactly true, and -`labels` equals that output's manifest class-label array by canonical byte equality. -Types and prediction-column order are preserved. A regressor report has exactly -`kind`, `label`, `mappedTo`, `output`, with label `Predicted value` and no labels -or probability setting. This extends MLSuite's single-output report shape with -an output index and typed labels instead of display-string coercion. - -Inspection validates reports against the manifest; it cannot verify that manifest -against the model. Load also verifies native claims, so a changed manifest/report -pair cannot conceal disagreement with native kind, counts or labels. Label-free -ONNX mappings remain count-checked declarations, as in the ONNX contract. +One-hot groups have exactly `kind`, `label`, `required`, `options`, with no parent +mapping. At least two binary named columns sharing a prefix before the first +stored separator form a group. There are 2–100,000 options, each exactly +`{"label":SUFFIX,"value":SUFFIX,"mappedTo":MAPPING}`. Label/value are nonempty +strings of at most 1,024 characters and must agree. Resolve the mapping by origin; +the manifest feature at that index must equal group label + stored separator + +option value. Other fields can contain the separator; readers cannot attest +whether training values were binary. Options preserve feature order; fields +preserve the order of their first mapped feature. Non-adjacent groups are supported. + +Resolved field/option indices must cover every feature exactly once, with no +duplicates, missing features, declared target or extra columns. Groups can make +the field count smaller than the feature count. Readers check this without +mlschema; they cannot verify historical inference or the declared source. + +There is exactly one report per model output, in output order. `mappedTo` is the +strict zero-based output integer, including `0` for a single output. There is no +`output` key. Labels are `Predicted class` / `Predicted value` for one output; +multiple outputs append a space and the position, such as `Predicted class 0`. +A regressor has exactly `kind`, `label`, `mappedTo`. A classifier also requires +`labels` and `showClassProbabilities: true`. `labels` is an array of bounded +strings of at most 1,024 characters (including empty original strings), obtained by Python `str()` on that output's original labels. +Their order matches prediction columns. If two original labels in one output +have the same string form, export fails with bounded `InvalidSchemaError`. +The same string can occur in different outputs. Manifest labels retain their +original JSON types and are verified against the native model. + +Reports are recomputed through shared construction and compared by canonical +JSON bytes. Inspect checks against manifest claims; load also verifies those +claims against the native model. An internally consistent manifest/report lie +cannot conceal native disagreement. Label-free ONNX labels remain count-checked +declarations, as in its native contract. ## What leaves the training machine @@ -128,7 +156,8 @@ The member is a canonical JSON array of row objects, for example `[{"age":42,"active":true}]`. Each row must have exactly the declared feature names. JSON keys are sorted canonically; consumers use manifest feature order. An externally written empty array is valid, but public export selects N > 0. -Reader validation also rejects a declared target name that is a sample feature. +Reader validation rejects any declared target name that is a schema/sample feature; +export rejects the same overlap before publication, including synthetic defaults. Cell mappings: Python and NumPy boolean/integer/real/string scalars become JSON scalars; float32 promotes exactly to binary64. Categorical cells use their scalar diff --git a/docs/format.md b/docs/format.md index e6f4502..9d39ff7 100644 --- a/docs/format.md +++ b/docs/format.md @@ -5,7 +5,7 @@ This document defines the wire contract for `format_name = "mlbundle"` and "Must" denotes a compatibility or security requirement. This pre-release contract is implemented by the manifest, secure container, inspect/export/load, sklearn/XGBoost/CatBoost/ONNX adapters, local schema, samples and prediction-state checks. -Version 1 is not yet frozen. The inspect/verify CLI uses these public APIs. +Version 1 is not yet frozen; 0.1.0a2 requires schema version 2 and re-export of older bundles. The inspect/verify CLI uses these public APIs. The normative container rules are in [Container and resource limits](format-container.md). The normative structure, adapter and result rules are in [Adapters and structure](format-adapters.md). @@ -24,7 +24,7 @@ The normative schema/declaration rules are in [Schema and samples](format-schema | `model.ubj` | Standalone model member; XGBoost native UBJSON format. | | `model.cbm` | Standalone model member; CatBoost native CBM binary format. | | `model.onnx` | Standalone model member; original, self-contained ONNX bytes, unchanged. | -| `schema.json` | Optional; exact object with `schema_version`, `fields` and `reports` from local inference. See [Schema and samples](format-schema.md). No training DataFrame. | +| `schema.json` | Required; exact object with `schema_version: "2"`, `source`, `onehot_separator`, `fields` and `reports` from local/default inference. See [Schema and samples](format-schema.md). No training DataFrame. | | `structure.json` | Optional only for families without extractable structure; object as defined below. | | `samples.json` | Optional; array of row objects keyed by feature name, in declared feature order when consumed. Only explicit user intent may enable it. Values use the canonical JSON value rules. Samples are real user data. | @@ -131,7 +131,7 @@ its extracted names in order, even when a frame supplies training dtypes. Otherwise `dataframe` means columns from `training_data=`, and `declared` means `feature_names=` without a frame. Both are author declarations: the reader checks count/alignment, not historical truth. With neither, origin is `generated` and -names must be `feature_1` through `feature_N`. Claim extraction represents missing +names must be `feature_0` through `feature_{N-1}`. Claim extraction represents missing model feature names as null, so falsely declaring a named model's origin as `dataframe`, `declared` or `generated` also fails comparison. Model dtype extraction is independent of name origin and must be compared even when its claim is null. @@ -141,14 +141,14 @@ independent of name origin and must be compared even when its claim is null. | Field | Type | Meaning / verification | | --- | --- | --- | | `class_labels` | null or array of label arrays | Verified original labels in prediction-column order, **one array per target output**, even for a single output. Regressors use null. Each output has 1–100,000 unique labels and its length equals the corresponding class count. | -| `name` | optional nonempty string, at most 1,024 characters | Target name when known; an author declaration. | +| `names` | null or array of unique nonempty strings, each at most 1,024 characters | Required key; one target name per output in output order, or null when undeclared. Length must equal `n_outputs`; names are author declarations. | Labels are strict strings (including an empty original label), integers, finite binary64 numbers or booleans. Their types are preserved; `1`, `1.0`, `true` and `"1"` have distinct canonical bytes. NaN/infinite class labels are unsupported. String labels are at most 1,024 -characters. Do not convert labels to display strings in the bundle or silently -reorder them; a consumer may format them for display separately. +characters. Preserve manifest labels and order; schema reports use their string +forms for display, as specified in [Schema and samples](format-schema.md). `export(class_labels=...)` accepts output-major sequences, including a nested sequence for a single output. Supplied labels must use one scalar type per output, @@ -205,13 +205,13 @@ file, not who produced or trained it. `integrity` is an object mapping **every other present member** to exactly `{"sha256":HEX,"size":BYTES}`. The digest is 64 lowercase hexadecimal characters; size is an integer 0–512 MiB, further bounded by the specific member cap. -It has 1–4 keys: exactly the selected model and referenced optional members. +It has 2–5 keys: exactly the selected model member(s), schema and referenced optional members. There is no manifest hash, since the manifest cannot contain its own digest. Declared sizes must equal both actual uncompressed sizes and ZIP metadata. -`references` is a required object with optional `schema`, `structure` and -`samples` fields. Each is null/absent or respectively the literal `"schema.json"`, -`"structure.json"`, `"samples.json"`. There are no URLs or arbitrary paths. +`references` requires `schema: "schema.json"`; optional `structure` and `samples` +are null/absent or respectively `"structure.json"`, `"samples.json"`. +There are no URLs or arbitrary paths. `user_metadata` is a free JSON object for name, description, author, tags, training metrics, dataset description or other annotations. Maximum 1,024 @@ -274,23 +274,23 @@ metadata and the optional fitted-state commitment. Classifier extraction may lea carries none; the shared pipeline resolves the declaration as specified above. Library/serialization/member pairing is validated by the closed registry. Structure is compared separately using the same canonical encoding. -Schema, training dtypes, frame/declared-origin names, target name, training environment, +Schema, training dtypes, frame/declared-origin names, target names, training environment, user metadata, identity and samples are not verified against the model; they must still pass their own validation and integrity checks. Inspection verifies format and integrity only, and must never attest that model claims match the model. ## Versioning and integrity -Readers must reject unknown format versions rather than guess. Any change to +Readers must reject unknown format versions rather than guess. After format freeze, changes to required fields, accepted identifiers, member rules, claim semantics, numeric encoding or mandatory structure shapes requires a new format version and an -explicit reader implementation. Tightening caller-selected resource limits does +explicit reader implementation. Before freeze, incompatible v1 revisions require re-export. Tightening caller-selected resource limits does not change the format. Corrections to documentation that preserve wire behavior and implementation changes behind these contracts do not require a version bump. -The unreleased version 1 includes schema vocabulary, `declared` origin, +The unfrozen version 1 includes required schema v2, positional mappings, `declared` origin, prediction-state records, missing tree routing, native XGBoost routing/dropout and closed state/work rules, scalar dtype/type/function records and composition -references. It remains subject to change before format freeze; older sklearn +references. It remains subject to change before format freeze; schema v1/absent schemas and older sklearn pipeline parameter projections require re-export under this revised v1 contract. Hashes detect corruption, not deliberate tampering: an attacker can replace a diff --git a/docs/publishing.md b/docs/publishing.md index 69b8691..3910af9 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -1,7 +1,7 @@ # Manual pre-alpha publishing -Version `0.1.0a1` reserves the PyPI name `mlbundle`. Format version 1 is not -frozen, so bundles may need re-export. The security design has not had an +Version `0.1.0a2` is the next unpublished pre-alpha; `0.1.0a1` reserved the PyPI name. +Format version 1 is not frozen; schema v2 requires older bundles to be re-exported. The security design has not had an independent review; do not accept bundles from untrusted parties. The owner, or an agent on the owner's explicit request for that specific version, @@ -22,10 +22,10 @@ The owner, or an agent on the owner's explicit request, follows these steps: 3. Open [Publish to PyPI](https://github.com/UlloaSP/mlbundle/actions/workflows/publish.yml), choose **Run workflow**, select **main**, and run it once. 4. Wait for the build/CI jobs to succeed. Review the run's commit and downloaded - `distributions` artifact, containing the `0.1.0a1` wheel and source archive. + `distributions` artifact, containing the `0.1.0a2` wheel and source archive. Approve the `pypi_env` deployment when satisfied. This approval permits upload. -5. Check [mlbundle on PyPI](https://pypi.org/project/mlbundle/0.1.0a1/) and install - with `python -m pip install --pre "mlbundle[sklearn]"` on Python 3.14. +5. Check [mlbundle on PyPI](https://pypi.org/project/mlbundle/0.1.0a2/) and install + with `python -m pip install "mlbundle[sklearn,schema]==0.1.0a2"` on Python 3.14. A pending publisher does not reserve a name until a successful upload. The [PyPI pending-publisher documentation](https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc/) diff --git a/pyproject.toml b/pyproject.toml index 2e053f3..e8c54e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mlbundle" -version = "0.1.0a1" +version = "0.1.0a2" description = "Pickle-free, verifiable model bundles with a manifest, schema and structure." readme = "README.md" license = "MIT" diff --git a/scripts/package_smoke.py b/scripts/package_smoke.py index e3b4de4..12c4c5c 100644 --- a/scripts/package_smoke.py +++ b/scripts/package_smoke.py @@ -52,7 +52,7 @@ def _base(path: Path) -> None: import mlbundle - assert mlbundle.__version__ == "0.1.0a1" + assert mlbundle.__version__ == "0.1.0a2" view = mlbundle.inspect(path) assert view.manifest.model.library == "sklearn" assert "cannot attest" in _console("--help") @@ -164,7 +164,14 @@ def main() -> None: _base(args.path) else: if args.action == "sklearn": - absent = _OPTIONAL - {"sklearn", "skops", "numpy", "scipy"} + absent = _OPTIONAL - { + "sklearn", + "skops", + "numpy", + "scipy", + "pandas", + "mlschema", + } assert all(find_spec(name) is None for name in absent) _sklearn(args.path) if args.action == "create": diff --git a/src/mlbundle/_container_rules.py b/src/mlbundle/_container_rules.py index ed625ba..b584f0f 100644 --- a/src/mlbundle/_container_rules.py +++ b/src/mlbundle/_container_rules.py @@ -41,6 +41,10 @@ def check_members( present = set(names) if MemberName.MANIFEST not in present: raise InvalidContainerError("Missing manifest.json.") + if MemberName.SCHEMA not in present: + raise InvalidContainerError( + "Bundle predates the required schema.json and must be re-exported." + ) models = {pair[1] for pair in MODEL_MEMBERS.values()} selected = present & models if len(selected) != 1 and selected not in ( diff --git a/src/mlbundle/_export_manifest.py b/src/mlbundle/_export_manifest.py index a2f610c..5f92f06 100644 --- a/src/mlbundle/_export_manifest.py +++ b/src/mlbundle/_export_manifest.py @@ -1,6 +1,6 @@ """Manifest construction shared by standalone and composite export.""" -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from datetime import UTC, datetime from importlib.metadata import version from uuid import uuid4 @@ -11,7 +11,6 @@ from .contract import ( FORMAT_NAME, FORMAT_VERSION, - FeatureOrigin, Library, MemberName, SerializationFormat, @@ -26,26 +25,10 @@ def manifest_parts( serialization: SerializationFormat, *, structure: dict[str, JsonValue] | None, - feature_names: Sequence[str] | None, - training: TrainingDeclaration | None, - target_name: str | None, + training: TrainingDeclaration, + target_names: list[str] | None, metadata: Mapping[str, JsonValue] | None, ) -> dict[str, object]: - names = claims.feature_names - origin = FeatureOrigin.MODEL - if names is None: - origin = ( - FeatureOrigin.DECLARED - if feature_names is not None - else FeatureOrigin.GENERATED - ) - names = ( - list(feature_names) - if feature_names is not None - else [f"feature_{index}" for index in range(1, claims.n_features + 1)] - ) - if training is not None: - names, origin = training.names, FeatureOrigin.DATAFRAME model = claims.model_dump( mode="json", exclude={"feature_names", "feature_dtypes", "class_labels", "onnx"} ) @@ -58,21 +41,17 @@ def manifest_parts( "writer_version": version("mlbundle"), "model": model, "features": { - "names": names, - "dtypes": [None] * claims.n_features - if training is None - else training.dtypes, + "names": training.names, + "dtypes": training.dtypes, "model_dtypes": claims.feature_dtypes, - "origin": origin.value, + "origin": training.origin.value, }, - "target": {"class_labels": claims.class_labels, "name": target_name}, + "target": {"class_labels": claims.class_labels, "names": target_names}, "environment": capture_environment(onnx=claims.onnx).model_dump(mode="json"), "references": { "structure": None if structure is None else MemberName.STRUCTURE.value, - "schema": None if training is None else MemberName.SCHEMA.value, - "samples": None - if training is None or training.samples is None - else MemberName.SAMPLES.value, + "schema": MemberName.SCHEMA.value, + "samples": None if training.samples is None else MemberName.SAMPLES.value, }, "user_metadata": {} if metadata is None else dict(metadata), } diff --git a/src/mlbundle/_schema_fields.py b/src/mlbundle/_schema_fields.py index 50fd57f..e56ef94 100644 --- a/src/mlbundle/_schema_fields.py +++ b/src/mlbundle/_schema_fields.py @@ -10,12 +10,13 @@ from .limits import MAX_FEATURES FieldLabel = Annotated[str, Field(min_length=1, max_length=100)] +MappedTo = Name | Annotated[int, Field(ge=0, lt=MAX_FEATURES)] class FieldBase(ContractModel): label: FieldLabel required: bool - mappedTo: Name + mappedTo: MappedTo class SimpleField(FieldBase): @@ -52,7 +53,7 @@ class SeriesField(FieldBase): class OneHotOption(ContractModel): label: Name value: Name - mappedTo: Name + mappedTo: MappedTo class OneHotField(ContractModel): diff --git a/src/mlbundle/cli.py b/src/mlbundle/cli.py index 8483880..372041d 100644 --- a/src/mlbundle/cli.py +++ b/src/mlbundle/cli.py @@ -54,6 +54,7 @@ def _summary(view: MetadataView) -> str: f"Counts: features={model.n_features}, outputs={model.n_outputs}, classes={_display(model.n_classes)}", f"Features ({manifest.features.origin.value}): {_display(manifest.features.names)}", f"Class labels (output-major): {_display(manifest.target.class_labels)}", + f"Target names (output-major): {_display(manifest.target.names)}", ] lines.extend(_pipeline_summary(view)) lines.append("Members:") diff --git a/src/mlbundle/declarations.py b/src/mlbundle/declarations.py index 4729e05..91a2b8a 100644 --- a/src/mlbundle/declarations.py +++ b/src/mlbundle/declarations.py @@ -1,15 +1,40 @@ """Export declarations and resolution for models without native class labels.""" from collections.abc import Sequence -from typing import cast +from typing import Annotated, cast -from pydantic import ValidationError +from pydantic import Field, TypeAdapter, ValidationError -from ._base import ContractModel +from ._base import ContractModel, Name from .canonical import canonical_dumps from .claims import ClassLabels, Label, ModelClaims, validate_labels from .contract import ModelKind -from .errors import UnsupportedModelError +from .errors import InvalidSchemaError, UnsupportedModelError +from .limits import MAX_OUTPUTS + +TargetNames = Annotated[list[Name], Field(min_length=1, max_length=MAX_OUTPUTS)] + + +def validate_target_names(names: list[str] | None, n_outputs: int) -> None: + if names is not None and (len(names) != n_outputs or len(set(names)) != len(names)): + raise ValueError("Target names must be unique and match the output count.") + + +def export_target_names( + names: Sequence[str] | None, n_outputs: int +) -> list[str] | None: + if isinstance(names, (str, bytes)): + raise InvalidSchemaError("target_names requires a list of names.") + try: + values = TypeAdapter(TargetNames | None).validate_python( + None if names is None else list(names), strict=True + ) + validate_target_names(values, n_outputs) + return values + except (ValidationError, TypeError, ValueError): + raise InvalidSchemaError( + "target_names must contain one unique bounded nonempty name per output." + ) from None class ExportDeclarations(ContractModel): diff --git a/src/mlbundle/manifest.py b/src/mlbundle/manifest.py index b934a1a..303739c 100644 --- a/src/mlbundle/manifest.py +++ b/src/mlbundle/manifest.py @@ -9,7 +9,7 @@ from pydantic import Field, JsonValue, ValidationError, field_validator, model_validator -from ._base import ContractModel, Name, ShortString +from ._base import ContractModel, ShortString from ._pipeline_contract import PipelineInfo, expected_model_members from .canonical import canonical_dumps, canonical_loads from .claims import ( @@ -29,6 +29,7 @@ MemberName, SerializationFormat, ) +from .declarations import TargetNames, validate_target_names from .environment import TrainingEnvironment from .errors import ( InvalidJSONError, @@ -91,7 +92,7 @@ def aligned_features(self) -> FeatureInfo: if self.model_dtypes is not None and len(self.model_dtypes) != len(self.names): raise ValueError("Model dtypes must align with feature names.") if self.origin == FeatureOrigin.GENERATED and self.names != [ - f"feature_{index}" for index in range(1, len(self.names) + 1) + f"feature_{index}" for index in range(len(self.names)) ]: raise ValueError("Generated names must follow the format convention.") return self @@ -99,11 +100,11 @@ def aligned_features(self) -> FeatureInfo: class TargetInfo(ContractModel): class_labels: ClassLabels | None - name: Name | None = None + names: TargetNames | None class MemberReferences(ContractModel): - schema_: Literal["schema.json"] | None = Field(default=None, alias="schema") + schema_: Literal["schema.json"] = Field(alias="schema") structure: Literal["structure.json"] | None = None samples: Literal["samples.json"] | None = None @@ -157,6 +158,7 @@ def consistent_manifest(self) -> Manifest: if len(self.features.names) != self.model.n_features: raise ValueError("Feature count and names must agree.") validate_labels(self.model, self.target.class_labels) + validate_target_names(self.target.names, self.model.n_outputs) expected = expected_model_members(self.model.library, self.model.pipeline) if self.model.pipeline is not None and ( canonical_dumps(self.target.class_labels) diff --git a/src/mlbundle/model_io.py b/src/mlbundle/model_io.py index 715036e..b8526f8 100644 --- a/src/mlbundle/model_io.py +++ b/src/mlbundle/model_io.py @@ -12,13 +12,18 @@ from .claims import Label, verify_claims, verify_structure_bytes from .container import read_validated_container from .contract import Library, MemberName -from .declarations import ExportDeclarations, export_declarations, resolve_labels +from .declarations import ( + ExportDeclarations, + export_declarations, + export_target_names, + resolve_labels, +) from .errors import ClaimMismatchError from .inspection import metadata_view from .limits import ArchiveLimits from .registry import find_adapter, get_adapter, is_composite from .results import LoadedBundle, MetadataView -from .schema import read_declarations +from .schema import read_declarations, validate_separator from .training import prepare_training from .writing import write_validated_container @@ -30,8 +35,9 @@ def export( training_data: object | None = None, feature_names: Sequence[str] | None = None, class_labels: Sequence[Sequence[Label]] | None = None, - target_name: str | None = None, + target_names: Sequence[str] | None = None, sample_rows: int | None = None, + onehot_separator: str = "__", metadata: Mapping[str, JsonValue] | None = None, overwrite: bool = False, limits: ArchiveLimits | None = None, @@ -39,12 +45,15 @@ def export( """Publish a native bundle only after its claims survive serialization/reloading. Model-exposed feature names take precedence over supplied names. Otherwise - supplied names are declarations; omitted names become feature_1 … feature_N. + supplied names are declarations; omitted names become feature_0 … feature_{N-1}. Class labels are output-major declarations handled by the native adapter; omitted labels use native labels or generated encoded indices. - training_data requires mlbundle[schema] and a nonempty pandas DataFrame. + Export requires mlbundle[schema]. With no training_data, mlschema infers a + default schema from one row of 1.0 values; training dtypes remain null. + training_data must be a nonempty pandas DataFrame. Its columns must match native names and order; feature_names must agree. - With a frame, target_name must name its target column, which is excluded; + target_names declares one unique name per output, in output order. + With a frame, all target columns must exist and are excluded; otherwise the frame contains features only. Dtypes and locally inferred schema fields are declarations. Category values and nullability leave the training machine in the schema, but rows stay local by default. @@ -54,6 +63,7 @@ def export( categorical/NumPy scalars JSON scalars and infinities canonical float tags. """ selected = effective_limits(limits) + separator = validate_separator(onehot_separator) declarations = export_declarations(class_labels) composite = None if is_composite(model): @@ -63,28 +73,28 @@ def export( else: prepared = prepare_native(model, find_adapter(model), declarations, selected) claims = prepared.claims + targets = export_target_names(target_names, claims.n_outputs) training = prepare_training( training_data, claims, feature_names=feature_names, - target_name=target_name, + target_names=targets, sample_rows=sample_rows, + onehot_separator=separator, limits=selected, ) members = prepared.members structure = prepared.structure - if training is not None: - members[MemberName.SCHEMA] = training.schema - if training.samples is not None: - members[MemberName.SAMPLES] = training.samples + members[MemberName.SCHEMA] = training.schema + if training.samples is not None: + members[MemberName.SAMPLES] = training.samples parts = manifest_parts( claims, prepared.library, prepared.serialization, structure=structure, - feature_names=feature_names, training=training, - target_name=target_name, + target_names=targets, metadata=metadata, ) if composite is not None: diff --git a/src/mlbundle/schema.py b/src/mlbundle/schema.py index c8088f1..b18106d 100644 --- a/src/mlbundle/schema.py +++ b/src/mlbundle/schema.py @@ -2,41 +2,53 @@ from typing import Annotated, Literal, cast -from pydantic import Field, JsonValue, ValidationError +from pydantic import Field, JsonValue, TypeAdapter, ValidationError from ._base import ContractModel from ._schema_fields import OneHotField, SchemaField, SeriesField from .canonical import DecodedJSON, canonical_dumps -from .claims import Label, ModelClaims -from .contract import MemberName, ModelKind +from .claims import ModelClaims +from .contract import FeatureOrigin, MemberName, ModelKind from .errors import InvalidSamplesError, InvalidSchemaError from .limits import MAX_CLASSES, MAX_FEATURES, MAX_OUTPUTS, ArchiveLimits from .manifest import Manifest -ONEHOT_SEPARATOR = "__" +OneHotSeparator = Annotated[str, Field(min_length=1, max_length=16, strict=True)] +OutputPosition = Annotated[int, Field(ge=0, lt=MAX_OUTPUTS)] +ReportLabel = Annotated[str, Field(min_length=1, max_length=32)] +DisplayLabel = Annotated[str, Field(max_length=1024)] + + +def validate_separator(value: object) -> str: + try: + return TypeAdapter(OneHotSeparator).validate_python(value) + except ValidationError: + raise InvalidSchemaError( + "onehot_separator must be a nonempty string of at most 16 characters." + ) from None class ClassifierReport(ContractModel): kind: Literal["classifier"] - label: Literal["Predicted class"] - mappedTo: Literal["classifier"] - output: Annotated[int, Field(ge=0, lt=MAX_OUTPUTS)] - labels: Annotated[list[Label], Field(min_length=1, max_length=MAX_CLASSES)] + label: ReportLabel + mappedTo: OutputPosition + labels: Annotated[list[DisplayLabel], Field(min_length=1, max_length=MAX_CLASSES)] showClassProbabilities: Literal[True] class RegressorReport(ContractModel): kind: Literal["regressor"] - label: Literal["Predicted value"] - mappedTo: Literal["regressor"] - output: Annotated[int, Field(ge=0, lt=MAX_OUTPUTS)] + label: ReportLabel + mappedTo: OutputPosition Report = Annotated[ClassifierReport | RegressorReport, Field(discriminator="kind")] class SchemaDeclaration(ContractModel): - schema_version: Literal["1"] + schema_version: Literal["2"] + source: Literal["training_data", "default"] + onehot_separator: OneHotSeparator fields: Annotated[list[SchemaField], Field(min_length=1, max_length=MAX_FEATURES)] reports: Annotated[list[Report], Field(min_length=1, max_length=MAX_OUTPUTS)] @@ -44,58 +56,115 @@ class SchemaDeclaration(ContractModel): def reports_for(claims: ModelClaims) -> list[dict[str, JsonValue]]: classifier = claims.kind == ModelKind.CLASSIFIER reports: list[dict[str, JsonValue]] = [] + label = "Predicted class" if classifier else "Predicted value" for output in range(claims.n_outputs): report: dict[str, JsonValue] = { "kind": claims.kind.value, - "label": "Predicted class" if classifier else "Predicted value", - "mappedTo": claims.kind.value, - "output": output, + "label": label if claims.n_outputs == 1 else f"{label} {output}", + "mappedTo": output, } if classifier: + labels = [str(value) for value in (claims.class_labels or [])[output]] + if len(set(labels)) != len(labels): + raise InvalidSchemaError( + "Class labels must have unique string forms per output." + ) report.update( - labels=list((claims.class_labels or [])[output]), - showClassProbabilities=True, + labels=cast(list[JsonValue], labels), showClassProbabilities=True ) reports.append(report) return reports -def _field_names(field: SchemaField) -> list[str]: +def positional_fields(fields: list[dict], names: list[str]) -> None: + """Rewrite inferred name mappings, retaining mlschema's labels and grouping.""" + positions = {name: index for index, name in enumerate(names)} + + def rewrite(field: dict) -> None: + if "mappedTo" in field: + field["mappedTo"] = positions[field["mappedTo"]] + for key in ("field1", "field2"): + if key in field: + rewrite(field[key]) + for option in field.get("options", []): + if isinstance(option, dict): + rewrite(option) + + for field in fields: + rewrite(field) + + +def _mapping_index( + mapping: str | int, positions: dict[str, int], origin: FeatureOrigin +) -> int: + if origin == FeatureOrigin.MODEL: + if type(mapping) is not str: + raise ValueError + return positions[mapping] + if type(mapping) is not int or not 0 <= mapping < len(positions): + raise ValueError + return mapping + + +def _field_indices( + field: SchemaField, + names: list[str], + positions: dict[str, int], + origin: FeatureOrigin, + separator: str, +) -> list[int]: if isinstance(field, OneHotField): - names = [] + indices = [] for option in field.options: - if option.mappedTo.split(ONEHOT_SEPARATOR, 1) != [ - field.label, - option.value, - ]: + index = _mapping_index(option.mappedTo, positions, origin) + if names[index].split(separator, 1) != [field.label, option.value]: raise ValueError if option.label != option.value: raise ValueError - names.append(option.mappedTo) - return names - if field.label != field.mappedTo: + indices.append(index) + return indices + index = _mapping_index(field.mappedTo, positions, origin) + if field.label != names[index]: raise ValueError if isinstance(field, SeriesField) and any( - sub.mappedTo != field.mappedTo for sub in (field.field1, field.field2) + _mapping_index(sub.mappedTo, positions, origin) != index + for sub in (field.field1, field.field2) ): raise ValueError - return [field.mappedTo] + return [index] def validate_schema( - value: JsonValue, names: list[str], claims: ModelClaims, *, max_bytes: int + value: JsonValue, + names: list[str], + claims: ModelClaims, + *, + origin: FeatureOrigin, + target_names: list[str] | None, + max_bytes: int, ) -> dict[str, JsonValue]: """Check declarations against resolved claims, without inferring historical truth.""" + if set(target_names or []) & set(names): + raise InvalidSchemaError("Schema features cannot contain declared targets.") + if isinstance(value, dict) and value.get("schema_version") != "2": + raise InvalidSchemaError( + "Unsupported schema version; bundle must be re-exported." + ) try: declaration = SchemaDeclaration.model_validate(value) positions = {name: index for index, name in enumerate(names)} - mappings = [_field_names(field) for field in declaration.fields] - flattened = [name for group in mappings for name in group] - if len(flattened) != len(names) or set(flattened) != set(names): + mappings = [ + _field_indices( + field, names, positions, origin, declaration.onehot_separator + ) + for field in declaration.fields + ] + flattened = [index for group in mappings for index in group] + if len(flattened) != len(names) or set(flattened) != set(range(len(names))): raise ValueError - if any(group != sorted(group, key=positions.__getitem__) for group in mappings): + if any(group != sorted(group) for group in mappings): raise ValueError - first_positions = [positions[group[0]] for group in mappings] + first_positions = [group[0] for group in mappings] if first_positions != sorted(first_positions): raise ValueError schema = cast(dict[str, JsonValue], value) @@ -119,24 +188,20 @@ def validate_samples(value: JsonValue, names: list[str]) -> list[dict[str, JsonV def read_declarations( manifest: Manifest, members: dict[MemberName, DecodedJSON], limits: ArchiveLimits -) -> tuple[dict[str, JsonValue] | None, list[dict[str, JsonValue]] | None]: - schema = samples = None - if ( - MemberName.SAMPLES in members - and manifest.target.name in manifest.features.names +) -> tuple[dict[str, JsonValue], list[dict[str, JsonValue]] | None]: + samples = None + if MemberName.SAMPLES in members and set(manifest.target.names or []) & set( + manifest.features.names ): raise InvalidSamplesError("Samples cannot contain the declared target.") - if MemberName.SCHEMA in members: - if manifest.target.name in manifest.features.names: - raise InvalidSchemaError( - "Schema features cannot contain the declared target." - ) - schema = validate_schema( - members[MemberName.SCHEMA].value, - manifest.features.names, - manifest.verified_claims(), - max_bytes=limits.member_bytes(MemberName.SCHEMA), - ) + schema = validate_schema( + members[MemberName.SCHEMA].value, + manifest.features.names, + manifest.verified_claims(), + origin=manifest.features.origin, + target_names=manifest.target.names, + max_bytes=limits.member_bytes(MemberName.SCHEMA), + ) if MemberName.SAMPLES in members: samples = validate_samples( members[MemberName.SAMPLES].value, diff --git a/src/mlbundle/training.py b/src/mlbundle/training.py index acc7b7a..e93b24a 100644 --- a/src/mlbundle/training.py +++ b/src/mlbundle/training.py @@ -7,18 +7,22 @@ from importlib import import_module from typing import TYPE_CHECKING, cast +from pydantic import ValidationError + from .canonical import canonical_dumps, canonical_value from .claims import ModelClaims -from .contract import MemberName +from .contract import FeatureOrigin, MemberName from .errors import ( + InvalidManifestError, InvalidSamplesError, InvalidSchemaError, MissingOptionalDependencyError, MLBundleError, ) from .limits import ArchiveLimits +from .manifest import FeatureInfo from .samples import sample_bytes -from .schema import ONEHOT_SEPARATOR, reports_for, validate_schema +from .schema import positional_fields, reports_for, validate_schema, validate_separator if TYPE_CHECKING: from pandas import DataFrame @@ -27,16 +31,17 @@ @dataclass(frozen=True) class TrainingDeclaration: names: list[str] - dtypes: list[str] + dtypes: list[str | None] schema: bytes samples: bytes | None + origin: FeatureOrigin def _feature_frame( data: DataFrame, claims: ModelClaims, feature_names: Sequence[str] | None, - target_name: str | None, + target_names: list[str] | None, ) -> DataFrame: if data.empty: raise InvalidSchemaError("training_data must be a nonempty pandas DataFrame.") @@ -45,9 +50,12 @@ def _feature_frame( raise InvalidSchemaError("Training columns must be bounded nonempty strings.") if len(set(columns)) != len(columns): raise InvalidSchemaError("Training column names must be unique.") - if target_name is not None and target_name not in columns: - raise InvalidSchemaError("target_name must identify a training column.") - names = [name for name in columns if name != target_name] + targets = set(target_names or []) + if not targets <= set(columns): + raise InvalidSchemaError( + "Every target_names entry must identify a training column." + ) + names = [name for name in columns if name not in targets] if len(names) != claims.n_features: raise InvalidSchemaError("Training column count disagrees with the model.") if claims.feature_names is not None and names != claims.feature_names: @@ -62,35 +70,80 @@ def prepare_training( claims: ModelClaims, *, feature_names: Sequence[str] | None, - target_name: str | None, + target_names: list[str] | None, sample_rows: int | None, + onehot_separator: str, limits: ArchiveLimits, -) -> TrainingDeclaration | None: - if data is None: - if sample_rows is not None: - raise InvalidSamplesError("sample_rows requires training_data.") - return None +) -> TrainingDeclaration: + separator = validate_separator(onehot_separator) + if data is None and sample_rows is not None: + raise InvalidSamplesError("sample_rows requires training_data.") try: pd = import_module("pandas") mlschema = import_module("mlschema") except ImportError: raise MissingOptionalDependencyError("schema") from None - if not isinstance(data, pd.DataFrame): - raise InvalidSchemaError("training_data must be a nonempty pandas DataFrame.") - frame = _feature_frame(cast("DataFrame", data), claims, feature_names, target_name) - names = list(frame.columns) + dtypes: list[str | None] + if data is not None: + if not isinstance(data, pd.DataFrame): + raise InvalidSchemaError( + "training_data must be a nonempty pandas DataFrame." + ) + frame = _feature_frame( + cast("DataFrame", data), claims, feature_names, target_names + ) + names = list(frame.columns) + origin = ( + FeatureOrigin.MODEL + if claims.feature_names is not None + else FeatureOrigin.DATAFRAME + ) + dtypes = [str(dtype) for dtype in frame.dtypes] + else: + names = claims.feature_names + origin = FeatureOrigin.MODEL + if names is None: + origin = ( + FeatureOrigin.DECLARED + if feature_names is not None + else FeatureOrigin.GENERATED + ) + names = ( + list(feature_names) + if feature_names is not None + else [f"feature_{index}" for index in range(claims.n_features)] + ) + try: + FeatureInfo(names=names, origin=origin, dtypes=[None] * claims.n_features) + except ValidationError: + raise InvalidManifestError from None + columns = ( + range(claims.n_features) if origin == FeatureOrigin.GENERATED else names + ) + frame = pd.DataFrame([[1.0] * claims.n_features], columns=columns) + dtypes = [None] * claims.n_features cap = limits.member_bytes(MemberName.SCHEMA) try: - fields = mlschema.infer_schema(frame, onehot_separator=ONEHOT_SEPARATOR) + fields = mlschema.infer_schema(frame, onehot_separator=separator) + if origin in (FeatureOrigin.DATAFRAME, FeatureOrigin.DECLARED): + positional_fields(fields, names) schema = canonical_value( - {"schema_version": "1", "fields": fields, "reports": reports_for(claims)}, + { + "schema_version": "2", + "source": "default" if data is None else "training_data", + "onehot_separator": separator, + "fields": fields, + "reports": reports_for(claims), + }, max_bytes=cap, ) except MLBundleError: raise except Exception: raise InvalidSchemaError("Local mlschema inference failed.") from None - validate_schema(schema, names, claims, max_bytes=cap) + validate_schema( + schema, names, claims, origin=origin, target_names=target_names, max_bytes=cap + ) samples = ( None if sample_rows is None @@ -98,7 +151,8 @@ def prepare_training( ) return TrainingDeclaration( names=names, - dtypes=[str(dtype) for dtype in frame.dtypes], + dtypes=dtypes, schema=canonical_dumps(schema, max_bytes=cap), samples=samples, + origin=origin, ) diff --git a/tests/catboost_helpers.py b/tests/catboost_helpers.py index be71e36..4d07eb4 100644 --- a/tests/catboost_helpers.py +++ b/tests/catboost_helpers.py @@ -42,11 +42,15 @@ def regressor(loss="RMSE", **params): ).fit(x, y + 1.0) -def rewrite(path, change): +def rewrite(path, change, *, schema=False): manifest, members = read_container(path) parts = manifest.model_dump(mode="json", by_alias=True) del members[MemberName.MANIFEST] change(parts, members) + if schema: + from conftest import align_schema + + align_schema(parts, members) return write_container(parts, members) diff --git a/tests/conftest.py b/tests/conftest.py index 506b20a..8d79221 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,14 +8,51 @@ import pytest -from mlbundle import __version__ +from mlbundle import MemberName, __version__ from mlbundle.canonical import canonical_dumps from mlbundle.integrity import hash_member +def schema_for_manifest(parts): + model, features = parts["model"], parts["features"] + reports = [] + label = "Predicted class" if model["kind"] == "classifier" else "Predicted value" + for index in range(model["n_outputs"]): + report = { + "kind": model["kind"], + "label": label if model["n_outputs"] == 1 else f"{label} {index}", + "mappedTo": index, + } + if model["kind"] == "classifier": + report.update( + labels=[str(value) for value in parts["target"]["class_labels"][index]], + showClassProbabilities=True, + ) + reports.append(report) + return { + "schema_version": "2", + "source": "default", + "onehot_separator": "__", + "fields": [ + { + "kind": "text", + "label": name, + "mappedTo": name if features["origin"] == "model" else index, + "required": True, + } + for index, name in enumerate(features["names"]) + ], + "reports": reports, + } + + +def align_schema(parts, members): + members[MemberName.SCHEMA] = canonical_dumps(schema_for_manifest(parts)) + + @pytest.fixture def manifest_data() -> dict: - return { + value = { "format_name": "mlbundle", "format_version": "1", "bundle_id": "f0d87d83-8cf9-4cf5-af08-e1c885156513", @@ -38,7 +75,7 @@ def manifest_data() -> dict: "model_dtypes": None, "origin": "model", }, - "target": {"class_labels": [[1, "positive"]], "name": "outcome"}, + "target": {"class_labels": [[1, "positive"]], "names": ["outcome"]}, "environment": { "python_version": "3.14.0", "python_implementation": "CPython", @@ -49,9 +86,13 @@ def manifest_data() -> dict: "onnx": None, }, "integrity": {"model.skops": hash_member(b"native model bytes").model_dump()}, - "references": {"schema": None, "structure": None, "samples": None}, + "references": {"schema": "schema.json", "structure": None, "samples": None}, "user_metadata": {"name": "example", "metrics": {"accuracy": 0.95}}, } + value["integrity"]["schema.json"] = hash_member( + canonical_dumps(schema_for_manifest(value)) + ).model_dump() + return value def build_zip(parts, compression=0, *, archive_comment=b"", **attributes): @@ -73,6 +114,7 @@ def parts(manifest_data): return [ ("manifest.json", canonical_dumps(manifest_data)), ("model.skops", b"native model bytes"), + ("schema.json", canonical_dumps(schema_for_manifest(manifest_data))), ] diff --git a/tests/onnx_helpers.py b/tests/onnx_helpers.py index 69a595b..b35234d 100644 --- a/tests/onnx_helpers.py +++ b/tests/onnx_helpers.py @@ -121,11 +121,15 @@ def regressor(*, columns=False, batch=None): return model(nodes, inputs, [h.make_tensor_value_info("Y", T.FLOAT, [batch, 1])]) -def rewrite(path, change): +def rewrite(path, change, *, schema=False): manifest, members = read_container(path) parts = manifest.model_dump(mode="json", by_alias=True) del members[MemberName.MANIFEST] change(parts, members) + if schema: + from conftest import align_schema + + align_schema(parts, members) return write_container(parts, members) diff --git a/tests/test_catboost.py b/tests/test_catboost.py index 129884f..030bc68 100644 --- a/tests/test_catboost.py +++ b/tests/test_catboost.py @@ -122,7 +122,7 @@ def test_schema_samples_and_lowered_limits(tmp_path): mlbundle.export(model, path, training_data=x, sample_rows=2) loaded = mlbundle.load(path) assert loaded.samples == x.head(2).to_dict("records") - assert loaded.schema["reports"][0]["labels"] == [0, 1, 2] + assert loaded.schema["reports"][0]["labels"] == ["0", "1", "2"] assert set(loaded.members) == { mlbundle.MemberName.MANIFEST, mlbundle.MemberName.CBM, diff --git a/tests/test_catboost_claims.py b/tests/test_catboost_claims.py index 73d5e19..83ad0b2 100644 --- a/tests/test_catboost_claims.py +++ b/tests/test_catboost_claims.py @@ -73,7 +73,7 @@ def test_lying_manifest(change, claim, tmp_path): path = tmp_path / "model.mlbundle" mlbundle.export(classifier(), path) with pytest.raises(mlbundle.ClaimMismatchError) as error: - mlbundle.load(rewrite(path, change)) + mlbundle.load(rewrite(path, change, schema=True)) assert error.value.claim == claim @@ -99,7 +99,7 @@ def change(manifest, members): members[mlbundle.MemberName.STRUCTURE] = canonical_dumps(value) with pytest.raises(mlbundle.ClaimMismatchError, match="structure"): - mlbundle.load(rewrite(path, change)) + mlbundle.load(rewrite(path, change, schema=True)) def test_native_leaf_state_changed_with_fresh_integrity(tmp_path): @@ -112,7 +112,7 @@ def change(manifest, members): ) with pytest.raises(mlbundle.ClaimMismatchError, match="state_sha256"): - mlbundle.load(rewrite(path, change)) + mlbundle.load(rewrite(path, change, schema=True)) def test_regressor_kind_is_native(tmp_path): @@ -124,4 +124,4 @@ def change(manifest, members): manifest["target"].update(class_labels=[[0, 1]]) with pytest.raises(mlbundle.ClaimMismatchError): - mlbundle.load(rewrite(path, change)) + mlbundle.load(rewrite(path, change, schema=True)) diff --git a/tests/test_cli.py b/tests/test_cli.py index 20747f8..f0fa012 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -54,7 +54,7 @@ def test_inspect_and_verify(bundle_path, capsys): "Class labels (output-major):", "Training:", "Packages:", - "schema=no", + "schema=yes", "samples=no", ): assert text in output.out @@ -128,7 +128,7 @@ def test_io_and_rejected_bundle(command, tmp_path, capsys): def test_verify_rejects_lie_that_inspect_accepts(bundle_path, capsys): manifest, members = read_container(bundle_path) - parts = manifest.model_dump(mode="json") + parts = manifest.model_dump(mode="json", by_alias=True) parts["model"]["estimator_class"] = "secret\x1b[31m" del members[MemberName.MANIFEST] bundle_path.write_bytes(write_container(parts, members)) diff --git a/tests/test_container.py b/tests/test_container.py index b6f3fbc..08dc853 100644 --- a/tests/test_container.py +++ b/tests/test_container.py @@ -129,8 +129,8 @@ def test_corruption_and_polyglots(parts, transform): ) def test_member_set(parts, manifest_data, change): if change == "absent_reference": - manifest_data["references"]["schema"] = "schema.json" - manifest_data["integrity"]["schema.json"] = hash_member(b"{}").model_dump() + manifest_data["references"]["structure"] = "structure.json" + manifest_data["integrity"]["structure.json"] = hash_member(b"{}").model_dump() parts[0] = ("manifest.json", canonical_dumps(manifest_data)) else: parts = { @@ -168,7 +168,7 @@ def test_integrity_mismatch(parts, manifest_data, field, value): ) def test_invalid_manifest(parts, data, error): with pytest.raises(error): - read_container(build_zip([("manifest.json", data), parts[1]])) + read_container(build_zip([("manifest.json", data), *parts[1:]])) def test_unknown_field_and_noncanonical_json(parts, manifest_data): @@ -209,7 +209,7 @@ def test_optional_caps_count_and_json(parts, manifest_data, name): bundle = build_zip( [ ("manifest.json", canonical_dumps(manifest_data)), - parts[1], + *[part for part in parts[1:] if part[0] != member], (member, value), ] ) @@ -218,7 +218,9 @@ def test_optional_caps_count_and_json(parts, manifest_data, name): def test_honest_and_lying_bombs(parts): - bundle = build_zip([parts[0], ("model.skops", b"0" * 4096)], compression=8) + bundle = build_zip( + [parts[0], ("model.skops", b"0" * 4096), parts[2]], compression=8 + ) with pytest.raises(LimitExceededError): read_container(bundle, limits=ArchiveLimits(max_compression_ratio=10)) with pytest.raises(InvalidContainerError, match="Actual member size"): @@ -269,10 +271,8 @@ def test_total_compressed_size(parts): limits = ArchiveLimits(max_compressed_bytes=len(parts[0][1]) + 1) with pytest.raises(LimitExceededError): read_container(build_zip(parts), limits=limits) - assert ( - read_container(reverse_directory(build_zip(parts)))[1][MemberName.SKOPS] - == parts[1][1] - ) + actual = read_container(reverse_directory(build_zip(parts)))[1] + assert actual[MemberName.SKOPS] == parts[1][1] def test_bounded_sources(parts, tmp_path, monkeypatch): diff --git a/tests/test_environment.py b/tests/test_environment.py index ad521d3..d92f40d 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -20,7 +20,7 @@ def test_installed_environment(): def test_missing_distributions_and_onnx(monkeypatch): def version(name): if name == "mlbundle": - return "0.1.0a1" + return "0.1.0a2" raise metadata.PackageNotFoundError(name) monkeypatch.setattr(module.metadata, "version", version) @@ -31,7 +31,7 @@ def version(name): producer_version="", ) environment = module.capture_environment(onnx=graph) - assert environment.packages == {"mlbundle": "0.1.0a1"} + assert environment.packages == {"mlbundle": "0.1.0a2"} assert environment.onnx == graph diff --git a/tests/test_inspection.py b/tests/test_inspection.py index e7d3de8..d773a98 100644 --- a/tests/test_inspection.py +++ b/tests/test_inspection.py @@ -22,7 +22,9 @@ def test_inspect_and_optional_json(manifest_data): manifest_data["references"]["schema"] = "schema.json" schema = { - "schema_version": "1", + "schema_version": "2", + "source": "training_data", + "onehot_separator": "__", "fields": [ {"kind": "text", "label": name, "required": True, "mappedTo": name} for name in ("a", "b") @@ -31,9 +33,8 @@ def test_inspect_and_optional_json(manifest_data): { "kind": "classifier", "label": "Predicted class", - "mappedTo": "classifier", - "output": 0, - "labels": [1, "positive"], + "mappedTo": 0, + "labels": ["1", "positive"], "showClassProbabilities": True, } ], @@ -51,8 +52,18 @@ def test_inspect_and_optional_json(manifest_data): def test_inspect_no_model_imports(manifest_data, tmp_path): + from conftest import schema_for_manifest + from mlbundle.canonical import canonical_dumps + target = tmp_path / "model.mlbundle" - write_container(manifest_data, {MemberName.SKOPS: b"model"}, destination=target) + write_container( + manifest_data, + { + MemberName.SKOPS: b"model", + MemberName.SCHEMA: canonical_dumps(schema_for_manifest(manifest_data)), + }, + destination=target, + ) script = """ import sys from mlbundle import inspect @@ -83,5 +94,30 @@ def test_inspect_rejections(manifest_data, parts): with pytest.raises(UnsupportedFormatVersionError): inspect( - build_zip([("manifest.json", canonical_dumps(manifest_data)), parts[1]]) + build_zip([("manifest.json", canonical_dumps(manifest_data)), *parts[1:]]) ) + + +def test_load_without_schema_libraries(tmp_path): + from sklearn.tree import DecisionTreeClassifier + + import mlbundle + + path = tmp_path / "model.mlbundle" + mlbundle.export(DecisionTreeClassifier().fit([[0], [1]], ["", "positive"]), path) + script = """ +import sys +class WithoutSchema: + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] in {'pandas', 'mlschema'}: + raise ModuleNotFoundError(name=fullname) +sys.meta_path.insert(0, WithoutSchema()) +from mlbundle import load +bundle = load(sys.argv[1]) +assert bundle.predict_proba([[0], [1]]) == [[1.0, 0.0], [0.0, 1.0]] +assert bundle.schema['reports'][0]['labels'] == ['', 'positive'] +assert not {'pandas', 'mlschema'} & sys.modules.keys() +""" + subprocess.run( + [sys.executable, "-I", "-c", script, str(path)], check=True, capture_output=True + ) diff --git a/tests/test_json_reuse.py b/tests/test_json_reuse.py index 89e7eb7..b0022bc 100644 --- a/tests/test_json_reuse.py +++ b/tests/test_json_reuse.py @@ -34,7 +34,9 @@ def native(monkeypatch, manifest_data): monkeypatch.setattr(model_io, "find_adapter", lambda model: adapter) monkeypatch.setattr(model_io, "get_adapter", lambda library: adapter) schema = { - "schema_version": "1", + "schema_version": "2", + "source": "training_data", + "onehot_separator": "__", "fields": [ {"kind": "text", "label": name, "mappedTo": name, "required": True} for name in extracted.feature_names @@ -47,6 +49,7 @@ def native(monkeypatch, manifest_data): ["object", "object"], canonical_dumps(schema), canonical_dumps(samples), + mlbundle.FeatureOrigin.MODEL, ) monkeypatch.setattr(model_io, "prepare_training", lambda *args, **kw: training) return structure, schema, samples diff --git a/tests/test_loading.py b/tests/test_loading.py index 2e87597..61f29a9 100644 --- a/tests/test_loading.py +++ b/tests/test_loading.py @@ -44,11 +44,15 @@ def test_load_sources_and_native_predictions(data, tmp_path): assert loaded.structure["family"] == "decision_tree" -def _rewrite(path, change): +def _rewrite(path, change, *, schema=False): manifest, members = read_container(path) value = manifest.model_dump(mode="json", by_alias=True) del members[MemberName.MANIFEST] change(value, members) + if schema: + from conftest import align_schema + + align_schema(value, members) return write_container(value, members) @@ -102,7 +106,7 @@ def test_lying_manifest(change, claim, data, tmp_path): DecisionTreeClassifier().fit(pd.DataFrame(x, columns=list("abcd")), y), path ) with pytest.raises(ClaimMismatchError) as error: - mlbundle.load(_rewrite(path, change)) + mlbundle.load(_rewrite(path, change, schema=True)) assert error.value.claim == claim @@ -132,7 +136,7 @@ def change(_, members): members[MemberName.STRUCTURE] = canonical_dumps(structure) with pytest.raises(ClaimMismatchError, match="structure"): - mlbundle.load(_rewrite(path, change)) + mlbundle.load(_rewrite(path, change, schema=True)) def test_added_structure_and_swapped_model(data, tmp_path): @@ -226,7 +230,7 @@ def test_invalid_or_non_estimator_payload(payload, error, data, tmp_path): "member,value,error", [ (MemberName.STRUCTURE, None, ClaimMismatchError), - (MemberName.SAMPLES, [{"feature_1": 0}], None), + (MemberName.SAMPLES, [{"feature_0": 0}], None), ], ) def test_optional_members(member, value, error, tmp_path): diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 8d6ec2c..7ddec9e 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -18,7 +18,7 @@ def read(data: dict) -> Manifest: return Manifest.from_bytes(canonical_dumps(data)) -@pytest.mark.parametrize("writer_version", ["0.1.0", "0.1.0a1"]) +@pytest.mark.parametrize("writer_version", ["0.1.0", "0.1.0a2"]) def test_writer_version_is_independent_of_reader(manifest_data, writer_version): manifest_data["writer_version"] = writer_version manifest_data["environment"]["packages"]["mlbundle"] = writer_version @@ -156,9 +156,10 @@ def test_bad_labels(manifest_data, labels): def test_multioutput_and_nonmodel_features(manifest_data): manifest_data["model"].update(n_outputs=2, n_classes=[2, 3]) + manifest_data["target"]["names"] = ["first", "second"] manifest_data["target"]["class_labels"] = [[False, True], [0.0, 1.0, "other"]] manifest_data["features"].update( - names=["feature_1", "feature_2"], origin="generated" + names=["feature_0", "feature_1"], origin="generated" ) manifest = read(manifest_data) assert manifest.verified_claims().feature_names is None @@ -247,7 +248,8 @@ def test_environment_invariants_and_onnx(manifest_data): "writer_version" ] manifest_data["model"].update(library="onnx", serialization="onnx") - manifest_data["integrity"] = {"model.onnx": hash_member(b"onnx").model_dump()} + manifest_data["integrity"].pop("model.skops") + manifest_data["integrity"]["model.onnx"] = hash_member(b"onnx").model_dump() with pytest.raises(InvalidManifestError): read(manifest_data) manifest_data["environment"]["onnx"] = { diff --git a/tests/test_model_io.py b/tests/test_model_io.py index 8767afd..09b50b6 100644 --- a/tests/test_model_io.py +++ b/tests/test_model_io.py @@ -152,11 +152,14 @@ def _check_graph_mismatch(path): def _check_declared_count_mismatch(path): + from conftest import align_schema + manifest, members = read_container(path) parts = manifest.model_dump(mode="json", by_alias=True) parts["model"]["n_classes"] = [1] parts["target"]["class_labels"] = [["one"]] del members[MemberName.MANIFEST] + align_schema(parts, members) with pytest.raises(ClaimMismatchError, match="n_classes"): mlbundle.load(write_container(parts, members)) diff --git a/tests/test_onnx.py b/tests/test_onnx.py index 0c4f78c..fa5dd09 100644 --- a/tests/test_onnx.py +++ b/tests/test_onnx.py @@ -100,7 +100,7 @@ def test_regressor(columns, tmp_path): assert np.array_equal(loaded.predict(values), expected) assert loaded.feature_origin == ("model" if columns else "generated") assert loaded.feature_names == ( - ("a", "b") if columns else ("feature_1", "feature_2") + ("a", "b") if columns else ("feature_0", "feature_1") ) assert loaded.manifest.features.model_dtypes == ( ["int64", "float64"] if columns else ["float32"] * 2 diff --git a/tests/test_onnx_claims.py b/tests/test_onnx_claims.py index eaf9b70..ce67e26 100644 --- a/tests/test_onnx_claims.py +++ b/tests/test_onnx_claims.py @@ -30,7 +30,7 @@ lambda m, b: ( m["model"].update(n_features=1), m["features"].update( - names=["feature_1"], dtypes=[None], model_dtypes=["float32"] + names=["feature_0"], dtypes=[None], model_dtypes=["float32"] ), ), "n_features", @@ -88,7 +88,7 @@ def test_lying_manifest(change, claim, tmp_path): path = tmp_path / "model.mlbundle" mlbundle.export(classifier(), path) with pytest.raises(ClaimMismatchError) as error: - mlbundle.load(rewrite(path, change)) + mlbundle.load(rewrite(path, change, schema=True)) assert error.value.claim == claim @@ -97,7 +97,9 @@ def test_lying_feature_names(tmp_path): mlbundle.export(regressor(columns=True), path) with pytest.raises(ClaimMismatchError, match="feature_names"): mlbundle.load( - rewrite(path, lambda m, b: m["features"].update(names=["b", "a"])) + rewrite( + path, lambda m, b: m["features"].update(names=["b", "a"]), schema=True + ) ) @@ -128,7 +130,11 @@ def test_label_free_declarations_are_count_checked_not_authenticated(tmp_path): path = tmp_path / "model.mlbundle" mlbundle.export(softmax(), path, class_labels=[["cold", "warm"]]) result = mlbundle.load( - rewrite(path, lambda m, b: m["target"].update(class_labels=[["red", "blue"]])) + rewrite( + path, + lambda m, b: m["target"].update(class_labels=[["red", "blue"]]), + schema=True, + ) ) assert result.class_labels == [["red", "blue"]] with pytest.raises(ClaimMismatchError, match="n_classes"): @@ -139,5 +145,6 @@ def test_label_free_declarations_are_count_checked_not_authenticated(tmp_path): m["model"].update(n_classes=[1]), m["target"].update(class_labels=[["red"]]), ), + schema=True, ) ) diff --git a/tests/test_pipeline_tampering.py b/tests/test_pipeline_tampering.py index 0d5f64c..78b6040 100644 --- a/tests/test_pipeline_tampering.py +++ b/tests/test_pipeline_tampering.py @@ -7,7 +7,7 @@ import skops.io as sio import mlbundle -from conftest import build_zip +from conftest import align_schema, build_zip from mlbundle._sklearn import transformers from mlbundle.canonical import canonical_dumps, canonical_loads from mlbundle.writing import write_container @@ -130,7 +130,7 @@ def test_recomputed_component_claims(composite, path, value, claim): ], ) def test_invalid_member_candidates(composite, member_set): - record, _ = composite + record, members = composite with pytest.raises( mlbundle.InvalidContainerError, match="Invalid model member combination" ): @@ -138,6 +138,7 @@ def test_invalid_member_candidates(composite, member_set): build_zip( [ ("manifest.json", canonical_dumps(record)), + ("schema.json", members[mlbundle.MemberName.SCHEMA]), *[(name, b"native") for name in member_set], ] ) @@ -214,6 +215,8 @@ def test_labels_counts_missing_structure_and_unknown_version(composite): changed["target"]["class_labels"] = changed["model"]["pipeline"]["estimator"][ "claims" ]["class_labels"] = [["no", "yes"]] + members = dict(members) + align_schema(changed, members) with pytest.raises(mlbundle.ClaimMismatchError, match="class_labels"): mlbundle.load(write_container(changed, members)) changed = copy.deepcopy(record) diff --git a/tests/test_publish.py b/tests/test_publish.py index f36194a..75a925d 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -15,20 +15,20 @@ def preflight(): return runpy.run_path(str(SCRIPT))["check"] -@pytest.mark.parametrize("ref", ["refs/heads/develop", "refs/tags/v0.1.0a1", ""]) +@pytest.mark.parametrize("ref", ["refs/heads/develop", "refs/tags/v0.1.0a2", ""]) def test_other_refs_fail_before_network(preflight, monkeypatch, ref): def unexpected(*args, **kwargs): raise AssertionError("Network must not be used for other refs") monkeypatch.setitem(preflight.__globals__, "urlopen", unexpected) with pytest.raises(SystemExit, match="requires refs/heads/main"): - preflight(ref, {"version": "0.1.0a1"}) + preflight(ref, {"version": "0.1.0a2"}) @pytest.mark.parametrize("code", [200, 404, 403, 429, 500]) def test_exact_version_response(preflight, monkeypatch, capsys, code): def response(url, timeout): - assert url == "https://pypi.org/pypi/mlbundle/0.1.0a1/json" + assert url == "https://pypi.org/pypi/mlbundle/0.1.0a2/json" assert timeout == 30 if code != 200: raise HTTPError(url, code, "response", None, None) @@ -36,12 +36,12 @@ def response(url, timeout): monkeypatch.setitem(preflight.__globals__, "urlopen", response) if code == 404: - preflight("refs/heads/main", {"version": "0.1.0a1"}) + preflight("refs/heads/main", {"version": "0.1.0a2"}) assert "HTTP 404" in capsys.readouterr().out else: message = "already exists" if code == 200 else f"HTTP {code}" with pytest.raises(SystemExit, match=message): - preflight("refs/heads/main", {"version": "0.1.0a1"}) + preflight("refs/heads/main", {"version": "0.1.0a2"}) @pytest.mark.parametrize("error", [URLError("offline"), TimeoutError("timeout")]) @@ -51,4 +51,4 @@ def response(*args, **kwargs): monkeypatch.setitem(preflight.__globals__, "urlopen", response) with pytest.raises(SystemExit, match="PyPI version check failed"): - preflight("refs/heads/main", {"version": "0.1.0a1"}) + preflight("refs/heads/main", {"version": "0.1.0a2"}) diff --git a/tests/test_samples.py b/tests/test_samples.py index 6933ed7..c9514e7 100644 --- a/tests/test_samples.py +++ b/tests/test_samples.py @@ -67,7 +67,7 @@ def model(data): def test_privacy_without_samples(data, model, tmp_path): path = tmp_path / "private.mlbundle" - view = mlbundle.export(model, path, training_data=data, target_name="target") + view = mlbundle.export(model, path, training_data=data, target_names=["target"]) _, members = read_container(path) assert MemberName.SAMPLES not in members and view.samples is None contents = b"\n".join(members.values()) @@ -96,7 +96,7 @@ def test_privacy_without_samples(data, model, tmp_path): def test_exact_sample_selection_and_conversion(data, model, tmp_path): path = tmp_path / "samples.mlbundle" view = mlbundle.export( - model, path, training_data=data, target_name="target", sample_rows=2 + model, path, training_data=data, target_names=["target"], sample_rows=2 ) expected = [ { @@ -135,7 +135,7 @@ def test_exact_sample_selection_and_conversion(data, model, tmp_path): model, tmp_path / "full.mlbundle", training_data=data, - target_name="target", + target_names=["target"], sample_rows=3, ) assert full.samples[-1]["number"] == {"$float": "+inf"} @@ -150,7 +150,7 @@ def test_schema_structure_samples_coexist(tmp_path): model, tmp_path / "all.mlbundle", training_data=frame, - target_name="target", + target_names=["target"], sample_rows=1, ) assert len(view.members) == 5 @@ -163,7 +163,7 @@ def test_invalid_sample_count(count, data, model, tmp_path): path = tmp_path / "never.mlbundle" with pytest.raises(InvalidSamplesError, match="positive"): mlbundle.export( - model, path, training_data=data, target_name="target", sample_rows=count + model, path, training_data=data, target_names=["target"], sample_rows=count ) assert not path.exists() @@ -177,7 +177,7 @@ def test_samples_require_frame_and_respect_byte_limit(data, model, tmp_path): model, path, training_data=data, - target_name="target", + target_names=["target"], sample_rows=2, limits=mlbundle.ArchiveLimits(max_samples_bytes=20), ) @@ -192,7 +192,7 @@ def test_unsupported_sample_values(value, data, model, tmp_path): model, tmp_path / "never.mlbundle", training_data=data, - target_name="target", + target_names=["target"], sample_rows=1, ) @@ -249,7 +249,7 @@ def test_nested_dates_scalars_and_sample_depth(tmp_path): def test_lying_samples(change, data, model, tmp_path): path = tmp_path / "samples.mlbundle" mlbundle.export( - model, path, training_data=data, target_name="target", sample_rows=1 + model, path, training_data=data, target_names=["target"], sample_rows=1 ) manifest, members = read_container(path) samples = canonical_loads(members[MemberName.SAMPLES]) @@ -265,11 +265,11 @@ def test_lying_samples(change, data, model, tmp_path): def test_manifest_target_cannot_be_a_sample_feature(data, model, tmp_path): path = tmp_path / "samples.mlbundle" mlbundle.export( - model, path, training_data=data, target_name="target", sample_rows=1 + model, path, training_data=data, target_names=["target"], sample_rows=1 ) manifest, members = read_container(path) parts = manifest.model_dump(mode="json", by_alias=True) - parts["target"]["name"] = "text" + parts["target"]["names"] = ["text"] del members[MemberName.MANIFEST] bundle = write_container(parts, members) for reader in (mlbundle.inspect, mlbundle.load): @@ -292,7 +292,7 @@ def test_numpy_timestamp_precision(fraction, unit, data, model, tmp_path): ) path = tmp_path / "precise.mlbundle" view = mlbundle.export( - model, path, training_data=data, target_name="target", sample_rows=2 + model, path, training_data=data, target_names=["target"], sample_rows=2 ) assert view.samples[0]["text"] == expected assert view.samples[1]["text"] is None diff --git a/tests/test_schema.py b/tests/test_schema.py index d3f16ed..6f0d28a 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -53,22 +53,35 @@ def test_adapter_schema_roundtrip(library, kind, native_names, frame, tmp_path): ) loaded = mlbundle.load(path) assert view.schema == loaded.schema == mlbundle.inspect(path).schema - assert view.schema["schema_version"] == "1" - assert view.schema["fields"] == infer_schema(frame, onehot_separator="__") + assert view.schema["schema_version"] == "2" + assert view.schema["source"] == "training_data" + assert view.schema["onehot_separator"] == "__" assert view.samples is None and loaded.samples is None assert MemberName.SCHEMA in view.members and MemberName.SAMPLES not in view.members features = view.manifest.features assert features.names == ["a", "b"] and features.dtypes == ["int64", "float64"] named = native_names and (library != "onnx" or kind == "regressor") assert features.origin == ("model" if named else "dataframe") + fields = infer_schema(frame, onehot_separator="__") + if not named: + for index, field in enumerate(fields): + field["mappedTo"] = index + values = frame.to_numpy() + actual = ( + loaded.predict_proba(values) + if kind == "classifier" + else loaded.predict(values) + ) + assert len(actual) == len(frame) + assert view.schema["fields"] == fields assert features.model_dtypes == loaded.manifest.features.model_dtypes report = view.schema["reports"][0] - assert report["kind"] == report["mappedTo"] == kind and report["output"] == 0 + assert report["kind"] == kind and report["mappedTo"] == 0 if kind == "classifier": - assert report["labels"] == loaded.class_labels[0] + assert report["labels"] == [str(label) for label in loaded.class_labels[0]] assert report["showClassProbabilities"] is True else: - assert set(report) == {"kind", "label", "mappedTo", "output"} + assert set(report) == {"kind", "label", "mappedTo"} pd.testing.assert_frame_equal(frame, original) @@ -102,7 +115,7 @@ def test_mixed_schema_and_series(tmp_path): "date", "series", ] - assert view.schema["reports"][0]["labels"] == [False, True] + assert view.schema["reports"][0]["labels"] == ["False", "True"] assert view.manifest.features.dtypes == [str(dtype) for dtype in frame.dtypes] @@ -117,19 +130,23 @@ def test_onehot_nonadjacent_columns_and_multioutput(tmp_path): "color__red", "color__blue", ] - assert [report["output"] for report in view.schema["reports"]] == [0, 1] - assert [ - report["labels"] for report in view.schema["reports"] - ] == view.manifest.target.class_labels + assert [report["mappedTo"] for report in view.schema["reports"]] == [0, 1] + assert [report["label"] for report in view.schema["reports"]] == [ + "Predicted class 0", + "Predicted class 1", + ] + assert [report["labels"] for report in view.schema["reports"]] == [ + [str(label) for label in output] for output in view.manifest.target.class_labels + ] reg = DecisionTreeRegressor().fit(frame, [[0.0, 1.0], [1.0, 0.0]]) - assert ( - len( - mlbundle.export(reg, tmp_path / "reg.mlbundle", training_data=frame).schema[ - "reports" - ] - ) - == 2 - ) + reports = mlbundle.export( + reg, tmp_path / "reg.mlbundle", training_data=frame + ).schema["reports"] + assert [report["mappedTo"] for report in reports] == [0, 1] + assert [report["label"] for report in reports] == [ + "Predicted value 0", + "Predicted value 1", + ] @pytest.mark.parametrize( @@ -146,7 +163,7 @@ def test_onehot_nonadjacent_columns_and_multioutput(tmp_path): (lambda df: df.set_axis([0, "b"], axis=1), {}, "strings"), (lambda df: df.set_axis(["", "b"], axis=1), {}, "strings"), (lambda df: df.set_axis(["x" * 1025, "b"], axis=1), {}, "bounded"), - (lambda df: df, {"target_name": "absent"}, "target_name"), + (lambda df: df, {"target_names": ["absent"]}, "target_names"), (lambda df: df, {"feature_names": ["b", "a"]}, "feature_names"), ], ) @@ -177,23 +194,34 @@ def test_frame_origin_target_exclusion_and_missing_extra(frame, tmp_path, monkey model = DecisionTreeClassifier().fit(frame.to_numpy(), [0, 1, 0, 1]) path = tmp_path / "frame.mlbundle" data = frame.assign(target=["secret"] * 4) - view = mlbundle.export(model, path, training_data=data, target_name="target") + view = mlbundle.export(model, path, training_data=data, target_names=["target"]) assert view.manifest.features.origin == "dataframe" - assert [field["mappedTo"] for field in view.schema["fields"]] == ["a", "b"] - assert view.manifest.target.name == "target" + assert [field["mappedTo"] for field in view.schema["fields"]] == [0, 1] + assert view.manifest.target.names == ["target"] declared = mlbundle.export( model, tmp_path / "declared.mlbundle", feature_names=["a", "b"] ) - assert declared.manifest.features.origin == "declared" and declared.schema is None + assert declared.manifest.features.origin == "declared" + assert declared.schema["source"] == "default" + assert declared.manifest.features.dtypes == [None, None] + + original_import = training.import_module + for dependency in ("pandas", "mlschema"): - def missing(name): - raise ModuleNotFoundError(name=name) + def missing(name, dependency=dependency): + if name == dependency: + raise ModuleNotFoundError(name=name) + return original_import(name) - monkeypatch.setattr(training, "import_module", missing) - with pytest.raises( - mlbundle.MissingOptionalDependencyError, match=r"mlbundle\[schema\]" - ): - mlbundle.export(model, tmp_path / "missing.mlbundle", training_data=frame) + monkeypatch.setattr(training, "import_module", missing) + for data in (frame, None): + with pytest.raises( + mlbundle.MissingOptionalDependencyError, match=r"mlbundle\[schema\]" + ): + mlbundle.export( + model, tmp_path / "missing.mlbundle", training_data=data + ) + assert not (tmp_path / "missing.mlbundle").exists() def test_inference_failure_and_schema_budget(frame, tmp_path, monkeypatch): @@ -223,14 +251,27 @@ def fail(*args, **kwargs): lambda s: s["reports"][0].update(labels=["liar", "labels"]), lambda s: s["reports"][0].update(kind="regressor"), lambda s: s["reports"][0].update(output=1), + lambda s: s["reports"][0].update(mappedTo=1), + lambda s: s["reports"][0].update(mappedTo=True), + lambda s: s["reports"].append(s["reports"][0].copy()), + lambda s: s["reports"][0].update(labels=[0, 1]), + lambda s: s["reports"][0].update(label="Predicted class 0"), lambda s: s["reports"][0].update(showClassProbabilities=1), lambda s: s["fields"][0].update(mappedTo="renamed", label="renamed"), + lambda s: s["fields"][0].update(mappedTo=0), lambda s: s["fields"].append(s["fields"][0].copy()), lambda s: s["fields"].pop(), lambda s: s["fields"].reverse(), lambda s: s["fields"][0].update(required="true"), lambda s: s["fields"][0].update(defaultValue="private"), - lambda s: s.update(schema_version="2"), + lambda s: s.update(schema_version="1"), + lambda s: s.update(schema_version="99"), + lambda s: s.update(source="unknown"), + lambda s: s.pop("source"), + lambda s: s.update(onehot_separator=""), + lambda s: s.update(onehot_separator="x" * 17), + lambda s: s.update(onehot_separator=1), + lambda s: s.pop("onehot_separator"), lambda s: s.update(extra="unknown"), lambda s: s.update(fields="invalid"), lambda s: s.pop("reports"), @@ -250,27 +291,3 @@ def test_lying_schema(change, frame, tmp_path): for reader in (mlbundle.inspect, mlbundle.load): with pytest.raises(InvalidSchemaError): reader(bundle) - - -@pytest.mark.parametrize("missing", [True, False]) -def test_schema_reference_presence(missing, frame, tmp_path): - from conftest import build_zip - - path = tmp_path / "schema.mlbundle" - mlbundle.export( - DecisionTreeClassifier().fit(frame, [0, 1, 0, 1]), path, training_data=frame - ) - manifest, members = read_container(path) - parts = manifest.model_dump(mode="json", by_alias=True) - del members[MemberName.MANIFEST] - if missing: - del members[MemberName.SCHEMA] - else: - parts["references"]["schema"] = None - del parts["integrity"]["schema.json"] - with pytest.raises(mlbundle.InvalidManifestError): - write_container(parts, members) - bundle = build_zip([("manifest.json", canonical_dumps(parts)), *members.items()]) - for reader in (mlbundle.inspect, mlbundle.load): - with pytest.raises(mlbundle.InvalidContainerError): - reader(bundle) diff --git a/tests/test_schema_fields.py b/tests/test_schema_fields.py index 934da1e..6d9c9ff 100644 --- a/tests/test_schema_fields.py +++ b/tests/test_schema_fields.py @@ -1,6 +1,7 @@ """Closed field vocabulary: native inference shapes and malicious alterations.""" import copy +from typing import get_args import pandas as pd import pytest @@ -35,7 +36,7 @@ def bundle(tmp_path): ] ).fit(frame, [0, 1]) path = tmp_path / "fields.mlbundle" - view = mlbundle.export(model, path, training_data=frame, target_name=None) + view = mlbundle.export(model, path, training_data=frame, target_names=None) assert view.schema["fields"] == infer_schema(frame, onehot_separator="__") return path @@ -100,7 +101,7 @@ def test_matching_manifest_and_report_lie_still_fails_native_claims(bundle): def test_declared_target_cannot_be_a_schema_feature(bundle): manifest, members = read_container(bundle) parts = manifest.model_dump(mode="json", by_alias=True) - parts["target"]["name"] = "age" + parts["target"]["names"] = ["age"] del members[MemberName.MANIFEST] data = write_container(parts, members) for reader in (mlbundle.inspect, mlbundle.load): @@ -116,7 +117,7 @@ def test_report_comparison_uses_schema_budget(tmp_path): rng = random.Random(7) labels = [ - f"{index:07d}" + "".join(rng.choices(string.ascii_letters, k=1000)) + f"{index:07d}" + "".join(rng.choices(string.ascii_letters, k=1001)) for index in range(8) ] frame = pd.DataFrame({"a": range(8)}) @@ -129,3 +130,170 @@ def test_report_comparison_uses_schema_budget(tmp_path): assert len(canonical_dumps(view.schema["reports"], max_bytes=cap)) > MANIFEST_BYTES assert len(view.manifest.to_bytes()) < MANIFEST_BYTES assert mlbundle.inspect(path).schema == mlbundle.load(path).schema == view.schema + + +def test_field_vocabulary_matches_mlschema_models(): + from mlschema.strategies.app import builtin_kinds + from mlschema.strategies.domain.onehot_category_field import OneHotCategoryOption + + from mlbundle._schema_fields import OneHotOption, SchemaField + + ours = { + kind: set(model.model_fields) + for model in get_args(get_args(SchemaField)[0]) + for kind in get_args(model.model_fields["kind"].annotation) + } + theirs = {item.name: set(item.model.model_fields) for item in builtin_kinds()} + assert ours.keys() == theirs.keys() + for kind, keys in ours.items(): + assert keys <= theirs[kind], kind + assert set(OneHotOption.model_fields) <= set(OneHotCategoryOption.model_fields) + + +@pytest.mark.parametrize("origin", ["model", "declared", "generated"]) +def test_default_inference_and_positional_predictions(origin, tmp_path): + frame = pd.DataFrame( + {"color::red": [1.0, 0.0], "age": [20.0, 30.0], "color::blue": [0.0, 1.0]} + ) + model = DecisionTreeClassifier().fit( + frame if origin == "model" else frame.to_numpy(), [0, 1] + ) + path = tmp_path / "default.mlbundle" + view = mlbundle.export( + model, + path, + feature_names=list(frame.columns) if origin == "declared" else None, + onehot_separator="::", + ) + loaded = mlbundle.load(path) + assert view.schema == mlbundle.inspect(path).schema == loaded.schema + assert view.schema["source"] == "default" and view.samples is None + assert view.manifest.features.origin == origin + assert view.manifest.features.dtypes == [None] * 3 + columns = range(3) if origin == "generated" else frame.columns + expected = infer_schema( + pd.DataFrame([[1.0] * 3], columns=columns), onehot_separator="::" + ) + if origin == "declared": + expected[0]["options"][0]["mappedTo"] = 0 + expected[0]["options"][1]["mappedTo"] = 2 + expected[1]["mappedTo"] = 1 + assert view.schema["fields"] == expected + if origin == "generated": + assert view.manifest.features.names == ["feature_0", "feature_1", "feature_2"] + assert all(field["step"] == 0.1 for field in expected) + rows = frame if origin == "model" else frame.to_numpy() + assert loaded.predict_proba(rows) == model.predict_proba(rows).tolist() + if origin != "model": + assert loaded.predict_proba(rows.tolist()) == model.predict_proba(rows).tolist() + + +@pytest.mark.parametrize("mapping", ["color::red", True, -1, 3, 100_000, 0.0]) +def test_positional_series_and_onehot_mapping_rejections(mapping, tmp_path): + frame = pd.DataFrame( + {"color::red": [1, 0], "pair": [(1, 2), (3, 4)], "color::blue": [0, 1]} + ) + model = DecisionTreeClassifier().fit([[1, 2, 0], [0, 3, 1]], [0, 1]) + path = tmp_path / "positions.mlbundle" + view = mlbundle.export(model, path, training_data=frame, onehot_separator="::") + assert [option["mappedTo"] for option in view.schema["fields"][0]["options"]] == [ + 0, + 2, + ] + series = view.schema["fields"][1] + assert ( + series["mappedTo"] + == series["field1"]["mappedTo"] + == series["field2"]["mappedTo"] + == 1 + ) + manifest, members = read_container(path) + del members[MemberName.MANIFEST] + for target in (view.schema["fields"][0]["options"][0], series["field1"]): + original = target["mappedTo"] + target["mappedTo"] = mapping + members[MemberName.SCHEMA] = canonical_dumps(view.schema) + data = write_container(manifest, members) + for reader in (mlbundle.inspect, mlbundle.load): + with pytest.raises(InvalidSchemaError): + reader(data) + target["mappedTo"] = original + + +@pytest.mark.parametrize("separator", [None, True, 1, [], b"__", "", "x" * 17]) +def test_export_separator_bounds(separator, tmp_path): + path = tmp_path / "never.mlbundle" + with pytest.raises(InvalidSchemaError, match="onehot_separator"): + mlbundle.export(object(), path, onehot_separator=separator) + assert not path.exists() + + +def test_separator_limit_and_reader_first_group_boundary(tmp_path): + separator = "x" * 16 + frame = pd.DataFrame( + { + f"color{separator}shade{separator}red": [1, 0], + f"color{separator}shade{separator}blue": [0, 1], + } + ) + path = tmp_path / "separator.mlbundle" + view = mlbundle.export( + DecisionTreeClassifier().fit(frame, [0, 1]), + path, + training_data=frame, + onehot_separator=separator, + ) + assert mlbundle.load(path).schema == view.schema + manifest, members = read_container(path) + del members[MemberName.MANIFEST] + group = view.schema["fields"][0] + group["label"] += separator + "shade" + for option in group["options"]: + option["label"] = option["value"] = option["value"].split(separator, 1)[1] + members[MemberName.SCHEMA] = canonical_dumps(view.schema) + for reader in (mlbundle.inspect, mlbundle.load): + with pytest.raises(InvalidSchemaError): + reader(write_container(manifest, members)) + + +def test_report_string_collision_fails_before_publication(tmp_path, manifest_data): + from xgboost import XGBClassifier + + from mlbundle.schema import reports_for + + manifest_data["target"]["class_labels"] = [[1, "1"]] + with pytest.raises(InvalidSchemaError, match="unique string forms"): + reports_for( + mlbundle.Manifest.from_bytes( + canonical_dumps(manifest_data) + ).verified_claims() + ) + + path = tmp_path / "never.mlbundle" + model = XGBClassifier(n_estimators=1, n_jobs=1).fit([[0], [1]], [0, 1]) + with pytest.raises(mlbundle.UnsupportedModelError, match="class_labels"): + mlbundle.export(model, path, class_labels=[[1, "1"]]) + assert not path.exists() + + +@pytest.mark.parametrize("missing", [True, False]) +def test_schema_reference_presence(missing, tmp_path): + from conftest import build_zip + + path = tmp_path / "schema.mlbundle" + mlbundle.export(DecisionTreeClassifier().fit([[0], [1]], [0, 1]), path) + manifest, members = read_container(path) + parts = manifest.model_dump(mode="json", by_alias=True) + del members[MemberName.MANIFEST] + if missing: + del members[MemberName.SCHEMA] + else: + parts["references"]["schema"] = None + del parts["integrity"]["schema.json"] + with pytest.raises((mlbundle.InvalidManifestError, mlbundle.InvalidContainerError)): + write_container(parts, members) + bundle = build_zip([("manifest.json", canonical_dumps(parts)), *members.items()]) + error = mlbundle.InvalidContainerError if missing else mlbundle.InvalidManifestError + for reader in (mlbundle.inspect, mlbundle.load): + with pytest.raises(error, match="re-exported" if missing else "manifest"): + reader(bundle) diff --git a/tests/test_sklearn.py b/tests/test_sklearn.py index 3d4aa94..2d9a35e 100644 --- a/tests/test_sklearn.py +++ b/tests/test_sklearn.py @@ -75,7 +75,7 @@ def test_roundtrip(model, data, tmp_path): x, y = data model.fit(x, y) path = tmp_path / "model.mlbundle" - view = mlbundle.export(model, path, target_name="species", metadata={"run": 1}) + view = mlbundle.export(model, path, target_names=["species"], metadata={"run": 1}) loaded = mlbundle.load(path) assert type(loaded.native_model) is type(model) assert np.array_equal(model.predict(x), loaded.native_model.predict(x)) @@ -89,11 +89,10 @@ def test_roundtrip(model, data, tmp_path): assert canonical_dumps(loaded.structure) == canonical_dumps( adapter.extract_structure(model) ) - assert loaded.feature_names == tuple(f"feature_{i}" for i in range(1, 5)) - assert ( - loaded.manifest.target.name == "species" - and loaded.manifest.user_metadata == {"run": 1} - ) + assert loaded.feature_names == tuple(f"feature_{i}" for i in range(4)) + assert loaded.manifest.target.names == [ + "species" + ] and loaded.manifest.user_metadata == {"run": 1} @pytest.mark.parametrize( diff --git a/tests/test_targets.py b/tests/test_targets.py new file mode 100644 index 0000000..47f238d --- /dev/null +++ b/tests/test_targets.py @@ -0,0 +1,180 @@ +"""Output-major target declarations, exclusion and hostile reader records.""" + +import inspect + +import pandas as pd +import pytest +from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor + +import mlbundle +from conftest import align_schema +from mlbundle.canonical import canonical_dumps +from mlbundle.cli import main +from mlbundle.container import read_container +from mlbundle.writing import write_container + + +@pytest.mark.parametrize("kind", ["classifier", "regressor"]) +@pytest.mark.parametrize("outputs", [1, 2]) +@pytest.mark.parametrize("training", [False, True]) +def test_target_roundtrip_exclusion_and_reports( + kind, outputs, training, tmp_path, capsys +): + features = pd.DataFrame({"a": [0.0, 1.0], "b": [1.0, 0.0]}) + targets = pd.DataFrame({"first": [0, 1], "second": [10, 20]}) + names = list(targets.columns[:outputs]) + y = targets[names].to_numpy() + model_type = ( + DecisionTreeClassifier if kind == "classifier" else DecisionTreeRegressor + ) + model = model_type().fit(features, y[:, 0] if outputs == 1 else y) + frame = features.assign(**targets.to_dict("list"))[["second", "a", "first", "b"]] + if outputs == 1: + frame = frame.drop(columns="second") + path = tmp_path / "targets.mlbundle" + view = mlbundle.export( + model, + path, + target_names=tuple(names), + training_data=frame if training else None, + sample_rows=2 if training else None, + ) + loaded = mlbundle.load(path) + assert ( + view.manifest.target.names + == mlbundle.inspect(path).manifest.target.names + == loaded.manifest.target.names + == names + ) + assert loaded.feature_names == ("a", "b") + assert loaded.schema == view.schema + assert loaded.samples == (features.to_dict("records") if training else None) + label = "Predicted class" if kind == "classifier" else "Predicted value" + assert [report["label"] for report in loaded.schema["reports"]] == [ + label if outputs == 1 else f"{label} {index}" for index in range(outputs) + ] + assert [report["mappedTo"] for report in loaded.schema["reports"]] == list( + range(outputs) + ) + actual = ( + loaded.predict_proba(features) + if kind == "classifier" + else loaded.predict(features) + ) + expected = ( + model.predict_proba(features) + if kind == "classifier" + else model.predict(features) + ) + assert actual == ( + [value.tolist() for value in expected] + if outputs > 1 and kind == "classifier" + else expected.tolist() + ) + assert main(["inspect", str(path)]) == 0 + assert ( + "Target names (output-major): " + str(names).replace("'", '"') + in capsys.readouterr().out + ) + + +@pytest.mark.parametrize( + "names,outputs,match", + [ + ("target", 1, "list"), + (b"target", 1, "list"), + ([], 1, "per output"), + (["a", "b"], 1, "per output"), + (["a"], 2, "per output"), + (["same", "same"], 2, "unique"), + ([""], 1, "nonempty"), + (["x" * 1025], 1, "bounded"), + ([0], 1, "bounded"), + ([True], 1, "bounded"), + ], +) +def test_export_target_errors_do_not_publish(names, outputs, match, tmp_path): + model = DecisionTreeClassifier().fit( + [[0], [1]], [0, 1] if outputs == 1 else [[0, 10], [1, 20]] + ) + path = tmp_path / "never.mlbundle" + with pytest.raises(mlbundle.InvalidSchemaError, match=match) as error: + mlbundle.export(model, path, target_names=names) + assert len(str(error.value)) < 160 and not path.exists() + + +def test_missing_training_target_and_removed_alias(tmp_path): + frame = pd.DataFrame({"a": [0, 1], "first": [0, 1]}) + model = DecisionTreeClassifier().fit(frame[["a"]], [0, 1]) + path = tmp_path / "never.mlbundle" + with pytest.raises(mlbundle.InvalidSchemaError, match="training column"): + mlbundle.export(model, path, training_data=frame, target_names=["missing"]) + assert "target_name" not in inspect.signature(mlbundle.export).parameters + with pytest.raises(TypeError, match="target_name"): + mlbundle.export(model, path, target_name="first") + assert not path.exists() + + +@pytest.mark.parametrize("origin", ["model", "declared", "generated"]) +@pytest.mark.parametrize("overwrite", [False, True]) +def test_target_feature_overlap_preserves_destination(origin, overwrite, tmp_path): + frame = pd.DataFrame({"a": [0, 1], "b": [1, 0]}) + model = DecisionTreeClassifier().fit( + frame if origin == "model" else frame.to_numpy(), [0, 1] + ) + path = tmp_path / "never.mlbundle" + if overwrite: + path.write_bytes(b"original") + names = ["feature_0"] if origin == "generated" else ["a"] + with pytest.raises(mlbundle.InvalidSchemaError, match="targets"): + mlbundle.export( + model, + path, + target_names=names, + feature_names=["a", "b"] if origin == "declared" else None, + overwrite=overwrite, + ) + assert path.read_bytes() == b"original" if overwrite else not path.exists() + assert list(tmp_path.glob(".mlbundle-*")) == [] + + +@pytest.mark.parametrize("names", [[], ["a", "b"], "outcome", [""], ["x" * 1025], [0]]) +def test_manifest_target_bounds_and_count(names, manifest_data): + manifest_data["target"]["names"] = names + with pytest.raises(mlbundle.InvalidManifestError): + mlbundle.Manifest.from_bytes(canonical_dumps(manifest_data)) + + +def test_manifest_target_duplicates_and_old_key(manifest_data): + manifest_data["model"].update(n_outputs=2, n_classes=[2, 2]) + manifest_data["target"].update( + names=["same", "same"], class_labels=[[0, 1], [10, 20]] + ) + with pytest.raises(mlbundle.InvalidManifestError): + mlbundle.Manifest.from_bytes(canonical_dumps(manifest_data)) + manifest_data["target"].pop("names") + manifest_data["target"]["name"] = "old" + with pytest.raises(mlbundle.InvalidManifestError): + mlbundle.Manifest.from_bytes(canonical_dumps(manifest_data)) + + +@pytest.mark.parametrize("samples", [False, True]) +def test_reader_checks_every_declared_target(samples, tmp_path): + frame = pd.DataFrame({"a": [0, 1], "b": [1, 0]}) + model = DecisionTreeClassifier().fit(frame, [[0, 10], [1, 20]]) + path = tmp_path / "model.mlbundle" + mlbundle.export( + model, path, training_data=frame, sample_rows=1 if samples else None + ) + manifest, members = read_container(path) + parts = manifest.model_dump(mode="json", by_alias=True) + parts["target"]["names"] = ["first", "b"] + del members[mlbundle.MemberName.MANIFEST] + align_schema(parts, members) + bundle = write_container(parts, members) + for reader in (mlbundle.inspect, mlbundle.load): + with pytest.raises( + mlbundle.InvalidSamplesError if samples else mlbundle.InvalidSchemaError, + match="target", + ): + reader(bundle) diff --git a/tests/test_writing.py b/tests/test_writing.py index 8e42e92..4ab479a 100644 --- a/tests/test_writing.py +++ b/tests/test_writing.py @@ -44,7 +44,7 @@ def test_roundtrip_and_determinism( "producer_name": "", "producer_version": "", } - members = {member: b"dummy native bytes"} + members = {member: b"dummy native bytes", MemberName.SCHEMA: b"{}"} if optional: for name in (MemberName.SCHEMA, MemberName.STRUCTURE, MemberName.SAMPLES): members[name] = b"[]" if name == MemberName.SAMPLES else b"{}" @@ -74,7 +74,7 @@ def test_atomic_write_and_failure(manifest_data, tmp_path, monkeypatch, overwrit target = tmp_path / "model.mlbundle" args = ( Manifest.from_bytes(canonical_dumps(manifest_data)), - {MemberName.SKOPS: b"model"}, + {MemberName.SKOPS: b"model", MemberName.SCHEMA: b"{}"}, ) first = write_container(*args, destination=target) assert read_container(str(target)) == read_container(io.BytesIO(first)) @@ -102,7 +102,7 @@ def fail(*args, **kwargs): ({"../model.skops": b"x"}, InvalidContainerError), ({"mödél.skops": b"x"}, InvalidContainerError), ({}, InvalidContainerError), - ({MemberName.SKOPS: "text"}, TypeError), + ({MemberName.SKOPS: "text", MemberName.SCHEMA: b"{}"}, TypeError), ], ) def test_invalid_writer_inputs(manifest_data, members, error): @@ -129,7 +129,7 @@ def test_writer_limits_leave_destination_untouched( with pytest.raises(LimitExceededError): write_container( manifest_data, - {MemberName.SKOPS: data}, + {MemberName.SKOPS: data, MemberName.SCHEMA: b"{}"}, destination=target, overwrite=True, limits=limits, @@ -147,14 +147,20 @@ def race(temporary, destination): monkeypatch.setattr("os.link", race) with pytest.raises(FileExistsError): - write_container(manifest_data, {MemberName.SKOPS: b"model"}, destination=target) + write_container( + manifest_data, + {MemberName.SKOPS: b"model", MemberName.SCHEMA: b"{}"}, + destination=target, + ) assert target.read_bytes() == b"concurrent writer" assert list(tmp_path.glob(".mlbundle-*")) == [] def test_chunked_model_and_empty_model(manifest_data): for data in (b"", random.Random(0).randbytes(200_000)): - bundle = write_container(manifest_data, {MemberName.SKOPS: data}) + bundle = write_container( + manifest_data, {MemberName.SKOPS: data, MemberName.SCHEMA: b"{}"} + ) assert read_container(bundle)[1][MemberName.SKOPS] == data @@ -163,31 +169,33 @@ def test_chunked_model_and_empty_model(manifest_data): [ ( b"native model bytes", - "8e4bea2f61364aa5247df42588f6ad89065bcf87dc65a96d0cc185b602b27fe8", + "e6fe466bcfcc6bbc8ac7310eb4a77a21e34a0545abd7ec9bc84149ba34e29a70", ), ( random.Random(0).randbytes(200_000), - "f32df46fe39f515d09403af12a2e64bbee1fe4c579ad4229c14f187d397eb473", + "adf3678cb1037c802bcf95688065302f6c11a16a057bbe29784a6a0235a6992e", ), ( b"hello world " * 10_000, - "2cfab1749bdca7b97cfd6eba990e16054f4b7404ac8a8a2c3a818c95d04150c8", + "b5219cc6066a71271bd993e7bb89462c927372349bedaba73549df61b2ed55ad", ), ( bytes(range(256)) * 500, - "a2d903caee80044799e2640cd5672e26c2fe13d0a0eddf1a5a9d061f12b9c942", + "509b61925106409488d5420eb97145d0b07770224144c6c5101d59e74fc72b10", ), ( b"0" * 1000, - "5e92b884593597b822b7a72326b06084fe4c85ee028c164869115895f52dbc41", + "f98eb7aa07287110dd50cf6d8f04cb60e5c82ae72428c45911e7a81653d84c7a", ), ], ids=["native", "noise", "text", "uniform", "runs"], ) def test_fixed_archive_bytes_on_both_platforms(manifest_data, data, expected): - manifest_data["writer_version"] = "0.1.0a1" - manifest_data["environment"]["packages"]["mlbundle"] = "0.1.0a1" - bundle = write_container(manifest_data, {MemberName.SKOPS: data}) + manifest_data["writer_version"] = "0.1.0a2" + manifest_data["environment"]["packages"]["mlbundle"] = "0.1.0a2" + bundle = write_container( + manifest_data, {MemberName.SKOPS: data, MemberName.SCHEMA: b"{}"} + ) assert hashlib.sha256(bundle).hexdigest() == expected @@ -202,11 +210,16 @@ def test_writer_optional_member_errors(manifest_data, name): ]: with pytest.raises(error): write_container( - manifest_data, {MemberName.SKOPS: b"model", member: data}, limits=limits + manifest_data, + {MemberName.SKOPS: b"model", MemberName.SCHEMA: b"{}", member: data}, + limits=limits, ) manifest_data["references"][name] = None with pytest.raises(InvalidManifestError): - write_container(manifest_data, {MemberName.SKOPS: b"model", member: b"{}"}) + write_container( + manifest_data, + {MemberName.SKOPS: b"model", MemberName.SCHEMA: b"{}", member: b"{}"}, + ) def test_remaining_compression_budget(manifest_data, monkeypatch): @@ -223,7 +236,10 @@ def observed(data, cap): with pytest.raises(LimitExceededError): write_container( manifest_data, - {MemberName.SKOPS: random.Random(0).randbytes(2000)}, + { + MemberName.SKOPS: random.Random(0).randbytes(2000), + MemberName.SCHEMA: b"{}", + }, limits=ArchiveLimits(max_archive_bytes=2048, max_compressed_bytes=1000), ) assert len(caps) == 2 and 0 < caps[1] < caps[0] == 1000 @@ -257,18 +273,24 @@ def fail_second(fd): if race or fail_copy: with pytest.raises(FileExistsError if race else OSError): write_container( - manifest_data, {MemberName.SKOPS: b"model"}, destination=target + manifest_data, + {MemberName.SKOPS: b"model", MemberName.SCHEMA: b"{}"}, + destination=target, ) assert target.read_bytes() == b"other writer" if race else not target.exists() else: result = write_container( - manifest_data, {MemberName.SKOPS: b"model"}, destination=target + manifest_data, + {MemberName.SKOPS: b"model", MemberName.SCHEMA: b"{}"}, + destination=target, ) assert target.read_bytes() == result if os.name != "nt": assert stat.S_IMODE(target.stat().st_mode) == 0o600 with pytest.raises(FileExistsError): write_container( - manifest_data, {MemberName.SKOPS: b"model"}, destination=target + manifest_data, + {MemberName.SKOPS: b"model", MemberName.SCHEMA: b"{}"}, + destination=target, ) assert not list(tmp_path.glob(".mlbundle-*")) diff --git a/tests/test_xgboost_claims.py b/tests/test_xgboost_claims.py index 4ff052b..8d988ec 100644 --- a/tests/test_xgboost_claims.py +++ b/tests/test_xgboost_claims.py @@ -66,7 +66,7 @@ def test_lying_manifest(change, claim, xgb_data, tmp_path): class_labels=[["a", "b", "c"]], ) with pytest.raises(ClaimMismatchError) as error: - mlbundle.load(rewrite(path, change)) + mlbundle.load(rewrite(path, change, schema=True)) assert error.value.claim == claim @@ -89,7 +89,7 @@ def change(m, b): b[MemberName.STRUCTURE] = canonical_dumps(value) with pytest.raises(ClaimMismatchError, match="structure"): - mlbundle.load(rewrite(path, change)) + mlbundle.load(rewrite(path, change, schema=True)) def test_swapped_model(xgb_data, tmp_path): @@ -120,4 +120,4 @@ def change(manifest, members): manifest["target"].update(class_labels=[[False, True]]) with pytest.raises(ClaimMismatchError, match="kind"): - mlbundle.load(rewrite(path, change)) + mlbundle.load(rewrite(path, change, schema=True)) diff --git a/tests/xgboost_helpers.py b/tests/xgboost_helpers.py index 6416cc7..af1287b 100644 --- a/tests/xgboost_helpers.py +++ b/tests/xgboost_helpers.py @@ -14,11 +14,15 @@ def classifier(x, y, **params): return xgb.XGBClassifier(n_estimators=3, max_depth=2, n_jobs=1, **params).fit(x, y) -def rewrite(path, change): +def rewrite(path, change, *, schema=False): manifest, members = read_container(path) parts = manifest.model_dump(mode="json", by_alias=True) del members[MemberName.MANIFEST] change(parts, members) + if schema: + from conftest import align_schema + + align_schema(parts, members) return write_container(parts, members) diff --git a/uv.lock b/uv.lock index 251f4eb..8f895fa 100644 --- a/uv.lock +++ b/uv.lock @@ -459,7 +459,7 @@ wheels = [ [[package]] name = "mlbundle" -version = "0.1.0a1" +version = "0.1.0a2" source = { editable = "." } dependencies = [ { name = "pydantic" },