From bdd88e9c41733c9f43f8e236f12dd3bc89d53a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Ulloa=20Sant=C3=ADn?= Date: Tue, 13 Oct 2026 16:47:16 +0200 Subject: [PATCH] feat: add record predictions and native decisions --- AGENTS.md | 122 +++++------ README.md | 91 ++++---- SECURITY.md | 17 +- docs/format-container.md | 6 +- docs/format-onnx.md | 7 + docs/format-schema.md | 63 +++++- docs/format-state.md | 9 +- docs/format.md | 2 +- docs/publishing.md | 12 +- pyproject.toml | 2 +- scripts/package_smoke.py | 2 +- src/mlbundle/__init__.py | 4 + src/mlbundle/_catboost/__init__.py | 6 +- src/mlbundle/_catboost/prediction.py | 28 ++- src/mlbundle/_composite/io.py | 39 ++-- src/mlbundle/_composite/prediction.py | 9 + src/mlbundle/_display.py | 9 + src/mlbundle/_load_diagnostics.py | 28 +++ src/mlbundle/_onnx/__init__.py | 4 +- src/mlbundle/_onnx/decisions.py | 72 +++++++ src/mlbundle/_onnx/description.py | 46 ++-- src/mlbundle/_onnx/labels.py | 30 +++ src/mlbundle/_onnx/prediction.py | 56 ++++- src/mlbundle/_prediction_decisions.py | 50 +++++ src/mlbundle/_schema_fields.py | 52 ++++- src/mlbundle/_schema_inference.py | 28 +++ src/mlbundle/_sklearn/__init__.py | 6 +- src/mlbundle/_sklearn/prediction.py | 27 ++- src/mlbundle/_sklearn/record_input.py | 44 ++++ src/mlbundle/_versions.py | 30 +++ src/mlbundle/_xgboost/__init__.py | 6 +- src/mlbundle/_xgboost/prediction.py | 15 +- src/mlbundle/adapters.py | 9 +- src/mlbundle/cli.py | 10 +- src/mlbundle/errors.py | 10 +- src/mlbundle/manifest.py | 13 +- src/mlbundle/model_io.py | 45 ++-- src/mlbundle/records.py | 157 ++++++++++++++ src/mlbundle/results.py | 13 ++ src/mlbundle/schema.py | 10 +- src/mlbundle/training.py | 3 +- tests/record_helpers.py | 172 +++++++++++++++ tests/test_catboost_decisions.py | 94 ++++++++ tests/test_environment.py | 4 +- tests/test_json_reuse.py | 2 +- tests/test_manifest.py | 2 +- tests/test_model_io.py | 2 +- tests/test_onnx_decisions.py | 270 +++++++++++++++++++++++ tests/test_pipeline_decisions.py | 65 ++++++ tests/test_publish.py | 12 +- tests/test_readme.py | 14 +- tests/test_records.py | 297 ++++++++++++++++++++++++++ tests/test_schema.py | 2 +- tests/test_schema_fields.py | 34 +-- tests/test_series_schema.py | 112 ++++++++++ tests/test_sklearn_decisions.py | 88 ++++++++ tests/test_version_diagnostics.py | 166 ++++++++++++++ tests/test_writing.py | 14 +- tests/test_xgboost_decisions.py | 76 +++++++ uv.lock | 2 +- 60 files changed, 2326 insertions(+), 294 deletions(-) create mode 100644 src/mlbundle/_display.py create mode 100644 src/mlbundle/_load_diagnostics.py create mode 100644 src/mlbundle/_onnx/decisions.py create mode 100644 src/mlbundle/_onnx/labels.py create mode 100644 src/mlbundle/_prediction_decisions.py create mode 100644 src/mlbundle/_schema_inference.py create mode 100644 src/mlbundle/_sklearn/record_input.py create mode 100644 src/mlbundle/_versions.py create mode 100644 src/mlbundle/records.py create mode 100644 tests/record_helpers.py create mode 100644 tests/test_catboost_decisions.py create mode 100644 tests/test_onnx_decisions.py create mode 100644 tests/test_pipeline_decisions.py create mode 100644 tests/test_records.py create mode 100644 tests/test_series_schema.py create mode 100644 tests/test_sklearn_decisions.py create mode 100644 tests/test_version_diagnostics.py create mode 100644 tests/test_xgboost_decisions.py diff --git a/AGENTS.md b/AGENTS.md index bb89332..427cbfe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,11 +15,9 @@ Implementation claims require code and test proof. `docs/format.md` owns the wir ### 1. Reproducibility across the training/runtime boundary -MLSuite turns trained models into reviewable applications; it does not train -them. Its current model and training-DataFrame joblib uploads permit pickle code -execution, omit training versions, and send real data just to infer a schema. -mlbundle must replace that workflow with one self-describing file. Preserve the -identity and meaning of the model, schema, features, classes and outputs. +MLSuite turns trained models into reviewable applications; it does not train them. +Its runtime accepts only `.mlbundle` uploads. Preserve the identity and meaning +of the model, schema, features, classes and outputs across this boundary. ### 2. Metadata is a claim, even when it looks useful @@ -197,61 +195,60 @@ The conversation, issues, code, tests and final diff are the work record. ## How it works -The container writer accepts serialized member bytes and a manifest or its -JSON-ready parts, computes integrity records and produces a deterministic ZIP. -It enforces the same limits and validation as the container reader and publishes -paths atomically on hard-link filesystems, requiring explicit overwrite. The -exclusive-creation fallback removes partial destinations on failure but exposes -the file during copying. Public sklearn/XGBoost/CatBoost export extracts claims/structure, -serializes with skops/native UBJSON/CBM and verifies a native round trip -before publication. XGBoost supports exact XGBClassifier/XGBRegressor tree -wrappers with closed probability/regression objectives and one target. Original -class labels supplied through output-major `class_labels=` are bound to a native -copy through an mlbundle-owned booster attribute and verified on load. Omitted -labels use encoded indices; sklearn/CatBoost reject `class_labels=` because they own classes_. Every export requires the schema extra: mlschema infers from `training_data=` or one float-1.0 row, recording schema v2 `source` and configurable `onehot_separator` (default `__`). Real frames exclude every declared target and record training dtypes; default dtypes stay null. Only explicit `sample_rows=N` includes real rows. -Model-origin mappings use names; other origins use positions and generated names start at `feature_0`. `target_names` declares one unique bounded name per output; real frames exclude all target columns. Reports map by output position with unique string labels, while manifest labels retain their types. -Schema/report and sample validation is shared by export, inspect and load without schema libraries; missing schema and old schema versions require re-export. -ONNX preserves path/bytes unchanged (ModelProto is serialized once), validates -bytes once per export/load with bounded scalar scans, and rejects external tensors, -custom domains, functions and training/runtime configuration before a CPU session. It verifies graph signatures, metadata and environment -claims. `native_model` is an OnnxModel with original bytes and a CPU session. -Kind is inferred from supported outputs; native labels are verified, absent -labels use count-checked declarations. Structure is null. See the normative -ONNX section linked from `docs/format.md` for exact layouts and resource bounds. -CatBoost supports exact numeric single-target symmetric classifiers/regressors, with native state commitments and in-memory loading. Exact outer sklearn Pipelines ending in XGBoost/CatBoost use separately checked skops/native components; other native placements remain rejected. -The closed member list is: required `manifest.json`, one -native member, or declared `model.skops` + `model.ubj`/`model.cbm` pair, required `schema.json`, and -`structure.json` when the supported model has extractable structure. Optional -`samples.json` requires explicit user intent. The ceiling is six members; no others are allowed. - -The manifest must record format name/version, bundle id, UTC creation time and -writer version; model library, estimator class, classifier/regressor kind, -hyperparameters and feature/class/output counts; feature names, order, dtypes -and origin; original class labels and output-major target names; training Python, OS, -architecture and package versions; member SHA-256 hashes and sizes; and -free-form user metadata. Sample rows are allowed only on explicit request. -All of this is untrusted input, and model-derived fields must be verified. -Class counts and original labels are output-major arrays, including for a single -output. Model-exposed dtypes are verified separately from declared training dtypes. -The writer must test serialization/reloading and reject claims or structure that -cannot survive the native round trip. Default safety ceilings may only be lowered. - -The container reader validates the archive, manifest, JSON bounds and integrity -without loading a model. `inspect` exposes validated metadata and a member list -without importing model libraries; it cannot attest that model-derived claims -match the model. Public `load` uses a closed adapter and recomputes claims and -structure before returning the native model and validated predictor. - -An adapter module provides `Adapter` with library/member identity, recognition, export preparation, serialization/deserialization, `ModelClaims`, structure extraction -and a `Predictor`. Extraction uses only the native model, shared by export/load; -ONNX graph fields belong in `ModelClaims.onnx`. Missing names are null claims; -the shared pipeline owns supplied/generated names and environment declarations. -Native errors must become bounded `MLBundleError` subclasses. `prepare_export` -receives the strict `ExportDeclarations` label record and leaves the user's -object unchanged. Classifier claims may use null labels only for a native model -without labels; `declarations.py` then resolves count-validated supplied/generated -labels. ONNX can use that declaration path without changing its original bytes; -its kind/count inference remains ONNX-owned. Keep native behavior in adapters; follow `adapters.py`. +The writer accepts member bytes and manifest parts, computes integrity records, +validates through the reader and produces a deterministic ZIP. Publication uses +atomic hard links or replacement, requiring explicit overwrite. Exclusive-create +fallback removes partial destinations on failure but exposes the file while copying. +Sklearn/XGBoost/CatBoost exports verify shared claims/structure after a native +skops/UBJSON/CBM round trip before publication. Safety ceilings may only be lowered. + +Every export requires the schema extra. mlschema infers from `training_data=` or +one float-1.0 row, recording schema v2 source/separator (default `__`). Real frames +exclude every declared target and record training dtypes; default dtypes are null. +Only explicit `sample_rows=N` includes real rows. Model-origin mappings use names; +others use positions; generated names start at `feature_0`. Unique `target_names` +map by output position. Reports use string labels; manifest labels retain types. +Series pairs are adapted once to MLForm columns until mlschema emits that shape. +Export/inspect/load share schema/report/sample validation without reader-side mlschema. + +XGBoost supports exact single-target tree wrappers with closed objectives. Supplied +output-major labels bind to a native copy via an owned booster attribute and are +verified on load; omitted labels use encoded indices. Sklearn/CatBoost own classes_ +and reject `class_labels=`. CatBoost supports exact numeric single-target symmetric +models, complete native state/structure and in-memory CBM loading. Exact outer +sklearn Pipelines ending in XGBoost/CatBoost use checked skops/native components; +other native placements remain rejected. + +ONNX preserves path/bytes unchanged (ModelProto serialized once), validates once +per export/load with bounded scalar scans, and rejects external tensors, custom +domains/functions and runtime/training configuration before a CPU session. It +verifies signatures, metadata/environment claims and native labels; absent labels +use count-checked declarations. OnnxModel retains bytes/session; structure is null. +Read the normative ONNX section linked from `docs/format.md` for layouts and bounds. + +The closed member set is manifest, required schema, one native member (or declared +skops + UBJSON/CBM pair), required extractable structure, and explicit samples: +at most six members. The manifest records identity/time/writer, model metadata, +ordered features/origin/dtypes, output-major targets/classes, training environment, +member hashes/sizes and user metadata. Model-exposed dtypes are separately verified. +Every declaration is untrusted; hashes do not establish authenticity. + +Inspect validates archive/JSON bounds, manifest, integrity and schema/samples +without model imports/deserialization; it cannot attest model-derived claims. +Load uses closed adapters and shared extraction to recompute claims and structure. +`row_from_record`/`predict_record` share schema-key lookups, bounded numeric-string +and bool conversion, optional None-to-NaN handling and escaped key errors without +values. Extra keys are never iterated. Predictions are frozen output-major report +results, including the native class decision's report label (never inferred from +probabilities). Safe version hints diagnose failed manifests/skops loads without new gates. + +Adapters own recognition, preparation, serialization/loading, claims, structure +and prediction; extraction uses only native state and is shared by export/load. +`prepare_export` receives strict ExportDeclarations and leaves the user's object +unchanged. Missing native names are null; shared code owns declared/generated names +and environment. Null native classifier labels permit count-checked declarations, +including ONNX without byte changes. Native errors become bounded MLBundleErrors. +Keep native behavior in adapters; follow `adapters.py`. ## Where code lives @@ -265,6 +262,7 @@ its kind/count inference remains ONNX-owned. Keep native behavior in adapters; f - `src/mlbundle/_composite/`, `_pipeline_contract.py` — strict recipe, separate I/O, closed reconstruction/claims/prediction and input/inner boundaries. Read `docs/format-pipeline.md` when changing composites. - `src/mlbundle/training.py` — lazy local/default inference and feature origin/dtype resolution. - `src/mlbundle/schema.py` and `_schema_fields.py` — closed schema/report declarations and reader cross-checks without mlschema. +- `src/mlbundle/_schema_inference.py` — temporary mlschema-to-MLForm series adaptation. - `src/mlbundle/samples.py` — explicit feature-row selection and bounded cell conversion. - `src/mlbundle/_sklearn/` — owned trust/skops/state/parameters/structure, transformer route and closed fitted-width/name proofs. Composite commitments/native structure bind the rebuilt Pipeline; all-sklearn structure stays null. - `src/mlbundle/_xgboost/` — closed wrappers/objectives, native UBJSON preflight/loading, label binding, recoverable config, complete tree structure and validated prediction. @@ -275,7 +273,7 @@ its kind/count inference remains ONNX-owned. Keep native behavior in adapters; f - `src/mlbundle/manifest.py` — manifest sections, cross-field validation and JSON boundaries. - `src/mlbundle/integrity.py` — member SHA-256/size records and verification. - `src/mlbundle/environment.py` — training declarations and distribution metadata capture. -- `src/mlbundle/adapters.py` and `_prediction_input.py` — adapter/predictor protocols, output types and shared input checks. +- `src/mlbundle/adapters.py`, `_prediction_input.py`, `_prediction_decisions.py` — adapter/predictor protocols, input checks and bounded native class mapping. - `src/mlbundle/registry.py` — closed lazy resolution of `_sklearn`, `_xgboost`, `_catboost` and `_onnx` adapters. - `src/mlbundle/_container_rules.py` — shared source type, closed member sets/order and ceilings. - `src/mlbundle/_zip_directory.py` — bounded ZIP/ZIP64 directory and extra-field validation. @@ -283,6 +281,8 @@ its kind/count inference remains ONNX-owned. Keep native behavior in adapters; f - `src/mlbundle/container.py` — source bounds, validated in-memory reading and integrity checks. - `src/mlbundle/writing.py` — integrity ownership, deterministic ZIP and atomic publication. - `src/mlbundle/inspection.py`, `results.py` — metadata-only inspection and result records without adapter resolution. +- `src/mlbundle/records.py`, `_display.py` — record input/output and shared safe display. +- `src/mlbundle/_versions.py`, `_load_diagnostics.py` — bounded post-failure version hints. - `docs/format.md` — version 1 wire specification; linked container, adapter/structure, ONNX and schema/sample sections are normative. - `tests/` — one file per feature, with success and error cases together. - `scripts/package_smoke.py`, `publish_check.py` — artifact checks and publish preflight. diff --git a/README.md b/README.md index cc335c2..cca674a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ mlbundle replaces three gaps in separate model/DataFrame uploads: - **Training data leaving the machine:** schema inference runs locally; training rows are included only when explicitly requested as samples. -**Pre-alpha v0.1.0a2, not yet published.** Version 0.1.0a1 reserved the PyPI name. Format version 1 is +**Pre-alpha v0.1.0a3, not yet published.** Version 0.1.0a2 is on PyPI. Format version 1 is not frozen; bundles may need re-export. The security design has not had an independent review: do not accept bundles from untrusted parties. Requires **Python >=3.14**; tested on Windows/Linux. Schema requires Python 3.14. @@ -20,7 +20,7 @@ Requires **Python >=3.14**; tested on Windows/Linux. Schema requires Python 3.14 ## Install and try scikit-learn Install the pre-release with -`python -m pip install "mlbundle[sklearn,schema]==0.1.0a2"` once published. +`python -m pip install "mlbundle[sklearn,schema]==0.1.0a3"` once published. Replace `sklearn` with `xgboost`, `catboost` or `onnx`, or use `[all]` for all dependencies. Export needs the `schema` extra; reading does not. Omit the brackets for metadata-only inspection; load/verify needs the matching model extra. @@ -49,6 +49,39 @@ Native names win; otherwise origin is `declared`, `dataframe`, or `generated` (` For models without native names, pass a 2-D NumPy array or nested numeric rows in manifest feature order; declared names do not add name-based prediction. `target_names=` and `metadata=` remain author declarations. +## Runtime records + +Load with the matching model extra, then use the bundle's own record mapping: + +```python +from mlbundle import load + +bundle = load("trained.mlbundle") +record = {"0": "0.0", "1": "true"} +print(bundle.predict_record(record)[0].probabilities) # (1.0, 0.0) +assert bundle.row_from_record(record) == [0.0, 1.0] # Also usable for explanations. +``` + +Model-origin keys are native feature names. Other origins use zero-based positions +as integers or canonical decimal strings; supplying both forms is an error. +Every feature is required as a key. Extra keys are ignored without iterating the +record, so work depends on feature count rather than record size. +Values accept finite Python numbers, booleans and decimal or true/false strings. +Strings strip ASCII whitespace and are limited to 64 characters; decimal forms +include `.5`, `1.` and exponents. `None` becomes NaN only when the schema field +permits missing values, using the group's requirement for one-hot options. +`PredictionInputError` names invalid keys without echoing values. Array/DataFrame +prediction retains its existing rules. Results are frozen `OutputPrediction` +records in output order, with classifier labels/probabilities or a regression value. +`predicted_label` is the report string for the native model's own class decision, +which may differ from the largest probability. It is `None` for regressors and +ONNX classifiers without a graph label output; mlbundle never substitutes argmax. +Name-selecting sklearn pipelines use a DataFrame and require pandas (included in +`schema`); other models use a numeric row, preserving native categorical handling. +`EnvironmentMismatchError` diagnoses a failed skops load with differing sklearn +versions. `InvalidManifestError` explains safe, different writer versions on a +failed manifest check. Neither diagnostic adds a compatibility gate. + ## XGBoost Install the `xgboost` extra. Train on encoded indices and supply the original @@ -227,40 +260,16 @@ dependency rejection (`MLBundleError`), **2** usage/I/O failure. ## Supported models and limits -- **Sklearn:** fitted classifiers exposing `predict_proba` and regressors with - recoverable feature/output counts. Tested families include trees/ensembles, - linear/regularized/robust/GLM models, SVMs, PLSRegression, naive Bayes, LDA/QDA, - dense brute-force neighbors, NearestCentroid, MLP and MultiOutput/RegressorChain. - SimpleImputer/KNNImputer, scalers, KBinsDiscretizer, categorical encoders, - univariate selectors and closed NumPy FunctionTransformer functions round-trip, - including nested ColumnTransformer pipelines; see [the exact lists](https://github.com/UlloaSP/mlbundle/blob/main/docs/format-state.md). - All-sklearn Pipeline structure stays null; composite trees expose the inner boundary. See +- **Sklearn:** fitted classifiers exposing `predict_proba` and regressors with recoverable feature/output counts. Tested families include trees/ensembles, linear/regularized/robust/GLM models, SVMs, PLSRegression, naive Bayes, LDA/QDA, dense brute-force neighbors, NearestCentroid, MLP and MultiOutput/RegressorChain. SimpleImputer/KNNImputer, scalers, KBinsDiscretizer, categorical encoders, univariate selectors and closed NumPy FunctionTransformer functions round-trip, including nested ColumnTransformer pipelines; see [the exact lists](https://github.com/UlloaSP/mlbundle/blob/main/docs/format-state.md). All-sklearn Pipeline structure stays null; composite trees expose the inner boundary. See [tested configurations](https://github.com/UlloaSP/mlbundle/blob/main/tests/test_sklearn_families.py) and [core round trips/rejections](https://github.com/UlloaSP/mlbundle/blob/main/tests/test_sklearn.py). -- **Sklearn rejections:** custom estimators/subclasses, functions/types outside the - closed lists (including lambdas and user functions), compound fitted dtypes, classifiers without - probabilities, multilabel/ClassifierChain layouts, unsupported native placements, - and untrusted nested storage such as fitted sparse neighbor arrays. - HistGradientBoosting uses untrusted `TreePredictor` storage and is rejected. -- **Sklearn versions:** the closed class/attribute catalog is audited/tested - against **1.9.1 with skops 0.16.0**, not every version allowed by `>=1.6`. - Unknown state fails; export/load sklearn versions must match. Configurations - must satisfy the catalog and round trip; see [state ownership](https://github.com/UlloaSP/mlbundle/blob/main/docs/format-state.md). +- **Sklearn rejections:** custom estimators/subclasses, functions/types outside the closed lists (including lambdas and user functions), compound fitted dtypes, classifiers without probabilities, multilabel/ClassifierChain layouts, unsupported native placements, and untrusted nested storage such as fitted sparse neighbor arrays. HistGradientBoosting uses untrusted `TreePredictor` storage and is rejected. +- **Sklearn versions:** the closed class/attribute catalog is audited/tested against **1.9.1 with skops 0.16.0**, not every version allowed by `>=1.6`. Unknown state fails; matching export/load sklearn versions are recommended. Configurations must satisfy the catalog and round trip; see [state ownership](https://github.com/UlloaSP/mlbundle/blob/main/docs/format-state.md). - **XGBoost:** exact XGBClassifier/XGBRegressor, one target, native gbtree - (including XGBoost 3.4.1 dart normalized to gbtree). Closed probability/regression - objectives; raw boosters, RF wrappers, subclasses, ranking/survival/custom - objectives and unsupported storage are rejected. At most 100,000 trees/depth 14, - with complete state/dumps; see [the mapping](https://github.com/UlloaSP/mlbundle/blob/main/docs/format-adapters.md#implemented-xgboost-mapping). -- **ONNX:** fixed-width rank-two numeric inputs; one regression/probability target, - optional native labels and dynamic/one-row batches. Unsupported layouts, - external data/custom domains/functions and runtime/training configuration fail - before CPU sessions. Tested with ONNX 1.23.0/Runtime 1.30.0; see [ONNX bounds](https://github.com/UlloaSP/mlbundle/blob/main/docs/format-onnx.md). + (including XGBoost 3.4.1 dart normalized to gbtree). Closed probability/regression objectives; raw boosters, RF wrappers, subclasses, ranking/survival/custom objectives and unsupported storage are rejected. At most 100,000 trees/depth 14, with complete state/dumps; see [the mapping](https://github.com/UlloaSP/mlbundle/blob/main/docs/format-adapters.md#implemented-xgboost-mapping). +- **ONNX:** fixed-width rank-two numeric inputs; one regression/probability target, optional native labels and dynamic/one-row batches. Unsupported layouts, external data/custom domains/functions and runtime/training configuration fail before CPU sessions. Tested with ONNX 1.23.0/Runtime 1.30.0; see [ONNX bounds](https://github.com/UlloaSP/mlbundle/blob/main/docs/format-onnx.md). Archive ceilings: **6 members**, **512 MiB** archive/compressed/member bytes, -**1 GiB** aggregate uncompressed bytes, ratio **200**, JSON depth **32**. -Manifest: **8 MiB**; structure: **256 MiB**; schema/samples: **16 MiB each**. -Features/classes: at most **100,000**; outputs: **1,024** (XGBoost/CatBoost/ONNX: one). -Callers may only lower `ArchiveLimits`. Native [state](https://github.com/UlloaSP/mlbundle/blob/main/docs/format-state.md)/[ONNX](https://github.com/UlloaSP/mlbundle/blob/main/docs/format-onnx.md) -bounds do not cap process memory, prediction rows or CPU. +**1 GiB** aggregate uncompressed bytes, ratio **200**, JSON depth **32**. Manifest: **8 MiB**; structure: **256 MiB**; schema/samples: **16 MiB each**. Features/classes: at most **100,000**; outputs: **1,024** (XGBoost/CatBoost/ONNX: one). Callers may only lower `ArchiveLimits`. Native [state](https://github.com/UlloaSP/mlbundle/blob/main/docs/format-state.md)/[ONNX](https://github.com/UlloaSP/mlbundle/blob/main/docs/format-onnx.md) bounds do not cap process memory, prediction rows or CPU. ## Bundle contents and security @@ -283,18 +292,6 @@ maintained dependencies. See [SECURITY.md](https://github.com/UlloaSP/mlbundle/b ## Development -From the root: `uv python install 3.14`, `uv sync --locked --all-extras`, -`uv run pre-commit install`. Default dev installs every extra; base: `uv sync --no-dev`. -Pydantic v2 is the sole core dependency; model/schema imports are lazy. Schema uses -mlschema `>=0.2.1,<0.3` on 3.14 only (0.2.1 excludes 3.15). `uv.lock` fixes tested -versions; dependency edits require `uv sync` and committing the updated lockfile. - -Run `uv run ruff check .`, `uv run ruff format --check .`, `uv run pyright`, -`uv run pytest` and `uv build`. Tests execute every fenced README example and -write terminal/XML coverage with an 80% branch-inclusive minimum. -CI checks lint/types and tests the same built wheel on Ubuntu/Windows, including -clean base installs. The owner, or an agent on the owner's explicit request for -that version, runs manual [publishing](https://github.com/UlloaSP/mlbundle/blob/main/docs/publishing.md) from `main` with `pypi_env` approval. Read [AGENTS.md](https://github.com/UlloaSP/mlbundle/blob/main/AGENTS.md). -Agents never run or approve it on their own initiative or add publish paths, tokens or secrets. -Releases, version tags, conversions, per-model environments, signing and additional -model libraries remain outside scope. The project is MIT licensed. +From the root: `uv python install 3.14`, `uv sync --locked --all-extras`, `uv run pre-commit install`. Default dev installs every extra; base: `uv sync --no-dev`. Pydantic v2 is the sole core dependency; model/schema imports are lazy. Schema uses mlschema `>=0.2.1,<0.3` on 3.14 only (0.2.1 excludes 3.15). `uv.lock` fixes tested versions; dependency edits require `uv sync` and committing the updated lockfile. + +Run `uv run ruff check .`, `uv run ruff format --check .`, `uv run pyright`, `uv run pytest` and `uv build`. Tests execute every fenced README example and write terminal/XML coverage with an 80% branch-inclusive minimum. CI checks lint/types and tests the same built wheel on Ubuntu/Windows, including clean base installs. The owner, or an agent on the owner's explicit request for that version, runs manual [publishing](https://github.com/UlloaSP/mlbundle/blob/main/docs/publishing.md) from `main` with `pypi_env` approval. Read [AGENTS.md](https://github.com/UlloaSP/mlbundle/blob/main/AGENTS.md). Agents never run or approve it on their own initiative or add publish paths, tokens or secrets. Releases, version tags, conversions, per-model environments, signing and additional model libraries remain outside scope. The project is MIT licensed. diff --git a/SECURITY.md b/SECURITY.md index 0838459..f91dc70 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,6 +1,6 @@ # Security policy -The v0.1.0a2 development pre-alpha has not had an independent security review; +The v0.1.0a3 development pre-alpha has not had an independent security review; do not accept bundles from untrusted parties. Format version 1 is not frozen. It implements strict manifest validation, secure container reading/writing, metadata-only `inspect`, export/load, sklearn, XGBoost, CatBoost and ONNX adapters, local schema inference, @@ -270,6 +270,19 @@ manifest data within format limits. Errors are one bounded stderr line with a failure category, not bundle values. Inspect still does not attest model claims; verify requires the matching extra and performs full loading and comparison. +Record input looks up only the expected schema keys and ignores all extra keys +without iterating the record. It accepts finite Python numbers, booleans and +bounded decimal/true/false strings after ASCII whitespace stripping. `None` +becomes NaN only for schema-declared optional features. Errors escape and bound +feature names and never echo input values. These conveniences belong only to +`row_from_record`/`predict_record`; array/DataFrame validation is unchanged. +Version diagnostics use strict version text of at most 64 characters and appear +only after failure; environment declarations remain unauthenticated claims. +Classifier record results validate native class decisions against the output's +classes before returning the report label. Invalid dimensions, values or classes +raise bounded errors without values. A missing ONNX label output yields None; +mlbundle never derives a decision from probabilities. + ## Reporting a vulnerability Use [GitHub private vulnerability reporting for UlloaSP/mlbundle](https://github.com/UlloaSP/mlbundle/security/advisories/new). @@ -278,5 +291,5 @@ behavior, and the impact. Share malicious bundles or sensitive details through that private report rather than a public issue or pull request. Remove personal or confidential training data from any reproducer you submit. -Version 0.1.0a1 is published; 0.1.0a2 is not yet published. Report the affected +Version 0.1.0a2 is published; 0.1.0a3 is not yet published. Report the affected version and commit you tested. diff --git a/docs/format-container.md b/docs/format-container.md index fa0f1a9..6a1b05d 100644 --- a/docs/format-container.md +++ b/docs/format-container.md @@ -150,4 +150,8 @@ ceilings raise `LimitExceededError`. Invalid manifest JSON or typed fields raise raise `UnsupportedFormatVersionError` before validating version-specific fields. Invalid member JSON raises `InvalidJSONError`. Member hash or integrity-size disagreement raises `IntegrityMismatchError` naming the allowed member. -Messages do not include untrusted names, JSON blobs or ZIP exception text. +Manifest-validation failures may name a strict version string of at most 64 +characters when `writer_version` differs from the installed reader, explain that +format 1 is not frozen and require re-export. Absent/malformed/equal versions +retain the generic error. Inspect/load share this manifest path; valid manifests +from a different writer still pass. Messages omit JSON blobs and ZIP exception text. diff --git a/docs/format-onnx.md b/docs/format-onnx.md index 3571c62..8a2f3d4 100644 --- a/docs/format-onnx.md +++ b/docs/format-onnx.md @@ -69,6 +69,13 @@ types and native column order. Identity/Cast can preserve a classifier's probabi columns; unrepresented transformations of labeled probabilities are rejected. Final ZipMap labels own the map keys, permitting a different mapping from an upstream classifier's encoded labels. Unrelated graph nodes cannot supply labels. +Record predictions read the optional graph label tensor. When the label producer +and final probability path preserve the same columns, its native class order maps +to final ZipMap keys, including renamed strings. A graph ArgMax on the class axis +can supply encoded indices on that same path; mlbundle never computes argmax. +Unrelated heads must return known labels, and invalid results raise bounded errors. +Without a label output, `predicted_label` is None. These decisions are independent +of the probability values and do not add an array-level method to LoadedBundle. Fixed probability width determines class count; otherwise native labels determine it. Width and native labels must agree when both exist. Labels must satisfy the shared uniqueness/count/bounds rules. diff --git a/docs/format-schema.md b/docs/format-schema.md index a938c28..0e319e5 100644 --- a/docs/format-schema.md +++ b/docs/format-schema.md @@ -25,7 +25,7 @@ feature order, separate from verified `model_dtypes`. Without `training_data`, inference uses a synthetic one-row pandas DataFrame of float `1.0` values, one column per feature. Native or declared names are its columns; generated origin uses columns `range(n_features)`. mlschema supplies -all field shapes, including numeric `step: 0.1` and one-hot grouping for named +field shapes, including numeric `step: 0.1` and one-hot grouping for named binary columns. Synthetic dtypes are not training evidence: manifest training dtypes remain null. Generated names/labels are `feature_0` through `feature_{N-1}`. @@ -35,8 +35,8 @@ without builders, overrides or custom kinds. The separator is stored and read from the schema; consumers must not assume the default. Model-origin fields retain name mappings. Dataframe/declared inference preserves -named columns for grouping, then rewrites every `mappedTo`, including series -subfields and one-hot options, to its zero-based feature position. Generated +named columns for grouping, then rewrites parent/one-hot `mappedTo` values +to their zero-based feature position. Series columns have no mappings. Generated origin is inferred on positional columns directly. Labels remain feature names. For models without native names, prediction accepts 2-D NumPy arrays or nested numeric rows in manifest feature order. Adapter-supported sparse arrays and @@ -96,9 +96,24 @@ a top-level ordinary label must equal the feature name at that index. | `text`, `boolean`, `date` | None. | | `number` | `step`: finite positive number; integers infer `1.0`, floats `0.1`. | | `category` | `options`: 1–100,000 unique strings, retaining categorical order and unused categories. | -| `series` | `field1`, `field2`: scalar fields from the same feature column; no nested series/one-hot groups. Subfield labels describe pair axes; mappings resolve to the parent's index. | +| `series` | `columns`: exactly two scalar subfields, with `id` equal to `field1` then `field2`, and no `mappedTo`; no nested series/one-hot groups. Labels describe pair axes. | | `onehot-category` | Special shape below. | +In 0.1.0a3 the `series` shape changed from `field1`/`field2` to MLForm's +`columns` while format 1 is not frozen. Schema version stays `"2"`: no bundle +written before this change contains a series field, so a version bump would +invalidate every 0.1.0a2 bundle for a shape none uses. Readers reject the old shape. +The exporter adapts mlschema's pairs in one small function until mlschema emits +this shape. Parent `label`, `required` and `mappedTo` remain unchanged: + +```json +{"kind":"series","label":"pair","mappedTo":0,"required":true, + "columns":[ + {"id":"field1","kind":"number","label":"field1","required":true,"step":1.0}, + {"id":"field2","kind":"number","label":"field2","required":true,"step":1.0} + ]} +``` + One-hot groups have exactly `kind`, `label`, `required`, `options`, with no parent mapping. At least two binary named columns sharing a prefix before the first stored separator form a group. There are 2–100,000 options, each exactly @@ -132,6 +147,46 @@ claims against the native model. An internally consistent manifest/report lie cannot conceal native disagreement. Label-free ONNX labels remain count-checked declarations, as in its native contract. +## Runtime record input + +`LoadedBundle.row_from_record` returns one numeric row in feature order; +`predict_record` returns a tuple of frozen `OutputPrediction` dataclasses, one +per output, using report `mappedTo` and string labels. Classifiers return labels, +probabilities and `predicted_label`: the report string at the index of the native +model's own class decision. Regressors return a value and `predicted_label=None`. +The decision comes from sklearn estimator/Pipeline `predict`, XGBoost wrapper +`predict`, CatBoost `predict(prediction_type="Class")` (including its threshold), +or the ONNX graph's label output. Composites transform inputs and call the native +leaf. Native dimensions, scalar bounds and membership in each output's classes +are validated before mapping to report strings; malformed results raise bounded +`InvalidModelError` without values. Multi-output decisions preserve output order. +ONNX classifiers with no label output return `predicted_label=None`; no argmax +fallback is computed. Decisions may disagree with probabilities and are never +checked against them. There is no new public array-level class-decision method. +The existing predictor validates the numeric row and its output as usual. +Sklearn/composite predictors supply a one-row pandas DataFrame only when their +fitted name-based column selectors require one; missing pandas raises +`MissingOptionalDependencyError` naming the existing schema extra. Other models +receive a nested numeric row, preserving native categorical behavior. +This optional native-owned `record_input` hook does not change array/DataFrame +validation. `row_from_record` itself has no optional imports. + +Records must be mappings. Model-origin keys are feature names; every other +origin accepts integer positions or their canonical decimal strings (`"0"`, +`"12"`), never both forms for a position. Lookups touch only expected keys; +extra keys are ignored without iteration, independent of the record's size. +Missing-key errors show at most five keys, each escaped and capped at 128 +characters, followed by the count of additional missing keys. + +Finite Python int/float and bool values convert to floats. Strings strip only +ASCII whitespace and must then contain at most 64 characters: case-insensitive +`true`/`false` or `[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?`. +Parsing must yield a finite float. `None` becomes NaN only when the schema's +parent `required` is false, including one-hot options and series fields. +Other types, nonfinite numbers and invalid strings raise `PredictionInputError` +with a bounded, escaped key and reason, never the value. Array/DataFrame paths +keep their existing validation; schema nullability remains an author declaration. + ## What leaves the training machine Default mlschema inference carries feature names/labels, field kinds, nullability diff --git a/docs/format-state.md b/docs/format-state.md index 258c043..977ec78 100644 --- a/docs/format-state.md +++ b/docs/format-state.md @@ -23,7 +23,14 @@ dependency floor is not a promise of compatibility with every later version. There is no version whitelist. Other versions can introduce unknown classes or state fields and fail export/load. `_sklearn_version` also participates in the commitment: a different installed sklearn version changes reserialized state, -so full loading rejects it as `InvalidModelError`. Export and load should use +so loading may reject it as `InvalidModelError`. After a sklearn/skops +`InvalidModelError`, safe differing declared/installed versions produce +`EnvironmentMismatchError` +with both versions and re-export/load guidance. Declarations must match a strict +version pattern of at most 64 characters; otherwise the original error remains. +Successful loads remain successful: this is diagnosis, not a compatibility gate. +It also covers a composite's skops prefix, without relabeling native leaf failures. +Export and load should use the same sklearn version. Other matching-version configurations must satisfy all catalog, trust, dimensions and round-trip checks; they are not a tested range. diff --git a/docs/format.md b/docs/format.md index 9d39ff7..7a5d1ad 100644 --- a/docs/format.md +++ b/docs/format.md @@ -5,7 +5,7 @@ This document defines the wire contract for `format_name = "mlbundle"` and "Must" denotes a compatibility or security requirement. This pre-release contract is implemented by the manifest, secure container, inspect/export/load, sklearn/XGBoost/CatBoost/ONNX adapters, local schema, samples and prediction-state checks. -Version 1 is not yet frozen; 0.1.0a2 requires schema version 2 and re-export of older bundles. The inspect/verify CLI uses these public APIs. +Version 1 is not frozen. Releases 0.1.0a2/a3 require schema v2; incompatible manifest/schema shapes require re-export. Valid a2 bundles without series remain compatible. The inspect/verify CLI uses these public APIs. The normative container rules are in [Container and resource limits](format-container.md). The normative structure, adapter and result rules are in [Adapters and structure](format-adapters.md). diff --git a/docs/publishing.md b/docs/publishing.md index 3910af9..6a8071a 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -1,7 +1,9 @@ # Manual pre-alpha publishing -Version `0.1.0a2` is the next unpublished pre-alpha; `0.1.0a1` reserved the PyPI name. -Format version 1 is not frozen; schema v2 requires older bundles to be re-exported. The security design has not had an +Version `0.1.0a3` is the next unpublished pre-alpha; `0.1.0a2` is on PyPI. +Format version 1 is not frozen; incompatible manifest/schema shapes require +re-export. Valid a2 bundles remain compatible, with the series exception documented +in [the schema contract](format-schema.md). The security design has not had an independent review; do not accept bundles from untrusted parties. The owner, or an agent on the owner's explicit request for that specific version, @@ -22,10 +24,10 @@ The owner, or an agent on the owner's explicit request, follows these steps: 3. Open [Publish to PyPI](https://github.com/UlloaSP/mlbundle/actions/workflows/publish.yml), choose **Run workflow**, select **main**, and run it once. 4. Wait for the build/CI jobs to succeed. Review the run's commit and downloaded - `distributions` artifact, containing the `0.1.0a2` wheel and source archive. + `distributions` artifact, containing the `0.1.0a3` wheel and source archive. Approve the `pypi_env` deployment when satisfied. This approval permits upload. -5. Check [mlbundle on PyPI](https://pypi.org/project/mlbundle/0.1.0a2/) and install - with `python -m pip install "mlbundle[sklearn,schema]==0.1.0a2"` on Python 3.14. +5. Check [mlbundle on PyPI](https://pypi.org/project/mlbundle/0.1.0a3/) and install + with `python -m pip install "mlbundle[sklearn,schema]==0.1.0a3"` on Python 3.14. A pending publisher does not reserve a name until a successful upload. The [PyPI pending-publisher documentation](https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc/) diff --git a/pyproject.toml b/pyproject.toml index e8c54e5..4655c57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mlbundle" -version = "0.1.0a2" +version = "0.1.0a3" description = "Pickle-free, verifiable model bundles with a manifest, schema and structure." readme = "README.md" license = "MIT" diff --git a/scripts/package_smoke.py b/scripts/package_smoke.py index 12c4c5c..5578ac5 100644 --- a/scripts/package_smoke.py +++ b/scripts/package_smoke.py @@ -52,7 +52,7 @@ def _base(path: Path) -> None: import mlbundle - assert mlbundle.__version__ == "0.1.0a2" + assert mlbundle.__version__ == "0.1.0a3" view = mlbundle.inspect(path) assert view.manifest.model.library == "sklearn" assert "cannot attest" in _console("--help") diff --git a/src/mlbundle/__init__.py b/src/mlbundle/__init__.py index f063dcc..6950961 100644 --- a/src/mlbundle/__init__.py +++ b/src/mlbundle/__init__.py @@ -16,6 +16,7 @@ from .errors import ( AdapterNotImplementedError, ClaimMismatchError, + EnvironmentMismatchError, IntegrityMismatchError, InvalidContainerError, InvalidJSONError, @@ -35,6 +36,7 @@ from .limits import ArchiveLimits from .manifest import FeatureInfo, Manifest, MemberReferences, ModelInfo, TargetInfo from .model_io import export, load +from .records import OutputPrediction from .results import LoadedBundle, MetadataView __version__ = _version("mlbundle") @@ -45,6 +47,7 @@ "AdapterNotImplementedError", "ArchiveLimits", "ClaimMismatchError", + "EnvironmentMismatchError", "FeatureInfo", "FeatureOrigin", "IntegrityMismatchError", @@ -69,6 +72,7 @@ "ModelKind", "OnnxInfo", "Opset", + "OutputPrediction", "PredictionInputError", "SerializationFormat", "TargetInfo", diff --git a/src/mlbundle/_catboost/__init__.py b/src/mlbundle/_catboost/__init__.py index 0883f20..2fb4b4f 100644 --- a/src/mlbundle/_catboost/__init__.py +++ b/src/mlbundle/_catboost/__init__.py @@ -155,5 +155,7 @@ def extract_structure(self, model: Any) -> dict[str, JsonValue]: except Exception: raise InvalidModelError("Invalid CatBoost model structure.") from None - def predictor(self, model: object) -> Predictor: - return Predictor(model, self.extract_claims(model)) + def predictor(self, model: object, claims: ModelClaims | None = None) -> Predictor: + return Predictor( + model, self.extract_claims(model) if claims is None else claims + ) diff --git a/src/mlbundle/_catboost/prediction.py b/src/mlbundle/_catboost/prediction.py index a84eed0..9a56357 100644 --- a/src/mlbundle/_catboost/prediction.py +++ b/src/mlbundle/_catboost/prediction.py @@ -4,6 +4,7 @@ import numpy as np +from mlbundle._prediction_decisions import class_indices from mlbundle._prediction_input import tabular from mlbundle.adapters import ProbabilityOutput, RegressionOutput from mlbundle.claims import ModelClaims @@ -15,7 +16,9 @@ class Predictor: def __init__(self, model: Any, claims: ModelClaims) -> None: self.model, self.claims = model, claims - def _call(self, data: object, *, probabilities: bool) -> tuple[np.ndarray, int]: + def _call( + self, data: object, *, probabilities: bool, decisions: bool = False + ) -> tuple[np.ndarray, int]: try: inputs, rows = tabular(data, self.claims.n_features) if rows < 1: @@ -29,17 +32,28 @@ def _call(self, data: object, *, probabilities: bool) -> tuple[np.ndarray, int]: raise PredictionInputError( "CatBoost prediction columns must match native feature order." ) - raw = ( - self.model.predict_proba(inputs, thread_count=1) - if probabilities - else self.model.predict(inputs, thread_count=1) - ) - return np.asarray(raw, dtype=float), rows + if decisions: + raw = self.model.predict( + inputs, prediction_type="Class", thread_count=1 + ) + else: + raw = ( + self.model.predict_proba(inputs, thread_count=1) + if probabilities + else self.model.predict(inputs, thread_count=1) + ) + return np.asarray(raw, dtype=object if decisions else float), rows except MLBundleError: raise except Exception: raise InvalidModelError("CatBoost prediction failed.") from None + def predict_class_indices(self, data: object) -> list[list[int]]: + if self.claims.kind != ModelKind.CLASSIFIER: + raise ValueError("Class prediction requires a classifier.") + raw, rows = self._call(data, probabilities=False, decisions=True) + return class_indices(raw, self.claims.class_labels or [], rows) + def predict(self, data: object) -> RegressionOutput: if self.claims.kind != ModelKind.REGRESSOR: raise ValueError("predict requires a regressor.") diff --git a/src/mlbundle/_composite/io.py b/src/mlbundle/_composite/io.py index 0b914a0..ab0274f 100644 --- a/src/mlbundle/_composite/io.py +++ b/src/mlbundle/_composite/io.py @@ -1,11 +1,13 @@ """Composite orchestration; each byte member belongs to its existing loader.""" +from contextlib import nullcontext from dataclasses import dataclass from typing import Any from pydantic import JsonValue from sklearn.pipeline import Pipeline +from mlbundle._load_diagnostics import sklearn_environment from mlbundle._native_io import NativeExport, prepare_native from mlbundle._pipeline_contract import PipelineInfo from mlbundle._sklearn import transformers @@ -120,29 +122,32 @@ def load_components( members: dict[MemberName, bytes], names: list[str], verbose: bool, + declared_sklearn: str | None, ) -> tuple[Pipeline, Predictor, ModelClaims, dict[str, JsonValue]]: adapter = get_adapter(recipe.estimator.library) transformer_adapter = transformers.Adapter() leaf = adapter.deserialize(members[adapter.model_member]) - prefix = ( - None - if recipe.transformers is None - else transformer_adapter.deserialize(members[MemberName.SKOPS]) - ) + with sklearn_environment(declared_sklearn): + prefix = ( + None + if recipe.transformers is None + else transformer_adapter.deserialize(members[MemberName.SKOPS]) + ) native_claims = adapter.extract_claims(leaf) - model = rebuild(prefix, leaf, recipe.step_names, verbose) - validate_components(model) - actual = extract( - prefix, - native_claims, - adapter.library, - recipe.step_names, - verbose, - names=names, - transformer_adapter=transformer_adapter, - ) + with sklearn_environment(declared_sklearn) if prefix is not None else nullcontext(): + model = rebuild(prefix, leaf, recipe.step_names, verbose) + validate_components(model) + actual = extract( + prefix, + native_claims, + adapter.library, + recipe.step_names, + verbose, + names=names, + transformer_adapter=transformer_adapter, + ) + claims = outer_claims(prefix, native_claims, verbose) verify(recipe, actual) - claims = outer_claims(prefix, native_claims, verbose) extracted = structure(adapter.extract_structure(leaf), actual) settings = native_claims.hyperparameters.get("prediction") categorical = ( diff --git a/src/mlbundle/_composite/prediction.py b/src/mlbundle/_composite/prediction.py index ac7fe8d..b600a30 100644 --- a/src/mlbundle/_composite/prediction.py +++ b/src/mlbundle/_composite/prediction.py @@ -7,6 +7,7 @@ from mlbundle._prediction_input import tabular from mlbundle._sklearn.prediction import _string_inputs +from mlbundle._sklearn.record_input import record_input from mlbundle.adapters import ( Predictor as NativePredictor, ProbabilityOutput, @@ -30,6 +31,9 @@ def __init__( self.strings = prefix is not None and _string_inputs(prefix) self.categorical = categorical + def record_input(self, row: list[float], names: list[str]) -> object: + return record_input(self.prefix, row, names) + def _transform(self, data: Any) -> Any: # A single-step composite preserves the leaf's categorical admission. if self.prefix is None: @@ -75,3 +79,8 @@ def predict_proba(self, data: object) -> ProbabilityOutput: if self.claims.kind != ModelKind.CLASSIFIER: raise ValueError("predict_proba requires a classifier.") return self.native.predict_proba(self._transform(data)) + + def predict_class_indices(self, data: object) -> list[list[int]] | None: + if self.claims.kind != ModelKind.CLASSIFIER: + raise ValueError("Class prediction requires a classifier.") + return self.native.predict_class_indices(self._transform(data)) diff --git a/src/mlbundle/_display.py b/src/mlbundle/_display.py new file mode 100644 index 0000000..6ed312a --- /dev/null +++ b/src/mlbundle/_display.py @@ -0,0 +1,9 @@ +"""Bounded ASCII display shared by diagnostics and the CLI.""" + +import json + + +def display(value: object, *, limit: int = 512) -> str: + # ASCII JSON escapes terminal controls, bidi controls and line separators. + rendered = json.dumps(value, ensure_ascii=True) + return rendered if len(rendered) <= limit else rendered[: limit - 3] + "..." diff --git a/src/mlbundle/_load_diagnostics.py b/src/mlbundle/_load_diagnostics.py new file mode 100644 index 0000000..8e056c6 --- /dev/null +++ b/src/mlbundle/_load_diagnostics.py @@ -0,0 +1,28 @@ +"""Diagnose sklearn failures after closed loading has rejected the model.""" + +from collections.abc import Iterator +from contextlib import contextmanager +from importlib.metadata import PackageNotFoundError, version + +from ._display import display +from ._versions import safe_version +from .errors import EnvironmentMismatchError, InvalidModelError + + +@contextmanager +def sklearn_environment(declared_version: object) -> Iterator[None]: + try: + yield + except InvalidModelError: + declared = safe_version(declared_version) + try: + installed = safe_version(version("scikit-learn")) + except PackageNotFoundError: + installed = None + if declared is None or installed is None or declared == installed: + raise + raise EnvironmentMismatchError( + f"Bundle declares scikit-learn {display(declared)}; installed version is " + f"{display(installed)}. The bundle must be re-exported with scikit-learn " + f"{display(installed)} or loaded where scikit-learn {display(declared)} is installed." + ) from None diff --git a/src/mlbundle/_onnx/__init__.py b/src/mlbundle/_onnx/__init__.py index 37bf0cb..badcbc9 100644 --- a/src/mlbundle/_onnx/__init__.py +++ b/src/mlbundle/_onnx/__init__.py @@ -111,5 +111,5 @@ def extract_claims(self, model: Any) -> ModelClaims: def extract_structure(self, model: object) -> dict[str, JsonValue] | None: # noqa: ARG002 return None - def predictor(self, model: Any) -> Predictor: - return Predictor(model.session, self._describe(model.data)) + def predictor(self, model: Any, claims: ModelClaims | None = None) -> Predictor: + return Predictor(model.session, self._describe(model.data), claims) diff --git a/src/mlbundle/_onnx/decisions.py b/src/mlbundle/_onnx/decisions.py new file mode 100644 index 0000000..49e1f4e --- /dev/null +++ b/src/mlbundle/_onnx/decisions.py @@ -0,0 +1,72 @@ +"""Graph-owned label positions, including final ZipMap renaming.""" + +from typing import Any + +from mlbundle.claims import ClassLabels +from mlbundle.errors import MLBundleError + +from .labels import LABEL_OPS, labels_for + +_LABEL_PRESERVING = frozenset({"Identity", "Cast", "Reshape", "Squeeze", "Unsqueeze"}) + + +def _label_node(producers: dict, output: str, steps: int) -> Any: + for _ in range(steps): + node = producers.get(output) + if node is None: + return None + if ( + node.domain in ("", "ai.onnx") + and node.op_type in _LABEL_PRESERVING + and node.input + ): + output = node.input[0] + else: + return node, output + return None + + +def _same_columns(producers: dict, output: str, source: str, steps: int) -> bool: + for _ in range(steps + 1): + if output == source: + return True + node = producers.get(output) + if node is None or len(node.input) != 1: + return False + if not ( + (node.domain in ("", "ai.onnx") and node.op_type in ("Identity", "Cast")) + or (node.domain == "ai.onnx.ml" and node.op_type == "ZipMap") + ): + return False + output = node.input[0] + return False + + +def decision_labels( + graph: Any, output: str, probability: str, count: int +) -> ClassLabels | None: + producers = {name: node for node in graph.node for name in node.output if name} + steps = len(graph.node) + found = _label_node(producers, output, steps) + if found is None: + return None + node, source_output = found + if node.domain == "ai.onnx.ml" and node.op_type in LABEL_OPS - {"ZipMap"}: + if ( + len(node.output) != 2 + or source_output != node.output[0] + or not _same_columns(producers, probability, node.output[1], steps) + ): + return None + try: + return labels_for([node]) + except (MLBundleError, UnicodeError): + # Unrecoverable identities still undergo ordinary known-class checks. + return None + if node.domain in ("", "ai.onnx") and node.op_type == "ArgMax" and node.input: + axis = next((attr.i for attr in node.attribute if attr.name == "axis"), 0) + if axis in (1, -1) and _same_columns( + producers, probability, node.input[0], steps + ): + return [list(range(count))] + return None diff --git a/src/mlbundle/_onnx/description.py b/src/mlbundle/_onnx/description.py index fd78cd4..bd1a44f 100644 --- a/src/mlbundle/_onnx/description.py +++ b/src/mlbundle/_onnx/description.py @@ -12,12 +12,10 @@ from mlbundle.errors import InvalidModelError, LimitExceededError from mlbundle.limits import HYPERPARAMETER_BYTES, MAX_CLASSES +from .decisions import decision_labels +from .labels import LABEL_OPS, labels_for as _labels from .signatures import _batch, _inputs, signature -LABEL_OPS = frozenset( - {"ZipMap", "LinearClassifier", "TreeEnsembleClassifier", "SVMClassifier"} -) - @dataclass(frozen=True) class Description: @@ -25,6 +23,8 @@ class Description: inputs: tuple[dict[str, Any], ...] output: str maps: bool + decision_output: dict[str, Any] | None = None + decision_labels: ClassLabels | None = None def _ancestors(graph: Any, output: str) -> list[Any]: @@ -42,25 +42,6 @@ def _ancestors(graph: Any, output: str) -> list[Any]: return result -def _labels(nodes: list[Any]) -> ClassLabels | None: - found = [] - for node in nodes: - if node.domain != "ai.onnx.ml" or node.op_type not in LABEL_OPS: - continue - for attr in node.attribute: - if attr.name == "classlabels_strings": - found.append([value.decode("utf-8") for value in attr.strings]) - elif attr.name in ("classlabels_ints", "classlabels_int64s"): - found.append(list(attr.ints)) - if not found: - return None - if any( - canonical_dumps(labels) != canonical_dumps(found[0]) for labels in found[1:] - ): - raise InvalidModelError("ONNX native label records disagree.") - return cast(ClassLabels, [found[0]]) - - def _native_labels(graph: Any, output: str) -> ClassLabels | None: producers = {name: node for node in graph.node for name in node.output if name} original = output @@ -249,4 +230,21 @@ def describe(model: Any) -> Description: producer_version=model.producer_version, ), ) - return Description(claims, inputs, output["name"], output["shape"] is None) + decision = ( + next((value for value in outputs if value["name"] != output["name"]), None) + if kind == ModelKind.CLASSIFIER + else None + ) + labels_for_decision = ( + None + if decision is None + else decision_labels(graph, decision["name"], output["name"], cast(int, count)) + ) + return Description( + claims, + inputs, + output["name"], + output["shape"] is None, + decision, + labels_for_decision, + ) diff --git a/src/mlbundle/_onnx/labels.py b/src/mlbundle/_onnx/labels.py new file mode 100644 index 0000000..53bd206 --- /dev/null +++ b/src/mlbundle/_onnx/labels.py @@ -0,0 +1,30 @@ +"""One extraction owner for native ONNX class-label attributes.""" + +from typing import Any, cast + +from mlbundle.canonical import canonical_dumps +from mlbundle.claims import ClassLabels +from mlbundle.errors import InvalidModelError + +LABEL_OPS = frozenset( + {"ZipMap", "LinearClassifier", "TreeEnsembleClassifier", "SVMClassifier"} +) + + +def labels_for(nodes: list[Any]) -> ClassLabels | None: + found = [] + for node in nodes: + if node.domain != "ai.onnx.ml" or node.op_type not in LABEL_OPS: + continue + for attr in node.attribute: + if attr.name == "classlabels_strings": + found.append([value.decode("utf-8") for value in attr.strings]) + elif attr.name in ("classlabels_ints", "classlabels_int64s"): + found.append(list(attr.ints)) + if not found: + return None + if any( + canonical_dumps(labels) != canonical_dumps(found[0]) for labels in found[1:] + ): + raise InvalidModelError("ONNX native label records disagree.") + return cast(ClassLabels, [found[0]]) diff --git a/src/mlbundle/_onnx/prediction.py b/src/mlbundle/_onnx/prediction.py index 177ebb2..fce1708 100644 --- a/src/mlbundle/_onnx/prediction.py +++ b/src/mlbundle/_onnx/prediction.py @@ -6,7 +6,9 @@ import numpy as np +from mlbundle._prediction_decisions import class_indices from mlbundle.adapters import ProbabilityOutput, RegressionOutput +from mlbundle.claims import ModelClaims from mlbundle.contract import ModelKind from mlbundle.errors import InvalidModelError, PredictionInputError @@ -15,6 +17,16 @@ _InputShapeError = PredictionInputError +def _check_decision_dtype(values: np.ndarray, dtype: str) -> None: + if dtype == "int64" and values.dtype.kind not in "iu": + raise ValueError + if dtype == "string" and ( + values.dtype.kind not in "OU" + or any(not isinstance(value, str) for value in values.flat) + ): + raise ValueError + + def _array(data: Any, dtype: str) -> np.ndarray: raw = np.asarray(data, dtype=object) if np.dtype(dtype).kind in "iu": @@ -45,10 +57,12 @@ def _array(data: Any, dtype: str) -> np.ndarray: class Predictor: - def __init__(self, session: Any, description: Description) -> None: + def __init__( + self, session: Any, description: Description, claims: ModelClaims | None = None + ) -> None: self.session = session self.description = description - self.claims = description.claims + self.claims = description.claims if claims is None else claims def _feeds(self, data: object) -> tuple[dict[str, np.ndarray], int]: frame = None @@ -85,10 +99,12 @@ def _feeds(self, data: object) -> tuple[dict[str, np.ndarray], int]: feeds[value["name"]] = _array(column, value["dtype"]) return feeds, rows - def _run(self, data: object) -> tuple[Any, int]: + def _run(self, data: object, *, output: str | None = None) -> tuple[Any, int]: try: feeds, rows = self._feeds(data) - return self.session.run([self.description.output], feeds)[0], rows + return self.session.run( + [self.description.output if output is None else output], feeds + )[0], rows except (InvalidModelError, _InputShapeError): raise except Exception: @@ -96,6 +112,38 @@ def _run(self, data: object) -> tuple[Any, int]: "ONNX prediction failed or input values are invalid." ) from None + def predict_class_indices(self, data: object) -> list[list[int]] | None: + if self.claims.kind != ModelKind.CLASSIFIER: + raise ValueError("Class prediction requires a classifier.") + output = self.description.decision_output + if output is None: + return None + raw, rows = self._run(data, output=output["name"]) + try: + values = np.asarray(raw) + labels = self.claims.class_labels + if self.description.decision_labels is not None: + labels = self.description.decision_labels + if len(labels[0]) != (self.claims.n_classes or [0])[0]: + raise ValueError + labels = ( + [[str(label) for label in labels[0]]] + if output["dtype"] == "string" + else [[int(label) for label in labels[0]]] + ) + elif ( + self.description.claims.class_labels is None + and output["dtype"] == "int64" + ): + labels = [list(range((self.claims.n_classes or [0])[0]))] + if labels is None: + raise ValueError + indices = class_indices(raw, labels, rows) + _check_decision_dtype(values, output["dtype"]) + return indices + except Exception: + raise InvalidModelError("Invalid ONNX class prediction output.") from None + def predict(self, data: object) -> RegressionOutput: if self.claims.kind != ModelKind.REGRESSOR: raise ValueError("predict requires a regressor.") diff --git a/src/mlbundle/_prediction_decisions.py b/src/mlbundle/_prediction_decisions.py new file mode 100644 index 0000000..ac5e717 --- /dev/null +++ b/src/mlbundle/_prediction_decisions.py @@ -0,0 +1,50 @@ +"""Bound native class results to output-major labels without inventing decisions.""" + +import math +from collections.abc import Sequence +from typing import Any + +import numpy as np + +from .claims import Label +from .errors import InvalidModelError + + +def _scalar(value: Any) -> Any: + normalized = value.item() if isinstance(value, np.generic) else value + if type(normalized) is str: + if len(normalized) > 1024: + raise ValueError + elif type(normalized) not in (int, float, bool) or not math.isfinite(normalized): + raise ValueError + return normalized + + +def class_indices( + raw: object, labels: Sequence[Sequence[Label]], rows: int +) -> list[list[int]]: + try: + values = np.asarray(raw, dtype=object) + if len(labels) == 1 and values.shape == (rows,): + values = values.reshape(rows, 1) + if values.shape != (rows, len(labels)): + raise ValueError + mappings = [ + {label: index for index, label in enumerate(output)} for output in labels + ] + if any( + len(mapping) != len(output) + for mapping, output in zip(mappings, labels, strict=True) + ): + raise ValueError + result = [] + for row in values: + indices = [] + for value, mapping in zip(row, mappings, strict=True): + indices.append(mapping[_scalar(value)]) + result.append(indices) + return result + except Exception: + raise InvalidModelError( + "Invalid native class prediction dimensions or labels." + ) from None diff --git a/src/mlbundle/_schema_fields.py b/src/mlbundle/_schema_fields.py index e56ef94..25ba5e8 100644 --- a/src/mlbundle/_schema_fields.py +++ b/src/mlbundle/_schema_fields.py @@ -1,4 +1,4 @@ -"""Closed inference-only mlschema 0.2.1 wire vocabulary; no optional imports.""" +"""Closed inferred vocabulary with MLForm series columns; no optional imports.""" from __future__ import annotations @@ -13,41 +13,73 @@ MappedTo = Name | Annotated[int, Field(ge=0, lt=MAX_FEATURES)] -class FieldBase(ContractModel): +class ScalarBase(ContractModel): label: FieldLabel required: bool + + +class FieldBase(ScalarBase): mappedTo: MappedTo -class SimpleField(FieldBase): +class SimpleProperties(ScalarBase): kind: Literal["text", "boolean", "date"] -class NumberField(FieldBase): +class NumberProperties(ScalarBase): kind: Literal["number"] step: Annotated[float, Field(gt=0)] -class CategoryField(FieldBase): +class CategoryProperties(ScalarBase): kind: Literal["category"] options: Annotated[list[str], Field(min_length=1, max_length=MAX_FEATURES)] @model_validator(mode="after") - def unique_options(self) -> CategoryField: + def unique_options(self) -> CategoryProperties: if len(set(self.options)) != len(self.options): raise ValueError("Category options must be unique.") return self -ScalarField = Annotated[ - SimpleField | NumberField | CategoryField, Field(discriminator="kind") +class SimpleField(SimpleProperties): + mappedTo: MappedTo + + +class NumberField(NumberProperties): + mappedTo: MappedTo + + +class CategoryField(CategoryProperties): + mappedTo: MappedTo + + +class SimpleColumn(SimpleProperties): + id: Literal["field1", "field2"] + + +class NumberColumn(NumberProperties): + id: Literal["field1", "field2"] + + +class CategoryColumn(CategoryProperties): + id: Literal["field1", "field2"] + + +SeriesColumn = Annotated[ + SimpleColumn | NumberColumn | CategoryColumn, Field(discriminator="kind") ] class SeriesField(FieldBase): kind: Literal["series"] - field1: ScalarField - field2: ScalarField + columns: Annotated[list[SeriesColumn], Field(min_length=2, max_length=2)] + + @model_validator(mode="after") + def ordered_columns(self) -> SeriesField: + if [column.id for column in self.columns] != ["field1", "field2"]: + raise ValueError("Series columns must be field1 then field2.") + return self class OneHotOption(ContractModel): diff --git a/src/mlbundle/_schema_inference.py b/src/mlbundle/_schema_inference.py new file mode 100644 index 0000000..dd27903 --- /dev/null +++ b/src/mlbundle/_schema_inference.py @@ -0,0 +1,28 @@ +"""The temporary adaptation between local mlschema inference and MLForm.""" + + +def mlform_fields(fields: list[dict]) -> list[dict]: + # Adapt mlschema 0.2.1 to MLForm 0.1.25 until mlschema emits series columns. + return [ + { + **{ + key: value + for key, value in field.items() + if key not in ("field1", "field2") + }, + "columns": [ + { + **{ + key: value + for key, value in field[axis].items() + if key != "mappedTo" + }, + "id": axis, + } + for axis in ("field1", "field2") + ], + } + if field.get("kind") == "series" + else field + for field in fields + ] diff --git a/src/mlbundle/_sklearn/__init__.py b/src/mlbundle/_sklearn/__init__.py index 1e14fa5..ac14da0 100644 --- a/src/mlbundle/_sklearn/__init__.py +++ b/src/mlbundle/_sklearn/__init__.py @@ -179,5 +179,7 @@ def extract_structure(self, model: object) -> dict[str, JsonValue] | None: except Exception: raise InvalidModelError("Invalid sklearn model structure.") from None - def predictor(self, model: object) -> Predictor: - return Predictor(model, self.extract_claims(model)) + def predictor(self, model: object, claims: ModelClaims | None = None) -> Predictor: + return Predictor( + model, self.extract_claims(model) if claims is None else claims + ) diff --git a/src/mlbundle/_sklearn/prediction.py b/src/mlbundle/_sklearn/prediction.py index 1896a3f..1e458b1 100644 --- a/src/mlbundle/_sklearn/prediction.py +++ b/src/mlbundle/_sklearn/prediction.py @@ -8,12 +8,15 @@ from sklearn.pipeline import FeatureUnion, Pipeline from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, TargetEncoder +from mlbundle._prediction_decisions import class_indices from mlbundle._prediction_input import tabular from mlbundle.adapters import ProbabilityOutput, RegressionOutput from mlbundle.claims import ModelClaims from mlbundle.contract import ModelKind from mlbundle.errors import InvalidModelError, MLBundleError +from .record_input import record_input + def _string_inputs(model: Any) -> bool: pending = [model] @@ -42,20 +45,29 @@ def __init__(self, model: Any, claims: ModelClaims) -> None: self.claims = claims self.strings = isinstance(self.model, Pipeline) and _string_inputs(self.model) + def record_input(self, row: list[float], names: list[str]) -> object: + return record_input(self.model, row, names) + def _data(self, data: object) -> tuple[object, int]: return tabular(data, self.claims.n_features, strings=self.strings) - def _call(self, data: object, *, probabilities: bool) -> Any: + def _call( + self, data: object, *, probabilities: bool, decisions: bool = False + ) -> Any: # Positional arrays/lists remain supported for named models; sklearn # still enforces DataFrame name/order when those names are supplied. try: - return self._native_call(data, probabilities=probabilities) + return self._native_call( + data, probabilities=probabilities, decisions=decisions + ) except MLBundleError: raise except Exception: raise InvalidModelError("Sklearn prediction failed.") from None - def _native_call(self, data: object, *, probabilities: bool) -> Any: + def _native_call( + self, data: object, *, probabilities: bool, decisions: bool = False + ) -> Any: with warnings.catch_warnings(): warnings.filterwarnings( "ignore", @@ -70,6 +82,8 @@ def _native_call(self, data: object, *, probabilities: bool) -> Any: if probabilities else self.model.predict(data) ) + if decisions: + return raw if probabilities: return ( [np.asarray(item, dtype=float) for item in raw] @@ -78,6 +92,13 @@ def _native_call(self, data: object, *, probabilities: bool) -> Any: ) return np.asarray(raw, dtype=float) + def predict_class_indices(self, data: object) -> list[list[int]]: + if self.claims.kind != ModelKind.CLASSIFIER: + raise ValueError("Class prediction requires a classifier.") + data, rows = self._data(data) + raw = self._call(data, probabilities=False, decisions=True) + return class_indices(raw, self.claims.class_labels or [], rows) + def predict(self, data: object) -> RegressionOutput: if self.claims.kind != ModelKind.REGRESSOR: raise ValueError("predict requires a regressor.") diff --git a/src/mlbundle/_sklearn/record_input.py b/src/mlbundle/_sklearn/record_input.py new file mode 100644 index 0000000..0e11d94 --- /dev/null +++ b/src/mlbundle/_sklearn/record_input.py @@ -0,0 +1,44 @@ +"""Preserve fitted name-based selectors only when a record needs a DataFrame.""" + +from importlib import import_module +from typing import Any + +import numpy as np +from sklearn.compose import ColumnTransformer +from sklearn.pipeline import FeatureUnion, Pipeline + +from mlbundle.errors import MissingOptionalDependencyError + + +def _named(columns: Any) -> bool: + if isinstance(columns, str): + return True + if isinstance(columns, slice): + return isinstance(columns.start, str) or isinstance(columns.stop, str) + return isinstance(columns, (list, tuple, np.ndarray)) and any( + isinstance(column, str) for column in columns + ) + + +def record_input(model: Any, row: list[float], names: list[str]) -> object: + pending = [model] + seen = set() + while pending: + value = pending.pop() + if id(value) in seen: + continue + seen.add(id(value)) + if isinstance(value, ColumnTransformer): + for _, step, columns in value.transformers_: + if step != "drop" and _named(columns): + try: + pandas = import_module("pandas") + except ImportError: + raise MissingOptionalDependencyError("schema") from None + return pandas.DataFrame([row], columns=names) + pending.append(step) + elif isinstance(value, Pipeline): + pending.extend(step for _, step in value.steps) + elif isinstance(value, FeatureUnion): + pending.extend(step for _, step in value.transformer_list) + return [row] diff --git a/src/mlbundle/_versions.py b/src/mlbundle/_versions.py new file mode 100644 index 0000000..70fee20 --- /dev/null +++ b/src/mlbundle/_versions.py @@ -0,0 +1,30 @@ +"""Strict bounded version text for failure diagnostics, never compatibility gates.""" + +import re +from importlib.metadata import version + +from ._display import display + +_VERSION = re.compile( + r"[0-9]+(?:\.[0-9]+)*(?:(?:a|b|rc)[0-9]+)?" + r"(?:\.post[0-9]+)?(?:\.dev[0-9]+)?(?:\+[a-z0-9]+(?:[.-][a-z0-9]+)*)?" +) + + +def safe_version(value: object) -> str | None: + if isinstance(value, str) and len(value) <= 64 and _VERSION.fullmatch(value): + return value + return None + + +def writer_hint(writer: object) -> str: + declared = safe_version(writer) + if declared is None: + return "" + reader = version("mlbundle") + if declared == reader: + return "" + return ( + f" Bundle was written by mlbundle {display(declared)}; this reader is " + f"{display(reader)}. Format 1 is not frozen; the model must be re-exported." + ) diff --git a/src/mlbundle/_xgboost/__init__.py b/src/mlbundle/_xgboost/__init__.py index 1347501..e16d3bb 100644 --- a/src/mlbundle/_xgboost/__init__.py +++ b/src/mlbundle/_xgboost/__init__.py @@ -248,8 +248,10 @@ def extract_structure(self, model: Any) -> dict[str, JsonValue]: except Exception: raise InvalidModelError("Invalid XGBoost model structure.") from None - def predictor(self, model: object) -> Predictor: - return Predictor(model, self.extract_claims(model)) + def predictor(self, model: object, claims: ModelClaims | None = None) -> Predictor: + return Predictor( + model, self.extract_claims(model) if claims is None else claims + ) def _binding(booster: Any) -> dict[str, Any]: diff --git a/src/mlbundle/_xgboost/prediction.py b/src/mlbundle/_xgboost/prediction.py index 664bdec..c65f285 100644 --- a/src/mlbundle/_xgboost/prediction.py +++ b/src/mlbundle/_xgboost/prediction.py @@ -4,6 +4,7 @@ import numpy as np +from mlbundle._prediction_decisions import class_indices from mlbundle._prediction_input import tabular from mlbundle.adapters import ProbabilityOutput, RegressionOutput from mlbundle.claims import ModelClaims @@ -16,7 +17,9 @@ def __init__(self, model: Any, claims: ModelClaims) -> None: self.model = model self.claims = claims - def _call(self, data: object, *, probabilities: bool) -> tuple[np.ndarray, int]: + def _call( + self, data: object, *, probabilities: bool, decisions: bool = False + ) -> tuple[np.ndarray, int]: try: data, rows = tabular( data, self.claims.n_features, categorical=self.model.enable_categorical @@ -26,12 +29,20 @@ def _call(self, data: object, *, probabilities: bool) -> tuple[np.ndarray, int]: if probabilities else self.model.predict(data) ) - return np.asarray(raw, dtype=float), rows + return np.asarray(raw, dtype=object if decisions else float), rows except MLBundleError: raise except Exception: raise InvalidModelError("XGBoost prediction failed.") from None + def predict_class_indices(self, data: object) -> list[list[int]]: + if self.claims.kind != ModelKind.CLASSIFIER: + raise ValueError("Class prediction requires a classifier.") + raw, rows = self._call(data, probabilities=False, decisions=True) + # The closed wrapper predicts encoded indices, including with bound labels. + labels = [list(range((self.claims.n_classes or [0])[0]))] + return class_indices(raw, labels, rows) + def predict(self, data: object) -> RegressionOutput: if self.claims.kind != ModelKind.REGRESSOR: raise ValueError("predict requires a regressor.") diff --git a/src/mlbundle/adapters.py b/src/mlbundle/adapters.py index b4ed051..c546dda 100644 --- a/src/mlbundle/adapters.py +++ b/src/mlbundle/adapters.py @@ -13,8 +13,12 @@ class Predictor(Protocol): + # Optional record_input(row, names) hook preserves native selector semantics; + # without that hook, records use one numeric row in manifest feature order. def predict(self, data: object) -> RegressionOutput: ... def predict_proba(self, data: object) -> ProbabilityOutput: ... + # Row-major native decisions as indices, or None when no native decision exists. + def predict_class_indices(self, data: object) -> list[list[int]] | None: ... class Adapter(Protocol): @@ -30,4 +34,7 @@ def serialize(self, model: object) -> bytes: ... def deserialize(self, data: bytes) -> object: ... def extract_claims(self, model: object) -> ModelClaims: ... def extract_structure(self, model: object) -> dict[str, JsonValue] | None: ... - def predictor(self, model: object) -> Predictor: ... + # Shared load supplies verified/resolved claims, including declared ONNX labels. + def predictor( + self, model: object, claims: ModelClaims | None = None + ) -> Predictor: ... diff --git a/src/mlbundle/cli.py b/src/mlbundle/cli.py index 372041d..5840096 100644 --- a/src/mlbundle/cli.py +++ b/src/mlbundle/cli.py @@ -9,11 +9,11 @@ from typing import TextIO from . import inspect, load +from ._display import display as _display from .contract import MemberName from .errors import MissingOptionalDependencyError, MLBundleError from .results import LoadedBundle, MetadataView -_DISPLAY_LIMIT = 512 _NOTICE = ( "inspect validates metadata and integrity; it cannot attest that metadata " "matches the model. Only verify loads the model and checks that agreement." @@ -32,14 +32,6 @@ def print_help(self, file: TextIO | None = None) -> None: super().print_help(file) -def _display(value: object) -> str: - # ASCII JSON escapes terminal controls, bidi controls and line separators. - rendered = json.dumps(value, ensure_ascii=True) - if len(rendered) > _DISPLAY_LIMIT: - return rendered[: _DISPLAY_LIMIT - 3] + "..." - return rendered - - def _summary(view: MetadataView) -> str: manifest = view.manifest model = manifest.model diff --git a/src/mlbundle/errors.py b/src/mlbundle/errors.py index 5d312c7..e2108ba 100644 --- a/src/mlbundle/errors.py +++ b/src/mlbundle/errors.py @@ -1,5 +1,7 @@ """Bounded errors at bundle trust boundaries.""" +from ._versions import writer_hint + def _identifier(value: str) -> str: if ( @@ -29,8 +31,8 @@ def __init__(self) -> None: class InvalidManifestError(MLBundleError): - def __init__(self) -> None: - super().__init__("Invalid mlbundle manifest.") + def __init__(self, *, writer_version: object = None) -> None: + super().__init__("Invalid mlbundle manifest." + writer_hint(writer_version)) class InvalidJSONError(MLBundleError): @@ -66,6 +68,10 @@ class InvalidModelError(MLBundleError): """Native model content is malformed or violates the adapter contract.""" +class EnvironmentMismatchError(InvalidModelError): + """A sklearn load failed under a different declared package version.""" + + class PredictionInputError(InvalidModelError, ValueError): """Bounded invalid-input error retaining the shape-error ValueError contract.""" diff --git a/src/mlbundle/manifest.py b/src/mlbundle/manifest.py index 303739c..84b0f3f 100644 --- a/src/mlbundle/manifest.py +++ b/src/mlbundle/manifest.py @@ -212,13 +212,22 @@ def to_bytes(self) -> bytes: @classmethod def from_bytes(cls, data: bytes) -> Manifest: + value = None try: value = canonical_loads(data) if not isinstance(value, dict) or value.get("format_name") != FORMAT_NAME: - raise InvalidManifestError + raise InvalidManifestError( + writer_version=value.get("writer_version") + if isinstance(value, dict) + else None + ) version = value.get("format_version") if isinstance(version, str) and version != FORMAT_VERSION: raise UnsupportedFormatVersionError return cls.model_validate_json(canonical_dumps(value)) except (ValidationError, InvalidJSONError): - raise InvalidManifestError from None + raise InvalidManifestError( + writer_version=value.get("writer_version") + if isinstance(value, dict) + else None + ) from None diff --git a/src/mlbundle/model_io.py b/src/mlbundle/model_io.py index b8526f8..757eaf5 100644 --- a/src/mlbundle/model_io.py +++ b/src/mlbundle/model_io.py @@ -1,6 +1,7 @@ """Shared export and validating load; adapters own all native-model behavior.""" from collections.abc import Mapping, Sequence +from contextlib import nullcontext from os import PathLike from typing import cast @@ -8,6 +9,7 @@ from ._container_rules import MEMBER_ORDER, BundleSource, effective_limits from ._export_manifest import manifest_parts +from ._load_diagnostics import sklearn_environment from ._native_io import prepare_native from .claims import Label, verify_claims, verify_structure_bytes from .container import read_validated_container @@ -123,28 +125,35 @@ def load(source: BundleSource, *, limits: ArchiveLimits | None = None) -> Loaded members, manifest.features.names, cast(bool, manifest.model.hyperparameters["verbose"]), + manifest.environment.packages.get("scikit-learn"), ) verify_claims(manifest.verified_claims(), actual) else: adapter = get_adapter(manifest.model.library) - model = adapter.deserialize(members[adapter.model_member]) - actual = adapter.extract_claims(model) - expected = manifest.verified_claims() - # Check native dimensions before using any label declaration from the bundle. - verify_claims( - expected.model_copy(update={"class_labels": actual.class_labels}), actual - ) - actual = resolve_labels( - actual, - ExportDeclarations( - class_labels=manifest.target.class_labels - if actual.class_labels is None - else None - ), - ) - verify_claims(expected, actual) - predictor = adapter.predictor(model) - extracted = adapter.extract_structure(model) + with ( + sklearn_environment(manifest.environment.packages.get("scikit-learn")) + if adapter.model_member == MemberName.SKOPS + else nullcontext() + ): + model = adapter.deserialize(members[adapter.model_member]) + actual = adapter.extract_claims(model) + expected = manifest.verified_claims() + # Check native dimensions before using any label declaration from the bundle. + verify_claims( + expected.model_copy(update={"class_labels": actual.class_labels}), + actual, + ) + actual = resolve_labels( + actual, + ExportDeclarations( + class_labels=manifest.target.class_labels + if actual.class_labels is None + else None + ), + ) + verify_claims(expected, actual) + predictor = adapter.predictor(model, actual) + extracted = adapter.extract_structure(model) decoded = container.json.get(MemberName.STRUCTURE) structure = None if decoded is None else decoded.value if (MemberName.STRUCTURE in members) != (extracted is not None): diff --git a/src/mlbundle/records.py b/src/mlbundle/records.py new file mode 100644 index 0000000..3f801f9 --- /dev/null +++ b/src/mlbundle/records.py @@ -0,0 +1,157 @@ +"""Single-record input and output contracts without optional library imports.""" + +import math +import re +from collections.abc import Mapping +from dataclasses import dataclass +from typing import cast + +from pydantic import JsonValue + +from ._display import display +from .adapters import Predictor +from .contract import FeatureOrigin, ModelKind +from .errors import PredictionInputError +from .manifest import Manifest + +_DECIMAL = re.compile(r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?") +_ASCII_WHITESPACE = " \t\n\r\v\f" +_MISSING_LIMIT = 5 + + +@dataclass(frozen=True) +class OutputPrediction: + """One output's report labels and validated numeric prediction.""" + + output: int + kind: ModelKind + labels: tuple[str, ...] | None + probabilities: tuple[float, ...] | None + value: float | None + predicted_label: str | None + + +def _value(value: object, key: str | int, *, required: bool) -> float: + prefix = f"Feature {display(key, limit=128)}: " + if value is None: + if required: + raise PredictionInputError(prefix + "a required feature cannot be None.") + return math.nan + if isinstance(value, str): + value = value.strip(_ASCII_WHITESPACE) + if len(value) > 64: + raise PredictionInputError(prefix + "string exceeds 64 characters.") + if value.lower() in ("true", "false"): + return float(value.lower() == "true") + if _DECIMAL.fullmatch(value) is None: + raise PredictionInputError( + prefix + "expected a decimal or true/false string." + ) + elif not isinstance(value, (int, float)): + raise PredictionInputError( + prefix + "expected a number, boolean or numeric string." + ) + try: + number = float(value) + except (ValueError, OverflowError): + raise PredictionInputError(prefix + "number must be finite.") from None + if not math.isfinite(number): + raise PredictionInputError(prefix + "number must be finite.") + return number + + +def _schema(schema: dict[str, JsonValue] | None) -> dict: + if schema is None: + raise PredictionInputError("Record input requires a validated schema.") + return cast(dict, schema) + + +def row_from_record( + manifest: Manifest, schema: dict[str, JsonValue] | None, record: object +) -> list[float]: + """Look up only expected keys; extra keys are ignored without iterating record.""" + if not isinstance(record, Mapping): + raise PredictionInputError("Prediction record must be a mapping.") + named = manifest.features.origin == FeatureOrigin.MODEL + required = {} + for field in _schema(schema)["fields"]: + mappings = ( + [option["mappedTo"] for option in field["options"]] + if field["kind"] == "onehot-category" + else [field["mappedTo"]] + ) + for mapping in mappings: + required[mapping] = field["required"] + row = [] + missing = [] + missing_count = 0 + for position, name in enumerate(manifest.features.names): + key = name if named else position + alternate = key if named else str(position) + present = key in record + if not named and alternate in record: + if present: + raise PredictionInputError( + f"Feature {position}: both integer and string keys are present." + ) + present = True + key = alternate + if not present: + missing_count += 1 + if len(missing) < _MISSING_LIMIT: + missing.append(display(alternate, limit=128)) + continue + row.append( + _value(record[key], key, required=required[name if named else position]) + ) + if missing_count: + more = ( + f"; {missing_count - len(missing)} more" + if missing_count > len(missing) + else "" + ) + raise PredictionInputError(f"Missing features: {', '.join(missing)}{more}.") + return row + + +def predict_record( + manifest: Manifest, + schema: dict[str, JsonValue] | None, + predictor: Predictor, + record: object, +) -> tuple[OutputPrediction, ...]: + row = row_from_record(manifest, schema, record) + # Owned predictors may preserve names needed by their native transformers. + record_input = getattr(predictor, "record_input", None) + data = [row] if record_input is None else record_input(row, manifest.features.names) + kind = manifest.model.kind + if kind == ModelKind.CLASSIFIER: + raw = predictor.predict_proba(data) + decisions = predictor.predict_class_indices(data) + outputs = [raw] if manifest.model.n_outputs == 1 else raw + return tuple( + OutputPrediction( + output=report["mappedTo"], + kind=kind, + labels=tuple(report["labels"]), + probabilities=tuple(cast(list[list[float]], matrix)[0]), + value=None, + predicted_label=None + if decisions is None + else report["labels"][decisions[0][report["mappedTo"]]], + ) + for report, matrix in zip(_schema(schema)["reports"], outputs, strict=True) + ) + values = predictor.predict(data) + result = [values[0]] if manifest.model.n_outputs == 1 else values[0] + return tuple( + OutputPrediction( + output=output, + kind=kind, + labels=None, + probabilities=None, + value=value, + predicted_label=None, + ) + for output, value in enumerate(cast(list[float], result)) + ) diff --git a/src/mlbundle/results.py b/src/mlbundle/results.py index 47e773f..d03d57a 100644 --- a/src/mlbundle/results.py +++ b/src/mlbundle/results.py @@ -8,6 +8,7 @@ from .claims import ClassLabels from .contract import FeatureOrigin, Library, MemberName, ModelKind from .manifest import Manifest +from .records import OutputPrediction, predict_record, row_from_record @dataclass(frozen=True) @@ -66,3 +67,15 @@ def predict(self, data: object) -> RegressionOutput: if self.kind != ModelKind.REGRESSOR: raise ValueError("predict requires a regressor.") return self.predictor.predict(data) + + def row_from_record(self, record: object) -> list[float]: + """Validate expected record keys and values in model feature order. + + Extra keys are ignored without iterating the record. Only this path + parses numeric strings and schema-permitted None values. + """ + return row_from_record(self.manifest, self.schema, record) + + def predict_record(self, record: object) -> tuple[OutputPrediction, ...]: + """Predict one record, returning one frozen result per output in order.""" + return predict_record(self.manifest, self.schema, self.predictor, record) diff --git a/src/mlbundle/schema.py b/src/mlbundle/schema.py index b18106d..fc3f616 100644 --- a/src/mlbundle/schema.py +++ b/src/mlbundle/schema.py @@ -5,7 +5,7 @@ from pydantic import Field, JsonValue, TypeAdapter, ValidationError from ._base import ContractModel -from ._schema_fields import OneHotField, SchemaField, SeriesField +from ._schema_fields import OneHotField, SchemaField from .canonical import DecodedJSON, canonical_dumps from .claims import ModelClaims from .contract import FeatureOrigin, MemberName, ModelKind @@ -83,9 +83,6 @@ def positional_fields(fields: list[dict], names: list[str]) -> None: def rewrite(field: dict) -> None: if "mappedTo" in field: field["mappedTo"] = positions[field["mappedTo"]] - for key in ("field1", "field2"): - if key in field: - rewrite(field[key]) for option in field.get("options", []): if isinstance(option, dict): rewrite(option) @@ -126,11 +123,6 @@ def _field_indices( index = _mapping_index(field.mappedTo, positions, origin) if field.label != names[index]: raise ValueError - if isinstance(field, SeriesField) and any( - _mapping_index(sub.mappedTo, positions, origin) != index - for sub in (field.field1, field.field2) - ): - raise ValueError return [index] diff --git a/src/mlbundle/training.py b/src/mlbundle/training.py index e93b24a..4909c9d 100644 --- a/src/mlbundle/training.py +++ b/src/mlbundle/training.py @@ -9,6 +9,7 @@ from pydantic import ValidationError +from ._schema_inference import mlform_fields from .canonical import canonical_dumps, canonical_value from .claims import ModelClaims from .contract import FeatureOrigin, MemberName @@ -124,7 +125,7 @@ def prepare_training( dtypes = [None] * claims.n_features cap = limits.member_bytes(MemberName.SCHEMA) try: - fields = mlschema.infer_schema(frame, onehot_separator=separator) + fields = mlform_fields(mlschema.infer_schema(frame, onehot_separator=separator)) if origin in (FeatureOrigin.DATAFRAME, FeatureOrigin.DECLARED): positional_fields(fields, names) schema = canonical_value( diff --git a/tests/record_helpers.py b/tests/record_helpers.py new file mode 100644 index 0000000..d812005 --- /dev/null +++ b/tests/record_helpers.py @@ -0,0 +1,172 @@ +"""Small fitted record fixture without input or prediction logic.""" + +import math +import warnings +from collections.abc import Mapping + +import pandas as pd +import pytest +from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor + +import mlbundle + + +class LookupOnly(Mapping): + def __getitem__(self, key): + assert key in (0, 1, "0", "1") + return {"0": 0, "1": 1}[key] + + def __iter__(self): + pytest.fail("record iterated") + + def __len__(self) -> int: + raise AssertionError("record size requested") + + +def threshold_classifier(): + from catboost import CatBoostClassifier + + model = CatBoostClassifier( + iterations=2, + depth=1, + learning_rate=0.5, + thread_count=1, + verbose=False, + allow_writing_files=False, + ).fit([[0], [1], [0], [1]], ["negative", "positive", "negative", "positive"]) + model.set_probability_threshold(0.8) + return model + + +def disagreement_classifier(): + from sklearn.svm import SVC + + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="The `probability` parameter was deprecated", + category=FutureWarning, + ) + return SVC(probability=True, random_state=3).fit( + [[index] for index in range(10)], [0, 0, 0, 0, 1, 1, 1, 1, 1, 1] + ) + + +def missing_pandas(_name): + raise ModuleNotFoundError("pandas") + + +def named_pipeline(kind): + from sklearn.compose import ColumnTransformer + from sklearn.pipeline import make_pipeline + from xgboost import XGBClassifier + + frame = pd.DataFrame({"a": [0, 1], "b": [1, 0]}) + estimator = { + "classifier": DecisionTreeClassifier(), + "regressor": DecisionTreeRegressor(), + "composite": XGBClassifier(n_estimators=1, n_jobs=1), + }[kind] + return make_pipeline( + ColumnTransformer([("select", "passthrough", ["a"])]), estimator + ).fit(frame, [0, 1]), frame + + +def categorical_model(composite): + from sklearn.pipeline import Pipeline + from xgboost import XGBClassifier + + frame = pd.DataFrame( + {"category": pd.Categorical([0, 1, 0, 1]), "amount": [0.0, 1.0, 2.0, 3.0]} + ) + estimator = XGBClassifier(enable_categorical=True, n_estimators=1, n_jobs=1) + model = ( + Pipeline([("identity", "passthrough"), ("leaf", estimator)]) + if composite + else estimator + ) + return model.fit(frame, [0, 1, 0, 1]) + + +def native_decision_model(library): + from catboost import CatBoostClassifier + from xgboost import XGBClassifier + + from onnx_helpers import ROWS, classifier + + if library == "onnx": + return classifier(("cold", "warm")), {} + if library == "xgboost": + return XGBClassifier(n_estimators=2, n_jobs=1).fit(ROWS, [0, 1, 0, 1]), { + "class_labels": [["cold", "warm"]] + } + model = CatBoostClassifier( + iterations=2, verbose=False, thread_count=1, allow_writing_files=False + ) + return model.fit(ROWS, ["cold", "warm"] * 2), {} + + +def bundle_at(path, *, origin="generated", kind="classifier", outputs=1, names=None): + frame = pd.DataFrame([[0, 1], [1, 0]], columns=names or ["age", "active"]) + cls = DecisionTreeClassifier if kind == "classifier" else DecisionTreeRegressor + targets = [0, 1] if outputs == 1 else [[0, 2], [1, 3]] + model = cls(random_state=7).fit( + frame if origin == "model" else frame.to_numpy(), targets + ) + options = {"training_data": frame} if origin == "dataframe" else {} + if origin == "declared": + options["feature_names"] = list(frame.columns) + mlbundle.export(model, path, **options) + return mlbundle.load(path) + + +ACCEPTED_VALUES = [ + (0, 0.0), + (-40, -40.0), + (1.5, 1.5), + (True, 1.0), + (False, 0.0), + ("40", 40.0), + ("-40", -40.0), + ("+40", 40.0), + ("1.5e3", 1500.0), + ("1E-3", 0.001), + (".5", 0.5), + ("1.", 1.0), + ("01", 1.0), + ("true", 1.0), + ("FALSE", 0.0), + ("TrUe", 1.0), + (" \t\r\n\v\f1.5 \t", 1.5), + ("0" * 64, 0.0), + (" " * 80 + "1", 1.0), +] + + +REJECTED_VALUES = [ + ("", "decimal"), + (" \t", "decimal"), + ("private text", "decimal"), + ("NaN", "decimal"), + ("inf", "decimal"), + ("0x10", "decimal"), + ("1_000", "decimal"), + ("1,2", "decimal"), + ("1e", "decimal"), + ("1e+", "decimal"), + ("1 2", "decimal"), + ("\u0661", "decimal"), + ("\u00a01\u00a0", "decimal"), + ("1\x00", "decimal"), + ("0" * 65, "64 characters"), + ("1e309", "finite"), + (math.inf, "finite"), + (-math.inf, "finite"), + (math.nan, "finite"), + (10**400, "finite"), + ([], "expected"), + ({}, "expected"), + (b"40", "expected"), + (1j, "expected"), + (object(), "expected"), +] diff --git a/tests/test_catboost_decisions.py b/tests/test_catboost_decisions.py new file mode 100644 index 0000000..fac2dbc --- /dev/null +++ b/tests/test_catboost_decisions.py @@ -0,0 +1,94 @@ +"""CatBoost class calls preserve native thresholds, labels and bounded errors.""" + +import catboost as cb +import numpy as np +import pytest + +import mlbundle +from mlbundle._catboost import Adapter +from record_helpers import threshold_classifier + + +@pytest.mark.parametrize( + "labels", [["cold", "warm"], [10, 20], [0.5, 1.5], ["a", "b", "c"]] +) +def test_native_decisions(labels, tmp_path): + rows = [[index] for index in range(9)] + targets = [labels[index % len(labels)] for index in range(9)] + model = cb.CatBoostClassifier( + iterations=2, depth=1, verbose=False, thread_count=1, allow_writing_files=False + ).fit(rows, targets) + path = tmp_path / "classes.mlbundle" + mlbundle.export(model, path) + loaded = mlbundle.load(path) + raw = np.asarray(model.predict(rows)).reshape(-1) + assert loaded.predictor.predict_class_indices(rows) == [ + [model.classes_.tolist().index(value)] for value in raw + ] + assert [ + loaded.predict_record({"0": row[0]})[0].predicted_label for row in rows + ] == [str(value) for value in raw] + + +def test_threshold_is_native_after_roundtrip(tmp_path): + model = threshold_classifier() + path = tmp_path / "threshold.mlbundle" + mlbundle.export(model, path) + loaded = mlbundle.load(path) + result = loaded.predict_record({"0": 1})[0] + assert loaded.native_model.get_probability_threshold() == 0.8 + assert 0.5 < result.probabilities[1] < 0.8 + assert result.predicted_label == str(model.predict([[1]])[0]) == "negative" + assert loaded.predictor.predict_class_indices([[0], [1]]) == [[0], [0]] + + +@pytest.mark.parametrize( + "raw", + [ + [], + ["private"], + [["negative", "positive"]], + [None], + [np.nan], + [np.inf], + [object()], + ["x" * 1025], + ], +) +def test_invalid_decision_output(raw, monkeypatch): + model = threshold_classifier() + predictor = Adapter().predictor(model) + monkeypatch.setattr(model, "predict", lambda *a, **kw: raw) + with pytest.raises(mlbundle.InvalidModelError) as error: + predictor.predict_class_indices([[1]]) + assert len(str(error.value)) < 160 and "private" not in str(error.value) + + +def test_class_call_and_failure_boundaries(monkeypatch): + model = threshold_classifier() + predictor = Adapter().predictor(model) + original = model.predict + seen = [] + + def predict(data, **kwargs): + seen.append(kwargs) + return original(data, **kwargs) + + monkeypatch.setattr(model, "predict", predict) + monkeypatch.setattr(model, "predict_proba", lambda *a: pytest.fail("argmax")) + assert predictor.predict_class_indices([[1]]) == [[0]] + assert seen == [{"prediction_type": "Class", "thread_count": 1}] + + def fail(*a, **kw): + raise ValueError("private native decision") + + monkeypatch.setattr(model, "predict", fail) + with pytest.raises( + mlbundle.InvalidModelError, match=r"^CatBoost prediction failed\.$" + ): + predictor.predict_class_indices([[1]]) + regressor = cb.CatBoostRegressor( + iterations=1, verbose=False, allow_writing_files=False + ).fit([[0], [1]], [0, 1]) + with pytest.raises(ValueError, match="classifier"): + Adapter().predictor(regressor).predict_class_indices([[1]]) diff --git a/tests/test_environment.py b/tests/test_environment.py index d92f40d..6705a04 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -20,7 +20,7 @@ def test_installed_environment(): def test_missing_distributions_and_onnx(monkeypatch): def version(name): if name == "mlbundle": - return "0.1.0a2" + return "0.1.0a3" raise metadata.PackageNotFoundError(name) monkeypatch.setattr(module.metadata, "version", version) @@ -31,7 +31,7 @@ def version(name): producer_version="", ) environment = module.capture_environment(onnx=graph) - assert environment.packages == {"mlbundle": "0.1.0a2"} + assert environment.packages == {"mlbundle": "0.1.0a3"} assert environment.onnx == graph diff --git a/tests/test_json_reuse.py b/tests/test_json_reuse.py index b0022bc..feaf24e 100644 --- a/tests/test_json_reuse.py +++ b/tests/test_json_reuse.py @@ -29,7 +29,7 @@ def native(monkeypatch, manifest_data): deserialize=lambda data: "native", extract_claims=lambda model: extracted, extract_structure=lambda model: structure, - predictor=lambda model: object(), + predictor=lambda model, claims=None: object(), ) monkeypatch.setattr(model_io, "find_adapter", lambda model: adapter) monkeypatch.setattr(model_io, "get_adapter", lambda library: adapter) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 7ddec9e..db2e3d8 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -18,7 +18,7 @@ def read(data: dict) -> Manifest: return Manifest.from_bytes(canonical_dumps(data)) -@pytest.mark.parametrize("writer_version", ["0.1.0", "0.1.0a2"]) +@pytest.mark.parametrize("writer_version", ["0.1.0", "0.1.0a3"]) def test_writer_version_is_independent_of_reader(manifest_data, writer_version): manifest_data["writer_version"] = writer_version manifest_data["environment"]["packages"]["mlbundle"] = writer_version diff --git a/tests/test_model_io.py b/tests/test_model_io.py index 09b50b6..52a99fe 100644 --- a/tests/test_model_io.py +++ b/tests/test_model_io.py @@ -119,7 +119,7 @@ def extract_claims(self, model): def extract_structure(self, model): return None - def predictor(self, model): + def predictor(self, model, claims=None): return SimpleNamespace(predict_proba=lambda data: [[0.2, 0.8]]) FakeAdapter.library = library diff --git a/tests/test_onnx_decisions.py b/tests/test_onnx_decisions.py new file mode 100644 index 0000000..c4f3def --- /dev/null +++ b/tests/test_onnx_decisions.py @@ -0,0 +1,270 @@ +"""ONNX uses an actual graph label output or reports no class decision.""" + +from types import SimpleNamespace + +import numpy as np +import pytest +from onnx import TensorProto as T, helper as h + +import mlbundle +from mlbundle._onnx import Adapter +from onnx_helpers import ROWS, classifier, regressor, softmax + + +@pytest.mark.parametrize("labels", [(10, 20), ("cold", "warm")]) +@pytest.mark.parametrize("zipmap", [False, True]) +def test_native_graph_label_output(labels, zipmap, tmp_path): + path = tmp_path / "label.mlbundle" + mlbundle.export(classifier(labels, zipmap=zipmap), path) + loaded = mlbundle.load(path) + expected = loaded.native_model.session.run( + ["label"], {"X": np.asarray(ROWS, dtype=np.float32)} + )[0] + indices = [[list(labels).index(value)] for value in expected] + assert loaded.predictor.predict_class_indices(ROWS) == indices + for row, index in zip(ROWS, indices, strict=True): + assert loaded.predict_record(dict(enumerate(row)))[0].predicted_label == str( + labels[index[0]] + ) + + +@pytest.mark.parametrize("strings", [False, True]) +@pytest.mark.parametrize("rank_two", [False, True]) +def test_declared_labels_and_native_encoded_decisions(strings, rank_two, tmp_path): + native = softmax(two_outputs=True) + if rank_two: + native.graph.node[-1].ClearField("attribute") + native.graph.node[-1].attribute.extend( + [h.make_attribute("axis", 1), h.make_attribute("keepdims", 1)] + ) + native.graph.output[0].CopyFrom( + h.make_tensor_value_info("label", T.INT64, [None, 1]) + ) + if strings: + native.graph.node.append( + h.make_node( + "LabelEncoder", + ["label"], + ["named"], + domain="ai.onnx.ml", + keys_int64s=[0, 1], + values_strings=["cold", "warm"], + ) + ) + native.graph.output[0].CopyFrom( + h.make_tensor_value_info( + "named", T.STRING, [None, 1] if rank_two else [None] + ) + ) + path = tmp_path / "declared.mlbundle" + mlbundle.export(native, path, class_labels=[["cold", "warm"]]) + loaded = mlbundle.load(path) + name = "named" if strings else "label" + expected = loaded.native_model.session.run( + [name], {"X": np.asarray(ROWS, dtype=np.float32)} + )[0].reshape(-1) + labels = [ + str(value) if strings else ["cold", "warm"][int(value)] for value in expected + ] + assert [ + loaded.predict_record(dict(enumerate(row)))[0].predicted_label for row in ROWS + ] == labels + + +@pytest.mark.parametrize( + "native", + [softmax(), classifier(map_only=True), classifier(zipmap=True, map_only=True)], +) +def test_probabilities_without_label_output(native, tmp_path): + path = tmp_path / "no-decision.mlbundle" + mlbundle.export(native, path) + loaded = mlbundle.load(path) + assert loaded.predictor.predict_class_indices(ROWS) is None + assert loaded.predict_record({"0": 0, "1": 1})[0].predicted_label is None + + +def test_native_decision_may_disagree_with_probability(tmp_path): + native = softmax(two_outputs=True) + native.graph.node.pop() + native.graph.node.append( + h.make_node( + "Constant", + [], + ["label"], + value=h.make_tensor("decision", T.INT64, [1], [1]), + ) + ) + path = tmp_path / "independent.mlbundle" + mlbundle.export(native, path, class_labels=[["cold", "warm"]]) + result = mlbundle.load(path).predict_record({"0": 0, "1": 1})[0] + assert ( + result.predicted_label == "warm" + and result.probabilities[0] > result.probabilities[1] + ) + + +@pytest.mark.parametrize( + "raw", + [ + [], + [10, 20], + [[10, 20]], + [True], + [10.0], + [np.nan], + [np.inf], + [999], + [0], + ["private"], + [None], + ], +) +def test_invalid_native_label_output(raw): + adapter = Adapter() + predictor = adapter.predictor(adapter.deserialize(classifier().SerializeToString())) + seen = [] + + def run(outputs, _feeds): + seen.append(outputs) + return [raw] + + predictor.session = SimpleNamespace(run=run) + with pytest.raises(mlbundle.InvalidModelError) as error: + predictor.predict_class_indices([[0, 1]]) + assert seen == [["label"]] + assert "private" not in str(error.value) and len(str(error.value)) < 160 + + +@pytest.mark.parametrize("raw", [["private"], [None], ["x" * 1025], [complex(1)]]) +def test_invalid_string_label_output(raw): + adapter = Adapter() + predictor = adapter.predictor( + adapter.deserialize(classifier(("cold", "warm")).SerializeToString()) + ) + predictor.session = SimpleNamespace(run=lambda *args: [raw]) + with pytest.raises(mlbundle.InvalidModelError) as error: + predictor.predict_class_indices([[0, 1]]) + assert "private" not in str(error.value) and len(str(error.value)) < 160 + + +def test_native_failure_and_regression_boundary(): + adapter = Adapter() + predictor = adapter.predictor(adapter.deserialize(classifier().SerializeToString())) + + def fail(*args): + raise ValueError("private native exception") + + predictor.session = SimpleNamespace(run=fail) + with pytest.raises(mlbundle.InvalidModelError) as error: + predictor.predict_class_indices([[0, 1]]) + assert "private" not in str(error.value) + with pytest.raises(ValueError, match="classifier"): + adapter.predictor( + adapter.deserialize(regressor().SerializeToString()) + ).predict_class_indices([[0, 1]]) + + +@pytest.mark.parametrize("source", [(0, 1), (10, 20), ("encoded-a", "encoded-b")]) +@pytest.mark.parametrize("renamed", [("cold", "warm"), (100, 200)]) +def test_zipmap_renaming_preserves_native_decision_positions(source, renamed, tmp_path): + native = classifier(source, zipmap=True) + node = native.graph.node[1] + node.ClearField("attribute") + strings = isinstance(renamed[0], str) + node.attribute.append( + h.make_attribute( + "classlabels_strings" if strings else "classlabels_int64s", list(renamed) + ) + ) + native.graph.output[1].type.sequence_type.elem_type.map_type.key_type = ( + T.STRING if strings else T.INT64 + ) + native.graph.node.append(h.make_node("Identity", ["label"], ["decision"])) + native.graph.output[0].name = "decision" + native.graph.node.insert(1, h.make_node("Identity", ["P"], ["scores"])) + node.input[0] = "scores" + path = tmp_path / "renamed.mlbundle" + mlbundle.export(native, path) + loaded = mlbundle.load(path) + expected = loaded.native_model.session.run( + ["decision"], {"X": np.asarray(ROWS, dtype=np.float32)} + )[0] + for row, label in zip(ROWS, expected, strict=True): + assert loaded.predict_record(dict(enumerate(row)))[0].predicted_label == str( + renamed[list(source).index(label)] + ) + + +def test_zipmap_with_graph_encoded_argmax(tmp_path): + native = softmax(two_outputs=True) + native.graph.node.append( + h.make_node( + "ZipMap", + ["P"], + ["probabilities"], + domain="ai.onnx.ml", + classlabels_strings=["cold", "warm"], + ) + ) + native.graph.output[1].CopyFrom( + h.make_value_info( + "probabilities", + h.make_sequence_type_proto( + h.make_map_type_proto(T.STRING, h.make_tensor_type_proto(T.FLOAT, [])) + ), + ) + ) + path = tmp_path / "mapped-softmax.mlbundle" + mlbundle.export(native, path) + loaded = mlbundle.load(path) + expected = loaded.native_model.session.run( + ["label"], {"X": np.asarray(ROWS, dtype=np.float32)} + )[0] + assert [ + loaded.predict_record(dict(enumerate(row)))[0].predicted_label for row in ROWS + ] == [["cold", "warm"][int(value)] for value in expected] + + +@pytest.mark.parametrize("label", [0, 20]) +def test_independent_head_still_requires_a_known_native_label(label, tmp_path): + native = classifier((10, 20), zipmap=True) + native.graph.node[0].output[0] = "unused" + native.graph.node.append( + h.make_node( + "Constant", + [], + ["label"], + value=h.make_tensor("decision", T.INT64, [1], [label]), + ) + ) + path = tmp_path / "independent-label.mlbundle" + mlbundle.export(native, path) + loaded = mlbundle.load(path) + if label == 0: + with pytest.raises(mlbundle.InvalidModelError): + loaded.predict_record({"0": 0, "1": 1}) + else: + assert loaded.predict_record({"0": 0, "1": 1})[0].predicted_label == "20" + + +def test_string_tensor_rejects_object_array_numeric_cells(): + native = classifier((10, 20)) + native.graph.node[0].output[0] = "unused" + native.graph.node.append( + h.make_node( + "Constant", + [], + ["label"], + value=h.make_tensor("decision", T.STRING, [1], ["10"]), + ) + ) + native.graph.output[0].CopyFrom(h.make_tensor_value_info("label", T.STRING, [None])) + adapter = Adapter() + predictor = adapter.predictor(adapter.deserialize(native.SerializeToString())) + predictor.session = SimpleNamespace( + run=lambda *args: [np.array([10], dtype=object)] + ) + with pytest.raises( + mlbundle.InvalidModelError, match=r"^Invalid ONNX class prediction output\.$" + ): + predictor.predict_class_indices([[0, 1]]) diff --git a/tests/test_pipeline_decisions.py b/tests/test_pipeline_decisions.py new file mode 100644 index 0000000..a6b904c --- /dev/null +++ b/tests/test_pipeline_decisions.py @@ -0,0 +1,65 @@ +"""Composite decisions transform the original row and call the native leaf.""" + +import numpy as np +import pytest +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler + +import mlbundle +from pipeline_helpers import pipeline +from record_helpers import threshold_classifier + +pytestmark = [ + pytest.mark.filterwarnings("ignore:X does not have valid feature names"), + pytest.mark.filterwarnings("ignore:X has feature names"), +] + + +@pytest.mark.parametrize("library", ["xgboost", "catboost"]) +@pytest.mark.parametrize("prefix", [False, True]) +def test_pipeline_native_class_decisions(library, prefix, tmp_path): + model, frame = pipeline(library, False, prefix) + path = tmp_path / "composite.mlbundle" + labels = [["cold", "warm"]] if library == "xgboost" else None + mlbundle.export(model, path, class_labels=labels) + loaded = mlbundle.load(path) + expected = model.predict(frame).reshape(-1) + classes = model.steps[-1][1].classes_.tolist() + assert loaded.predictor.predict_class_indices(frame) == [ + [classes.index(value)] for value in expected + ] + for row, label in zip(frame.to_numpy(), expected, strict=True): + keys = ( + loaded.feature_names + if loaded.feature_origin == "model" + else map(str, range(len(row))) + ) + result = loaded.predict_record(dict(zip(keys, row, strict=True)))[0] + assert result.predicted_label == ( + labels[0][int(label)] if labels else str(label) + ) + + +def test_composite_preserves_catboost_threshold(tmp_path): + model = make_pipeline(StandardScaler().fit([[0], [1]]), threshold_classifier()) + path = tmp_path / "threshold-pipeline.mlbundle" + mlbundle.export(model, path) + loaded = mlbundle.load(path) + result = loaded.predict_record({"0": 1})[0] + assert result.predicted_label == str(model.predict([[1]])[0]) == "negative" + assert result.probabilities[1] > result.probabilities[0] + + +def test_composite_decision_errors_are_bounded(tmp_path, monkeypatch): + model, frame = pipeline() + path = tmp_path / "invalid.mlbundle" + mlbundle.export(model, path) + predictor = mlbundle.load(path).predictor + monkeypatch.setattr(predictor.native.model, "predict", lambda _: np.array([999])) + with pytest.raises(mlbundle.InvalidModelError) as error: + predictor.predict_class_indices(frame.iloc[:1]) + assert len(str(error.value)) < 160 + regressor, frame = pipeline(regression=True) + mlbundle.export(regressor, path, overwrite=True) + with pytest.raises(ValueError, match="classifier"): + mlbundle.load(path).predictor.predict_class_indices(frame) diff --git a/tests/test_publish.py b/tests/test_publish.py index 75a925d..b8f4bae 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -15,20 +15,20 @@ def preflight(): return runpy.run_path(str(SCRIPT))["check"] -@pytest.mark.parametrize("ref", ["refs/heads/develop", "refs/tags/v0.1.0a2", ""]) +@pytest.mark.parametrize("ref", ["refs/heads/develop", "refs/tags/v0.1.0a3", ""]) def test_other_refs_fail_before_network(preflight, monkeypatch, ref): def unexpected(*args, **kwargs): raise AssertionError("Network must not be used for other refs") monkeypatch.setitem(preflight.__globals__, "urlopen", unexpected) with pytest.raises(SystemExit, match="requires refs/heads/main"): - preflight(ref, {"version": "0.1.0a2"}) + preflight(ref, {"version": "0.1.0a3"}) @pytest.mark.parametrize("code", [200, 404, 403, 429, 500]) def test_exact_version_response(preflight, monkeypatch, capsys, code): def response(url, timeout): - assert url == "https://pypi.org/pypi/mlbundle/0.1.0a2/json" + assert url == "https://pypi.org/pypi/mlbundle/0.1.0a3/json" assert timeout == 30 if code != 200: raise HTTPError(url, code, "response", None, None) @@ -36,12 +36,12 @@ def response(url, timeout): monkeypatch.setitem(preflight.__globals__, "urlopen", response) if code == 404: - preflight("refs/heads/main", {"version": "0.1.0a2"}) + preflight("refs/heads/main", {"version": "0.1.0a3"}) assert "HTTP 404" in capsys.readouterr().out else: message = "already exists" if code == 200 else f"HTTP {code}" with pytest.raises(SystemExit, match=message): - preflight("refs/heads/main", {"version": "0.1.0a2"}) + preflight("refs/heads/main", {"version": "0.1.0a3"}) @pytest.mark.parametrize("error", [URLError("offline"), TimeoutError("timeout")]) @@ -51,4 +51,4 @@ def response(*args, **kwargs): monkeypatch.setitem(preflight.__globals__, "urlopen", response) with pytest.raises(SystemExit, match="PyPI version check failed"): - preflight("refs/heads/main", {"version": "0.1.0a2"}) + preflight("refs/heads/main", {"version": "0.1.0a3"}) diff --git a/tests/test_readme.py b/tests/test_readme.py index 1883aba..c7cf74b 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -20,10 +20,20 @@ @pytest.mark.parametrize( "code", PYTHON, - ids=["sklearn", "xgboost", "catboost", "composite", "onnx", "schema-samples"], + ids=[ + "sklearn", + "records", + "xgboost", + "catboost", + "composite", + "onnx", + "schema-samples", + ], ) def test_python_example(code, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) + if 'record = {"0"' in code: + exec(compile(PYTHON[0], str(README), "exec"), {}) if 'Path("trained.onnx")' in code: # The user's prerequisite file: a self-contained, two-column classifier. Path("trained.onnx").write_bytes(softmax().SerializeToString()) @@ -31,7 +41,7 @@ def test_python_example(code, tmp_path, monkeypatch): def test_console_examples(tmp_path, monkeypatch): - assert len(PYTHON) == 6 + assert len(PYTHON) == 7 assert all(language in {"python", "shell"} for language, _ in BLOCKS) monkeypatch.chdir(tmp_path) exec(compile(PYTHON[0], str(README), "exec"), {}) diff --git a/tests/test_records.py b/tests/test_records.py new file mode 100644 index 0000000..176cbc5 --- /dev/null +++ b/tests/test_records.py @@ -0,0 +1,297 @@ +"""Record keys, scalar conversion, schema nullability and output normalization.""" + +import copy +import math +from dataclasses import FrozenInstanceError, replace + +import pandas as pd +import pytest +from sklearn.tree import DecisionTreeClassifier + +import mlbundle +from mlbundle.canonical import canonical_dumps +from mlbundle.container import read_container +from mlbundle.writing import write_container +from record_helpers import ( + ACCEPTED_VALUES, + REJECTED_VALUES, + LookupOnly, + bundle_at, + categorical_model, + disagreement_classifier, + missing_pandas, + named_pipeline, + native_decision_model, + threshold_classifier, +) + + +@pytest.fixture +def bundle(tmp_path): + return bundle_at(tmp_path / "record.mlbundle") + + +@pytest.mark.parametrize("origin", ["model", "generated", "declared", "dataframe"]) +@pytest.mark.parametrize("strings", [False, True]) +def test_keys_order_and_ignored_extras(origin, strings, tmp_path): + loaded = bundle_at(tmp_path / "keys.mlbundle", origin=origin, names=["12", "0"]) + keys = ["12", "0"] if origin == "model" else (["0", "1"] if strings else [0, 1]) + record = {"ignored": object(), keys[1]: 1, keys[0]: 0} + assert loaded.row_from_record(record) == [0.0, 1.0] + assert loaded.predict_record(record)[0].probabilities == (1.0, 0.0) + + +@pytest.mark.parametrize("method", ["row_from_record", "predict_record"]) +@pytest.mark.parametrize("record", [None, [], [0, 1], "0", 1]) +def test_mapping_required(bundle, method, record): + with pytest.raises(mlbundle.PredictionInputError, match="mapping"): + getattr(bundle, method)(record) + + +@pytest.mark.parametrize("record", [{0: 0, "0": 0, 1: 1}, {0: 0, 1: 1, "1": 1}]) +def test_duplicate_position_forms(bundle, record): + with pytest.raises(mlbundle.PredictionInputError, match="both integer and string"): + bundle.predict_record(record) + + +@pytest.mark.parametrize("key", ["00", "+0", "-0", " 0", "0 ", "0.0", "\u0660"]) +def test_noncanonical_position_is_ignored_and_missing(bundle, key): + with pytest.raises(mlbundle.PredictionInputError, match='Missing features: "0"'): + bundle.row_from_record({key: 0, "1": 1}) + + +def test_lookup_only_mapping(bundle): + assert bundle.predict_record(LookupOnly())[0].probabilities == (1.0, 0.0) + + +@pytest.mark.parametrize("value,expected", ACCEPTED_VALUES) +def test_accepted_values(bundle, value, expected): + assert bundle.row_from_record({"0": value, "1": 1}) == [expected, 1.0] + + +@pytest.mark.parametrize("value,reason", REJECTED_VALUES) +def test_rejected_values_without_echo(bundle, value, reason): + with pytest.raises(mlbundle.PredictionInputError, match=reason) as error: + bundle.predict_record({"0": value, "1": 1}) + assert str(error.value).startswith('Feature "0": ') + assert "private text" not in str(error.value) + + +@pytest.mark.parametrize("kind", ["classifier", "regressor"]) +@pytest.mark.parametrize("outputs", [1, 2]) +@pytest.mark.parametrize("row", [[0, 1], [1, 0]]) +def test_outputs_equal_array_prediction_and_frozen(kind, outputs, row, tmp_path): + loaded = bundle_at(tmp_path / "outputs.mlbundle", kind=kind, outputs=outputs) + record = {str(index): str(value) for index, value in enumerate(row)} + row = loaded.row_from_record(record) + results = loaded.predict_record(record) + assert isinstance(results, tuple) and len(results) == outputs + if kind == "classifier": + raw = loaded.predict_proba([row]) + matrices = [raw] if outputs == 1 else raw + native = loaded.native_model.predict([row])[0] + for output, (result, matrix) in enumerate(zip(results, matrices, strict=True)): + assert ( + result.output == output and result.kind == mlbundle.ModelKind.CLASSIFIER + ) + assert result.labels == tuple(loaded.schema["reports"][output]["labels"]) + assert result.probabilities == tuple(matrix[0]) and result.value is None + assert result.predicted_label == str( + native if outputs == 1 else native[output] + ) + else: + raw = loaded.predict([row]) + values = [raw[0]] if outputs == 1 else raw[0] + for output, (result, value) in enumerate(zip(results, values, strict=True)): + assert ( + result.output == output and result.kind == mlbundle.ModelKind.REGRESSOR + ) + assert ( + result.labels is result.probabilities is None and result.value == value + ) + assert result.predicted_label is None + assert isinstance(results[0], mlbundle.OutputPrediction) + assert not hasattr(loaded, "predict_class_indices") + with pytest.raises(FrozenInstanceError): + results[0].output = 99 + + +@pytest.mark.parametrize("field_kind", ["number", "onehot-category", "series"]) +@pytest.mark.parametrize("required", [True, False]) +def test_none_uses_parent_required(field_kind, required, tmp_path): + path = tmp_path / "nullable.mlbundle" + names = ( + ["color__red", "color__blue"] if field_kind == "onehot-category" else ["a", "b"] + ) + loaded = bundle_at(path, origin="model", names=names) + manifest, members = read_container(path) + schema = copy.deepcopy(loaded.schema) + parent = schema["fields"][0] + parent["required"] = required + if field_kind == "series": + parent.pop("step") + parent.update( + kind="series", + columns=[ + { + "id": axis, + "kind": "number", + "label": axis, + "required": not required, + "step": 1.0, + } + for axis in ("field1", "field2") + ], + ) + members[mlbundle.MemberName.SCHEMA] = canonical_dumps(schema) + del members[mlbundle.MemberName.MANIFEST] + loaded = mlbundle.load(write_container(manifest, members)) + record = {names[0]: None, names[1]: 1} + if required: + with pytest.raises( + mlbundle.PredictionInputError, match="required feature cannot be None" + ): + loaded.predict_record(record) + else: + assert math.isnan(loaded.row_from_record(record)[0]) + assert loaded.predict_record(record)[0].probabilities is not None + + +def test_missing_keys_and_untrusted_names_are_bounded(tmp_path): + names = [f"private\n\x1b[31m{index}" + "\x1b" * 70 for index in range(12)] + frame = pd.DataFrame([[0] * 12, [1] * 12], columns=names) + path = tmp_path / "escaped.mlbundle" + mlbundle.export(DecisionTreeClassifier().fit(frame, [0, 1]), path) + loaded = mlbundle.load(path) + with pytest.raises(mlbundle.PredictionInputError) as error: + loaded.row_from_record({}) + assert "7 more" in str(error.value) and "\\n\\u001b" in str(error.value) + assert len(str(error.value)) < 700 + assert "\n" not in str(error.value) and "\x1b" not in str(error.value) + with pytest.raises(mlbundle.PredictionInputError) as error: + loaded.row_from_record(dict.fromkeys(names, "secret value")) + assert "\\n\\u001b" in str(error.value) and "secret value" not in str(error.value) + + +def test_array_path_stays_strict_and_schema_is_required(bundle): + with pytest.raises(mlbundle.InvalidModelError): + bundle.predict_proba([["0", "1"]]) + with pytest.raises(mlbundle.PredictionInputError, match="required"): + bundle.row_from_record({"0": None, "1": 1}) + with pytest.raises(mlbundle.PredictionInputError, match="validated schema"): + replace(bundle, schema=None).row_from_record({"0": 0, "1": 1}) + + +@pytest.mark.parametrize("kind", ["classifier", "regressor", "composite"]) +@pytest.mark.parametrize("position", [0, 1]) +def test_named_pipeline_record(kind, position, tmp_path, monkeypatch): + from mlbundle._sklearn import record_input + + model, frame = named_pipeline(kind) + path = tmp_path / "named-pipeline.mlbundle" + mlbundle.export(model, path) + loaded = mlbundle.load(path) + record = {name: str(value) for name, value in frame.iloc[position].items()} + result = loaded.predict_record(record)[0] + if kind == "regressor": + assert result.value == loaded.predict(frame.iloc[position : position + 1])[0] + assert result.predicted_label is None + else: + assert result.probabilities == tuple( + loaded.predict_proba(frame.iloc[position : position + 1])[0] + ) + assert result.predicted_label == str( + model.predict(frame.iloc[position : position + 1])[0] + ) + monkeypatch.setattr(record_input, "import_module", missing_pandas) + with pytest.raises(mlbundle.MissingOptionalDependencyError) as error: + loaded.predict_record({"a": 0, "b": 1}) + assert error.value.extra == "schema" + + +def test_named_record_without_pandas(tmp_path, monkeypatch): + from mlbundle._sklearn import record_input + + loaded = bundle_at(tmp_path / "named.mlbundle", origin="model") + + monkeypatch.setattr(record_input, "import_module", pytest.fail) + assert loaded.row_from_record({"age": "0", "active": "1"}) == [0.0, 1.0] + assert loaded.predict_record({"age": "0", "active": "1"})[0].probabilities == ( + 1.0, + 0.0, + ) + + +@pytest.mark.parametrize("composite", [False, True]) +def test_numeric_record_preserves_xgboost_categories(composite, tmp_path): + model = categorical_model(composite) + path = tmp_path / "categorical.mlbundle" + mlbundle.export(model, path) + loaded = mlbundle.load(path) + assert loaded.predict_record({"category": "0", "amount": "0.0"})[ + 0 + ].probabilities == tuple(loaded.predict_proba([[0.0, 0.0]])[0]) + + +@pytest.mark.parametrize("library", ["xgboost", "catboost", "onnx"]) +def test_native_record_labels_on_several_rows(library, tmp_path): + import numpy as np + + from onnx_helpers import ROWS + + native, options = native_decision_model(library) + path = tmp_path / "decision.mlbundle" + mlbundle.export(native, path, **options) + loaded = mlbundle.load(path) + expected = ( + loaded.native_model.session.run( + ["label"], {"X": np.asarray(ROWS, dtype="float32")} + )[0] + if library == "onnx" + else loaded.native_model.predict(ROWS).reshape(-1) + ) + for row, label in zip(ROWS, expected, strict=True): + result = loaded.predict_record( + {str(index): value for index, value in enumerate(row)} + )[0] + assert result.predicted_label == ( + ["cold", "warm"][int(label)] if library == "xgboost" else str(label) + ) + + +def test_decisions_can_disagree_with_probabilities(tmp_path): + for name, model, row in [ + ("threshold", threshold_classifier(), [1]), + ("svc", disagreement_classifier(), [3.5]), + ]: + path = tmp_path / f"{name}.mlbundle" + mlbundle.export(model, path) + loaded = mlbundle.load(path) + result = loaded.predict_record({"0": row[0]})[0] + assert result.predicted_label == str(model.predict([row])[0]) + assert ( + result.predicted_label + != result.labels[result.probabilities.index(max(result.probabilities))] + ) + if name == "threshold": + assert ( + 0.5 < result.probabilities[1] < 0.8 + and result.predicted_label == "negative" + ) + + +def test_onnx_without_decision_returns_none(tmp_path): + from onnx_helpers import softmax + + path = tmp_path / "probabilities-only.mlbundle" + mlbundle.export(softmax(), path) + assert ( + mlbundle.load(path).predict_record({"0": 0, "1": 1})[0].predicted_label is None + ) + + +def test_record_invalid_native_decision_is_bounded(bundle, monkeypatch): + monkeypatch.setattr(bundle.native_model, "predict", lambda _: ["private class"]) + with pytest.raises(mlbundle.InvalidModelError) as error: + bundle.predict_record({"0": 0, "1": 1}) + assert "private" not in str(error.value) and len(str(error.value)) < 160 diff --git a/tests/test_schema.py b/tests/test_schema.py index 6f0d28a..f35497b 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -106,7 +106,7 @@ def test_mixed_schema_and_series(tmp_path): ] ).fit(frame, [False, True]) view = mlbundle.export(model, tmp_path / "mixed.mlbundle", training_data=frame) - assert view.schema["fields"] == infer_schema(frame, onehot_separator="__") + assert view.schema["fields"][:-1] == infer_schema(frame, onehot_separator="__")[:-1] assert [field["kind"] for field in view.schema["fields"]] == [ "number", "category", diff --git a/tests/test_schema_fields.py b/tests/test_schema_fields.py index 6d9c9ff..3a1e34f 100644 --- a/tests/test_schema_fields.py +++ b/tests/test_schema_fields.py @@ -1,8 +1,5 @@ """Closed field vocabulary: native inference shapes and malicious alterations.""" -import copy -from typing import get_args - import pandas as pd import pytest from mlschema import infer_schema @@ -37,7 +34,7 @@ def bundle(tmp_path): ).fit(frame, [0, 1]) path = tmp_path / "fields.mlbundle" view = mlbundle.export(model, path, training_data=frame, target_names=None) - assert view.schema["fields"] == infer_schema(frame, onehot_separator="__") + assert view.schema["fields"][:3] == infer_schema(frame, onehot_separator="__")[:3] return path @@ -51,8 +48,6 @@ def bundle(tmp_path): lambda s: s["fields"][0].update(label="color__red"), lambda s: s["fields"][1].update(label="wrong"), lambda s: s["fields"][2].update(options=["low", "low"]), - lambda s: s["fields"][3]["field1"].update(mappedTo="age"), - lambda s: s["fields"][3].update(field1=copy.deepcopy(s["fields"][3])), lambda s: s["fields"][1].update(kind="custom"), lambda s: s["fields"][1].update(step=-1), lambda s: s.update(fields=[], reports=[]), @@ -132,24 +127,6 @@ def test_report_comparison_uses_schema_budget(tmp_path): assert mlbundle.inspect(path).schema == mlbundle.load(path).schema == view.schema -def test_field_vocabulary_matches_mlschema_models(): - from mlschema.strategies.app import builtin_kinds - from mlschema.strategies.domain.onehot_category_field import OneHotCategoryOption - - from mlbundle._schema_fields import OneHotOption, SchemaField - - ours = { - kind: set(model.model_fields) - for model in get_args(get_args(SchemaField)[0]) - for kind in get_args(model.model_fields["kind"].annotation) - } - theirs = {item.name: set(item.model.model_fields) for item in builtin_kinds()} - assert ours.keys() == theirs.keys() - for kind, keys in ours.items(): - assert keys <= theirs[kind], kind - assert set(OneHotOption.model_fields) <= set(OneHotCategoryOption.model_fields) - - @pytest.mark.parametrize("origin", ["model", "declared", "generated"]) def test_default_inference_and_positional_predictions(origin, tmp_path): frame = pd.DataFrame( @@ -201,15 +178,10 @@ def test_positional_series_and_onehot_mapping_rejections(mapping, tmp_path): 2, ] series = view.schema["fields"][1] - assert ( - series["mappedTo"] - == series["field1"]["mappedTo"] - == series["field2"]["mappedTo"] - == 1 - ) + assert series["mappedTo"] == 1 manifest, members = read_container(path) del members[MemberName.MANIFEST] - for target in (view.schema["fields"][0]["options"][0], series["field1"]): + for target in (view.schema["fields"][0]["options"][0], series): original = target["mappedTo"] target["mappedTo"] = mapping members[MemberName.SCHEMA] = canonical_dumps(view.schema) diff --git a/tests/test_series_schema.py b/tests/test_series_schema.py new file mode 100644 index 0000000..3145c77 --- /dev/null +++ b/tests/test_series_schema.py @@ -0,0 +1,112 @@ +"""The sole mlschema shape adaptation and its closed reader contract.""" + +import copy +from typing import get_args + +import pandas as pd +import pytest +from mlschema import infer_schema +from sklearn.tree import DecisionTreeClassifier + +import mlbundle +from mlbundle._schema_fields import OneHotOption, SchemaField +from mlbundle.canonical import canonical_dumps +from mlbundle.container import read_container +from mlbundle.writing import write_container + + +def test_field_vocabulary_matches_mlschema_except_series(): + from mlschema.strategies.app import builtin_kinds + from mlschema.strategies.domain.onehot_category_field import OneHotCategoryOption + + ours = { + kind: set(model.model_fields) + for model in get_args(get_args(SchemaField)[0]) + for kind in get_args(model.model_fields["kind"].annotation) + } + theirs = {item.name: set(item.model.model_fields) for item in builtin_kinds()} + assert ours.keys() == theirs.keys() + for kind, keys in ours.items(): + if kind == "series": + # The MLForm columns adaptation is the only inferred shape difference. + assert keys - theirs[kind] == {"columns"} + assert {"field1", "field2"} <= theirs[kind] - keys + else: + assert keys <= theirs[kind], kind + assert set(OneHotOption.model_fields) <= set(OneHotCategoryOption.model_fields) + + +@pytest.fixture +def series_bundle(tmp_path): + frame = pd.DataFrame({"pair": [(1, 2), (3, 4)]}) + path = tmp_path / "series.mlbundle" + view = mlbundle.export( + DecisionTreeClassifier().fit([[1], [3]], [0, 1]), path, training_data=frame + ) + original = infer_schema(frame)[0] + field = view.schema["fields"][0] + assert field == { + **{ + key: value + for key, value in original.items() + if key not in ("field1", "field2", "mappedTo") + }, + "mappedTo": 0, + "columns": [ + { + **{ + key: value + for key, value in original[axis].items() + if key != "mappedTo" + }, + "id": axis, + } + for axis in ("field1", "field2") + ], + } + assert view.schema["schema_version"] == "2" + assert mlbundle.inspect(path).schema == mlbundle.load(path).schema == view.schema + return path, field + + +def test_series_export(series_bundle): + assert [column["id"] for column in series_bundle[1]["columns"]] == [ + "field1", + "field2", + ] + + +def old_shape(field): + for column in field.pop("columns"): + field[column.pop("id")] = {**column, "mappedTo": field["mappedTo"]} + + +@pytest.mark.parametrize( + "change", + [ + old_shape, + lambda f: f["columns"].reverse(), + lambda f: f["columns"].pop(), + lambda f: f["columns"].append(copy.deepcopy(f["columns"][0])), + lambda f: f["columns"][1].update(id="field1"), + lambda f: f["columns"][0].update(id="unknown"), + lambda f: f["columns"][0].pop("id"), + lambda f: f["columns"][0].update(mappedTo=0), + lambda f: f["columns"][0].update(kind="series"), + lambda f: f["columns"][0].update(step=0), + lambda f: f["columns"][0].update(required="false"), + lambda f: f["columns"][0].update(extra="private"), + ], +) +def test_series_rejections(change, series_bundle): + path, field = series_bundle + manifest, members = read_container(path) + schema = mlbundle.inspect(path).schema + change(field) + schema["fields"] = [field] + members[mlbundle.MemberName.SCHEMA] = canonical_dumps(schema) + del members[mlbundle.MemberName.MANIFEST] + data = write_container(manifest, members) + for reader in (mlbundle.inspect, mlbundle.load): + with pytest.raises(mlbundle.InvalidSchemaError): + reader(data) diff --git a/tests/test_sklearn_decisions.py b/tests/test_sklearn_decisions.py new file mode 100644 index 0000000..1df5790 --- /dev/null +++ b/tests/test_sklearn_decisions.py @@ -0,0 +1,88 @@ +"""Native sklearn class decisions, including calibrated SVC disagreement.""" + +import numpy as np +import pytest +from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor + +import mlbundle +from mlbundle._sklearn import Adapter +from record_helpers import disagreement_classifier + + +@pytest.mark.parametrize("outputs", [1, 2]) +def test_native_decisions_and_output_order(outputs, monkeypatch): + rows = [[0], [1], [2], [3]] + labels = ( + ["cold", "cold", "warm", "warm"] + if outputs == 1 + else [[0, 2], [0, 3], [1, 2], [1, 3]] + ) + model = DecisionTreeClassifier().fit(rows, labels) + predictor = Adapter().predictor(model) + raw = np.asarray(model.predict(rows)).reshape(len(rows), outputs) + classes = [model.classes_] if outputs == 1 else model.classes_ + expected = [ + [ + values.tolist().index(value) + for values, value in zip(classes, row, strict=True) + ] + for row in raw + ] + monkeypatch.setattr( + model, "predict_proba", lambda *a: pytest.fail("probability decision") + ) + assert predictor.predict_class_indices(rows) == expected + with pytest.raises(ValueError, match="classifier"): + Adapter().predictor( + DecisionTreeRegressor().fit(rows, [0, 0, 1, 1]) + ).predict_class_indices(rows) + + +def test_svc_uses_predict_with_disagreeing_probabilities(tmp_path): + model = disagreement_classifier() + path = tmp_path / "svc.mlbundle" + mlbundle.export(model, path) + loaded = mlbundle.load(path) + row = [[3.5]] + native = int(model.predict(row)[0]) + assert loaded.predictor.predict_class_indices(row) == [[native]] + result = loaded.predict_record({"0": 3.5})[0] + assert result.predicted_label == str(native) + assert result.probabilities.index(max(result.probabilities)) != native + + +@pytest.mark.parametrize( + "raw", + [ + [], + ["cold"], + [["cold", "warm"]], + ["private", "warm"], + [None, "warm"], + [np.nan, "warm"], + [complex(1), "warm"], + [{"private": 1}, "warm"], + ["x" * 1025, "warm"], + ], +) +def test_invalid_decision_output_is_bounded(raw, monkeypatch): + model = DecisionTreeClassifier().fit([[0], [1]], ["cold", "warm"]) + predictor = Adapter().predictor(model) + monkeypatch.setattr(model, "predict", lambda _: raw) + with pytest.raises(mlbundle.InvalidModelError) as error: + predictor.predict_class_indices([[0], [1]]) + assert "private" not in str(error.value) and len(str(error.value)) < 160 + + +def test_native_failure_is_bounded(monkeypatch): + model = DecisionTreeClassifier().fit([[0], [1]], [0, 1]) + predictor = Adapter().predictor(model) + + def fail(_): + raise ValueError("private decision") + + monkeypatch.setattr(model, "predict", fail) + with pytest.raises( + mlbundle.InvalidModelError, match=r"^Sklearn prediction failed\.$" + ): + predictor.predict_class_indices([[0]]) diff --git a/tests/test_version_diagnostics.py b/tests/test_version_diagnostics.py new file mode 100644 index 0000000..f9bb371 --- /dev/null +++ b/tests/test_version_diagnostics.py @@ -0,0 +1,166 @@ +"""Version hints diagnose existing rejection paths without adding a load gate.""" + +import importlib.metadata + +import pytest +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import StandardScaler +from sklearn.tree import DecisionTreeClassifier +from xgboost import XGBClassifier + +import mlbundle +from mlbundle import _load_diagnostics +from mlbundle._sklearn import Adapter, transformers +from mlbundle.canonical import canonical_dumps +from mlbundle.container import read_container +from mlbundle.writing import write_container + + +def failed_load(_self, _data): + raise mlbundle.InvalidModelError( + "Skops state does not survive native construction." + ) + + +@pytest.fixture +def path(tmp_path): + path = tmp_path / "version.mlbundle" + mlbundle.export(DecisionTreeClassifier().fit([[0], [1]], [0, 1]), path) + return path + + +def declared_package(path, declared): + manifest, members = read_container(path) + parts = manifest.model_dump(mode="json", by_alias=True) + if declared is None: + parts["environment"]["packages"].pop("scikit-learn", None) + else: + parts["environment"]["packages"]["scikit-learn"] = declared + del members[mlbundle.MemberName.MANIFEST] + return write_container(parts, members) + + +def test_environment_mismatch_after_failure(path, monkeypatch): + data = declared_package(path, "1.9.1") + monkeypatch.setattr(_load_diagnostics, "version", lambda _: "1.9.0") + assert mlbundle.load(data).predict_proba([[0]]) == [[1.0, 0.0]] + monkeypatch.setattr(Adapter, "deserialize", failed_load) + with pytest.raises(mlbundle.EnvironmentMismatchError) as error: + mlbundle.load(data) + assert isinstance(error.value, mlbundle.InvalidModelError) + assert str(error.value) == ( + 'Bundle declares scikit-learn "1.9.1"; installed version is "1.9.0". ' + 'The bundle must be re-exported with scikit-learn "1.9.0" or loaded ' + 'where scikit-learn "1.9.1" is installed.' + ) + + +@pytest.mark.parametrize( + "declared", + [None, "1.9.0", "private\n\x1b", "1.9.1 ", "9" * 65, "v1.9.1", "1.9.1;evil"], +) +def test_invalid_or_equal_environment_keeps_original(path, monkeypatch, declared): + data = declared_package(path, declared) + monkeypatch.setattr(_load_diagnostics, "version", lambda _: "1.9.0") + monkeypatch.setattr(Adapter, "deserialize", failed_load) + with pytest.raises(mlbundle.InvalidModelError) as error: + mlbundle.load(data) + assert type(error.value) is mlbundle.InvalidModelError + assert str(error.value) == "Skops state does not survive native construction." + + +def test_unavailable_installed_metadata_keeps_original(path, monkeypatch): + def absent(_): + raise importlib.metadata.PackageNotFoundError + + monkeypatch.setattr(_load_diagnostics, "version", absent) + monkeypatch.setattr(Adapter, "deserialize", failed_load) + with pytest.raises(mlbundle.InvalidModelError) as error: + mlbundle.load(declared_package(path, "1.9.1")) + assert type(error.value) is mlbundle.InvalidModelError + + +def test_claim_mismatch_is_not_environment_failure(path, monkeypatch): + data = declared_package(path, "0.0.1") + + def claims(*_): + raise mlbundle.ClaimMismatchError("n_features") + + monkeypatch.setattr(Adapter, "extract_claims", claims) + with pytest.raises(mlbundle.ClaimMismatchError): + mlbundle.load(data) + + +@pytest.mark.parametrize("sklearn_method", ["deserialize", "claims"]) +@pytest.mark.parametrize( + "native_method", ["deserialize", "extract_claims", "extract_structure"] +) +def test_composite_skops_only_failures( + tmp_path, monkeypatch, sklearn_method, native_method +): + path = tmp_path / "composite.mlbundle" + model = make_pipeline( + StandardScaler(), XGBClassifier(n_estimators=1, n_jobs=1) + ).fit([[0], [1]], [0, 1]) + mlbundle.export(model, path) + data = declared_package(path, "0.0.1") + assert mlbundle.load(data).predict_record({"0": 0}) + from mlbundle._xgboost import Adapter as XGBAdapter + + with monkeypatch.context() as patch: + patch.setattr(XGBAdapter, native_method, failed_load) + with pytest.raises(mlbundle.InvalidModelError) as error: + mlbundle.load(data) + assert type(error.value) is mlbundle.InvalidModelError + monkeypatch.setattr(transformers.Adapter, sklearn_method, failed_load) + with pytest.raises(mlbundle.EnvironmentMismatchError): + mlbundle.load(data) + + +@pytest.mark.parametrize( + "writer", ["0.1.0a1", "0.1.0a2", "1.0rc1", "1.0.post2.dev3+local"] +) +def test_old_manifest_diagnosis_shared_by_inspect_and_load(manifest_data, writer): + from conftest import build_zip, schema_for_manifest + + manifest_data["writer_version"] = writer + manifest_data["environment"]["packages"]["mlbundle"] = writer + manifest_data["target"]["name"] = manifest_data["target"].pop("names") + data = build_zip( + [ + ("manifest.json", canonical_dumps(manifest_data)), + ("model.skops", b"native model bytes"), + ("schema.json", canonical_dumps(schema_for_manifest(manifest_data))), + ] + ) + expected = ( + f'Invalid mlbundle manifest. Bundle was written by mlbundle "{writer}"; ' + f'this reader is "{mlbundle.__version__}". Format 1 is not frozen; ' + "the model must be re-exported." + ) + for reader in (mlbundle.inspect, mlbundle.load): + with pytest.raises(mlbundle.InvalidManifestError) as error: + reader(data) + assert str(error.value) == expected + + +@pytest.mark.parametrize( + "writer", [None, 1, [], "", "evil\n\x1b", "9" * 65, "1.0 ", mlbundle.__version__] +) +def test_manifest_without_safe_different_writer_keeps_message(manifest_data, writer): + manifest_data["unexpected"] = True + manifest_data["writer_version"] = writer + with pytest.raises(mlbundle.InvalidManifestError) as error: + mlbundle.Manifest.from_bytes(canonical_dumps(manifest_data)) + assert str(error.value) == "Invalid mlbundle manifest." + + +def test_malformed_json_and_unknown_format_keep_errors(): + with pytest.raises( + mlbundle.InvalidManifestError, match=r"^Invalid mlbundle manifest\.$" + ): + mlbundle.Manifest.from_bytes(b"not JSON") + with pytest.raises(mlbundle.UnsupportedFormatVersionError): + mlbundle.Manifest.from_bytes( + b'{"format_name":"mlbundle","format_version":"99","writer_version":"0.1.0a1"}' + ) diff --git a/tests/test_writing.py b/tests/test_writing.py index 4ab479a..57691c6 100644 --- a/tests/test_writing.py +++ b/tests/test_writing.py @@ -169,30 +169,30 @@ def test_chunked_model_and_empty_model(manifest_data): [ ( b"native model bytes", - "e6fe466bcfcc6bbc8ac7310eb4a77a21e34a0545abd7ec9bc84149ba34e29a70", + "04135705c8b64e3a492fc543b0cb150a747cc2214533c9425ba93d61870b7494", ), ( random.Random(0).randbytes(200_000), - "adf3678cb1037c802bcf95688065302f6c11a16a057bbe29784a6a0235a6992e", + "af4a7e26486a59a105f9363c4b8d034172c27a7058d80635d2dadadf26b8ac05", ), ( b"hello world " * 10_000, - "b5219cc6066a71271bd993e7bb89462c927372349bedaba73549df61b2ed55ad", + "a69ad3950fbf46b8524f38f65b92911215dc435060b4158b7be2c01adb6b79dc", ), ( bytes(range(256)) * 500, - "509b61925106409488d5420eb97145d0b07770224144c6c5101d59e74fc72b10", + "605ab1cb22b719643413342f342258cb92d81be9aa65bbfe19ebeccc27a9b1f3", ), ( b"0" * 1000, - "f98eb7aa07287110dd50cf6d8f04cb60e5c82ae72428c45911e7a81653d84c7a", + "1f1facf21a7f91d32984764a4efb4c8b9bf9992aa4c639c0abe2cd3b943e14d9", ), ], ids=["native", "noise", "text", "uniform", "runs"], ) def test_fixed_archive_bytes_on_both_platforms(manifest_data, data, expected): - manifest_data["writer_version"] = "0.1.0a2" - manifest_data["environment"]["packages"]["mlbundle"] = "0.1.0a2" + manifest_data["writer_version"] = "0.1.0a3" + manifest_data["environment"]["packages"]["mlbundle"] = "0.1.0a3" bundle = write_container( manifest_data, {MemberName.SKOPS: data, MemberName.SCHEMA: b"{}"} ) diff --git a/tests/test_xgboost_decisions.py b/tests/test_xgboost_decisions.py new file mode 100644 index 0000000..64c9fc1 --- /dev/null +++ b/tests/test_xgboost_decisions.py @@ -0,0 +1,76 @@ +"""Native encoded XGBoost decisions map through the bound original labels.""" + +from decimal import Decimal + +import numpy as np +import pytest +from xgboost import XGBClassifier, XGBRegressor + +import mlbundle +from mlbundle._xgboost import Adapter + + +@pytest.mark.parametrize("classes", [2, 3]) +def test_native_wrapper_decisions_and_bound_labels(classes, tmp_path): + rows = [[index] for index in range(9)] + model = XGBClassifier(n_estimators=2, max_depth=1, n_jobs=1).fit( + rows, [index % classes for index in range(9)] + ) + names = ["cold", "warm", "hot"][:classes] + path = tmp_path / "bound.mlbundle" + mlbundle.export(model, path, class_labels=[names]) + loaded = mlbundle.load(path) + native = model.predict(rows) + assert loaded.predictor.predict_class_indices(rows) == [ + [int(index)] for index in native + ] + assert [ + loaded.predict_record({"0": row[0]})[0].predicted_label for row in rows + ] == [names[int(index)] for index in native] + + +@pytest.mark.parametrize( + "raw", + [ + [], + [2], + [-1], + [0.5], + [np.nan], + [np.inf], + [[0, 1]], + ["private"], + ["0"], + [Decimal("1")], + np.array([1 + 4j]), + ], +) +def test_invalid_encoded_decision(raw, monkeypatch): + model = XGBClassifier(n_estimators=1, n_jobs=1).fit([[0], [1]], [0, 1]) + predictor = Adapter().predictor(model) + monkeypatch.setattr(model, "predict", lambda _: raw) + with pytest.raises(mlbundle.InvalidModelError) as error: + predictor.predict_class_indices([[0]]) + assert "private" not in str(error.value) and len(str(error.value)) < 160 + + +def test_native_call_without_probability_inference(monkeypatch): + rows = [[0], [1], [2], [3]] + model = XGBClassifier(n_estimators=2, n_jobs=1).fit(rows, [0, 1, 0, 1]) + predictor = Adapter().predictor(model) + expected = [[int(index)] for index in model.predict(rows)] + monkeypatch.setattr(model, "predict_proba", lambda *a: pytest.fail("argmax")) + assert predictor.predict_class_indices(rows) == expected + + def fail(_): + raise ValueError("private prediction") + + monkeypatch.setattr(model, "predict", fail) + with pytest.raises( + mlbundle.InvalidModelError, match=r"^XGBoost prediction failed\.$" + ): + predictor.predict_class_indices(rows) + with pytest.raises(ValueError, match="classifier"): + Adapter().predictor( + XGBRegressor(n_estimators=1, n_jobs=1).fit(rows, [0, 1, 0, 1]) + ).predict_class_indices(rows) diff --git a/uv.lock b/uv.lock index 8f895fa..c777b7d 100644 --- a/uv.lock +++ b/uv.lock @@ -459,7 +459,7 @@ wheels = [ [[package]] name = "mlbundle" -version = "0.1.0a2" +version = "0.1.0a3" source = { editable = "." } dependencies = [ { name = "pydantic" },