From 85f910fc1d440e0185211c2755bf256f1c0fa899 Mon Sep 17 00:00:00 2001 From: Sacha Braun <114589494+ElSacho@users.noreply.github.com> Date: Mon, 2 Mar 2026 19:16:58 +0100 Subject: [PATCH 01/11] feat: New variational losses for Lp calibration errors --- README.md | 144 +++++++++++- probmetrics/__about__.py | 2 +- probmetrics/calibrators.py | 275 +++++++++++++++++------ probmetrics/classifiers.py | 176 +++++++++++++++ probmetrics/distributions.py | 14 +- probmetrics/metrics.py | 423 ++++++++++++++++++++++++++++++----- probmetrics/utils.py | 33 +-- pyproject.toml | 7 +- tests/test_metrics.py | 4 +- 9 files changed, 936 insertions(+), 142 deletions(-) create mode 100644 probmetrics/classifiers.py diff --git a/README.md b/README.md index 9295213..887cf6d 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ metrics for assessing the quality of probabilistic predictions, and logistic-based calibration functions. It accompanies our papers -[Rethinking Early Stopping: Refine, Then Calibrate](https://arxiv.org/abs/2501.19195) and [Structured Matrix Scaling for Multi-Class Calibration](https://arxiv.org/abs/2511.03685). +[Rethinking Early Stopping: Refine, Then Calibrate](https://arxiv.org/abs/2501.19195) and [Structured Matrix Scaling for Multi-Class Calibration](https://arxiv.org/abs/2511.03685) +and [A Variational Estimator for Lp Calibration Errors](https://arxiv.org/abs/2602.24230). Please cite us if you use this repository for research purposes. The experiments from the papers can be found here: - Rethinking Early Stopping: @@ -23,6 +24,8 @@ The experiments from the papers can be found here: - [theory](https://github.com/eugeneberta/RefineThenCalibrate-Theory). - Structured Matrix Scaling: [all experiments](https://github.com/eugeneberta/LogisticCalibrationBenchmark). +- A Variational Estimator for Lp Calibration Errors: + [all experiments](https://github.com/ElSacho/Evaluating_Lp_Calibration_Errors). ## Installation @@ -31,10 +34,11 @@ Probmetrics is available via pip install probmetrics ``` To obtain all functionality, install `probmetrics[extra,dev,dirichletcal]`. -- extra installs more packages for smooth ECE, +- extra installs more packages for our CatBoost/LightGBM-based $L_p$ calibration + error metrics, smooth ECE (only works with scikit-learn versions <= 1.6), Venn-Abers calibration, centered isotonic regression, - the temperature scaling implementation in NetCal. + and the temperature scaling implementation in NetCal. - dev installs more packages for development (esp. documentation) - dirichletcal installs Dirichlet calibration, which however only works for Python 3.12 upwards. @@ -156,7 +160,10 @@ metrics = Metrics.from_names([ 'refinement_logloss_ts-mix_all', 'calib-err_logloss_ts-mix_all', 'refinement_brier_ts-mix_all', - 'calib-err_brier_ts-mix_all' + 'calib-err_brier_ts-mix_all', + 'calib-err_proper-L1-binary-as-1d_WS_CatboostClassifier_all', + 'calib-err_proper-L2-binary-as-1d_WS_CatboostClassifier_all', + 'calib-err_proper-Linf-binary-as-1d_WS_CatboostClassifier_all', ]) ``` @@ -168,12 +175,137 @@ Metrics.get_available_names(metric_type=MetricType.CLASS) While there are some classes for regression metrics, they are not implemented. +## Advanced calibration, confidence, and top-class metrics + +Beyond standard metrics, you can evaluate proper Lp calibration errors for +any p, as well as isolate specific types of errors like over-confidence, +under-confidence, and top-class errors. + +**Note:** Over- and under-confidence metrics are designed for binary classification. +To use those for multi-class, please use `TopClassLoss(OverConfidenceLoss(your_metric))`. + +```python +from probmetrics.metrics import ( + ProperLpLoss, + BrierLoss, + OverConfidenceLoss, + UnderConfidenceLoss, + TopClassLoss +) + +# Evaluate proper Lp calibration errors for any p +lp_loss_l1 = ProperLpLoss(p=1) # Evaluate E[ \| Y - E[Y|f(X)] \|_1 ] +lp_loss_l2 = ProperLpLoss(p=2) # Evaluate E[ \| Y - E[Y|f(X)] \|_2 ] + +# Evaluate over-confidence and under-confidence +# (Initialize via string name or by passing a metric object) +over_brier = OverConfidenceLoss.from_name("brier") +under_L1 = UnderConfidenceLoss.from_name("proper-L1") + +# Evaluate top-class error with any accompanying loss +topclass_brier = TopClassLoss(BrierLoss(binary_as_multiclass=False)) +topclass_L1 = TopClassLoss.from_name("proper-L1") + +# Compose wrappers (e.g., top-class with underconfidence for proper-L1) +under_topclass_l1 = TopClassLoss(UnderConfidenceLoss.from_name("proper-L1")) +over_topclass_brier = TopClassLoss(OverConfidenceLoss(BrierLoss())) + +# Some metrics are listed by default, here are some of them +metrics = metrics = Metrics.from_names([ + 'proper-L1-binary-as-1d', # use to estimate E[ \| Y - E[Y|f(X)] \|_1 ] and treat binary + # predictions as scalars with shapes (n,1) ) + 'proper-L2', # use to estimate E[ \| Y - E[Y|f(X)] \|_2 ] (and treat binary predictions + # as vector with shapes (n,2) ) + "topclass-proper-L1-binary-as-1d", # Estimate L1 calibration error of top class + "topclass-under-proper-L1-binary-as-1d", # Estimate L1-overconfidence of top class + "topclass-over-proper-L1-binary-as-1d", # Estimate L1-underconfidence of top class +]) + +``` + +Once those losses are defined, you can evaluate the calibration error by doing: + +```python +from probmetrics.metrics import MetricsWithCalibration, CombinedMetrics +from probmetrics.classifiers import WS_CatboostClassifier, WS_LGBMClassifier +from probmetrics.splitters import CVSplitter + +loss = ProperLpLoss(p=2) + +metrics = MetricsWithCalibration(loss, + calibrator=WS_CatboostClassifier(), # The classifier used to recalibrate the predictions + val_splitter=CVSplitter(n_cv=5) # cross-validation splitter + ) + +# or use combined metrics +combined_losses = CombinedMetrics( + [ + ProperLpLoss(p=1), + OverConfidenceLoss.from_name("brier"), + OverConfidenceLoss.from_name("proper-L1") , + UnderConfidenceLoss.from_name("proper-L1" ), + UnderConfidenceLoss( BrierLoss() ), + BrierLoss() + ] + ) + +metrics = MetricsWithCalibration(combined_losses, + calibrator=WS_LGBMClassifier(), + val_splitter=CVSplitter(n_cv=5) + ) + +y_true = torch.tensor(...) +y_prob = torch.tensor(...) +results = metrics.compute_all_from_labels_probs(y_true, y_prob) +``` + +The `calibrator` argument is a class used to recalibrate the original predictions. +Any estimator that inherits from sklearn.base.ClassifierMixin (i.e., follows the +scikit-learn classifier API) can be used. +We recommend using `WS_CatboostClassifier` with default parameters. +The "WS" stands for "Warm Start", as predictions are initialized at the +original predicted $f(x)$ values, (see the paper A Variational Estimator for Lp +Calibration Errors for additional information). + + +### Binary vs. multiclass formatting + +The library expects predictions in a multiclass format with shape `(n_samples, n_classes)`. +For binary classification, you can control whether to treat the output as a two-column distribution +or a single-column probability using the `binary_as_multiclass` parameter. + +Setting `binary_as_multiclass=False` tells the loss function to treat +`(n_samples, 2)` predictions as a single-column `(n_samples, 1)` probability. + It automatically transforms binary labels $Y \in {0, 1}$ and the probability + column $f(X) \in [0, 1]$ for the calculation. + +Those features are also valid with the `TopClassLoss`. +The `TopClassLoss` wrapper focuses the loss calculation on the class with +the highest predicted probability. The behavior changes based on your binary setting, +for instance: + +| Configuration | Estimate | Description | +|:---------------------------------------------------------------| :--- | :--- | +| `TopClassLoss( ProperLpLoss(p=1) )` | $\mathbb{E}[ \lvert Z - \mathbb{E}[Z \mid topf(X)] \rvert ]$ | Scalar probability: $topf(X)$ is the scalar probability of the top class of $f(X)$; $Z \in \{0, 1\}$ equals $1$ if the label is what the top-class predicted and $0$ otherwise. Evaluates the absolute error of the top-class prediction. | +| `TopClassLoss( ProperLpLoss(p=1, binary_as_multiclass=True) )` | $\mathbb{E}[ \Vert \mathbf{Z} - \mathbb{E}[\mathbf{Z} \mid topf(X)] \Vert_1 ]$ | Vectorized: $\mathbf{Z}$ is a one-hot vector. Calculates the $L_1$ norm of the error vector. | + + ## Contributors - David Holzmüller - Eugène Berta +- Sacha Braun ## Releases +- v1.2.0 by [@elsacho](https://github.com/elsacho): Added new proper loss functions: + - ProperLpLoss(p=p): Metrics to evaluate $E[ \Vert f(X) - E[Y|f(X)] \Vert_p ]$ where $f(X)$ are the + predictions of the classifier, $p >= 1$, including `p=float("inf")` + - TopClassLoss: A wrapper to variationally evaluate top-class errors. + - OverConfidenceLoss & UnderConfidenceLoss: Wrappers to variationally evaluate + over/under-confidence in binary predictors. + - New classifiers: Added `WS_CatboostClassifier` and `WS_LGBMClassifier` for + evaluating calibration errors. + - removed sklearn < 1.7 constraint. - v1.1.0 by [@eugeneberta](https://github.com/eugeneberta): Improvements to the SVS and SMS calibrators: - logit pre-processing with `'ts-mix'` is now automatic, and the global scaling parameter $\alpha$ is fixed to 1. This yields: @@ -197,4 +329,6 @@ While there are some classes for regression metrics, they are not implemented. - add TorchCal temperature scaling - minor fixes in AutoGluon temperature scaling that shouldn't affect the performance in practice -- v0.0.1 by [@dholzmueller](https://github.com/dholzmueller): Initial release +- v0.0.1 by [@dholzmueller](https://github.com/dholzmueller): + Initial release with classification metrics, + calibration/refinement metrics, and some post-hoc calibration methods. \ No newline at end of file diff --git a/probmetrics/__about__.py b/probmetrics/__about__.py index 6973895..881244a 100644 --- a/probmetrics/__about__.py +++ b/probmetrics/__about__.py @@ -1,2 +1,2 @@ # SPDX-License-Identifier: Apache-2.0 -__version__ = "1.1.0" +__version__ = "1.2.0" diff --git a/probmetrics/calibrators.py b/probmetrics/calibrators.py index 419fd81..ef01281 100644 --- a/probmetrics/calibrators.py +++ b/probmetrics/calibrators.py @@ -6,6 +6,7 @@ import logging import torchmin import numpy as np + from probmetrics.saga import ( fit_affine_scaling, warm_up_affine_scaling, @@ -17,6 +18,7 @@ warm_up_matrix_scaling, ) from probmetrics.utils import ( + process_binary_probs, binary_probs_to_logits, multiclass_probs_to_logits, validate_probabilities, @@ -24,6 +26,8 @@ from sklearn.base import BaseEstimator, ClassifierMixin from sklearn.calibration import CalibratedClassifierCV from sklearn.model_selection import StratifiedKFold, GridSearchCV +from sklearn.preprocessing import LabelEncoder + from torch import nn from probmetrics.distributions import ( @@ -255,7 +259,7 @@ class BinaryLogisticCalibrator(Calibrator): - 'quadratic': calibrated_logit = b + w1 * logit + w2 * logit^2 """ - VALID_TYPES = ["linear", "affine", "quadratic"] + VALID_TYPES = ["linear", "affine", "quadratic", "beta"] def __init__(self, type: str = "affine"): """ @@ -274,21 +278,32 @@ def __init__(self, type: str = "affine"): self.cal_ = None def _get_logits(self, X: np.ndarray) -> np.ndarray: - logits = binary_probs_to_logits(X) - logits = logits.reshape(-1, 1) + log_p, log_1_minus_p = binary_probs_to_logits(X) + logits = log_p - log_1_minus_p + + if self.type == "linear": + return logits + + ones = np.ones_like(logits) + if self.type == "affine": - # Adding a constant term to logit vectors. - logits = np.hstack([np.ones((len(X), 1)), logits]) + return np.hstack([ones, logits]) + elif self.type == "quadratic": - # Adding a constant and quadratic term to logit vectors. - logits = np.hstack([np.ones((len(X), 1)), logits, np.square(logits)]) - return logits + return np.hstack([ones, logits, np.square(logits)]) + + if self.type == "beta": # TODO Check that this is indeed beta + return np.hstack([ones, log_p, log_1_minus_p]) def _fit_impl(self, X: np.ndarray, y: np.ndarray) -> None: """Fits the logistic regression calibrator.""" from sklearn.linear_model import LogisticRegression - self.cal_ = LogisticRegression(penalty=None, fit_intercept=False) + self.cal_ = LogisticRegression( + penalty=None, + fit_intercept=False, + solver="lbfgs" + ) features = self._get_logits(X) self.cal_.fit(features, y) @@ -304,7 +319,7 @@ def predict_proba(self, X: np.ndarray) -> np.ndarray: return self.cal_.predict_proba(features) -class AffineScalingCalibrator(Calibrator): +class RegularizedAffineScalingCalibrator(Calibrator): """ Binary post-hoc calibration with Platt scaling (~binary logistic regression) $sigmoid(ax+b)$ on the logits x. Uses the SAGA algorithm to fit the scaling and intercept parameters a & b. @@ -332,22 +347,21 @@ def __init__( self.w = None - if not AffineScalingCalibrator._warmed_up: + if not RegularizedAffineScalingCalibrator._warmed_up: if self.print_init_info: print( - "First AffineScalingCalibrator instantiation - warming up Numba functions..." + "First RegularizedAffineScalingCalibrator instantiation - warming up Numba functions..." ) warm_up_affine_scaling() - AffineScalingCalibrator._warmed_up = True + RegularizedAffineScalingCalibrator._warmed_up = True if self.print_init_info: print("Warmed up!") def _get_logits(self, X: np.ndarray) -> np.ndarray: - logits = binary_probs_to_logits(X) - logits = logits.reshape(-1, 1) - # Adding a constant term to logit vectors to fit the intercept b. - logits = np.hstack([np.ones((len(X), 1)), logits]) - return logits + log_p, log_1_minus_p = binary_probs_to_logits(X) + logits = log_p - log_1_minus_p + ones = np.ones_like(logits) + return np.hstack([ones, logits]) def _fit_impl(self, X: np.ndarray, y: np.ndarray) -> None: logits = self._get_logits(X) @@ -372,7 +386,7 @@ def predict_proba(self, X: np.ndarray) -> np.ndarray: return np.hstack([1 - probs, probs]) -class QuadraticScalingCalibrator(Calibrator): +class RegularizedQuadraticScalingCalibrator(Calibrator): """ Binary post-hoc calibration with quadratic scaling $sigmoid(a + bx + cx^2)$ on the logits x. Uses the SAGA algorithm to fit the intercept, scaling and quadratic parameters a, b & c. @@ -403,22 +417,21 @@ def __init__( self.w = None - if not QuadraticScalingCalibrator._warmed_up: + if not RegularizedQuadraticScalingCalibrator._warmed_up: if self.print_init_info: print( - "First QuadraticScalingCalibrator instantiation - warming up Numba functions..." + "First RegularizedQuadraticScalingCalibrator instantiation - warming up Numba functions..." ) warm_up_quadratic_scaling() - QuadraticScalingCalibrator._warmed_up = True + RegularizedQuadraticScalingCalibrator._warmed_up = True if self.print_init_info: print("Warmed up!") def _get_logits(self, X: np.ndarray) -> np.ndarray: - logits = binary_probs_to_logits(X) - logits = logits.reshape(-1, 1) - # Adding a constant and quadratic term to logit vectors. - logits = np.hstack([np.ones((len(X), 1)), logits, np.square(logits)]) - return logits + log_p, log_1_minus_p = binary_probs_to_logits(X) + logits = log_p - log_1_minus_p + ones = np.ones_like(logits) + return np.hstack([ones, logits, np.square(logits)]) def _fit_impl(self, X: np.ndarray, y: np.ndarray) -> None: logits = self._get_logits(X) @@ -1051,16 +1064,92 @@ def __init__(self, init_val: float = 1, lr: float = 0.1, max_iter: int = 100): self.lr = lr self.max_iter = max_iter +class TabPFNCalibrator(Calibrator): + def __init__(self): + super().__init__() -class NetcalTemperatureScalingCalibrator(Calibrator): - # this one does nothing due to https://github.com/EFS-OpenSource/calibration-framework/issues/61 + def _fit_impl(self, X: np.ndarray, y: np.ndarray) -> None: + from tabpfn import TabPFNClassifier + + self.cal_ = TabPFNClassifier() + + probs = process_binary_probs(X) + + self.cal_.fit(probs.reshape(-1,1), y) + + def predict_proba(self, X: np.ndarray) -> np.ndarray: + + probs = process_binary_probs(X) + + result = self.cal_.predict_proba(probs.reshape(-1,1)) + if len(result.shape) == 1: + return np.stack([1.0 - result, result], axis=1) + elif result.shape[1] == 1: + return np.concatenate([1.0 - result, result], axis=1) + + return result + +class BetacalCalibrator(Calibrator): + # From https://github.com/betacal/python/ + def __init__(self, parameters="abm"): + super().__init__() + + from betacal import BetaCalibration + + self.cal_ = BetaCalibration(parameters=parameters) + + def _fit_impl(self, X: np.ndarray, y: np.ndarray) -> None: + probs = process_binary_probs(X) + + self.cal_.fit(probs, y) + + def predict_proba(self, X: np.ndarray) -> np.ndarray: + probs = process_binary_probs(X) + + result = self.cal_.predict(probs) + + if len(result.shape) == 1: + return np.stack([1.0 - result, result], axis=1) + elif result.shape[1] == 1: + return np.concatenate([1.0 - result, result], axis=1) + + return result + +class MLISplineCalibrator(Calibrator): + # From https://github.com/numeristical/introspective def __init__(self): super().__init__() + from ml_insights import SplineCalib + + self.cal_ = SplineCalib() + def _fit_impl(self, X: np.ndarray, y: np.ndarray) -> None: + probs = process_binary_probs(X) + + self.cal_.fit(probs, y) + + def predict_proba(self, X: np.ndarray) -> np.ndarray: + probs = process_binary_probs(X) + + result = self.cal_.calibrate(probs) + + if len(result.shape) == 1: + return np.stack([1.0 - result, result], axis=1) + elif result.shape[1] == 1: + return np.concatenate([1.0 - result, result], axis=1) + + return result + +class NetcalTemperatureScalingCalibrator(Calibrator): + # this one does nothing due to https://github.com/EFS-OpenSource/calibration-framework/issues/61 + def __init__(self): + super().__init__() from netcal.scaling import TemperatureScaling self.cal_ = TemperatureScaling() + + def _fit_impl(self, X: np.ndarray, y: np.ndarray) -> None: if X.shape[1] == 2: # binary, convert X = X[:, 1] @@ -1200,42 +1289,55 @@ def predict_proba_torch( class CenteredIsotonicRegressionCalibrator(Calibrator): + def __init__(self): + super().__init__() + + from cir_model import CenteredIsotonicRegression + + self.cal_ = CenteredIsotonicRegression() + def _fit_impl(self, X: np.ndarray, y: np.ndarray) -> None: - assert not np.any(np.isnan(X)) - # print(f'{np.unique(y)=}') - # print(f'{np.unique(X)=}') + # TODO # we have to use float64 since with float32 it can happen that somewhere internally # a rounding error occurs (?) and interp1d thinks that it is asked to extrapolate at the boundary value, # which causes it to output nan values, # which are seen as a separate possible value but (y==value) is empty because nan==nan is false, # so an error occurs because the method attempts to take an average of an empty set - X = X.astype(np.float64) + + probs = process_binary_probs(X) + + probs = probs.astype(np.float64) y = y.astype(np.float64) - from cir_model import CenteredIsotonicRegression - self.cal_ = CenteredIsotonicRegression() - self.cal_.fit(X[:, 1], y) - self.min_ = np.min(X[:, 1]) - self.max_ = np.max(X[:, 1]) + self.cal_.fit(probs, y) + self.min_ = np.min(probs) + self.max_ = np.max(probs) def predict_proba(self, X): + probs = process_binary_probs(X) + probs = probs.astype(np.float64) + # have to clip since CenteredIsotonicRegression refuses to extrapolate (?) - X = X.astype(np.float64) - X = np.clip(X, self.min_, self.max_) - pred_probs = self.cal_.transform(X[:, 1]) - pred_probs = np.clip(pred_probs, 0.0, 1.0) - return np.stack([1.0 - pred_probs, pred_probs], axis=-1) + probs = np.clip(probs, self.min_, self.max_) + + result = self.cal_.transform(probs) + result = np.clip(result, 0.0, 1.0) + return np.stack([1.0 - result, result], axis=-1) class BinaryVennAbersCalibrator(Calibrator): - def _fit_impl(self, X: np.ndarray, y: np.ndarray) -> None: + def __init__(self): + super().__init__() + from venn_abers import VennAbers - self.va_ = VennAbers() - self.va_.fit(X, y) + self.cal_ = VennAbers() + + def _fit_impl(self, X: np.ndarray, y: np.ndarray) -> None: + self.cal_.fit(X, y) def predict_proba(self, X): - return self.va_.predict_proba(X)[0] + return self.cal_.predict_proba(X)[0] class VennAbersCalibrator(Calibrator): @@ -1439,36 +1541,79 @@ def predict_proba_torch( class SklearnCalibrator(Calibrator): - def __init__(self, method: str, cv: str): + def __init__(self, method: str, cv: str = "prefit"): + assert cv == 'prefit', "Other methods than prefit not intended and not implemented for sklearn > 1.6" super().__init__() self.method = method self.cv = cv + @staticmethod + def _normalize_rows(A: np.ndarray) -> np.ndarray: + """Row-normalize so rows sum to 1; safe if a row sums to 0.""" + s = A.sum(axis=1, keepdims=True) + s = np.where(s == 0.0, 1.0, s) + return A / s + def _fit_impl(self, X: np.ndarray, y: np.ndarray) -> None: if X.ndim == 1: X = np.stack((1.0 - X, X), axis=1) - n_classes = X.shape[-1] - est = IdentityEstimator(n_classes) - est.fit(X, y) + y = np.asarray(y) - # tried this workaround for deprecation warnings, but it complains in case of missing classes because it still tries to fit the estimator - # try: - # # cv='prefit' option is deprecated from sklearn 1.6, FrozenEstimator was introduced in sklearn 1.6 - # from sklearn.frozen import FrozenEstimator - # self.calib_ = CalibratedClassifierCV(FrozenEstimator(est), method=self.method, cv=[(np.arange(0), np.arange(X.shape[0]))]) - # except ImportError: - # self.calib_ = CalibratedClassifierCV(est, method=self.method, cv=self.cv) + self.n_classes_ = X.shape[-1] + self.classes_ = np.arange(self.n_classes_) + + # Fit label encoder up-front (used for the missing-class path) + self.label_encoder_ = LabelEncoder() + y_mapped = self.label_encoder_.fit_transform(y) + self.present_classes_ = self.label_encoder_.classes_ + self.missing_classes_ = np.setdiff1d(self.classes_, self.present_classes_) + + # ---------- Case 1: no missing classes -> original formulation ---------- + if self.missing_classes_.size == 0: + est = IdentityEstimator(self.n_classes_) + est.fit(X, y) + + try: + # sklearn >= 1.6 + from sklearn.frozen import FrozenEstimator + self.calib_ = CalibratedClassifierCV(FrozenEstimator(est), method=self.method) + except ImportError: + # sklearn < 1.6 + self.calib_ = CalibratedClassifierCV(est, method=self.method, cv=self.cv) + + self.calib_.fit(X, y) + return - self.calib_ = CalibratedClassifierCV(est, method=self.method, cv=self.cv) - self.calib_.fit(X, y) - self.classes_ = est.classes_ + # ---------- Case 2: missing classes -> slice + normalize X_present, use y_mapped ---------- + X_present = self._normalize_rows(X[:, self.present_classes_]) - def predict_proba(self, X): + est = IdentityEstimator(len(self.present_classes_)) + est.fit(X_present, y_mapped) + + try: + from sklearn.frozen import FrozenEstimator + self.calib_ = CalibratedClassifierCV(FrozenEstimator(est), method=self.method) + except ImportError: + self.calib_ = CalibratedClassifierCV(est, method=self.method, cv=self.cv) + + self.calib_.fit(X_present, y_mapped) + + def predict_proba(self, X: np.ndarray) -> np.ndarray: if X.ndim == 1: X = np.stack((1.0 - X, X), axis=1) - return self.calib_.predict_proba(X) + # If fit saw all classes, keep original behavior + if self.missing_classes_.size == 0: + return self.calib_.predict_proba(X) + + # Missing classes: normalize the same sliced X_present and zero-fill missing classes (Option A) + X_present = self._normalize_rows(X[:, self.present_classes_]) + p_present = self.calib_.predict_proba(X_present) + + p_full = np.zeros((X.shape[0], self.n_classes_), dtype=p_present.dtype) + p_full[:, self.present_classes_] = p_present + return p_full class WrapCalibratorAsClassifier(BaseEstimator, ClassifierMixin): @@ -1631,6 +1776,10 @@ def get_calibrator( cal = BinaryLogisticCalibrator(type="affine") elif calibration_method == "quadratic-scaling": cal = BinaryLogisticCalibrator(type="quadratic") + elif calibration_method == "beta": + cal = BetacalCalibrator( + parameters=config.get("beta_parameters", "abm") + ) elif calibration_method == "svs": cal = SVSCalibrator( penalty=config.get("svs_penalty", "ridge"), diff --git a/probmetrics/classifiers.py b/probmetrics/classifiers.py new file mode 100644 index 0000000..1c6f43f --- /dev/null +++ b/probmetrics/classifiers.py @@ -0,0 +1,176 @@ +import numpy as np +import sklearn +import pandas as pd +import torch + +from probmetrics.calibrators import get_calibrator +from probmetrics.utils import multiclass_probs_to_logits + + +class WS_CatboostClassifier(sklearn.base.BaseEstimator, sklearn.base.ClassifierMixin): + def __init__(self, iterations=10, use_init_logits=True, early_stopping_rounds=300, thread_count=1, random_state=0, verbose=0): + self.iterations = iterations + self.early_stopping_rounds = early_stopping_rounds + self.thread_count = thread_count + self.random_state = random_state + self.use_init_logits = use_init_logits + self.verbose = verbose + + def _fit_model(self, idxs): + from catboost import CatBoostClassifier, Pool + train_idx, val_idx = idxs[0], idxs[1] + X_train, y_train = self.X_.take(train_idx, 0), self.y_[train_idx] + X_val, y_val = self.X_.take(val_idx, 0), self.y_[val_idx] + + train_pool = Pool(X_train, label=y_train) + val_pool = Pool(X_val, label=y_val) + + if self.init_logits_ is not None: + train_pool.set_baseline(self.init_logits_.take(train_idx, 0)) + val_pool.set_baseline(self.init_logits_.take(val_idx, 0)) + + m = CatBoostClassifier( + iterations=self.iterations, + random_state=self.random_state, + early_stopping_rounds=self.early_stopping_rounds, + thread_count=self.thread_count, + verbose=self.verbose, + loss_function='MultiClass', + classes_count=len(self.classes_), + ) + + return m.fit(train_pool, eval_set=val_pool) + + def _get_model_proba(self, model, X): + """Calculates probabilities by adding baseline to raw residuals.""" + from scipy.special import expit, softmax + raw_preds = model.predict(X, prediction_type='RawFormulaVal') + init_score = multiclass_probs_to_logits(X) if self.use_init_logits else None + if init_score is not None: + raw_preds = raw_preds + init_score + + return softmax(raw_preds, axis=1) + + def fit(self, X, y): + import multiprocessing as mp + if isinstance(X, (pd.DataFrame, pd.Series)): X = X.values + if isinstance(y, (pd.DataFrame, pd.Series)): y = y.values + + init_logits = multiclass_probs_to_logits(X) if self.use_init_logits else None + + self.init_logits_ = init_logits + self.le_ = sklearn.preprocessing.LabelEncoder().fit(y) + self.X_, self.y_, self.classes_ = X, self.le_.transform(y), self.le_.classes_ + + splits = list(sklearn.model_selection.StratifiedKFold( + n_splits=8, shuffle=True, random_state=self.random_state + ).split(X, y)) + + with mp.Pool(processes=min(len(splits), mp.cpu_count())) as pool: + self.models_ = pool.map(self._fit_model, splits) + + oof_preds_list = [] + for m, idxs in zip(self.models_, splits): + val_idx = idxs[1] + oof_preds_list.append(self._get_model_proba(m, X.take(val_idx, 0))) + + oof_preds = np.concatenate(oof_preds_list, axis=0) + val_indices = np.concatenate([idxs[1] for idxs in splits]) + oof_labels = y[val_indices] + + self.calib_ = get_calibrator('logistic', calibrate_with_mixture=True, + logistic_binary_type='quadratic').fit(oof_preds, oof_labels) + return self + + def predict_proba(self, X): + if isinstance(X, (pd.DataFrame, pd.Series)): X = X.values + avg_probas = np.mean([self._get_model_proba(m, X) for m in self.models_], axis=0) + return self.calib_.predict_proba(avg_probas) + + def predict(self, X): + return self.le_.inverse_transform(np.argmax(self.predict_proba(X), axis=1)) + + +class WS_LGBMClassifier(sklearn.base.BaseEstimator, sklearn.base.ClassifierMixin): + def __init__(self, n_estimators=10, use_init_logits=True, learning_rate=0.04, subsample=0.75, num_leaves=50, subsample_freq=1, random_state=0, early_stopping_round=100, min_child_samples=40, min_child_weight=1e-7, n_jobs=1): + self.n_estimators = n_estimators + self.learning_rate = learning_rate + self.subsample = subsample + self.num_leaves = num_leaves + self.subsample_freq = subsample_freq + self.random_state = random_state + self.early_stopping_round = early_stopping_round + self.min_child_samples = min_child_samples + self.min_child_weight = min_child_weight + self.n_jobs = n_jobs + self.use_init_logits = use_init_logits + + + def _fit_model(self, idxs): + from lightgbm import LGBMClassifier + train_idx, val_idx = idxs[0], idxs[1] + X_train = self.X_.take(train_idx, 0) + y_train = self.y_[train_idx] + X_val = self.X_.take(val_idx, 0) + y_val = self.y_[val_idx] + + fit_params = {} + if self.init_logits_ is not None: + fit_params['init_score'] = self.init_logits_.take(train_idx, 0) + fit_params['eval_init_score'] = [self.init_logits_.take(val_idx, 0)] + + m = LGBMClassifier(n_estimators=self.n_estimators, learning_rate=self.learning_rate, subsample=self.subsample, subsample_freq=self.subsample_freq, num_leaves=self.num_leaves, + random_state=self.random_state, early_stopping_round=self.early_stopping_round, min_child_samples=self.min_child_samples, min_child_weight=self.min_child_weight, + n_jobs=self.n_jobs, verbosity=self.verbose_, objective="multiclass", num_class=len(self.classes_)) + + return m.fit(X_train, y_train, eval_set=(X_val, y_val), **fit_params) + + def _get_model_proba(self, model, X): + from scipy.special import softmax + init_score = multiclass_probs_to_logits(X) if self.use_init_logits else None + + raw_preds = model.predict(X, raw_score=True) + if init_score is not None: raw_preds += init_score + + return softmax(raw_preds, axis=1) + + def fit(self, X, y, verbose=-1): + import multiprocessing as mp + if isinstance(X, torch.Tensor) or isinstance(X, pd.DataFrame) or isinstance(X, pd.Series): X = np.asarray(X) + if isinstance(y, torch.Tensor) or isinstance(y, pd.DataFrame) or isinstance(y, pd.Series): y = np.asarray(y) + + init_logits = multiclass_probs_to_logits(X) if self.use_init_logits else None + + self.verbose_ = verbose + self.init_logits_ = init_logits + self.le_ = sklearn.preprocessing.LabelEncoder().fit(y) + self.X_, self.y_, self.classes_ = X, self.le_.transform(y), self.le_.classes_ + + splits = list(sklearn.model_selection.StratifiedKFold(n_splits=8, shuffle=True, random_state=0).split(X, y)) + + with mp.Pool(processes=min(len(splits), mp.cpu_count())) as pool: + self.models_ = pool.map(self._fit_model, splits) + + oof_preds_list = [] + for m, idxs in zip(self.models_, splits): + val_idx = idxs[1] + oof_preds_list.append(self._get_model_proba(m, X.take(val_idx, 0))) + + oof_preds = np.concatenate(oof_preds_list, axis=0) + oof_labels = np.concatenate([y[idxs[1]] for idxs in splits], axis=0) + + self.calib_ = get_calibrator('logistic', calibrate_with_mixture=True, + logistic_binary_type='quadratic').fit(oof_preds, oof_labels) + return self + + def predict_proba(self, X): + if isinstance(X, torch.Tensor) or isinstance(X, pd.DataFrame) or isinstance(X, pd.Series): X = np.asarray(X) + + avg_probas = np.mean([self._get_model_proba(m, X) for m in self.models_], axis=0) + + return self.calib_.predict_proba(avg_probas) + + def predict(self, X): + if isinstance(X, torch.Tensor) or isinstance(X, pd.DataFrame) or isinstance(X, pd.Series): X = np.asarray(X) + return self.le_.inverse_transform(np.argmax(self.predict_proba(X), axis=1)) + diff --git a/probmetrics/distributions.py b/probmetrics/distributions.py index baa0919..db422c5 100644 --- a/probmetrics/distributions.py +++ b/probmetrics/distributions.py @@ -106,13 +106,16 @@ class CategoricalLogits(CategoricalDistribution): # or even more versions for different link functions? def __init__(self, logits: torch.Tensor): """ - :param logits: Logits of shape (n_samples, n_classes) or (n_samples,). - In the latter case, logits are interpreted as binary logits + :param logits: Logits of shape (n_samples, n_classes) or (n_samples,) or (n_samples, 1). + In the latter two cases, logits are interpreted as binary logits such that sigmoid(logits) is the probability for class 1. They will be converted to (n_samples, 2) shaped logits. """ if len(logits.shape) == 1: logits = torch.stack([torch.zeros_like(logits), logits], dim=1) + elif logits.shape[1] == 1: + logits = torch.cat([torch.zeros_like(logits), logits], dim=1) + assert logits.ndim == 2 self.logits = logits self.probs = None self.modes = None @@ -149,12 +152,15 @@ class CategoricalProbs(CategoricalDistribution): # or even more versions for different link functions? def __init__(self, probs: torch.Tensor): """ - :param probs: Probabilities of shape (n_samples, n_classes) or (n_samples,). - In the latter case, the vector is interpreted as probabilities for class 1. + :param probs: Probabilities of shape (n_samples, n_classes) or (n_samples,) or (n_samples, 1). + In the latter two cases, the vector is interpreted as probabilities for class 1. It will then be converted to a (n_samples, 2) shaped vector. """ if len(probs.shape) == 1: probs = torch.stack([1.-probs, probs], dim=1) + elif probs.shape[-1] == 1: + probs = torch.cat([1.-probs, probs], dim=1) + assert probs.ndim == 2 self.logits = None self.probs = probs self.modes = None diff --git a/probmetrics/metrics.py b/probmetrics/metrics.py index eaae0b8..bfd9784 100644 --- a/probmetrics/metrics.py +++ b/probmetrics/metrics.py @@ -1,16 +1,19 @@ -from typing import Optional, Dict, List, Callable, Literal +from collections import Counter +from typing import Optional, Dict, List, Callable, Literal, Union import sklearn +import numpy as np import torch import torch.nn.functional as F from sklearn.metrics import roc_auc_score from probmetrics import utils -from probmetrics.calibrators import Calibrator, TemperatureScalingCalibrator, get_calibrator +from probmetrics.calibrators import Calibrator, get_calibrator from probmetrics.distributions import Distribution, CategoricalDistribution, ContinuousDistribution, CategoricalProbs, \ CategoricalDirac, CategoricalLogits from probmetrics.splitters import Splitter, CVSplitter, AllSplitter from probmetrics.torch_utils import remove_missing_classes +from probmetrics.classifiers import WS_CatboostClassifier class MetricType: @@ -35,11 +38,14 @@ class MetricType: class Metrics: # todo: type-hinting for enum type? - def __init__(self, names: List[str], is_lower_better_list: List[bool], metric_type: str): + def __init__(self, names: List[str], is_lower_better_list: List[bool], metric_type: str, requires_f_x: bool = False, + binary_as_multiclass: bool = True): assert len(names) == len(is_lower_better_list) self.names = names self.is_lower_better_list = is_lower_better_list self.metric_type = metric_type + self.requires_f_x = requires_f_x + self.binary_as_multiclass = binary_as_multiclass def get_metric_type(self) -> str: return self.metric_type @@ -49,7 +55,7 @@ def get_names(self) -> List[str]: def compute_all(self, y_true: Distribution, y_pred: Distribution, other_metric_values: Optional[Dict[str, torch.Tensor]] = None, reduction: str = 'mean', - weights: Optional[torch.Tensor] = None) -> Dict[str, torch.Tensor]: + weights: Optional[torch.Tensor] = None, f_x: Optional[Distribution] = None) -> Dict[str, torch.Tensor]: """ Compute the metrics values. Should be overridden by subclasses. @@ -62,6 +68,7 @@ def compute_all(self, y_true: Distribution, y_pred: Distribution, Note that the option 'none' is not supported by all subclasses. :param weights: (Optional) sample weights of shape (n_samples,). Can only be used with 'mean' reduction. Weights are not implemented by all subclasses. + :param f_x: Values f(x) for calibration-related proper losses (these can be sample-dependent). :return: """ raise NotImplementedError() @@ -91,10 +98,21 @@ def compute_all_from_labels_logits(self, y_true: torch.Tensor, y_logits: torch.T @staticmethod def _get_candidate_metrics() -> List['Metrics']: - proper_metrics = CombinedMetrics([LogLoss(), BrierLoss(), ClippedLogLoss()]) + proper_metrics = CombinedMetrics([LogLoss(), BrierLoss(), ClippedLogLoss(), + ProperLpLoss(p=1), ProperLpLoss(p=2), ProperLpLoss(p=float('inf'))]) cand_list = [ClassError(), Accuracy(), LogLoss(), BrierLoss(), ClippedLogLoss(), AUROCOneVsRest(), AUROCOneVsOneSklearn(), AUROCOneVsRestSklearn(), OneMinusAUROCOneVsRest(), CalibrationError(norm='l1'), CalibrationError(norm='l2'), CalibrationError(norm='max'), + ProperLpLoss(p=1), ProperLpLoss(p=2), ProperLpLoss(p=float('inf')), + ProperLpLoss(p=1, binary_as_multiclass=True), + ProperLpLoss(p=2, binary_as_multiclass=True), + ProperLpLoss(p=float('inf'), binary_as_multiclass=True), + TopClassLoss(UnderConfidenceLoss(ProperLpLoss(p=1))), + TopClassLoss(OverConfidenceLoss(ProperLpLoss(p=1))), + TopClassLoss(ProperLpLoss(p=1)), + TopClassLoss(ProperLpLoss(p=2)), + TopClassLoss(ProperLpLoss(p=float('inf'))), + BrierLoss(binary_as_multiclass=False), SmoothCalibrationError(), MSE(), RMSE(), NRMSE(), MAE(), NMAE()] cand_list.extend([MeanProbNormalizedMetric(metric) for metric in @@ -109,6 +127,10 @@ def _get_candidate_metrics() -> List['Metrics']: get_calibrator('isotonic', calibrate_with_mixture=True), val_splitter=splitter, cal_name='isotonic-mix', random_state=0), + MetricsWithCalibration(proper_metrics, + WS_CatboostClassifier(), + val_splitter=splitter, + random_state=0), ]) return cand_list @@ -162,6 +184,13 @@ def __init__(self, metrics_list: List[Metrics], used_metric_names: Optional[List zip_dict = {n: i for n, i in zip(names, is_lower_better)} names = used_metric_names is_lower_better = [zip_dict[name] for name in names] + + # check for duplicates in the names + counts = Counter(names) + duplicates = [item for item, count in counts.items() if count > 1] + if duplicates: + raise ValueError(f"Multiple metrics generate results with the same name, duplicate names are: {duplicates}") + super().__init__(names=names, is_lower_better_list=is_lower_better, metric_type=metrics_list[0].metric_type @@ -170,19 +199,22 @@ def __init__(self, metrics_list: List[Metrics], used_metric_names: Optional[List def compute_all(self, y_true: Distribution, y_pred: Distribution, other_metric_values: Optional[Dict[str, torch.Tensor]] = None, reduction: str = 'mean', - weights: Optional[torch.Tensor] = None) -> Dict[str, torch.Tensor]: - metric_values = other_metric_values or dict() + weights: Optional[torch.Tensor] = None, f_x: Optional[Distribution] = None) -> Dict[str, torch.Tensor]: + metric_values = dict() + other_metric_values = dict() if other_metric_values is None else other_metric_values for metrics in self.metrics_list: - results = metrics.compute_all(y_true, y_pred, metric_values, reduction, weights) + results = metrics.compute_all(y_true, y_pred, metric_values, reduction, weights, f_x=f_x) metric_values = utils.join_dicts(metric_values, results, allow_overlap=False) + other_metric_values = utils.join_dicts(other_metric_values, results) # here we can have overlap return {key: value for key, value in metric_values.items() if key in self.get_names()} class Metric(Metrics): - def __init__(self, name: str, is_lower_better: bool, metric_type: str): - super().__init__(names=[name], is_lower_better_list=[is_lower_better], metric_type=metric_type) + def __init__(self, name: str, is_lower_better: bool, metric_type: str, requires_f_x: bool = False, binary_as_multiclass:bool = True): + name = name if binary_as_multiclass else name+"-binary-as-1d" + super().__init__(names=[name], is_lower_better_list=[is_lower_better], metric_type=metric_type, requires_f_x=requires_f_x, binary_as_multiclass=binary_as_multiclass) self.name = name self.is_lower_better = is_lower_better @@ -194,8 +226,8 @@ def get_is_lower_better(self) -> bool: def compute_all(self, y_true: Distribution, y_pred: Distribution, other_metric_values: Optional[Dict[str, torch.Tensor]] = None, reduction: str = 'mean', - weights: Optional[torch.Tensor] = None) -> Dict[str, torch.Tensor]: - return {self.get_name(): self.compute(y_true, y_pred, other_metric_values, reduction, weights)} + weights: Optional[torch.Tensor] = None, f_x: Optional[Distribution] = None) -> Dict[str, torch.Tensor]: + return {self.get_name(): self.compute(y_true, y_pred, other_metric_values, reduction, weights, f_x=f_x)} # may have parameters (e.g., quantile_alpha, or bin distribution for regression-as-classification) # alternative computations? @@ -203,7 +235,7 @@ def compute_all(self, y_true: Distribution, y_pred: Distribution, # batched computation? def compute(self, y_true: Distribution, y_pred: Distribution, other_metric_values: Optional[Dict[str, torch.Tensor]] = None, reduction: str = 'mean', - weights: Optional[torch.Tensor] = None) -> torch.Tensor: + weights: Optional[torch.Tensor] = None, **kwargs) -> torch.Tensor: # maybe use kwargs? # or have other functions for compute() and compute with other metric values? # maybe with the reduction we have the possibility to compute confidence intervals as well? @@ -233,17 +265,18 @@ def __init__(self, metrics: Metrics, name: str): def compute(self, y_true: Distribution, y_pred: Distribution, other_metric_values: Optional[Dict[str, torch.Tensor]] = None, reduction: str = 'mean', - weights: Optional[torch.Tensor] = None) -> torch.Tensor: - return self.metrics.compute_all(y_true, y_pred, other_metric_values, reduction, weights)[self.name] + weights: Optional[torch.Tensor] = None, **kwargs) -> torch.Tensor: + return self.metrics.compute_all(y_true, y_pred, other_metric_values, reduction, weights, **kwargs)[self.name] class ClassificationMetric(Metric): - def __init__(self, name: str, is_lower_better: bool): - super().__init__(name=name, is_lower_better=is_lower_better, metric_type=MetricType.CLASS) + def __init__(self, name: str, is_lower_better: bool, requires_f_x: bool = False, binary_as_multiclass: bool = True): + super().__init__(name=name, is_lower_better=is_lower_better, metric_type=MetricType.CLASS, + requires_f_x=requires_f_x, binary_as_multiclass=binary_as_multiclass) def compute(self, y_true: Distribution, y_pred: Distribution, other_metric_values: Optional[Dict[str, torch.Tensor]] = None, reduction: str = 'mean', - weights: Optional[torch.Tensor] = None) -> torch.Tensor: + weights: Optional[torch.Tensor] = None, **kwargs) -> torch.Tensor: if not isinstance(y_pred, CategoricalDistribution): raise ValueError(f'Classification metrics need a categorical distribution, but got {type(y_pred)=}') if not isinstance(y_true, CategoricalDistribution): @@ -259,13 +292,13 @@ def compute(self, y_true: Distribution, y_pred: Distribution, if result is not None: return result - result = self._compute_mean(y_true, y_pred) + result = self._compute_mean(y_true, y_pred, **kwargs) if result is None: raise NotImplementedError() return result else: - result = self._compute_indiv(y_true, y_pred) + result = self._compute_indiv(y_true, y_pred, **kwargs) if result is None: raise NotImplementedError() if reduction == 'mean': @@ -290,7 +323,7 @@ def _try_use_cached(self, y_true: CategoricalDistribution, y_pred: CategoricalDi """ return None - def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution) -> Optional[torch.Tensor]: + def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[torch.Tensor]: """ Compute a metric value for the entire dataset. By default, this tries to use _compute_indiv(). Override this method if the metric doesn't support _compute_indiv(). @@ -298,10 +331,10 @@ def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDist :param y_pred: :return: """ - indiv_results = self._compute_indiv(y_true, y_pred) + indiv_results = self._compute_indiv(y_true, y_pred, **kwargs) return None if indiv_results is None else indiv_results.mean(dim=-1) - def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution) -> Optional[ + def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[ torch.Tensor]: """ Compute metric values per sample. Override this method if applicable, else override _compute_mean(). @@ -319,7 +352,7 @@ def __init__(self, name: str, is_lower_better: bool): def compute(self, y_true: Distribution, y_pred: Distribution, other_metric_values: Optional[Dict[str, torch.Tensor]] = None, reduction: str = 'mean', - weights: Optional[torch.Tensor] = None) -> torch.Tensor: + weights: Optional[torch.Tensor] = None, **kwargs) -> torch.Tensor: # todo: do we really want y_true to be a distribution here? # todo: what about the multi-output setting? if not isinstance(y_pred, ContinuousDistribution): @@ -334,13 +367,13 @@ def compute(self, y_true: Distribution, y_pred: Distribution, if result is not None: return result - result = self._compute_mean(y_true, y_pred) + result = self._compute_mean(y_true, y_pred, **kwargs) if result is None: raise NotImplementedError() return result else: - result = self._compute_indiv(y_true, y_pred) + result = self._compute_indiv(y_true, y_pred, **kwargs) if result is None: raise NotImplementedError() if reduction == 'mean': @@ -356,11 +389,11 @@ def _try_use_cached(self, y_true: ContinuousDistribution, y_pred: ContinuousDist other_metric_values: Dict[str, torch.Tensor]) -> Optional[torch.Tensor]: return None - def _compute_mean(self, y_true: ContinuousDistribution, y_pred: ContinuousDistribution) -> Optional[torch.Tensor]: + def _compute_mean(self, y_true: ContinuousDistribution, y_pred: ContinuousDistribution, **kwargs) -> Optional[torch.Tensor]: indiv_results = self._compute_indiv(y_true, y_pred) return None if indiv_results is None else indiv_results.mean(dim=-1) - def _compute_indiv(self, y_true: ContinuousDistribution, y_pred: ContinuousDistribution) -> Optional[ + def _compute_indiv(self, y_true: ContinuousDistribution, y_pred: ContinuousDistribution, **kwargs) -> Optional[ torch.Tensor]: return None @@ -388,7 +421,7 @@ def _try_use_cached(self, y_true: CategoricalDistribution, y_pred: CategoricalDi return 1 - other_metric_values['accuracy'] return None - def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution) -> Optional[ + def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[ torch.Tensor]: return (y_pred.get_modes() != y_true.get_modes()).to(torch.float32) @@ -403,18 +436,20 @@ def _try_use_cached(self, y_true: CategoricalDistribution, y_pred: CategoricalDi return 1 - other_metric_values['class-error'] return None - def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution) -> Optional[ + def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[ torch.Tensor]: return (y_pred.get_modes() == y_true.get_modes()).to(torch.float32) class LogLoss(ClassificationMetric): def __init__(self): - super().__init__(name='logloss', is_lower_better=True) + super().__init__(name='logloss', is_lower_better=True, binary_as_multiclass=True) - def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution) -> Optional[ + def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[ torch.Tensor]: logits = y_pred.get_logits() + if not self.binary_as_multiclass: + raise NotImplementedError("This class does not suppoer binary_as_multiclass yet.") if y_true.is_dirac(): return -F.log_softmax(logits, dim=-1).gather(-1, y_true.get_modes().unsqueeze(-1)).squeeze(-1) else: @@ -423,11 +458,13 @@ def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDis class ClippedLogLoss(ClassificationMetric): def __init__(self, clip_threshold: float = 1e-6): - super().__init__(name=f'logloss-clip{clip_threshold:g}', is_lower_better=True) + super().__init__(name=f'logloss-clip{clip_threshold:g}', is_lower_better=True, binary_as_multiclass=True) self.clip_threshold = clip_threshold - def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution) -> Optional[ + def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[ torch.Tensor]: + if not self.binary_as_multiclass: + raise NotImplementedError("This class does not suppoer binary_as_multiclass yet.") probs = y_pred.get_probs() probs = probs.clamp(min=self.clip_threshold, max=1.0) probs = probs / probs.sum(dim=-1, keepdim=True) @@ -438,16 +475,288 @@ def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDis return (-log_probs * y_true.get_probs()).sum(dim=-1) -class BrierLoss(ClassificationMetric): # todo: also works as a regression metric - def __init__(self): - super().__init__(name='brier', is_lower_better=True) +class BrierLoss(ClassificationMetric): + def __init__(self, binary_as_multiclass: bool = True): + """ + Brier loss. + :param binary_as_multiclass: Whether to treat binary classification as a 2-dimensional multiclass problem + (yields twice the Brier score from 1D, which is also twice the value from scikit-learn). + """ + super().__init__(name='brier', is_lower_better=True, binary_as_multiclass=binary_as_multiclass) - def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution) -> Optional[ + def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[ torch.Tensor]: - # todo: adjust to sklearn? + if y_pred.get_n_classes() <= 2 and not self.binary_as_multiclass: + return (y_pred.get_probs()[:, 1] - y_true.get_probs()[:, 1]).square().sum(dim=-1) return (y_pred.get_probs() - y_true.get_probs()).square().sum(dim=-1) +def _clip_p_values(p: torch.Tensor, f_x: torch.Tensor, for_over: bool = False, for_under: bool = False) -> torch.Tensor: + """ + Rectifies probabilities based on the reference point f_x. + + Logic for_under: 1_{f>0.5} * max(p,f) + 1_{f<0.5} * min(p,f) + 1_{f=0.5} * 0.5 + Logic for_over: 1_{f>0.5} * min(p,f) + 1_{f<0.5} * max(p,f) + 1_{f=0.5} * 0.5 + + :param p: (Tensor) Predicted probability distribution. + :param f_x: (Tensor) Reference probability distribution. + :param for_over: (bool) Whether to clip for over-confidence. + :param for_under: (bool) Whether to clip for under-confidence. + """ + if for_over and for_under: + raise ValueError("Both 'for_over' and 'for_under' cannot be True simultaneously.") + if not (for_over or for_under): + import warnings + msg = ( + "Confidence clipping is initialized, but both 'for_over' and 'for_under' are set to False. " + "No values will be modified. Did you mean to enable one of these flags?" + ) + warnings.warn(msg, category=UserWarning, stacklevel=2) + + p = torch.as_tensor(p) + f_x = torch.as_tensor(f_x) + + if p.shape[-1] != 1: + import warnings + msg = ( + f"Input shape {p.shape} detected. Over/underconfidence metrics are " + "designed for binary tasks with shape (n, 1). For multi-class data, " + "consider using 'OverConfidenceLoss( TopClassLoss(DesiredMetric) )' or " + "OverConfidenceLoss( TopClassLoss(DesiredMetric) ) for confidence clipping." + ) + warnings.warn(msg, UserWarning) + + clipped_p = torch.full_like(p, 0.5) + + if for_under: + clipped_p = torch.where(f_x > 0.5, torch.maximum(p, f_x), clipped_p) + clipped_p = torch.where(f_x < 0.5, torch.minimum(p, f_x), clipped_p) + + elif for_over: + clipped_p = torch.where(f_x > 0.5, torch.minimum(p, f_x), clipped_p) + clipped_p = torch.where(f_x < 0.5, torch.maximum(p, f_x), clipped_p) + + return clipped_p + + +class ProperLpLoss(ClassificationMetric): + def __init__(self, p: float = 2., binary_as_multiclass: bool = False): + assert p >= 1 + super().__init__(name=f'proper-L{p:.10g}', is_lower_better=True, requires_f_x=True, + binary_as_multiclass=binary_as_multiclass) + self.p = p + + def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[ + torch.Tensor]: + """ + Calculates the proper loss l_{f(X)}(g_f_x, Y) for a given norm. + + Formula: l_{f(X)}(g, y) = h(g) + + where h(g) = -||g - f_x||_norm + g_f_x = y_pred + f_x is in kwargs + y = y_true + """ + f_x = kwargs["f_x"].get_probs() # values f(x) for calibration, that's where the loss is centered + y_pred_probs = y_pred.get_probs() + y_true_probs = y_true.get_probs() + if y_pred.get_n_classes() == 2 and not self.binary_as_multiclass: + y_pred_probs, y_true_probs = y_pred_probs[:, 1:], y_true_probs[:, 1:] + + g_f_x = y_pred_probs + y = y_true_probs + + diff = g_f_x - f_x + norm_val = torch.linalg.norm(diff, ord=self.p, dim=1, keepdim=True) + h_p = -norm_val.squeeze() + + if self.p == float('inf'): + abs_diff = torch.abs(diff) + subgradient = torch.zeros_like(diff) + indices = torch.argmax(abs_diff, dim=1, keepdim=True) + subgradient.scatter_(1, indices, 1.0) + subgradient = -subgradient * torch.sign(diff) + dot_product = torch.sum(subgradient * (y - g_f_x), dim=1) + return dot_product + h_p + + safe_norm = torch.where(norm_val == 0, torch.ones_like(norm_val), norm_val) + grad_norm = (torch.abs(diff) ** (self.p - 1) * torch.sign(diff)) / (safe_norm ** (self.p - 1)) + subgradient = -torch.where(norm_val == 0, torch.zeros_like(grad_norm), grad_norm) + dot_product = torch.sum(subgradient * (y - g_f_x), dim=1) + + return dot_product + h_p + + +class TopClassLoss(ClassificationMetric): + def __init__(self, metric: Metrics): + """ + Wrapper to calculate a given metric focusing only on the class + where f(x) has the highest confidence and treating it as binary. + If f_x is provided, then f_x is used to determine the top class, otherwise y_pred is used. + + :param metric: An instantiated single Metric object. + """ + if isinstance(metric, str): + raise TypeError("Expected a metric object, got a string. Use TopClassWrapper.from_name() instead.") + + if isinstance(metric, list) or metric.__class__.__name__ == 'CombinedMetrics': + raise ValueError("TopClassWrapper only supports a single metric object.") + + self.base_metric = metric + + base_name = self.base_metric.get_names()[0] + is_lower_better = self.base_metric.is_lower_better_list[0] + wrapper_name = f"topclass-{base_name}" + + super().__init__(name=wrapper_name, is_lower_better=is_lower_better, requires_f_x=False) + + @classmethod + def from_name(cls, metric_name: str) -> 'TopClassLoss': + """ + Initialize the wrapper directly from a metric's string name. + """ + base_metric = cls._resolve_single_metric(metric_name) + return cls(metric=base_metric) + + @staticmethod + def _resolve_single_metric(name: str) -> Metrics: + """Helper to pull the specific base class out of candidates.""" + return Metrics.from_names([name]).metrics_list[0] + + def _extract_top_class(self, + y_true: CategoricalDistribution, + y_pred: CategoricalDistribution, + f_x: CategoricalDistribution): + """ + Slices the distributions down to just the top class predicted by f_x. + Returns wrapped CategoricalProbs with two classes. + """ + y_true_probs = y_true.get_probs() + y_pred_probs = y_pred.get_probs() + f_x_probs = f_x.get_probs() + + top_indices = torch.argmax(f_x_probs, dim=-1) + row_indices = torch.arange(f_x_probs.size(0), device=f_x_probs.device) + + y_true_top = y_true_probs[row_indices, top_indices] + y_pred_top = y_pred_probs[row_indices, top_indices].unsqueeze(-1) + f_x_top = f_x_probs[row_indices, top_indices].unsqueeze(-1) + + y_pred_top = torch.cat([1 - y_pred_top, y_pred_top], dim=1) + f_x_top = torch.cat([1 - f_x_top, f_x_top], dim=1) + + return CategoricalDirac(torch.as_tensor(y_true_top).long(), n_classes=2), CategoricalProbs(y_pred_top), CategoricalProbs(f_x_top) + + def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[torch.Tensor]: + """ + Isolates the top class distributions, updates kwargs, and defers to base metric. + """ + f_x = kwargs.get("f_x", y_pred) # use predictions for choosing the top class if f_x is not provided + + y_true_top, y_pred_top, f_x_top = self._extract_top_class(y_true, y_pred, f_x) + + updated_kwargs = kwargs.copy() + updated_kwargs["f_x"] = f_x_top + + return self.base_metric._compute_indiv(y_true=y_true_top, y_pred=y_pred_top, **updated_kwargs) + + +class ConfidenceLoss(ClassificationMetric): + def __init__(self, metric: Metrics, direction: str): + """ + Wrapper to calculate under- and over-confidence for a single metric. + + :param metric: An instantiated single Metric object. + :param direction: 'over' or 'under' + """ + if isinstance(metric, str): + raise TypeError("Expected a metric object, got a string. Use ConfidenceWrapper.from_name() to initialize from a string.") + + if isinstance(metric, list) or metric.__class__.__name__ == 'CombinedMetrics': + raise ValueError("ConfidenceWrapper only supports a single metric object. Multiple metrics or CombinedMetrics are not supported.") + + assert direction in ['over', 'under'], "direction must be 'over' or 'under'" + + self.direction = direction + self.base_metric = metric + + base_name = self.base_metric.get_names()[0] + is_lower_better = self.base_metric.is_lower_better_list[0] + wrapper_name = f"{direction}-{base_name}" + super().__init__(name=wrapper_name, is_lower_better=is_lower_better, requires_f_x=True) + + @classmethod + def from_name(cls, metric_name: str, direction: str) -> 'ConfidenceLoss': + """ + Initialize the wrapper directly from a metric's string name. + """ + base_metric = cls._resolve_single_metric(metric_name) + return cls(metric=base_metric, direction=direction) + + @staticmethod + def _resolve_single_metric(name: str) -> Metrics: + """Helper to pull the specific base class out of candidates.""" + return Metrics.from_names([name]).metrics_list[0] + + def _clip_distribution(self, y_pred: CategoricalDistribution, f_x: CategoricalDistribution) -> CategoricalDistribution: + """Helper to handle the clipping logic dynamically based on direction.""" + if self.direction == 'over': + clipped_probs = _clip_p_values(y_pred.get_probs(), f_x.get_probs(), for_over=True) + elif self.direction == 'under': + clipped_probs = _clip_p_values(y_pred.get_probs(), f_x.get_probs(), for_under=True) + else: + raise ValueError("direction must be 'over' or 'under'") + + return CategoricalProbs(clipped_probs) + + def _compute_indiv(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[torch.Tensor]: + """ + Standard individual computation for single ClassificationMetrics. + """ + f_x = kwargs.get("f_x") + assert f_x is not None, f"f_x is required for {self.get_names()[0]}" + + clipped_y_pred = self._clip_distribution(y_pred, f_x) + + return self.base_metric._compute_indiv(y_true=y_true, y_pred=clipped_y_pred, **kwargs) + + +class OverConfidenceLoss(ConfidenceLoss): + def __init__(self, metric: Metrics): + """ + Specialized wrapper for calculating over-confidence for a single metric. + + :param metric: An instantiated single Metric object. + """ + # We explicitly set direction to 'over' + super().__init__(metric=metric, direction='over') + + @classmethod + def from_name(cls, metric_name: str) -> 'OverConfidenceLoss': + """ + Initialize the over-confidence wrapper directly from a metric's string name. + """ + return cls(metric=cls._resolve_single_metric(metric_name)) + + +class UnderConfidenceLoss(ConfidenceLoss): + def __init__(self, metric: Metrics): + """ + Specialized wrapper for calculating under-confidence for a single metric. + + :param metric: An instantiated single Metric object. + """ + # We explicitly set direction to 'under' + super().__init__(metric=metric, direction='under') + + @classmethod + def from_name(cls, metric_name: str) -> 'UnderConfidenceLoss': + """ + Initialize the under-confidence wrapper directly from a metric's string name. + """ + return cls(metric=cls._resolve_single_metric(metric_name)) + + class BalancedAccuracy(ClassificationMetric): pass # todo: could be implemented using SklearnClassificiationMetric but this should use threshold tuning @@ -460,7 +769,7 @@ def __init__(self, name: str, is_lower_better: bool, sklearn_func: Callable, use self.uses_probs = uses_probs self.two_class_single_column = two_class_single_column - def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution) -> Optional[torch.Tensor]: + def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[torch.Tensor]: # adapted from https://github.com/dholzmueller/pytabkit/blob/39086ac0621918de315c234d4719411705e13ea1/pytabkit/models/training/metrics.py#L518 # handle classes that don't occur in the test set y_probs = y_pred.get_probs() @@ -481,7 +790,6 @@ def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDist y_np = y.cpu().numpy() return torch.as_tensor(self.sklearn_func(y_np, y_pred_np)) - class AUROCOneVsRestSklearn(SklearnClassificationMetric): def __init__(self): super().__init__('auroc-ovr-sklearn', is_lower_better=False, @@ -507,7 +815,7 @@ def _get_torch_metric(self, n_classes: int): # can be overridden instead of specifying torch_metric in the constructor if more control is needed return self.torch_metric(task='binary' if n_classes == 2 else 'multiclass', num_classes=n_classes) - def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution) -> Optional[torch.Tensor]: + def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[torch.Tensor]: import torchmetrics y_probs = y_pred.get_probs() y = y_true.get_modes() @@ -546,7 +854,7 @@ def _try_use_cached(self, y_true: CategoricalDistribution, y_pred: CategoricalDi return 1. - other_metric_values['auroc_ovr'] return None - def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution) -> Optional[torch.Tensor]: + def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[torch.Tensor]: return 1. - AUROCOneVsRest().compute(y_true=y_true, y_pred=y_pred) @@ -575,7 +883,7 @@ class SmoothCalibrationError(ClassificationMetric): def __init__(self): super().__init__(name='smece', is_lower_better=True) - def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution) -> Optional[torch.Tensor]: + def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[torch.Tensor]: import relplot as rp labels_torch = y_true.get_modes() @@ -609,7 +917,7 @@ def __init__(self, metrics: Metrics, calibrator: sklearn.base.ClassifierMixin, v def compute_all(self, y_true: Distribution, y_pred: Distribution, other_metric_values: Optional[Dict[str, torch.Tensor]] = None, reduction: str = 'mean', - weights: Optional[torch.Tensor] = None) -> Dict[str, torch.Tensor]: + weights: Optional[torch.Tensor] = None, **kwargs) -> Dict[str, torch.Tensor]: assert isinstance(y_true, CategoricalDistribution) assert isinstance(y_pred, CategoricalDistribution) @@ -619,11 +927,23 @@ def compute_all(self, y_true: Distribution, y_pred: Distribution, cal_probs = [] true_probs = [] + # is_binary = y_pred.get_probs().shape[-1] == 1 + for train_idxs, val_idxs in splits: cal: Calibrator = sklearn.base.clone(self.calibrator) - cal.fit_torch(CategoricalProbs(y_pred.get_probs()[train_idxs]), y_true.get_modes()[train_idxs]) - y_val_probs = cal.predict_proba_torch(CategoricalProbs(y_pred.get_probs()[val_idxs])).get_probs() - + if isinstance(cal, Calibrator): + cal.fit_torch(CategoricalProbs(y_pred.get_probs()[train_idxs]), y_true.get_modes()[train_idxs]) + y_val_probs = cal.predict_proba_torch(CategoricalProbs(y_pred.get_probs()[val_idxs])).get_probs() + else: + cal.fit( y_pred.get_probs().detach().cpu().numpy()[train_idxs], y_true.get_modes().detach().cpu().numpy()[train_idxs]) + y_val_probs = torch.tensor( cal.predict_proba( y_pred.get_probs().detach().cpu().numpy()[val_idxs] ) ) + + # todo: fix this + # if is_binary: + # orig_probs.append(y_pred.get_probs()[val_idxs]) + # cal_probs.append(y_val_probs[:, [1]]) + # true_probs.append(y_true.get_modes()[val_idxs].reshape(-1, 1)) + # else: orig_probs.append(y_pred.get_probs()[val_idxs]) cal_probs.append(y_val_probs) true_probs.append(y_true.get_probs()[val_idxs]) @@ -632,9 +952,10 @@ def compute_all(self, y_true: Distribution, y_pred: Distribution, y_pred_cal_dist = CategoricalProbs(torch.cat(cal_probs, dim=-2)) y_true_dist = CategoricalProbs(torch.cat(true_probs, dim=-2)) - orig_results = self.metrics.compute_all(y_true=y_true_dist, y_pred=y_pred_orig_dist, + orig_results = self.metrics.compute_all(y_true=y_true_dist, y_pred=y_pred_orig_dist, f_x=y_pred_orig_dist, other_metric_values=other_metric_values) - ref_results = self.metrics.compute_all(y_true=y_true_dist, y_pred=y_pred_cal_dist) + ref_results = self.metrics.compute_all(y_true=y_true_dist, y_pred=y_pred_cal_dist, f_x=y_pred_orig_dist) + calerr_results = {f'calib-err_{key}{self.name_suffix}': orig_results[key] - ref_results[key] for key in orig_results} ref_results = {f'refinement_{key}{self.name_suffix}': value for key, value in ref_results.items()} @@ -660,7 +981,7 @@ def _try_use_cached(self, y_true: CategoricalDistribution, y_pred: CategoricalDi else: return None - def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution) -> Optional[torch.Tensor]: + def _compute_mean(self, y_true: CategoricalDistribution, y_pred: CategoricalDistribution, **kwargs) -> Optional[torch.Tensor]: unnorm_value = self.metric.compute(y_true, y_pred) ref_value = self.metric.compute(y_true, CategoricalProbs( y_pred.get_probs().mean(dim=0, keepdim=True).expand(y_pred.get_n_samples(), -1))) diff --git a/probmetrics/utils.py b/probmetrics/utils.py index 3d0deed..f7fb3b6 100644 --- a/probmetrics/utils.py +++ b/probmetrics/utils.py @@ -22,29 +22,36 @@ def validate_probabilities(X: np.ndarray) -> None: if not np.all(np.isclose(X.sum(axis=1), 1.0)): raise ValueError("Each row of the input array must sum to 1 to be a valid probability distribution.") -def binary_probs_to_logits(probs: np.ndarray) -> np.ndarray: - """Converts binary probabilities to logits using the logit function (inverse sigmoid). - Clips logits to the log of the tinyest normal float32 to avoid infinite logit values. - """ +def process_binary_probs(probs: np.ndarray) -> np.ndarray: probs = probs.astype(np.float32) - # 1D array of probabilities - if probs.ndim == 1: - logits = probs - # 2D array of probability pairs [(p0, p1), ...] - elif probs.ndim == 2 and probs.shape[1] == 2: - logits = probs[:, 1] + + if probs.ndim == 1: # 1D array of probabilities + return probs + elif probs.ndim == 2 and probs.shape[1] == 2: # 2D array of probability pairs [(p0, p1), ...] + return probs[:, 1] else: raise ValueError(f"Invalid input shape: {probs.shape}." f"1D array or 2D array with shape (n, 2)") + +def binary_probs_to_logits(probs: np.ndarray) -> np.ndarray: + """Converts binary probabilities to logits using the logit function (inverse sigmoid). + Clips logits to the log of the tiniest normal float32 to avoid infinite logit values. + """ + probs = process_binary_probs(probs) with np.errstate(divide="ignore"): - logits = np.log(logits) - np.log1p(-logits) + log_p = np.log(probs) + log_1_minus_p = np.log1p(-probs) + thresh = np.log(np.finfo(np.float32).tiny) - return np.clip(logits, a_min=thresh, a_max=-thresh) + log_p = np.clip(log_p, a_min=thresh, a_max=-thresh).reshape(-1, 1) + log_1_minus_p = np.clip(log_1_minus_p, a_min=thresh, a_max=-thresh).reshape(-1, 1) + + return log_p, log_1_minus_p def multiclass_probs_to_logits(probs: np.ndarray) -> np.ndarray: """Converts multiclass probabilities to logits using the log function. - Clips logits to the log of the tinyest normal float32 to avoid infinite logit values. + Clips logits to the log of the tiniest normal float32 to avoid infinite logit values. """ # probs = np.atleast_2d(probs) # Ensure we have a 2D array probs = probs.astype(np.float32) diff --git a/pyproject.toml b/pyproject.toml index 49f6af7..c1274ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,8 +27,6 @@ classifiers = [ dependencies = [ "torch>=2.0", "numpy>=1.25", - # scikit-learn==1.8 will deprecate the cv='prefit' option in CalibratedClassifierCV, - # for which the suggested solution doesn't work at the moment in case of missing classes "scikit-learn>=1.3", # older versions of torchmetrics (<1.2.1) have a bug that makes certain metrics slow: # https://github.com/Lightning-AI/torchmetrics/pull/2184 @@ -44,7 +42,10 @@ extra = [ "relplot", # for smooth ECE "pytorch-minimize", # for torchcal TS implementation # relplot fails for sklearn 1.7: ModuleNotFoundError: No module named 'sklearn.utils._estimator_html_repr' - "scikit-learn>=1.3,<1.7", + "scikit-learn>=1.3", + "catboost", # For the WS_CatboostClassifier + "lightgbm", # For the WS_LGBMClassifier + "scipy", # For the WS_LGBMClassifier to handle softmax ] dev = [ "matplotlib>=3.0", # plotting diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 0d2bf1b..254db52 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -48,9 +48,9 @@ def test_metrics_equal(metrics: List[Metric], calib_dataset: Tuple[np.ndarray, n ['logloss', 'brier', 'accuracy', 'class-error', 'auroc-ovr', 'auroc-ovr-sklearn', 'auroc-ovo-sklearn', 'logloss-clip1e-06', 'smece', 'ece-15', 'rmsce-15', 'mce-15', 'mpn_logloss', 'mpn_brier', 'mpn_accuracy']), - MetricsWithCalibration(Metrics.from_names(['logloss', 'brier', 'accuracy', 'class-error']), + MetricsWithCalibration(Metrics.from_names(['logloss', 'brier', 'accuracy', 'class-error', 'proper-L2', 'proper-Linf']), calibrator=get_calibrator('temp-scaling'), val_splitter=CVSplitter(n_cv=5)), - MetricsWithCalibration(Metrics.from_names(['logloss', 'brier', 'accuracy', 'class-error']), + MetricsWithCalibration(Metrics.from_names(['logloss', 'brier', 'accuracy', 'class-error', 'proper-L2', 'proper-Linf']), calibrator=get_calibrator('temp-scaling'), val_splitter=AllSplitter()), ]) def test_metrics_names(metrics: Metrics, calib_dataset: Tuple[np.ndarray, np.ndarray]): From eed39d05ae3f2255914070cfd75b89d915c248d2 Mon Sep 17 00:00:00 2001 From: Sacha Braun <114589494+ElSacho@users.noreply.github.com> Date: Mon, 2 Mar 2026 19:34:28 +0100 Subject: [PATCH 02/11] updated numba>=0.59.0 for compatibility reasons --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c1274ea..f5b0ebc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ # older versions of torchmetrics (<1.2.1) have a bug that makes certain metrics slow: # https://github.com/Lightning-AI/torchmetrics/pull/2184 "torchmetrics>=1.2.1", - "numba", + "numba>=0.59.0", ] [project.optional-dependencies] From 71ff592fbcfc20444bf531a10fb9e99f3d19cc44 Mon Sep 17 00:00:00 2001 From: Sacha Braun <114589494+ElSacho@users.noreply.github.com> Date: Mon, 2 Mar 2026 19:37:21 +0100 Subject: [PATCH 03/11] updated numba to python dependant versions --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f5b0ebc..327599c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,8 @@ dependencies = [ # older versions of torchmetrics (<1.2.1) have a bug that makes certain metrics slow: # https://github.com/Lightning-AI/torchmetrics/pull/2184 "torchmetrics>=1.2.1", - "numba>=0.59.0", + "numba<0.61; python_version < '3.10'", + "numba>=0.61; python_version >= '3.10'", ] [project.optional-dependencies] From 11d7a30cefc188149b7e724843a0791c13c188c7 Mon Sep 17 00:00:00 2001 From: Sacha Braun <114589494+ElSacho@users.noreply.github.com> Date: Mon, 2 Mar 2026 19:42:05 +0100 Subject: [PATCH 04/11] --vv test --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1ce5a19..900a0e6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -34,7 +34,7 @@ jobs: - name: Install swig run: uv pip install --system swig - name: Run tests - run: hatch test + run: hatch -vv test - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4.5.0 with: From be8f98e975028afce40c41607bb4e5ad6f854974 Mon Sep 17 00:00:00 2001 From: Sacha Braun <114589494+ElSacho@users.noreply.github.com> Date: Mon, 2 Mar 2026 19:47:24 +0100 Subject: [PATCH 05/11] torch python version dependance --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 327599c..b03b742 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,6 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", ] dependencies = [ - "torch>=2.0", "numpy>=1.25", "scikit-learn>=1.3", # older versions of torchmetrics (<1.2.1) have a bug that makes certain metrics slow: @@ -33,6 +32,8 @@ dependencies = [ "torchmetrics>=1.2.1", "numba<0.61; python_version < '3.10'", "numba>=0.61; python_version >= '3.10'", + "torch>=2.0,<2.10; python_version < '3.10'", + "torch>=2.0; python_version >= '3.10'", ] [project.optional-dependencies] From ff5876b097b83a20c1db2641f753cec8af51b582 Mon Sep 17 00:00:00 2001 From: Sacha Braun <114589494+ElSacho@users.noreply.github.com> Date: Mon, 2 Mar 2026 19:50:30 +0100 Subject: [PATCH 06/11] tool.hatch.envs.hatch-test.matrix --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index b03b742..3f42095 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,9 @@ features = ["extra", "dev"] installer = "uv" features = ["extra", "dev"] +[[tool.hatch.envs.hatch-test.matrix]] +python = ["3.12", "3.11", "3.10", "3.9"] + [tool.hatch.build.targets.sdist] package = ['probmetrics'] only-include = ['probmetrics'] From 646c7dd69eebbbb11df0163a9ec84910b32c6792 Mon Sep 17 00:00:00 2001 From: Sacha Braun <114589494+ElSacho@users.noreply.github.com> Date: Mon, 2 Mar 2026 19:59:16 +0100 Subject: [PATCH 07/11] run: python -m pip install -U pip hatch --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 900a0e6..0ad6a7d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -30,7 +30,7 @@ jobs: # Install a specific version of uv. version: "0.5.4" - name: Install hatch - run: uv pip install --system hatch + run: python -m pip install -U pip hatch - name: Install swig run: uv pip install --system swig - name: Run tests From d0d22abe13fe6698b401299237f53285da69defd Mon Sep 17 00:00:00 2001 From: Sacha Braun <114589494+ElSacho@users.noreply.github.com> Date: Mon, 2 Mar 2026 20:07:06 +0100 Subject: [PATCH 08/11] - name: Install dependencies --- .github/workflows/testing.yml | 9 ++++++--- pyproject.toml | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 0ad6a7d..210cdf3 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -29,12 +29,15 @@ jobs: with: # Install a specific version of uv. version: "0.5.4" - - name: Install hatch - run: python -m pip install -U pip hatch + # - name: Install hatch + # run: uv pip install --system hatch - name: Install swig run: uv pip install --system swig + - name: Install dependencies + run: uv pip install -e .[extra,dev] - name: Run tests - run: hatch -vv test + # run: hatch -vv test + run: pytest tests - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4.5.0 with: diff --git a/pyproject.toml b/pyproject.toml index 3f42095..60c179a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,12 +72,15 @@ path = "probmetrics/__about__.py" installer = "uv" features = ["extra", "dev"] -[tool.hatch.envs.hatch-test] +[tool.hatch.envs.test] installer = "uv" features = ["extra", "dev"] -[[tool.hatch.envs.hatch-test.matrix]] -python = ["3.12", "3.11", "3.10", "3.9"] +[tool.hatch.envs.test.scripts] +run-tests = "pytest {args}" + +[[tool.hatch.envs.test.matrix]] +python = ["3.9", "3.10", "3.11", "3.12"] [tool.hatch.build.targets.sdist] package = ['probmetrics'] From 4f5a1645b46488c7d9e6dac058093d0873ff50a7 Mon Sep 17 00:00:00 2001 From: Sacha Braun <114589494+ElSacho@users.noreply.github.com> Date: Mon, 2 Mar 2026 20:10:36 +0100 Subject: [PATCH 09/11] minimal yml --- .github/workflows/testing.yml | 40 ++++++++++++++--------------------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 210cdf3..17f3775 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,45 +1,37 @@ -name: 'test' +name: test on: push: - branches: - - "main" - - "dev" - - "test" + branches: [main, dev, test] pull_request: - branches: - - '*' + branches: ['*'] jobs: test: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] python-version: ['3.9', '3.10', '3.11', '3.12'] + runs-on: ${{ matrix.os }} + steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + + - name: Set up Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install uv uses: astral-sh/setup-uv@v3 - with: - # Install a specific version of uv. - version: "0.5.4" - # - name: Install hatch - # run: uv pip install --system hatch - - name: Install swig - run: uv pip install --system swig - - name: Install dependencies + + - name: Create virtual environment + run: uv venv + + - name: Install project with extras run: uv pip install -e .[extra,dev] + - name: Run tests - # run: hatch -vv test - run: pytest tests - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4.5.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - slug: dholzmueller/probmetrics + run: uv run pytest -q \ No newline at end of file From 0b19c75963e5f79e0a566aa2a4ab79c3ff325378 Mon Sep 17 00:00:00 2001 From: Sacha Braun <114589494+ElSacho@users.noreply.github.com> Date: Mon, 2 Mar 2026 20:14:53 +0100 Subject: [PATCH 10/11] run: hatch run pytest --- .github/workflows/testing.yml | 39 ++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 17f3775..e176585 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,37 +1,42 @@ -name: test +name: 'test' on: push: - branches: [main, dev, test] + branches: + - "main" + - "dev" + - "test" pull_request: - branches: ['*'] + branches: + - '*' jobs: test: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [windows-latest, ubuntu-latest, macos-latest] python-version: ['3.9', '3.10', '3.11', '3.12'] - runs-on: ${{ matrix.os }} - steps: - uses: actions/checkout@v4 - - - name: Set up Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install uv uses: astral-sh/setup-uv@v3 - - - name: Create virtual environment - run: uv venv - - - name: Install project with extras - run: uv pip install -e .[extra,dev] - + with: + # Install a specific version of uv. + version: "0.5.4" + - name: Install hatch + run: uv pip install --system hatch + - name: Install swig + run: uv pip install --system swig - name: Run tests - run: uv run pytest -q \ No newline at end of file + run: hatch run pytest + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.5.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: dholzmueller/probmetrics From f5ff9521131aca7587baabab2328b10e362f77f2 Mon Sep 17 00:00:00 2001 From: Sacha Braun <114589494+ElSacho@users.noreply.github.com> Date: Mon, 2 Mar 2026 20:18:45 +0100 Subject: [PATCH 11/11] run: uv pip install --system hatch "virtualenv<21" --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e176585..e819fe2 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -30,7 +30,7 @@ jobs: # Install a specific version of uv. version: "0.5.4" - name: Install hatch - run: uv pip install --system hatch + run: uv pip install --system hatch "virtualenv<21" - name: Install swig run: uv pip install --system swig - name: Run tests