Source code for caliber.binary_classification.metrics.ece

import numpy as np


[docs] def expected_calibration_error( targets: np.ndarray, probs: np.ndarray, n_bins: int = 10, min_prob_bin: float = 0.0 ) -> float: bin_edges = np.linspace(0, 1, n_bins + 1) bin_indices = np.digitize(probs, bin_edges) ece = 0 for i in range(1, n_bins + 1): mask = bin_indices == i prob_bin = np.mean(mask) if prob_bin > min_prob_bin: _probs = probs[mask] _targets = targets[mask] acc = np.mean((_probs >= 0.5) == _targets) conf = np.mean(np.maximum(1 - _probs, _probs)) ece += prob_bin * np.abs(acc - conf) return ece