API Reference¶

@ensure_numpy_array
@abstractmethod
def fit(
    self,
    x: pd.DataFrame | np.ndarray,
    y: np.ndarray | None = None,
    *,
    n_jobs: int | None = None,
) -> Self:
    """Fit the detector model(s) and compute calibration scores.

    Args:
        x: The dataset used for fitting the model(s) and determining
            calibration scores.
        y: Ignored. Present for sklearn API compatibility.
        n_jobs: Optional strategy-specific parallelism hint.
            Currently used by strategies that expose an ``n_jobs`` parameter
            (for example, ``JackknifeBootstrap``).

    Returns:
        The fitted detector instance.
    """
    raise NotImplementedError("Subclasses must implement fit()")

@ensure_numpy_array
def calibrate(
    self,
    x: pd.DataFrame | np.ndarray,
    y: np.ndarray | None = None,
) -> Self:
    """Calibrate a pre-fitted detector on separate calibration data.

    Args:
        x: Dataset used only to compute calibration scores.
        y: Ignored. Present for sklearn API compatibility.

    Returns:
        The calibrated detector instance.
    """
    raise NotImplementedError("Subclasses must implement calibrate()")

@abstractmethod
def compute_p_values(
    self,
    x: pd.DataFrame | pd.Series | np.ndarray,
    *,
    refit_weights: bool = True,
) -> np.ndarray | pd.Series:
    """Return conformal p-values for new data.

    Args:
        x: New data instances for anomaly estimation.
        refit_weights: Whether to refit the weight estimator for this batch
            in weighted mode. Ignored in standard mode.

    Returns:
        P-values as ndarray for numpy input, or pandas Series for pandas input.
    """
    raise NotImplementedError("Subclasses must implement compute_p_values()")

@abstractmethod
def score_samples(
    self,
    x: pd.DataFrame | pd.Series | np.ndarray,
    *,
    refit_weights: bool = True,
) -> np.ndarray | pd.Series:
    """Return aggregated raw anomaly scores for new data.

    Args:
        x: New data instances for anomaly estimation.
        refit_weights: Whether to refit the weight estimator for this batch
            in weighted mode. Ignored in standard mode.

    Returns:
        Raw scores as ndarray for numpy input, or pandas Series for pandas input.
    """
    raise NotImplementedError("Subclasses must implement score_samples()")

def get_params(self, deep: bool = True) -> dict[str, Any]:
    """Return estimator parameters following sklearn conventions.

    Notes:
        - ``deep=False`` returns constructor-facing parameters used for
          sklearn clone compatibility.
        - ``deep=True`` also includes nested ``component__param`` entries
          read from the current runtime components (effective/internal state),
          which may differ from originally passed constructor objects after
          adaptation/normalization.
    """
    params: dict[str, Any] = {
        "detector": self._init_detector,
        "strategy": self._init_strategy,
        "estimation": self._init_estimation,
        "weight_estimator": self._init_weight_estimator,
        "aggregation": self._init_aggregation,
        "score_polarity": self._init_score_polarity,
        "seed": self._init_seed,
        "verbose": self._init_verbose,
        "verify_prepared_batch_content": self._init_verify_prepared_batch_content,
    }
    if not deep:
        return params

    for component_name in self._NESTED_COMPONENTS:
        component = getattr(self, component_name)
        if component is None or not hasattr(component, "get_params"):
            continue
        try:
            component_params = component.get_params(deep=True)
        except TypeError:
            component_params = component.get_params()
        for key, value in component_params.items():
            params[f"{component_name}__{key}"] = value
    return params

def set_params(self, **params: Any) -> Self:
    """Set estimator parameters following sklearn conventions."""
    if not params:
        return self

    updated_params = self.get_params(deep=False)
    nested_updates: dict[str, dict[str, Any]] = {}

    for key, value in params.items():
        if "__" in key:
            component_name, nested_key = key.split("__", 1)
            if component_name not in self._NESTED_COMPONENTS:
                raise ValueError(f"Invalid parameter {component_name!r}.")
            nested_updates.setdefault(component_name, {})[nested_key] = value
            continue

        if key not in updated_params:
            raise ValueError(
                f"Invalid parameter {key!r} for estimator {type(self).__name__}."
            )
        updated_params[key] = value

    for component_name, component_params in nested_updates.items():
        component = updated_params[component_name]
        if component is None:
            raise ValueError(
                f"Cannot set nested parameters for {component_name!r}: "
                "component is None."
            )
        if not hasattr(component, "set_params"):
            raise ValueError(
                f"Cannot set nested parameters for {component_name!r}: "
                "component does not implement set_params()."
            )
        component.set_params(**component_params)

    self._configure(**updated_params)
    return self

@ensure_numpy_array
def fit(
    self,
    x: pd.DataFrame | np.ndarray,
    y: np.ndarray | None = None,
    *,
    n_jobs: int | None = None,
) -> Self:
    """Fit detector model(s) and compute calibration scores.

    Uses the specified strategy to train the base detector(s) and calculate
    non-conformity scores on the calibration set.

    Args:
        x: The dataset used for fitting and calibration.
        y: Ignored. Present for sklearn API compatibility.
        n_jobs: Optional strategy-specific parallelism hint. Supported by
            strategies whose ``fit_calibrate`` signature includes ``n_jobs``
            (for example, ``JackknifeBootstrap``).

    Returns:
        The fitted detector instance (for method chaining).
    """
    _ = y
    self._last_selection_result = None
    if self.strategy._uses_e_values:
        self._reset_fit_state()
    fit_kwargs: dict[str, Any] = {
        "x": x,
        "detector": self.detector,
        "weighted": self._is_weighted_mode,
        "seed": self.seed,
    }
    if n_jobs is not None:
        strategy_params = inspect.signature(self.strategy.fit_calibrate).parameters
        if "n_jobs" not in strategy_params:
            raise ValueError(
                f"Strategy {type(self.strategy).__name__} does not support n_jobs. "
                "Pass n_jobs only when using a strategy that exposes it, "
                "such as JackknifeBootstrap."
            )
        fit_kwargs["n_jobs"] = n_jobs

    self._detector_set, self._calibration_set = self.strategy.fit_calibrate(
        **fit_kwargs
    )
    self._calibration_mode = CalibrationMode.INTEGRATED
    self._n_features_in = int(x.shape[1])

    if (
        self._is_weighted_mode
        and self.strategy.calibration_ids is not None
        and len(self.strategy.calibration_ids) > 0
    ):
        self._calibration_samples = x[self.strategy.calibration_ids]
    else:
        self._calibration_samples = np.array([])

    self._prepared_weight_batch_size = None
    self._prepared_weight_batch_signature = None
    self._last_result = None
    return self

@ensure_numpy_array
def calibrate(
    self,
    x: pd.DataFrame | np.ndarray,
    y: np.ndarray | None = None,
) -> Self:
    """Calibrate a pre-fitted detector on separate calibration data.

    This detached workflow is currently supported only for ``Split`` strategy,
    where a single pre-fitted model is calibrated on a dedicated dataset.

    Args:
        x: Calibration dataset used to compute calibration scores.
        y: Ignored. Present for sklearn API compatibility.

    Returns:
        The calibrated detector instance (for method chaining).

    Raises:
        ValueError: If strategy is not ``Split``.
        NotFittedError: If the base detector appears unfitted.
    """
    _ = y
    self._last_selection_result = None
    if not isinstance(self.strategy, Split):
        raise ValueError(
            "calibrate() is supported only with Split strategy. "
            f"Got {type(self.strategy).__name__}. Use fit(x_reference) for "
            "integrated calibration."
        )

    try:
        calibration_set = np.asarray(
            self.detector.decision_function(x),
            dtype=float,
        ).ravel()
    except Exception as exc:
        message = str(exc).lower()
        if (
            isinstance(exc, NotFittedError)
            or "not fitted" in message
            or (isinstance(exc, AttributeError) and "has no attribute" in message)
        ):
            raise NotFittedError(
                "Base detector is not fitted. Fit the base detector before "
                "calling calibrate()."
            ) from exc
        raise

    if calibration_set.shape[0] != len(x):
        raise ValueError(
            "calibration scores must have one value per calibration sample. "
            f"Got {calibration_set.shape[0]} scores for {len(x)} samples."
        )

    self._detector_set = [self.detector]
    self._calibration_set = calibration_set
    self._calibration_mode = CalibrationMode.DETACHED
    self._n_features_in = int(x.shape[1])
    if self._is_weighted_mode:
        self._calibration_samples = x.copy()
    else:
        self._calibration_samples = np.array([])

    self._prepared_weight_batch_size = None
    self._prepared_weight_batch_signature = None
    self._last_result = None
    return self

def select(
    self,
    x: pd.DataFrame | pd.Series | np.ndarray,
    *,
    alpha: float = 0.05,
    pruning: Pruning = Pruning.DETERMINISTIC,
    seed: int | None = None,
    refit_weights: bool = True,
) -> np.ndarray | pd.Series:
    """Construct evidence and select anomalies from one fixed test batch.

    This is the single-call batch workflow. It combines
    ``compute_p_values()`` with Benjamini-Hochberg in standard mode or
    weighted conformalized selection in weighted mode. Validity still
    depends on the assumptions of both the p-value construction and the
    selected multiple-testing procedure.

    With DerandomizedSplits, this instead constructs per-split e-values,
    averages them uniformly, and applies e-BH once. Configure alpha_bh and
    tie_seed on that strategy. Automatic ties use a separate fitting-derived
    seed. Selection populates last_selection_result and clears last_result.

    Args:
        x: New data instances for anomaly estimation.
        alpha: Nominal FDR target in ``(0, 1)``. Defaults to ``0.05``.
        pruning: Pruning strategy for weighted FDR control. Ignored in
            standard (unweighted) mode. Defaults to
            ``Pruning.DETERMINISTIC``.
        seed: Optional random seed for weighted randomized pruning modes.
            When ``None``, falls back to detector ``seed``. Ignored in
            standard mode and deterministic pruning mode.
        refit_weights: Whether to refit the weight estimator for this batch
            in weighted mode. Ignored in standard mode. Defaults to True.

    Returns:
        Boolean selection mask of shape ``(n_test,)``. ``True`` entries are
        the selected anomaly discoveries. Returns a pandas Series when the
        input is a DataFrame or Series.

    Examples:
        Standard workflow (no weight estimator):

        ```python
        import numpy as np
        from sklearn.ensemble import IsolationForest

        from nonconform import ConformalDetector, Split

        rng = np.random.default_rng(42)
        x_reference = rng.normal(size=(300, 2))
        x_test = np.vstack(
            [rng.normal(size=(38, 2)), rng.normal(loc=5.0, size=(2, 2))]
        )
        detector = ConformalDetector(
            detector=IsolationForest(random_state=42),
            strategy=Split(n_calib=0.25),
            score_polarity="higher_is_normal",
            seed=42,
        ).fit(x_reference)
        selected = detector.select(x_test, alpha=0.10)
        print("Selected indices:", np.flatnonzero(selected))
        ```

        Weighted workflow:

        ```python
        import numpy as np
        from sklearn.ensemble import IsolationForest

        from nonconform import (
            ConformalDetector,
            Split,
            logistic_weight_estimator,
        )

        rng = np.random.default_rng(7)
        x_reference = rng.normal(size=(400, 2))
        x_test = rng.normal(loc=0.5, size=(40, 2))
        detector = ConformalDetector(
            detector=IsolationForest(random_state=7),
            strategy=Split(n_calib=0.25),
            weight_estimator=logistic_weight_estimator(),
            score_polarity="higher_is_normal",
            seed=7,
        ).fit(x_reference)
        selected = detector.select(x_test, alpha=0.10)
        print("Number selected:", int(selected.sum()))
        ```
    """
    self._last_selection_result = None
    if self.strategy._uses_e_values:
        self._last_result = None
    if not (0.0 < alpha < 1.0):
        raise ValueError(f"alpha must be in (0, 1), got {alpha}")

    from nonconform.fdr import weighted_false_discovery_control

    x_array, index = _as_numpy_with_index(x)
    if self.strategy._uses_e_values:
        scores = self._score_models(x_array)
        selection = self.strategy._select_e_values(
            scores, self._calibration_set, alpha=alpha
        )
        self._last_selection_result = selection
        mask = selection.selected.copy()
        if index is not None:
            return pd.Series(mask, index=index, name="selected")
        return mask

    self.compute_p_values(x_array, refit_weights=refit_weights)
    result = self._last_result
    if result is None or result.p_values is None:
        raise RuntimeError(
            "Internal error: select() expected p-values after compute_p_values()."
        )

    if self._is_weighted_mode:
        selection_seed = self.seed if seed is None else seed
        mask = weighted_false_discovery_control(
            result=result,
            alpha=alpha,
            pruning=pruning,
            seed=_derive_wcs_pruning_seed(selection_seed),
        )
    else:
        p_values = np.asarray(result.p_values, dtype=float)
        mask = false_discovery_control(p_values, method="bh") <= alpha

    if index is not None:
        return pd.Series(mask, index=index, name="selected")
    return mask

@ensure_numpy_array
def prepare_weights_for(self, x: pd.DataFrame | np.ndarray) -> Self:
    """Prepare weighted conformal state for a specific test batch.

    In weighted mode, this fits the weight estimator for the supplied batch
    without producing predictions. Use this for explicit state transitions in
    exploratory workflows.

    Args:
        x: Test batch for which weights should be prepared.

    Returns:
        The fitted detector instance (for method chaining).

    Raises:
        NotFittedError: If fit() has not been called.
        RuntimeError: If weighted mode is disabled.
    """
    if not self.is_fitted:
        raise NotFittedError("This ConformalDetector instance is not fitted yet.")
    if not self._is_weighted_mode or self.weight_estimator is None:
        raise RuntimeError(
            "prepare_weights_for() requires weighted mode with a weight_estimator."
        )

    self._fit_weights_for_batch(x)
    return self

def score_samples(
    self,
    x: pd.DataFrame | pd.Series | np.ndarray,
    *,
    refit_weights: bool = True,
) -> np.ndarray | pd.Series:
    """Return aggregated raw anomaly scores for new data.

    Clears last_selection_result. With DerandomizedSplits, raw aggregation
    is diagnostic only and the last_result snapshot has calib_scores=None;
    select() instead uses each model's separate score/calibration pair.

    Args:
        x: New data instances for anomaly estimation.
        refit_weights: Whether to refit the weight estimator for this batch
            in weighted mode. Defaults to True.

    Returns:
        Aggregated raw anomaly scores.
    """
    self._last_selection_result = None
    if self.strategy._uses_e_values:
        self._last_result = None
    x_array, index = _as_numpy_with_index(x)
    test_batch_signature = batch_signature(x_array)
    estimates = self._aggregate_scores(x_array)
    weights = self._resolve_weights(
        x_array,
        refit_weights=refit_weights,
        test_batch_signature=test_batch_signature,
    )
    calib_weights, test_weights = weights if weights else (None, None)

    result = ConformalResult(
        p_values=None,
        test_scores=estimates.copy(),
        calib_scores=(
            None if self.strategy._uses_e_values else self._calibration_set.copy()
        ),
        test_weights=_safe_copy(test_weights),
        calib_weights=_safe_copy(calib_weights),
        metadata={},
    )
    result._provenance = self._result_provenance(test_batch_signature)
    self._last_result = result
    if index is not None:
        return pd.Series(estimates, index=index, name="score")
    return estimates

def compute_p_value(self, x: pd.Series | np.ndarray) -> float:
    """Return one value from the configured estimation strategy.

    Unavailable for DerandomizedSplits; use select() on a fixed test batch
    and inspect last_selection_result instead.

    This is a single-sample convenience wrapper around
    :meth:`compute_p_values`. It updates :attr:`last_result` with the
    corresponding one-row result and does not update the calibration set.
    With the default ``Empirical`` estimator, the returned value is a
    rank-based conformal p-value. Other estimators define their own
    interpretation and assumptions.

    Args:
        x: One-dimensional feature vector with the same number of features
            used during fitting or detached calibration.

    Returns:
        The observation's p-value or score-tail estimate as a Python float.

    Raises:
        NotFittedError: If the detector has not been fitted or calibrated.
        RuntimeError: If weighted conformal mode is enabled. Density-ratio
            estimation requires a representative test batch.
        ValueError: If ``x`` is not one-dimensional or its feature count does
            not match the fitted data.

    Note:
        Repeated single-sample calls are batch-equivalent only when detector
        scoring and p-value estimation are sample-wise and deterministic.
        Randomized tie-breaking can produce different values from one batch
        call.
    """
    self._require_p_value_strategy()
    if not self.is_fitted:
        raise NotFittedError("This ConformalDetector instance is not fitted yet.")
    if self._is_weighted_mode:
        raise RuntimeError(
            "compute_p_value() is unavailable in weighted mode because "
            "density-ratio estimation requires a representative test batch. "
            "Use compute_p_values() with that batch instead."
        )

    x_array = np.asarray(x)
    if x_array.ndim != 1:
        raise ValueError(
            "x must be a one-dimensional feature vector; "
            f"got shape {x_array.shape}."
        )

    expected_features = self._n_features_in
    if expected_features is None:
        raise RuntimeError(
            "Fitted feature count is unavailable. Refit or recalibrate the "
            "detector before calling compute_p_value()."
        )
    if x_array.shape[0] != expected_features:
        raise ValueError(
            f"x has {x_array.shape[0]} features, but this ConformalDetector "
            f"was fitted with {expected_features} features."
        )

    p_values = self.compute_p_values(x_array[np.newaxis, :])
    return float(np.asarray(p_values).item())

def compute_p_values(
    self,
    x: pd.DataFrame | pd.Series | np.ndarray,
    *,
    refit_weights: bool = True,
) -> np.ndarray | pd.Series:
    """Return values from the configured estimation strategy for new data.

    Unavailable for DerandomizedSplits; use select() and inspect
    last_selection_result instead.

    Args:
        x: New data instances for anomaly estimation.
        refit_weights: Whether to refit the weight estimator for this batch
            in weighted mode. Defaults to True.

    Returns:
        P-values or score-tail estimates. Pandas input produces a Series
        named ``"p_value"``; NumPy input produces an ndarray.
    """
    self._require_p_value_strategy()
    x_array, index = _as_numpy_with_index(x)
    test_batch_signature = batch_signature(x_array)
    estimates = self._aggregate_scores(x_array)
    weights = self._resolve_weights(
        x_array,
        refit_weights=refit_weights,
        test_batch_signature=test_batch_signature,
    )
    calib_weights, test_weights = weights if weights else (None, None)

    p_values = self.estimation.compute_p_values(
        estimates, self._calibration_set, weights
    )

    metadata = self._result_metadata()
    if hasattr(self.estimation, "get_metadata"):
        meta = self.estimation.get_metadata()
        if meta:
            metadata.update(meta)

    result = ConformalResult(
        p_values=p_values.copy(),
        test_scores=estimates.copy(),
        calib_scores=self._calibration_set.copy(),
        test_weights=_safe_copy(test_weights),
        calib_weights=_safe_copy(calib_weights),
        metadata=metadata,
    )
    result._provenance = self._result_provenance(test_batch_signature)
    self._last_result = result
    if index is not None:
        return pd.Series(p_values, index=index, name="p_value")
    return p_values

def fdp_bounds(
    self,
    x: pd.DataFrame | pd.Series | np.ndarray,
    *,
    confidence: float = 0.95,
    method: str = "mc_thc",
    n_resamples: int | None = None,
    seed: int | None = None,
    boost: bool = True,
    lower: float | None = None,
    upper: float | None = None,
    beta: float | None = None,
    precision: float | None = None,
) -> FDPCertificate:
    """Compute p-values once and return a simultaneous FDP certificate.

    Supports unweighted empirical Split inference, including detached
    calibration. Choose the method before inspecting its curve and keep
    the testing family fixed. Confidence is coverage, not an FDR target.
    Scientific exchangeability remains the caller's responsibility.

    Options match :meth:`nonconform.fdr.FDPCertificate.from_p_values`.
    The seed controls certificate Monte Carlo sampling only and does not
    inherit the fitting seed. The returned certificate is independent of
    subsequent detector operations; its select() returns a NumPy mask.
    """
    from nonconform._internal.fdp_bounds import validate_scope

    if not self.is_fitted:
        raise NotFittedError("This ConformalDetector instance is not fitted yet.")
    validate_scope(self._result_provenance(None))
    self.compute_p_values(x)
    return self._last_result.fdp_bounds(
        confidence=confidence,
        method=method,
        n_resamples=n_resamples,
        seed=seed,
        boost=boost,
        lower=lower,
        upper=upper,
        beta=beta,
        precision=precision,
    )

@abc.abstractmethod
def fit_calibrate(
    self,
    x: pd.DataFrame | np.ndarray,
    detector: AnomalyDetector,
    seed: int | None = None,
    weighted: bool = False,
) -> tuple[list[AnomalyDetector], np.ndarray]:
    """Fits the detector and performs calibration.

    Args:
        x: The input data for fitting and calibration.
        detector: The anomaly detection model to be fitted and calibrated.
        seed: Random seed for reproducibility. Defaults to None.
        weighted: Whether to use weighted approach. Defaults to False.

    Returns:
        Tuple of (list of trained detectors, calibration scores array).
    """
    raise NotImplementedError(
        "The fit_calibrate() method must be implemented by subclasses."
    )

@ensure_numpy_array
def fit_calibrate(
    self,
    x: pd.DataFrame | np.ndarray,
    detector: AnomalyDetector,
    weighted: bool = False,
    seed: int | None = None,
) -> tuple[list[AnomalyDetector], np.ndarray]:
    """Fits detector and generates calibration scores using a data split.

    Args:
        x: The input data.
        detector: The detector instance to train.
        weighted: If True, stores calibration sample indices. Defaults to False.
        seed: Random seed for reproducibility. Defaults to None.

    Returns:
        Tuple of (list with trained detector, calibration scores array).
    """
    self._validate_n_calib(len(x))
    x_id = np.arange(len(x))
    train_id, calib_id = train_test_split(
        x_id, test_size=self._calib_size, shuffle=True, random_state=seed
    )

    _try_set_random_state(detector, seed)

    detector.fit(x[train_id])
    calibration_set = detector.decision_function(x[calib_id])

    if weighted:
        self._calibration_ids = calib_id.tolist()
    else:
        self._calibration_ids = None
    return [detector], calibration_set

def get_params(self, deep: bool = True) -> dict[str, Any]:
    """Return constructor parameters for inspection and sklearn cloning."""
    return {
        "n_repetitions": self.n_repetitions,
        "n_calib": self.n_calib,
        "alpha_bh": self.alpha_bh,
        "tie_seed": self.tie_seed,
    }

def set_params(self, **params: Any) -> Self:
    """Update constructor parameters and clear derived random state."""
    current = self.get_params()
    unknown = params.keys() - current.keys()
    if unknown:
        raise ValueError(
            f"Invalid DerandomizedSplits parameters: {sorted(unknown)}"
        )
    updated = type(self)(**(current | params))
    self.__dict__.update(updated.__dict__)
    return self

@ensure_numpy_array
def fit_calibrate(
    self,
    x: pd.DataFrame | np.ndarray,
    detector: AnomalyDetector,
    seed: int | None = None,
    weighted: bool = False,
) -> tuple[list[AnomalyDetector], np.ndarray]:
    """Fit independent replicas and return aligned calibration-score rows.

    Returns:
        Models and scores shaped (n_repetitions, n_calibration). Row i
        contains only held-out scores produced by model i.
    """
    self._effective_tie_seed = None
    x = np.asarray(x)
    if x.ndim != 2 or x.shape[1] == 0:
        raise ValueError(
            "x must be a two-dimensional reference batch with features."
        )
    if weighted:
        raise ValueError(
            "DerandomizedSplits does not support weighted calibration."
        )
    validate_optional_seed("seed", seed)
    validate_positive_integer("n_repetitions", self.n_repetitions)
    split = Split(n_calib=self.n_calib)
    split._validate_n_calib(len(x))
    n_calibration = (
        math.ceil(len(x) * self.n_calib)
        if isinstance(self.n_calib, float)
        else self.n_calib
    )
    split_stream, tie_stream = np.random.SeedSequence(seed).spawn(2)
    models = []
    calibration_rows = []
    for stream in split_stream.spawn(self.n_repetitions):
        split_seed = int(stream.generate_state(1)[0])
        replica = set_params(deepcopy(detector), split_seed)
        fitted, scores = split.fit_calibrate(x, replica, seed=split_seed)
        row = as_1d_numeric("calibration scores", scores)
        if row.size != n_calibration:
            raise ValueError(
                "Each model must return one score per calibration row."
            )
        validate_finite("calibration scores", row)
        models.append(fitted[0])
        calibration_rows.append(row.copy())
    calibration_scores = np.vstack(calibration_rows)
    self._effective_tie_seed = (
        int(tie_stream.generate_state(1, dtype=np.uint64)[0])
        if self.tie_seed is None
        else self.tie_seed
    )
    return models, calibration_scores