"""
Orchestrates the execution of the med3pa method and integrates the functionality of other modules to run comprehensive experiments.
It includes classes to manage and store results ``Med3paResults``, execute experiments like ``Med3paExperiment`` and ``Med3paDetectronExperiment``, and integrate results from the Detectron method ``Med3paDetectronResults``
"""
import json
import os
from typing import Any, Dict, List, Tuple, Type, Union
import numpy as np
from sklearn.model_selection import train_test_split
from MED3pa.datasets import DatasetsManager, MaskedDataset
from MED3pa.detectron.experiment import DetectronExperiment, DetectronResult, DetectronStrategy, EnhancedDisagreementStrategy
from MED3pa.med3pa.mdr import MDRCalculator
from MED3pa.med3pa.models import *
from MED3pa.med3pa.profiles import Profile, ProfilesManager
from MED3pa.med3pa.uncertainty import *
from MED3pa.models.base import BaseModelManager
from MED3pa.models.classification_metrics import *
from MED3pa.models.concrete_regressors import *
from MED3pa.med3pa.tree import TreeRepresentation
[docs]
def to_serializable(obj: Any, additional_arg: Any = None) -> Any:
"""
Convert an object to a JSON-serializable format.
Args:
obj (Any): The object to convert.
Returns:
Any: The JSON-serializable representation of the object.
"""
if isinstance(obj, np.ndarray):
return obj.tolist()
if isinstance(obj, (np.integer, np.floating)):
return obj.item()
if isinstance(obj, Profile):
if additional_arg is not None:
return obj.to_dict(additional_arg)
else:
return obj.to_dict()
if isinstance(obj, dict):
return {k: to_serializable(v) for k, v in obj.items()}
if isinstance(obj, list):
return [to_serializable(v) for v in obj]
return obj
[docs]
class Med3paRecord:
"""
Class to store and manage results from the MED3PA method on one set.
"""
def __init__(self) -> None:
self.metrics_by_dr: Dict[int, Dict] = {}
self.models_evaluation: Dict[str, Dict] = {}
self.profiles_manager: ProfilesManager = None
self.datasets: Dict[int, MaskedDataset] = {}
self.experiment_config = {}
self.tree = None
self.ipc_scores = None
self.apc_scores = None
self.mpc_scores = None
[docs]
def set_metrics_by_dr(self, metrics_by_dr: Dict) -> None:
"""
Set the calculated metrics by declaration rate.
Args:
metrics_by_dr (Dict): Dictionary of metrics by declaration rate.
"""
self.metrics_by_dr = metrics_by_dr
[docs]
def set_profiles_manager(self, profile_manager : ProfilesManager) -> None:
"""
Set the profile manager for this Med3paResults instance.
Args:
profile_manager (ProfilesManager): The ProfileManager instance.
"""
self.profiles_manager = profile_manager
[docs]
def set_models_evaluation(self, ipc_evaluation: Dict, apc_evaluation: Dict=None) -> None:
"""
Set models evaluation metrics.
Args:
ipc_evaluation (Dict): Evaluation metrics for IPC model.
apc_evaluation (Dict): Evaluation metrics for APC model.
"""
self.models_evaluation['IPC_evaluation'] = ipc_evaluation
if apc_evaluation is not None:
self.models_evaluation['APC_evaluation'] = apc_evaluation
[docs]
def set_tree(self, tree:TreeRepresentation):
"""
Sets the constructed tree
"""
self.tree = tree
[docs]
def set_dataset(self, mode: str, dataset: MaskedDataset) -> None:
"""
Saves the dataset for a given sample ratio.
Args:
samples_ratio (int): The sample ratio.
dataset (MaskedDataset): The MaskedDataset instance.
"""
self.datasets[mode] = dataset
[docs]
def save(self, file_path: str) -> None:
"""
Saves the experiment results.
Args:
file_path (str): The file path to save the JSON files.
"""
# Ensure the main directory exists
os.makedirs(file_path, exist_ok=True)
metrics_file_path = os.path.join(file_path, 'metrics_dr.json')
with open(metrics_file_path, 'w') as file:
json.dump(self.metrics_by_dr, file, default=to_serializable, indent=4)
if self.profiles_manager is not None:
profiles_file_path = os.path.join(file_path, 'profiles.json')
with open(profiles_file_path, 'w') as file:
json.dump(self.profiles_manager.get_profiles(), file, default=to_serializable, indent=4)
lost_profiles_file_path = os.path.join(file_path, 'lost_profiles.json')
with open(lost_profiles_file_path, 'w') as file:
json.dump(self.profiles_manager.get_lost_profiles(), file, default=lambda x: to_serializable(x, additional_arg=False), indent=4)
if self.models_evaluation is not None:
models_evaluation_file_path = os.path.join(file_path, 'models_evaluation.json')
with open(models_evaluation_file_path, 'w') as file:
json.dump(self.models_evaluation, file, default=to_serializable, indent=4)
for mode, dataset in self.datasets.items():
dataset_path = os.path.join(file_path, f'dataset_{mode}.csv')
dataset.save_to_csv(dataset_path)
if self.tree is not None:
tree_path = os.path.join(file_path, 'tree.json')
self.tree.save_tree(tree_path)
[docs]
def get_profiles_manager(self) -> ProfilesManager:
"""
Retrieves the profiles manager for this Med3paResults instance
"""
return self.profiles_manager
[docs]
def set_confidence_scores(self, scores:np.ndarray, mode = 'str') -> None:
if mode == 'ipc':
self.ipc_scores = scores
elif mode == "apc":
self.apc_scores = scores
elif mode=="mpc":
self.mpc_scores = scores
[docs]
def get_confidence_scores(self, mode = 'str') -> np.ndarray:
if mode == 'ipc':
return self.ipc_scores
elif mode == "apc":
return self.apc_scores
elif mode=="mpc":
return self.mpc_scores
[docs]
class Med3paResults:
"""
Class to store and manage results from the MED3PA complete experiment.
"""
def __init__(self, reference_record:Med3paRecord, test_record:Med3paRecord) -> None:
self.reference_record = reference_record
self.test_record = test_record
self.experiment_config = {}
self.detectron_results = None
[docs]
def set_detectron_results(self, detectron_results: DetectronResult=None) -> None:
"""
Sets the detectron results for the Med3paDetectron experiment.
Args:
detectron_results (DetectronResult): The structure holding the detectron results.
"""
self.detectron_results = detectron_results
[docs]
def set_experiment_config(self, config: Dict[str, Any]) -> None:
"""
Sets or updates the configuration for the Med3pa experiment.
Args:
config (Dict[str, Any]): A dictionary of experiment configuration.
"""
self.experiment_config.update(config)
[docs]
def set_models(self, ipc_model: IPCModel, apc_model:APCModel = None):
"""
Sets the models.
"""
self.ipc_model = ipc_model
self.apc_model = apc_model
[docs]
def save(self, file_path: str) -> None:
"""
Saves the experiment results.
Args:
file_path (str): The file path to save the JSON files.
"""
# Ensure the main directory exists
os.makedirs(file_path, exist_ok=True)
reference_path = os.path.join(file_path, 'reference')
test_path = os.path.join(file_path, 'test')
detectron_path = os.path.join(file_path, 'detectron')
self.reference_record.save(file_path=reference_path)
self.test_record.save(file_path=test_path)
if self.detectron_results is not None:
self.detectron_results.save(file_path=detectron_path, save_config=False)
experiment_config_path = os.path.join(file_path, 'experiment_config.json')
with open(experiment_config_path, 'w') as file:
json.dump(self.experiment_config, file, default=to_serializable, indent=4)
import os
[docs]
def save_models(self, file_path: str, mode: str = 'all', id: str = None) -> None:
"""
Saves the experiment ipc and apc models as .pkl files, alongside the tree structure for the test set.
Args:
file_path (str): The file path to save the pickled files.
mode (str): Defines the type of models to save, either ipc, apc, or both.
id (str): Optional identifier to append to the filenames.
"""
# Ensure the main directory exists
os.makedirs(file_path, exist_ok=True)
# Function to generate the file name with optional id
def generate_file_name(base_name, id):
return f"{id}_{base_name}" if id else base_name
if mode == 'all':
if self.ipc_model:
ipc_model_name = generate_file_name('ipc_model.pkl', id)
ipc_path = os.path.join(file_path, ipc_model_name)
self.ipc_model.save_model(ipc_path)
if self.apc_model:
apc_model_name = generate_file_name('apc_model.pkl', id)
apc_path = os.path.join(file_path, apc_model_name)
self.apc_model.save_model(apc_path)
if self.test_record.tree:
tree_structure_name = generate_file_name('tree.json', id)
tree_structure_path = os.path.join(file_path, tree_structure_name)
self.test_record.tree.save_tree(tree_structure_path)
elif mode == 'ipc':
if self.ipc_model:
ipc_model_name = generate_file_name('ipc_model.pkl', id)
ipc_path = os.path.join(file_path, ipc_model_name)
self.ipc_model.save_model(ipc_path)
elif mode == 'apc':
if self.apc_model:
apc_model_name = generate_file_name('apc_model.pkl', id)
apc_path = os.path.join(file_path, apc_model_name)
self.apc_model.save_model(apc_path)
if self.test_record.tree:
tree_structure_name = generate_file_name('tree.json', id)
tree_structure_path = os.path.join(file_path, tree_structure_name)
self.test_record.tree.save_tree(tree_structure_path)
[docs]
class Med3paExperiment:
"""
Class to run the MED3PA method experiment.
"""
[docs]
@staticmethod
def run(datasets_manager: DatasetsManager,
base_model_manager: BaseModelManager = None,
uncertainty_metric: str = 'absolute_error',
ipc_type: str = 'RandomForestRegressor',
ipc_params: Dict = None,
ipc_grid_params: Dict = None,
ipc_cv: int = 4,
pretrained_ipc: str = None,
apc_params: Dict = None,
apc_grid_params: Dict = None,
apc_cv: int = 4,
pretrained_apc: str = None,
samples_ratio_min: int = 0,
samples_ratio_max: int = 50,
samples_ratio_step: int = 5,
med3pa_metrics: List[str] = ['Accuracy', 'BalancedAccuracy', 'Precision', 'F1Score', 'Specificity', 'Sensitivity', 'Auc', 'LogLoss', 'Auprc', 'MCC'],
evaluate_models: bool = False,
use_ref_models: bool = False,
mode: str = 'mpc',
models_metrics: List[str] = ['MSE', 'RMSE', 'MAE']) -> Med3paResults:
"""
Runs the MED3PA experiment on both reference and testing sets.
Args:
datasets_manager (DatasetsManager): the datasets manager containing the dataset to use in the experiment.
base_model_manager (BaseModelManager, optional): Instance of BaseModelManager to get the base model, by default None.
uncertainty_metric (str, optional): the uncertainty metric ysed to calculate uncertainty, by default absolute_error.
ipc_type (str, optional): The regressor model to use for IPC, by default RandomForestRegressor.
ipc_params (dict, optional): Parameters for initializing the IPC regressor model, by default None.
ipc_grid_params (dict, optional): Grid search parameters for optimizing the IPC model, by default None.
ipc_cv (int, optional): Number of cross-validation folds for optimizing the IPC model, by default None.
pretrained_ipc (str, optional): path to a pretrained ipc, by default None.
apc_params (dict, optional): Parameters for initializing the APC regressor model, by default None.
apc_grid_params (dict, optional): Grid search parameters for optimizing the APC model, by default None.
apc_cv (int, optional): Number of cross-validation folds for optimizing the APC model, by default None.
pretrained_apc (str, optional): path to a pretrained apc, by default None.
use_ref_models (bool, optional): whether or not to use the trained IPC and APC models from the reference set on the test set.
samples_ratio_min (int, optional): Minimum sample ratio, by default 0.
samples_ratio_max (int, optional): Maximum sample ratio, by default 50.
samples_ratio_step (int, optional): Step size for sample ratio, by default 5.
med3pa_metrics (list of str, optional): List of metrics to calculate, by default ['Auc', 'Accuracy', 'BalancedAccuracy'].
evaluate_models (bool, optional): Whether to evaluate the models, by default False.
models_metrics (list of str, optional): List of metrics for model evaluation, by default ['MSE', 'RMSE'].
Returns:
Med3paResults: the results of the MED3PA experiment on the reference set and testing set.
"""
print("Running MED3pa Experiment on the reference set:")
results_reference, ipc_config, apc_config = Med3paExperiment._run_by_set(datasets_manager=datasets_manager,set= 'reference',base_model_manager= base_model_manager,
uncertainty_metric=uncertainty_metric,
ipc_type=ipc_type, ipc_params=ipc_params, ipc_grid_params=ipc_grid_params, ipc_cv=ipc_cv, pretrained_ipc=pretrained_ipc,
apc_params=apc_params,apc_grid_params=apc_grid_params, apc_cv=apc_cv, pretrained_apc=pretrained_apc,
samples_ratio_min=samples_ratio_min, samples_ratio_max=samples_ratio_max, samples_ratio_step=samples_ratio_step,
med3pa_metrics=med3pa_metrics, evaluate_models=evaluate_models, models_metrics=models_metrics, mode=mode)
print("Running MED3pa Experiment on the test set:")
if use_ref_models:
results_testing, ipc_config, apc_config = Med3paExperiment._run_by_set(datasets_manager=datasets_manager,set= 'testing',base_model_manager= base_model_manager,
uncertainty_metric=uncertainty_metric,
ipc_type=ipc_type, ipc_params=ipc_params, ipc_grid_params=ipc_grid_params, ipc_cv=ipc_cv, pretrained_ipc=pretrained_ipc, ipc_instance=ipc_config,
apc_params=apc_params,apc_grid_params=apc_grid_params, apc_cv=apc_cv, pretrained_apc=pretrained_apc, apc_instance=apc_config,
samples_ratio_min=samples_ratio_min, samples_ratio_max=samples_ratio_max, samples_ratio_step=samples_ratio_step,
med3pa_metrics=med3pa_metrics, evaluate_models=evaluate_models, models_metrics=models_metrics, mode=mode)
else:
results_testing, ipc_config, apc_config = Med3paExperiment._run_by_set(datasets_manager=datasets_manager,set= 'testing',base_model_manager= base_model_manager,
uncertainty_metric=uncertainty_metric,
ipc_type=ipc_type, ipc_params=ipc_params, ipc_grid_params=ipc_grid_params, ipc_cv=ipc_cv, pretrained_ipc=pretrained_ipc, ipc_instance=None,
apc_params=apc_params,apc_grid_params=apc_grid_params, apc_cv=apc_cv, pretrained_apc=pretrained_apc, apc_instance=None,
samples_ratio_min=samples_ratio_min, samples_ratio_max=samples_ratio_max, samples_ratio_step=samples_ratio_step,
med3pa_metrics=med3pa_metrics, evaluate_models=evaluate_models, models_metrics=models_metrics, mode=mode)
results = Med3paResults(results_reference, results_testing)
med3pa_params = {
'uncertainty_metric': uncertainty_metric,
'samples_ratio_min': samples_ratio_min,
'samples_ratio_max': samples_ratio_max,
'samples_ratio_step': samples_ratio_step,
'med3pa_metrics': med3pa_metrics,
'evaluate_models':evaluate_models,
'models_evaluation_metrics': models_metrics,
'mode':mode,
'ipc_model': ipc_config.get_info(),
'apc_model': apc_config.get_info() if apc_config is not None else None,
}
experiment_config = {
'experiment_name': "Med3paExperiment",
'datasets':datasets_manager.get_info(),
'base_model': base_model_manager.get_instance().get_info(),
'med3pa_params': med3pa_params
}
results.set_experiment_config(experiment_config)
results.set_models(ipc_config, apc_config)
return results
@staticmethod
def _run_by_set(datasets_manager: DatasetsManager,
set: str = 'reference',
base_model_manager: BaseModelManager = None,
uncertainty_metric: str = 'absolute_error',
ipc_type: str = 'RandomForestRegressor',
ipc_params: Dict = None,
ipc_grid_params: Dict = None,
ipc_cv: int = 4,
pretrained_ipc:str = None,
ipc_instance: IPCModel = None,
apc_params: Dict = None,
apc_grid_params: Dict = None,
apc_cv: int = 4,
apc_instance: APCModel = None,
pretrained_apc:str = None,
samples_ratio_min: int = 0,
samples_ratio_max: int = 50,
samples_ratio_step: int = 5,
med3pa_metrics: List[str] = ['Accuracy', 'BalancedAccuracy', 'Precision', 'Recall', 'F1Score', 'Specificity', 'Sensitivity', 'Auc', 'LogLoss', 'Auprc', 'NPV', 'PPV', 'MCC'],
evaluate_models: bool = False,
mode: str = 'mpc',
models_metrics: List[str] = ['MSE', 'RMSE', 'MAE']) -> Tuple[Med3paRecord, dict, dict]:
"""
Orchestrates the MED3PA experiment on one specific set of the dataset.
Args:
datasets_manager (DatasetsManager): the datasets manager containing the dataset to use in the experiment.
base_model_manager (BaseModelManager, optional): Instance of BaseModelManager to get the base model, by default None.
uncertainty_metric (str, optional): the uncertainty metric ysed to calculate uncertainty, by default absolute_error.
ipc_type (str, optional): The regressor model to use for IPC, by default RandomForestRegressor.
ipc_params (dict, optional): Parameters for initializing the IPC regressor model, by default None.
ipc_grid_params (dict, optional): Grid search parameters for optimizing the IPC model, by default None.
ipc_cv (int, optional): Number of cross-validation folds for optimizing the IPC model, by default None.
apc_params (dict, optional): Parameters for initializing the APC regressor model, by default None.
apc_grid_params (dict, optional): Grid search parameters for optimizing the APC model, by default None.
apc_cv (int, optional): Number of cross-validation folds for optimizing the APC model, by default None.
samples_ratio_min (int, optional): Minimum sample ratio, by default 0.
samples_ratio_max (int, optional): Maximum sample ratio, by default 50.
samples_ratio_step (int, optional): Step size for sample ratio, by default 5.
med3pa_metrics (list of str, optional): List of metrics to calculate, by default ['Auc', 'Accuracy', 'BalancedAccuracy'].
evaluate_models (bool, optional): Whether to evaluate the models, by default False.
models_metrics (list of str, optional): List of metrics for model evaluation, by default ['MSE', 'RMSE'].
Returns:
Med3paRecord: the results of the MED3PA experiment.
"""
# Step 1 : datasets and base model setting
# Retrieve the dataset based on the set type
if set == 'reference':
dataset = datasets_manager.get_dataset_by_type(dataset_type="reference", return_instance=True)
elif set == 'testing':
dataset = datasets_manager.get_dataset_by_type(dataset_type="testing", return_instance=True)
else:
raise ValueError("The set must be either the reference set or the testing set")
# retrieve different dataset components needed for the experiment
x = dataset.get_observations()
y_true = dataset.get_true_labels()
predicted_probabilities = dataset.get_pseudo_probabilities()
features = datasets_manager.get_column_labels()
# Initialize base model and predict probabilities if not provided
if base_model_manager is None and predicted_probabilities is None:
raise ValueError("Either the base model or the predicted probabilities should be provided!")
if predicted_probabilities is None:
base_model = base_model_manager.get_instance()
predicted_probabilities = base_model.predict(x, True)
dataset.set_pseudo_probs_labels(predicted_probabilities, 0.5)
# Step 2 : Mode and metrics setup
valid_modes = ['mpc', 'apc', 'ipc']
if mode not in valid_modes:
raise ValueError(f"Invalid mode '{mode}'. The mode must be one of {valid_modes}.")
if med3pa_metrics == []:
med3pa_metrics = ClassificationEvaluationMetrics.supported_metrics()
# Step 3 : Calculate uncertainty values
uncertainty_calc = UncertaintyCalculator(uncertainty_metric)
uncertainty_values = uncertainty_calc.calculate_uncertainty(x, predicted_probabilities, y_true)
# Step 4: Set up splits to evaluate the models
if evaluate_models:
_, x_test, _, uncertainty_test = train_test_split(x, uncertainty_values, test_size=0.1, random_state=42)
x_train = x
uncertainty_train = uncertainty_values
results = Med3paRecord()
# Step 5: Create and train IPCModel
if pretrained_ipc is None and ipc_instance is None:
IPC_model = IPCModel(model_name=ipc_type, params=ipc_params, pretrained_model=None)
IPC_model.train(x_train, uncertainty_train)
print("IPC Model training complete.")
# optimize IPC model if grid params were provided
if ipc_grid_params is not None:
IPC_model.optimize(ipc_grid_params, ipc_cv, x_train, uncertainty_train)
print("IPC Model optimization complete.")
elif pretrained_ipc is not None:
IPC_model = IPCModel(model_name=ipc_type, params=ipc_params, pretrained_model=pretrained_ipc)
print("Loaded a pretrained IPC model.")
else:
IPC_model = ipc_instance
print("Used a trained IPC instance.")
# Predict IPC values
IPC_values = IPC_model.predict(x)
print("Individualized confidence scores calculated.")
# Save the calculated confidence scores by the APCmodel
ipc_dataset = dataset.clone()
ipc_dataset.set_confidence_scores(IPC_values)
results.set_dataset(mode="ipc", dataset=ipc_dataset)
results.set_confidence_scores(IPC_values, "ipc")
metrics_by_dr = MDRCalculator.calc_metrics_by_dr(dataset=ipc_dataset, confidence_scores=IPC_values, metrics_list=med3pa_metrics)
results.set_metrics_by_dr(metrics_by_dr)
if mode in ['mpc', 'apc']:
# Step 6: Create and train APCModel
if pretrained_apc is None and apc_instance is None:
APC_model = APCModel(features=features, params=apc_params)
APC_model.train(x, IPC_values)
print("APC Model training complete.")
# optimize APC model if grid params were provided
if apc_grid_params is not None:
APC_model.optimize(apc_grid_params, apc_cv, x_train, uncertainty_train)
print("APC Model optimization complete.")
elif pretrained_apc is not None:
APC_model = APCModel(features=features, params=apc_params, pretrained_model=pretrained_apc)
APC_model.train(x, IPC_values)
print("Loaded a pretrained APC model.")
else:
APC_model = apc_instance
print("Used a trainde IPC instance.")
# Predict APC values
APC_values = APC_model.predict(x)
print("Aggregated confidence scores calculated.")
# Save the tree structure created by the APCModel
tree = APC_model.treeRepresentation
results.set_tree(tree=tree)
# Save the calculated confidence scores by the APCmodel
apc_dataset = dataset.clone()
apc_dataset.set_confidence_scores(APC_values)
results.set_dataset(mode="apc", dataset=apc_dataset)
results.set_confidence_scores(APC_values, "apc")
# Step 7: Create and train MPCModel
if mode == 'mpc':
# Create and predict MPC values
MPC_model = MPCModel(IPC_values=IPC_values, APC_values=APC_values)
MPC_values = MPC_model.predict()
# Save the calculated confidence scores by the MPCmodel
mpc_dataset = dataset.clone()
mpc_dataset.set_confidence_scores(MPC_values)
results.set_dataset(mode="mpc", dataset=mpc_dataset)
results.set_confidence_scores(MPC_values, "mpc")
else:
MPC_model = MPCModel(APC_values=APC_values)
MPC_values = MPC_model.predict()
mpc_dataset = dataset.clone()
mpc_dataset.set_confidence_scores(MPC_values)
print("Mixed confidence scores calculated.")
new_metrics_by_dr = MDRCalculator.update_min_confidence(metrics_by_dr, MPC_values)
results.set_metrics_by_dr(new_metrics_by_dr)
# Step 8: Calculate the profiles for the different samples_ratio and drs
profiles_manager = ProfilesManager(features)
for samples_ratio in range(samples_ratio_min, samples_ratio_max + 1, samples_ratio_step):
# Calculate profiles and their metrics by declaration rate
MDRCalculator.calc_profiles(profiles_manager, tree, mpc_dataset, features, MPC_values, samples_ratio)
MDRCalculator.calc_metrics_by_profiles(profiles_manager, mpc_dataset, features, MPC_values, samples_ratio, med3pa_metrics)
results.set_profiles_manager(profiles_manager)
print("Results extracted for minimum_samples_ratio = ", samples_ratio)
if mode in ['mpc', 'apc']:
ipc_config = IPC_model
apc_config = APC_model
if evaluate_models:
IPC_evaluation = IPC_model.evaluate(x_test, uncertainty_test, models_metrics)
APC_evaluation = APC_model.evaluate(x_test, uncertainty_test, models_metrics)
results.set_models_evaluation(IPC_evaluation, APC_evaluation)
else:
ipc_config = IPC_model
apc_config = None
if evaluate_models:
IPC_evaluation = IPC_model.evaluate(x_test, uncertainty_test, models_metrics)
results.set_models_evaluation(IPC_evaluation, None)
return results, ipc_config, apc_config
[docs]
class Med3paDetectronExperiment:
[docs]
@staticmethod
def run(datasets: DatasetsManager,
base_model_manager: BaseModelManager,
uncertainty_metric: str = 'absolute_error',
training_params: Dict =None,
samples_size: int = 20,
samples_size_profiles: int = 10,
ensemble_size: int = 10,
num_calibration_runs: int = 100,
patience: int = 3,
test_strategies: Union[str, List[str]] = "enhanced_disagreement_strategy",
allow_margin: bool = False,
margin: float = 0.05,
ipc_type: str = 'RandomForestRegressor',
ipc_params: Dict = None,
ipc_grid_params: Dict = None,
ipc_cv: int = None,
pretrained_ipc: str = None,
apc_params: Dict = None,
apc_grid_params: Dict = None,
apc_cv: int = None,
pretrained_apc: str = None,
samples_ratio_min: int = 0,
samples_ratio_max: int = 50,
samples_ratio_step: int = 5,
med3pa_metrics: List[str] = ['Accuracy', 'BalancedAccuracy', 'Precision', 'F1Score', 'Specificity', 'Sensitivity', 'Auc', 'LogLoss', 'Auprc', 'MCC'],
evaluate_models: bool = False,
use_ref_models: bool = False,
models_metrics: List[str] = ['MSE', 'RMSE', 'MAE'],
mode: str = 'mpc',
all_dr: bool = False) -> Med3paResults:
"""
Runs the MED3PA and Detectron experiment.
Args:
datasets (DatasetsManager): The datasets manager instance.
training_params (dict): Parameters for training the models.
base_model_manager (BaseModelManager): The base model manager instance.
uncertainty_metric (str, optional): the uncertainty metric ysed to calculate uncertainty, by default absolute_error.
samples_size (int, optional): Sample size for the Detectron experiment, by default 20.
samples_size_profiles (int, optional): Sample size for Profiles Detectron experiment, by default 10.
ensemble_size (int, optional): Number of models in the ensemble, by default 10.
num_calibration_runs (int, optional): Number of calibration runs, by default 100.
patience (int, optional): Patience for early stopping, by default 3.
test_strategies (Union[str, List[str]): strategies for testing disagreement, by default enhanced_disagreement_strategies.
allow_margin (bool, optional): Whether to allow a margin in the test, by default False.
margin (float, optional): Margin value for the test, by default 0.05.
ipc_type (str, optional): The regressor model to use for IPC, by default RandomForestRegressor.
ipc_params (dict, optional): Parameters for initializing the IPC regressor model, by default None.
ipc_grid_params (dict, optional): Grid search parameters for optimizing the IPC model, by default None.
ipc_cv (int, optional): Number of cross-validation folds for optimizing the IPC model, by default None.
pretrained_ipc (str, optional): path to a pretrained ipc, by default None.
apc_params (dict, optional): Parameters for initializing the APC regressor model, by default None.
apc_grid_params (dict, optional): Grid search parameters for optimizing the APC model, by default None.
pretrained_apc (str, optional): path to a pretrained apc, by default None.
apc_cv (int, optional): Number of cross-validation folds for optimizing the APC model, by default None.
samples_ratio_min (int, optional): Minimum sample ratio, by default 0.
samples_ratio_max (int, optional): Maximum sample ratio, by default 50.
samples_ratio_step (int, optional): Step size for sample ratio, by default 5.
med3pa_metrics (list of str, optional): List of metrics to calculate, by default ['Auc', 'Accuracy', 'BalancedAccuracy'].
evaluate_models (bool, optional): Whether to evaluate the models, by default False.
models_metrics (list of str, optional): List of metrics for model evaluation, by default ['MSE', 'RMSE'].
all_dr (bool, optional): Whether to run for all declaration rates, by default False.
Returns:
Med3paResults: Results of MED3pa on reference and testing sets, plus Detectron Results.
"""
valid_modes = ['mpc', 'apc']
if mode not in valid_modes:
raise ValueError(f"Invalid mode '{mode}'. The mode must be one of {valid_modes}.")
print(datasets.get_dataset_by_type("reference", True))
med3pa_results = Med3paExperiment.run(datasets_manager=datasets,
base_model_manager=base_model_manager, uncertainty_metric=uncertainty_metric,
ipc_params=ipc_params, ipc_grid_params=ipc_grid_params, ipc_cv=ipc_cv, ipc_type=ipc_type, pretrained_ipc=pretrained_ipc,
apc_params=apc_params, apc_grid_params=apc_grid_params, apc_cv=apc_cv, pretrained_apc=pretrained_apc,
evaluate_models=evaluate_models, models_metrics=models_metrics,
samples_ratio_min=samples_ratio_min, samples_ratio_max=samples_ratio_max, samples_ratio_step=samples_ratio_step,
med3pa_metrics=med3pa_metrics, mode=mode, use_ref_models=use_ref_models)
print("Running Global Detectron Experiment:")
detectron_results = DetectronExperiment.run(datasets=datasets, training_params=training_params, base_model_manager=base_model_manager,
samples_size=samples_size, num_calibration_runs=num_calibration_runs, ensemble_size=ensemble_size,
patience=patience, allow_margin=allow_margin, margin=margin)
detectron_results.analyze_results(test_strategies)
if med3pa_results.test_record.get_confidence_scores("mpc") is not None:
confidence_scores = med3pa_results.test_record.get_confidence_scores("mpc")
elif med3pa_results.test_record.get_confidence_scores("apc") is not None:
confidence_scores = med3pa_results.test_record.get_confidence_scores("apc")
else:
raise ValueError("the confidence scores were not calculated!")
print("Running Profiled Detectron Experiment:")
detectron_profiles_res = MDRCalculator.detectron_by_profiles(datasets=datasets, profiles_manager=med3pa_results.test_record.get_profiles_manager(),training_params=training_params,
base_model_manager=base_model_manager, confidence_scores = confidence_scores,
samples_size=samples_size_profiles, num_calibration_runs=num_calibration_runs, ensemble_size=ensemble_size,
patience=patience, strategies=test_strategies,
allow_margin=allow_margin, margin=margin, all_dr=all_dr)
med3pa_params = {
'uncertainty_metric': uncertainty_metric,
'samples_ratio_min': samples_ratio_min,
'samples_ratio_max': samples_ratio_max,
'samples_ratio_step': samples_ratio_step,
'med3pa_metrics': med3pa_metrics,
'evaluate_models':evaluate_models,
'models_evaluation_metrics': models_metrics,
'mode':mode,
}
detectron_params = {
'samples_size': samples_size,
'profiles_samples_size': samples_size_profiles,
'cdcs_ensemble_size': ensemble_size,
'num_runs': num_calibration_runs,
'patience': patience,
'allow_margin': allow_margin,
'margin': margin,
'additional_training_params': training_params,
}
experiment_config = {
'experiment_name': "Med3paDetectronExperiment",
'med3pa_detectron_params': {},
}
experiment_config['med3pa_detectron_params']['detectron_params'] = detectron_params
experiment_config['med3pa_detectron_params']['med3pa_params'] = med3pa_params
med3pa_results.set_detectron_results(detectron_results)
med3pa_results.set_experiment_config(experiment_config)
return med3pa_results