Source code for MED3pa.detectron.comparaison

"""
Compares between two ``DetectronExperiment``.
"""
import json
import os
from typing import Any, Dict, List, Tuple, Type, Union

import numpy as np
from sklearn.model_selection import train_test_split

[docs] class DetectronComparison: """ Class to compare the results of two Med3paExperiment instances. """ def __init__(self, results1_path: str, results2_path: str) -> None: self.results1_path = os.path.abspath(results1_path) self.results2_path = os.path.abspath(results2_path) self.detectron_results_comparaison = {} self.config_file = {} self.model_evaluation_comparaison = {} self.rejection_counts_comparaison = {} self._check_experiment_name()
[docs] def is_comparable(self) -> bool: """ Determines whether the two experiments can be compared based on the given criteria. Two experiments can be compared if either: - The datasets are different, but the base model and detectron_params are the same. - The base model is different, but the datasets and detectron_params are the same. Returns: bool: True if the experiments can be compared, False otherwise. """ # Load the config files if they haven't been compared yet if not self.config_file: self.compare_config() datasets_different = self.config_file['datasets']['different'] datasets_different_sets = self.config_file['datasets']['different_datasets'] base_model_different = self.config_file['base_model']['different'] detectron_params_different = self.config_file['detectron_params']['different'] # Check the conditions for comparability can_compare = False # First condition: params are the same, base model is the same, only the testing_set is different if not detectron_params_different and not base_model_different and datasets_different_sets == ['testing_set']: can_compare = True # Second condition: base model is different, params are the same, datasets are the same or only differ in training and validation sets elif base_model_different and not detectron_params_different and (not datasets_different or set(datasets_different_sets) <= {'training_set', 'validation_set'}): can_compare = True return can_compare
def _check_experiment_name(self) -> None: """ Checks if the experiment_name in the config_file of both results paths is the same. If not, raises a ValueError. Also sets the flag for Detectron comparison if applicable. """ config_file_1 = os.path.join(self.results1_path, 'experiment_config.json') config_file_2 = os.path.join(self.results2_path, 'experiment_config.json') with open(config_file_1, 'r') as f1, open(config_file_2, 'r') as f2: config1 = json.load(f1) config2 = json.load(f2) if config1['experiment_name'] != "DetectronExperiment": raise ValueError("Only DetectronExperiment can be compared using this class") if config1['experiment_name'] != config2['experiment_name']: raise ValueError("The two results are not from the same experiment.")
[docs] def compare_detectron_results(self): """ Compares detectron results two sets of results and stores them in a dictionary. """ combined = {} file_1 = os.path.join(self.results1_path, 'detectron_results.json') file_2 = os.path.join(self.results2_path, 'detectron_results.json') with open(file_1, 'r') as f1, open(file_2, 'r') as f2: detectron1 = json.load(f1) detectron2 = json.load(f2) combined['detectron_results1'] = detectron1 combined['detectron_results2'] = detectron2 comparison_results = {} for r1 in detectron1: for r2 in detectron2: if r1["Strategy"] == r2["Strategy"]: strategy = r1["Strategy"] strategy_dict = {} if "p_value" in r1 and "p_value" in r2: strategy_dict["detectron_results1"] = r1["p_value"] strategy_dict["detectron_results2"] = r2["p_value"] strategy_dict["comparison_criteria"] = "p_value" if r1["p_value"] == r2["p_value"]: strategy_dict["best"] = None else: strategy_dict["best"] = "detectron_results2" if r2["p_value"] > r1["p_value"] else "detectron_results1" if "shift_probability" in r1 and "shift_probability" in r2: strategy_dict["detectron_results1"] = r1["shift_probability"] strategy_dict["detectron_results2"] = r2["shift_probability"] strategy_dict["comparison_criteria"] = "shift_probability" if r1["shift_probability"] == r2["shift_probability"]: strategy_dict["best"] = None else: strategy_dict["best"] = "detectron_results2" if r2["shift_probability"] < r1["shift_probability"] else "detectron_results1" if strategy not in comparison_results: comparison_results[strategy] = strategy_dict self.detectron_results_comparaison = comparison_results
[docs] def compare_evaluation(self): """ Compares model evaluations two sets of results and stores them in a dictionary. """ file_1 = os.path.join(self.results1_path, 'model_evaluation.json') file_2 = os.path.join(self.results2_path, 'model_evaluation.json') with open(file_1, 'r') as f1, open(file_2, 'r') as f2: detectron1 = json.load(f1) detectron2 = json.load(f2) comparison = {} metrics = detectron1["test"].keys() # Assuming both experiments have the same metrics for metric in metrics: value1 = detectron1["test"][metric] value2 = detectron2["test"][metric] if value1 == value2: best = None else: best = "detectron_results1" if value1 > value2 else "detectron_results2" comparison[metric] = { "detectron_results1": value1, "detectron_results2": value2, "best": best } self.model_evaluation_comparaison = comparison
[docs] def compare_counts(self): """ Compares rejection two sets of results and stores them in a dictionary. """ combined = {} file_1 = os.path.join(self.results1_path, 'rejection_counts.json') file_2 = os.path.join(self.results2_path, 'rejection_counts.json') with open(file_1, 'r') as f1, open(file_2, 'r') as f2: detectron1 = json.load(f1) detectron2 = json.load(f2) combined['rejection_counts1'] = detectron1 combined['rejection_counts2'] = detectron2 self.rejection_counts_comparaison = combined
[docs] def compare_config(self): """ Compares the config files of the two experiments. """ combined = {} config_file_1 = os.path.join(self.results1_path, 'experiment_config.json') config_file_2 = os.path.join(self.results2_path, 'experiment_config.json') with open(config_file_1, 'r') as f1, open(config_file_2, 'r') as f2: config1 = json.load(f1) config2 = json.load(f2) combined['datasets'] = {} dataset_keys = ['training_set', 'validation_set', 'reference_set', 'testing_set'] different_datasets = [] if config1["datasets"] == config2["datasets"]: combined['datasets']['different'] = False else: combined['datasets']['different'] = True for key in dataset_keys: if config1["datasets"].get(key) != config2["datasets"].get(key): different_datasets.append(key) combined['datasets']['different_datasets'] = different_datasets combined['datasets']['datasets1'] = config1["datasets"] combined['datasets']['datasets2'] = config2["datasets"] combined['base_model'] = {} if config1["base_model"] == config2["base_model"]: combined['base_model']['different'] = False else: combined['base_model']['different'] = True combined['base_model']['base_model1'] = config1["base_model"] combined['base_model']['base_model2'] = config2["base_model"] combined['detectron_params'] = {} if config1["detectron_params"] == config2["detectron_params"]: combined['detectron_params']['different'] = False else: combined['detectron_params']['different'] = True combined['detectron_params']['detectron_params1'] = config1["detectron_params"] combined['detectron_params']['detectron_params2'] = config2["detectron_params"] self.config_file = combined
[docs] def compare_experiments(self): """ Compares the experiments by detectron_results. """ if not self.is_comparable(): raise ValueError("The two experiments cannot be compared based on the provided criteria.") self.compare_detectron_results() self.compare_counts() self.compare_evaluation() self.compare_config()
[docs] def save(self, directory_path: str) -> None: """ Saves the comparison results to a specified directory. Args: directory_path (str): The directory where the comparison results will be saved. """ if not self.is_comparable(): raise ValueError("The two experiments cannot be compared based on the provided criteria.") # Ensure the main directory exists os.makedirs(directory_path, exist_ok=True) global_comparaison_path = os.path.join(directory_path, 'detectron_results_comparaison.json') with open(global_comparaison_path, 'w') as f: json.dump(self.detectron_results_comparaison, f, indent=4) config_path = os.path.join(directory_path, 'experiment_config_comparaison.json') with open(config_path, 'w') as f: json.dump(self.config_file, f, indent=4) eval_path = os.path.join(directory_path, 'model_evaluation_comparaison.json') with open(eval_path, 'w') as f: json.dump(self.model_evaluation_comparaison, f, indent=4) counts_path = os.path.join(directory_path, 'rejection_counts_comparaison.json') with open(counts_path, 'w') as f: json.dump(self.rejection_counts_comparaison, f, indent=4)