Source code for dice_ml.explainer_interfaces.dice_pytorch

"""
Module to generate diverse counterfactual explanations based on PyTorch framework
"""
from dice_ml.explainer_interfaces.explainer_base import ExplainerBase
import torch

import numpy as np
import random
import timeit
import copy

from dice_ml import diverse_counterfactuals as exp

[docs]class DicePyTorch(ExplainerBase): def __init__(self, data_interface, model_interface): """Init method :param data_interface: an interface class to access data related params. :param model_interface: an interface class to access trained ML model. """ # initiating data related parameters super().__init__(data_interface) self.minx, self.maxx, self.encoded_categorical_feature_indexes, self.encoded_continuous_feature_indexes, self.cont_minx, self.cont_maxx, self.cont_precisions = self.data_interface.get_data_params_for_gradient_dice() # initializing model related variables self.model = model_interface self.model.load_model() # loading trained model ev = self.model.set_eval_mode() # set the model in evaluation mode if self.model.transformer.func is not None: # TODO: this error is probably too big - need to change it. raise ValueError("Gradient-based DiCE currently (1) accepts the data only in raw categorical and continuous formats, (2) does one-hot-encoding and min-max-normalization internally, (3) expects the ML model the accept the data in this same format. If your problem supports this, please initialize model class again with no custom transformation function.") self.num_output_nodes = self.model.get_num_output_nodes(len(self.data_interface.ohe_encoded_feature_names)).shape[1] # number of output nodes of ML model # variables required to generate CFs - see generate_counterfactuals() for more info self.cfs = [] self.features_to_vary = [] self.cf_init_weights = [] # total_CFs, algorithm, features_to_vary self.loss_weights = [] # yloss_type, diversity_loss_type, feature_weights self.feature_weights_input = '' self.hyperparameters = [1, 1, 1] # proximity_weight, diversity_weight, categorical_penalty self.optimizer_weights = [] # optimizer, learning_rate
[docs] def generate_counterfactuals(self, query_instance, total_CFs, desired_class="opposite", proximity_weight=0.5, diversity_weight=1.0, categorical_penalty=0.1, algorithm="DiverseCF", features_to_vary="all", permitted_range=None, yloss_type="hinge_loss", diversity_loss_type="dpp_style:inverse_dist", feature_weights="inverse_mad", optimizer="pytorch:adam", learning_rate=0.05, min_iter=500, max_iter=5000, project_iter=0, loss_diff_thres=1e-5, loss_converge_maxiter=1, verbose=False, init_near_query_instance=True, tie_random=False, stopping_threshold=0.5, posthoc_sparsity_param=0.1, posthoc_sparsity_algorithm="linear"): """Generates diverse counterfactual explanations :param query_instance: Test point of interest. A dictionary of feature names and values or a single row dataframe :param total_CFs: Total number of counterfactuals required. :param desired_class: Desired counterfactual class - can take 0 or 1. Default value is "opposite" to the outcome class of query_instance for binary classification. :param proximity_weight: A positive float. Larger this weight, more close the counterfactuals are to the query_instance. :param diversity_weight: A positive float. Larger this weight, more diverse the counterfactuals are. :param categorical_penalty: A positive float. A weight to ensure that all levels of a categorical variable sums to 1. :param algorithm: Counterfactual generation algorithm. Either "DiverseCF" or "RandomInitCF". :param features_to_vary: Either a string "all" or a list of feature names to vary. param permitted_range: Dictionary with continuous feature names as keys and permitted min-max range in list as values. Defaults to the range inferred from training data. If None, uses the parameters initialized in data_interface. :param yloss_type: Metric for y-loss of the optimization function. Takes "l2_loss" or "log_loss" or "hinge_loss". :param diversity_loss_type: Metric for diversity loss of the optimization function. Takes "avg_dist" or "dpp_style:inverse_dist". :param feature_weights: Either "inverse_mad" or a dictionary with feature names as keys and corresponding weights as values. Default option is "inverse_mad" where the weight for a continuous feature is the inverse of the Median Absolute Devidation (MAD) of the feature's values in the training set; the weight for a categorical feature is equal to 1 by default. :param optimizer: PyTorch optimization algorithm. Currently tested only with "pytorch:adam". :param learning_rate: Learning rate for optimizer. :param min_iter: Min iterations to run gradient descent for. :param max_iter: Max iterations to run gradient descent for. :param project_iter: Project the gradients at an interval of these many iterations. :param loss_diff_thres: Minimum difference between successive loss values to check convergence. :param loss_converge_maxiter: Maximum number of iterations for loss_diff_thres to hold to declare convergence. Defaults to 1, but we assigned a more conservative value of 2 in the paper. :param verbose: Print intermediate loss value. :param init_near_query_instance: Boolean to indicate if counterfactuals are to be initialized near query_instance. :param tie_random: Used in rounding off CFs and intermediate projection. :param stopping_threshold: Minimum threshold for counterfactuals target class probability. :param posthoc_sparsity_param: Parameter for the post-hoc operation on continuous features to enhance sparsity. :param posthoc_sparsity_algorithm: Perform either linear or binary search. Takes "linear" or "binary". Prefer binary search when a feature range is large (for instance, income varying from 10k to 1000k) and only if the features share a monotonic relationship with predicted outcome in the model. :return: A CounterfactualExamples object to store and visualize the resulting counterfactual explanations (see diverse_counterfactuals.py). """ # check feature MAD validity and throw warnings if feature_weights == "inverse_mad": self.data_interface.get_valid_mads(display_warnings=True, return_mads=False) # check permitted range for continuous features if permitted_range is not None: # if not self.data_interface.check_features_range(permitted_range): # raise ValueError( # "permitted range of features should be within their original range") # else: self.data_interface.permitted_range = permitted_range self.minx, self.maxx = self.data_interface.get_minx_maxx(normalized=True) self.cont_minx = [] self.cont_maxx = [] for feature in self.data_interface.continuous_feature_names: self.cont_minx.append(self.data_interface.permitted_range[feature][0]) self.cont_maxx.append(self.data_interface.permitted_range[feature][1]) if([total_CFs, algorithm, features_to_vary] != self.cf_init_weights): self.do_cf_initializations(total_CFs, algorithm, features_to_vary) if([yloss_type, diversity_loss_type, feature_weights] != self.loss_weights): self.do_loss_initializations(yloss_type, diversity_loss_type, feature_weights) if([proximity_weight, diversity_weight, categorical_penalty] != self.hyperparameters): self.update_hyperparameters(proximity_weight, diversity_weight, categorical_penalty) final_cfs_df, test_instance_df, final_cfs_df_sparse = self.find_counterfactuals(query_instance, desired_class, optimizer, learning_rate, min_iter, max_iter, project_iter, loss_diff_thres, loss_converge_maxiter, verbose, init_near_query_instance, tie_random, stopping_threshold, posthoc_sparsity_param, posthoc_sparsity_algorithm) return exp.CounterfactualExamples(data_interface=self.data_interface, final_cfs_df=final_cfs_df, test_instance_df=test_instance_df, final_cfs_df_sparse = final_cfs_df_sparse, posthoc_sparsity_param=posthoc_sparsity_param, desired_class=desired_class)
[docs] def get_model_output(self, input_instance): """get output probability of ML model""" return self.model.get_output(input_instance)[(self.num_output_nodes-1):]
[docs] def predict_fn(self, input_instance): """prediction function""" if not torch.is_tensor(input_instance): input_instance = torch.tensor(input_instance).float() return self.get_model_output(input_instance).data.numpy()
[docs] def predict_fn_for_sparsity(self, input_instance): """prediction function for sparsity correction""" input_instance = self.data_interface.get_ohe_min_max_normalized_data(input_instance).iloc[0].values return self.predict_fn(torch.tensor(input_instance).float())
[docs] def do_cf_initializations(self, total_CFs, algorithm, features_to_vary): """Intializes CFs and other related variables.""" self.cf_init_weights = [total_CFs, algorithm, features_to_vary] if algorithm == "RandomInitCF": # no. of times to run the experiment with random inits for diversity self.total_random_inits = total_CFs self.total_CFs = 1 # size of counterfactual set else: self.total_random_inits = 0 self.total_CFs = total_CFs # size of counterfactual set # freeze those columns that need to be fixed if features_to_vary != self.features_to_vary: self.features_to_vary = features_to_vary self.feat_to_vary_idxs = self.data_interface.get_indexes_of_features_to_vary(features_to_vary=features_to_vary) # CF initialization if len(self.cfs) != self.total_CFs: self.cfs = [] for ix in range(self.total_CFs): one_init = [] for jx in range(self.minx.shape[1]): one_init.append(np.random.uniform(self.minx[0][jx], self.maxx[0][jx])) self.cfs.append(torch.tensor(one_init).float()) self.cfs[ix].requires_grad = True
[docs] def do_loss_initializations(self, yloss_type, diversity_loss_type, feature_weights): """Intializes variables related to main loss function""" self.loss_weights = [yloss_type, diversity_loss_type, feature_weights] # define the loss parts self.yloss_type = yloss_type self.diversity_loss_type = diversity_loss_type # define feature weights if feature_weights != self.feature_weights_input: self.feature_weights_input = feature_weights if feature_weights == "inverse_mad": normalized_mads = self.data_interface.get_valid_mads(normalized=True) feature_weights = {} for feature in normalized_mads: feature_weights[feature] = round(1/normalized_mads[feature], 2) feature_weights_list = [] for feature in self.data_interface.ohe_encoded_feature_names: if feature in feature_weights: feature_weights_list.append(feature_weights[feature]) else: feature_weights_list.append(1.0) self.feature_weights_list = torch.tensor(feature_weights_list) # define different parts of loss function self.yloss_opt = torch.nn.BCEWithLogitsLoss()
[docs] def update_hyperparameters(self, proximity_weight, diversity_weight, categorical_penalty): """Update hyperparameters of the loss function""" self.hyperparameters = [proximity_weight, diversity_weight, categorical_penalty] self.proximity_weight = proximity_weight self.diversity_weight = diversity_weight self.categorical_penalty = categorical_penalty
[docs] def do_optimizer_initializations(self, optimizer, learning_rate): """Initializes gradient-based PyTorch optimizers.""" opt_library = optimizer.split(':')[0] opt_method = optimizer.split(':')[1] # optimizater initialization if opt_method == "adam": self.optimizer = torch.optim.Adam(self.cfs, lr=learning_rate) elif opt_method == "rmsprop": self.optimizer = torch.optim.RMSprop(self.cfs, lr=learning_rate)
[docs] def compute_yloss(self): """Computes the first part (y-loss) of the loss function.""" yloss = 0.0 for i in range(self.total_CFs): if self.yloss_type == "l2_loss": temp_loss = torch.pow((self.get_model_output(self.cfs[i]) - self.target_cf_class), 2)[0] elif self.yloss_type == "log_loss": temp_logits = torch.log((abs(self.get_model_output(self.cfs[i]) - 0.000001))/(1 - abs(self.get_model_output(self.cfs[i]) - 0.000001))) criterion = torch.nn.BCEWithLogitsLoss() temp_loss = criterion(temp_logits, torch.tensor([self.target_cf_class])) elif self.yloss_type == "hinge_loss": temp_logits = torch.log((abs(self.get_model_output(self.cfs[i]) - 0.000001))/(1 - abs(self.get_model_output(self.cfs[i]) - 0.000001))) criterion = torch.nn.ReLU() all_ones = torch.ones_like(self.target_cf_class) labels = 2 * self.target_cf_class - all_ones temp_loss = all_ones - torch.mul(labels, temp_logits) temp_loss = torch.norm(criterion(temp_loss)) yloss += temp_loss return yloss/self.total_CFs
[docs] def compute_dist(self, x_hat, x1): """Compute weighted distance between two vectors.""" return torch.sum(torch.mul((torch.abs(x_hat - x1)), self.feature_weights_list), dim=0)
[docs] def compute_proximity_loss(self): """Compute the second part (distance from x1) of the loss function.""" proximity_loss = 0.0 for i in range(self.total_CFs): proximity_loss += self.compute_dist(self.cfs[i], self.x1) return proximity_loss/(torch.mul(len(self.minx[0]), self.total_CFs))
[docs] def dpp_style(self, submethod): """Computes the DPP of a matrix.""" det_entries = torch.ones((self.total_CFs, self.total_CFs)) if submethod == "inverse_dist": for i in range(self.total_CFs): for j in range(self.total_CFs): det_entries[(i,j)] = 1.0/(1.0 + self.compute_dist(self.cfs[i], self.cfs[j])) if i == j: det_entries[(i,j)] += 0.0001 elif submethod == "exponential_dist": for i in range(self.total_CFs): for j in range(self.total_CFs): det_entries[(i,j)] = 1.0/(torch.exp(self.compute_dist(self.cfs[i], self.cfs[j]))) if i == j: det_entries[(i,j)] += 0.0001 diversity_loss = torch.det(det_entries) return diversity_loss
[docs] def compute_diversity_loss(self): """Computes the third part (diversity) of the loss function.""" if self.total_CFs == 1: return torch.tensor(0.0) if "dpp" in self.diversity_loss_type: submethod = self.diversity_loss_type.split(':')[1] return self.dpp_style(submethod) elif self.diversity_loss_type == "avg_dist": diversity_loss = 0.0 count = 0.0 # computing pairwise distance and transforming it to normalized similarity for i in range(self.total_CFs): for j in range(i+1, self.total_CFs): count += 1.0 diversity_loss += 1.0/(1.0 + self.compute_dist(self.cfs[i], self.cfs[j])) return 1.0 - (diversity_loss/count)
[docs] def compute_regularization_loss(self): """Adds a linear equality constraints to the loss functions - to ensure all levels of a categorical variable sums to one""" regularization_loss = 0.0 for i in range(self.total_CFs): for v in self.encoded_categorical_feature_indexes: regularization_loss += torch.pow((torch.sum(self.cfs[i][v[0]:v[-1]+1]) - 1.0), 2) return regularization_loss
[docs] def compute_loss(self): """Computes the overall loss""" self.yloss = self.compute_yloss() self.proximity_loss = self.compute_proximity_loss() if self.proximity_weight > 0 else 0.0 self.diversity_loss = self.compute_diversity_loss() if self.diversity_weight > 0 else 0.0 self.regularization_loss = self.compute_regularization_loss() self.loss = self.yloss + (self.proximity_weight * self.proximity_loss) - (self.diversity_weight * self.diversity_loss) + (self.categorical_penalty * self.regularization_loss) return self.loss
[docs] def initialize_CFs(self, query_instance, init_near_query_instance=False): """Initialize counterfactuals.""" for n in range(self.total_CFs): for i in range(len(self.minx[0])): if i in self.feat_to_vary_idxs: if init_near_query_instance: self.cfs[n].data[i] = query_instance[i]+(n*0.01) else: self.cfs[n].data[i] = np.random.uniform(self.minx[0][i], self.maxx[0][i]) else: self.cfs[n].data[i] = query_instance[i]
[docs] def round_off_cfs(self, assign=False): """function for intermediate projection of CFs.""" temp_cfs = [] for index, tcf in enumerate(self.cfs): cf = tcf.detach().clone().numpy() for i, v in enumerate(self.encoded_continuous_feature_indexes): org_cont = (cf[v]*(self.cont_maxx[i] - self.cont_minx[i])) + self.cont_minx[i] # continuous feature in orginal scale org_cont = round(org_cont, self.cont_precisions[i]) # rounding off normalized_cont = (org_cont - self.cont_minx[i])/(self.cont_maxx[i] - self.cont_minx[i]) cf[v] = normalized_cont # assign the projected continuous value for v in self.encoded_categorical_feature_indexes: maxs = np.argwhere( cf[v[0]:v[-1]+1] == np.amax(cf[v[0]:v[-1]+1])).flatten().tolist() if(len(maxs) > 1): if self.tie_random: ix = random.choice(maxs) else: ix = maxs[0] else: ix = maxs[0] for vi in range(len(v)): if vi == ix: cf[v[vi]] = 1.0 else: cf[v[vi]] = 0.0 temp_cfs.append(cf) if assign: for jx in range(len(cf)): self.cfs[index].data[jx] = torch.tensor(temp_cfs[index])[jx] if assign: return None else: return temp_cfs
[docs] def stop_loop(self, itr, loss_diff): """Determines the stopping condition for gradient descent.""" # intermediate projections if((self.project_iter > 0) & (itr > 0)): if((itr % self.project_iter) == 0): self.round_off_cfs(assign=True) # do GD for min iterations if itr < self.min_iter: return False # stop GD if max iter is reached if itr >= self.max_iter: return True # else stop when loss diff is small & all CFs are valid (less or greater than a stopping threshold) if loss_diff <= self.loss_diff_thres: self.loss_converge_iter += 1 if self.loss_converge_iter < self.loss_converge_maxiter: return False else: temp_cfs = self.round_off_cfs(assign=False) test_preds = [self.predict_fn(cf)[0] for cf in temp_cfs] if self.target_cf_class == 0 and all(i <= self.stopping_threshold for i in test_preds): self.converged = True return True elif self.target_cf_class == 1 and all(i >= self.stopping_threshold for i in test_preds): self.converged = True return True else: return False else: self.loss_converge_iter = 0 return False
[docs] def find_counterfactuals(self, query_instance, desired_class, optimizer, learning_rate, min_iter, max_iter, project_iter, loss_diff_thres, loss_converge_maxiter, verbose, init_near_query_instance, tie_random, stopping_threshold, posthoc_sparsity_param, posthoc_sparsity_algorithm): """Finds counterfactuals by graident-descent.""" # Prepares user defined query_instance for DiCE. # query_instance = self.data_interface.prepare_query_instance(query_instance=query_instance, encoding='one-hot') # query_instance = query_instance.iloc[0].values query_instance = self.data_interface.get_ohe_min_max_normalized_data(query_instance).iloc[0].values self.x1 = torch.tensor(query_instance) # find the predicted value of query_instance test_pred = self.predict_fn(torch.tensor(query_instance).float())[0] if desired_class == "opposite": desired_class = 1.0 - np.round(test_pred) self.target_cf_class = torch.tensor(desired_class).float() self.min_iter = min_iter self.max_iter = max_iter self.project_iter = project_iter self.loss_diff_thres = loss_diff_thres # no. of iterations to wait to confirm that loss has converged self.loss_converge_maxiter = loss_converge_maxiter self.loss_converge_iter = 0 self.converged = False self.stopping_threshold = stopping_threshold if self.target_cf_class == 0 and self.stopping_threshold > 0.5: self.stopping_threshold = 0.25 elif self.target_cf_class == 1 and self.stopping_threshold < 0.5: self.stopping_threshold = 0.75 # to resolve tie - if multiple levels of an one-hot-encoded categorical variable take value 1 self.tie_random = tie_random # running optimization steps start_time = timeit.default_timer() self.final_cfs = [] # looping the find CFs depending on whether its random initialization or not loop_find_CFs = self.total_random_inits if self.total_random_inits > 0 else 1 # variables to backup best known CFs so far in the optimization process - if the CFs dont converge in max_iter iterations, then best_backup_cfs is returned. self.best_backup_cfs = [0]*max(self.total_CFs, loop_find_CFs) self.best_backup_cfs_preds = [0]*max(self.total_CFs, loop_find_CFs) self.min_dist_from_threshold = [100]*loop_find_CFs # for backup CFs for loop_ix in range(loop_find_CFs): # CF init if self.total_random_inits > 0: self.initialize_CFs(query_instance, False) else: self.initialize_CFs(query_instance, init_near_query_instance) # initialize optimizer self.do_optimizer_initializations(optimizer, learning_rate) iterations = 0 loss_diff = 1.0 prev_loss = 0.0 while self.stop_loop(iterations, loss_diff) is False: # zero all existing gradients self.optimizer.zero_grad() self.model.model.zero_grad() # get loss and backpropogate loss_value = self.compute_loss() self.loss.backward() # freeze features other than feat_to_vary_idxs for ix in range(self.total_CFs): for jx in range(len(self.minx[0])): if jx not in self.feat_to_vary_idxs: self.cfs[ix].grad[jx] = 0.0 # update the variables self.optimizer.step() # projection step for ix in range(self.total_CFs): for jx in range(len(self.minx[0])): self.cfs[ix].data[jx] = torch.clamp(self.cfs[ix][jx], min=self.minx[0][jx], max=self.maxx[0][jx]) if verbose: if (iterations) % 50 == 0: print('step %d, loss=%g' % (iterations+1, loss_value)) loss_diff = abs(loss_value-prev_loss) prev_loss = loss_value iterations += 1 # backing up CFs if they are valid temp_cfs_stored = self.round_off_cfs(assign=False) test_preds_stored = [self.predict_fn(cf) for cf in temp_cfs_stored] if((self.target_cf_class == 0 and all(i <= self.stopping_threshold for i in test_preds_stored)) or (self.target_cf_class == 1 and all(i >= self.stopping_threshold for i in test_preds_stored))): avg_preds_dist = np.mean([abs(pred[0]-self.stopping_threshold) for pred in test_preds_stored]) if avg_preds_dist < self.min_dist_from_threshold[loop_ix]: self.min_dist_from_threshold[loop_ix] = avg_preds_dist for ix in range(self.total_CFs): self.best_backup_cfs[loop_ix+ix] = copy.deepcopy(temp_cfs_stored[ix]) self.best_backup_cfs_preds[loop_ix+ix] = copy.deepcopy(test_preds_stored[ix]) # rounding off final cfs - not necessary when inter_project=True self.round_off_cfs(assign=True) # storing final CFs for j in range(0, self.total_CFs): temp = self.cfs[j].detach().clone().numpy() self.final_cfs.append(temp) # max iterations at which GD stopped self.max_iterations_run = iterations self.elapsed = timeit.default_timer() - start_time self.cfs_preds = [self.predict_fn(cfs) for cfs in self.final_cfs] # update final_cfs from backed up CFs if valid CFs are not found if((self.target_cf_class == 0 and any(i[0] > self.stopping_threshold for i in self.cfs_preds)) or (self.target_cf_class == 1 and any(i[0] < self.stopping_threshold for i in self.cfs_preds))): for loop_ix in range(loop_find_CFs): if self.min_dist_from_threshold[loop_ix] != 100: for ix in range(self.total_CFs): self.final_cfs[loop_ix+ix] = copy.deepcopy(self.best_backup_cfs[loop_ix+ix]) self.cfs_preds[loop_ix+ix] = copy.deepcopy(self.best_backup_cfs_preds[loop_ix+ix]) # convert to the format that is consistent with dice_tensorflow query_instance = np.array([query_instance], dtype=np.float32) for tix in range(max(loop_find_CFs, self.total_CFs)): self.final_cfs[tix] = np.array([self.final_cfs[tix]], dtype=np.float32) self.cfs_preds[tix] = np.array([self.cfs_preds[tix]], dtype=np.float32) # if self.final_cfs_sparse is not None: # self.final_cfs_sparse[tix] = np.array([self.final_cfs_sparse[tix]], dtype=np.float32) # self.cfs_preds_sparse[tix] = np.array([self.cfs_preds_sparse[tix]], dtype=np.float32) # if isinstance(self.best_backup_cfs[0], np.ndarray): # checking if CFs are backed self.best_backup_cfs[tix] = np.array([self.best_backup_cfs[tix]], dtype=np.float32) self.best_backup_cfs_preds[tix] = np.array([self.best_backup_cfs_preds[tix]], dtype=np.float32) # do inverse transform of CFs to original user-fed format cfs = np.array([self.final_cfs[i][0] for i in range(len(self.final_cfs))]) final_cfs_df = self.data_interface.get_inverse_ohe_min_max_normalized_data(cfs) cfs_preds = [np.round(preds.flatten().tolist(), 3) for preds in self.cfs_preds] cfs_preds = [item for sublist in cfs_preds for item in sublist] final_cfs_df[self.data_interface.outcome_name] = np.array(cfs_preds) test_instance_df = self.data_interface.get_inverse_ohe_min_max_normalized_data(query_instance) test_instance_df[self.data_interface.outcome_name] = np.array(np.round(test_pred, 3)) # post-hoc operation on continuous features to enhance sparsity - only for public data if posthoc_sparsity_param != None and posthoc_sparsity_param > 0 and 'data_df' in self.data_interface.__dict__: final_cfs_df_sparse = final_cfs_df.copy() final_cfs_df_sparse = self.do_posthoc_sparsity_enhancement(final_cfs_df_sparse, test_instance_df, posthoc_sparsity_param, posthoc_sparsity_algorithm) else: final_cfs_df_sparse = None m, s = divmod(self.elapsed, 60) if((self.target_cf_class == 0 and all(i <= self.stopping_threshold for i in self.cfs_preds)) or (self.target_cf_class == 1 and all(i >= self.stopping_threshold for i in self.cfs_preds))): self.total_CFs_found = max(loop_find_CFs, self.total_CFs) valid_ix = [ix for ix in range(max(loop_find_CFs, self.total_CFs))] # indexes of valid CFs print('Diverse Counterfactuals found! total time taken: %02d' % m, 'min %02d' % s, 'sec') else: self.total_CFs_found = 0 valid_ix = [] # indexes of valid CFs for cf_ix, pred in enumerate(self.cfs_preds): if((self.target_cf_class == 0 and pred[0][0] < self.stopping_threshold) or (self.target_cf_class == 1 and pred[0][0] > self.stopping_threshold)): self.total_CFs_found += 1 valid_ix.append(cf_ix) if self.total_CFs_found == 0 : print('No Counterfactuals found for the given configuation, perhaps try with different values of proximity (or diversity) weights or learning rate...', '; total time taken: %02d' % m, 'min %02d' % s, 'sec') else: print('Only %d (required %d) Diverse Counterfactuals found for the given configuation, perhaps try with different values of proximity (or diversity) weights or learning rate...' % (self.total_CFs_found, max(loop_find_CFs, self.total_CFs)), '; total time taken: %02d' % m, 'min %02d' % s, 'sec') if final_cfs_df_sparse is not None: final_cfs_df_sparse = final_cfs_df_sparse.iloc[valid_ix].reset_index(drop=True) return final_cfs_df.iloc[valid_ix].reset_index(drop=True), test_instance_df, final_cfs_df_sparse # returning only valid CFs