From 933f2b7eca9d57e5d0b34205040c36d977fe18bb Mon Sep 17 00:00:00 2001 From: beckynevin Date: Tue, 14 May 2024 15:31:08 -0600 Subject: [PATCH 1/3] coeff and loss_type are now args for analysis --- src/analyze/analyze.py | 12 +- src/models/models.py | 6 +- src/scripts/Aleatoric.py | 54 +++-- src/scripts/AleatoricandEpistemic.py | 243 +++++++++------------ src/scripts/Epistemic.py | 305 +++++++++++++++++++++++++++ src/train/train.py | 6 + src/utils/defaults.py | 13 +- 7 files changed, 471 insertions(+), 168 deletions(-) create mode 100644 src/scripts/Epistemic.py diff --git a/src/analyze/analyze.py b/src/analyze/analyze.py index e75c393..7507515 100644 --- a/src/analyze/analyze.py +++ b/src/analyze/analyze.py @@ -13,11 +13,12 @@ def load_checkpoint( model_name, noise, epoch, - beta, device, - loss="SDER", - nmodel=None, path="models/", + BETA=0.5, + nmodel=None, + COEFF=0.5, + loss="SDER", ): """ Load PyTorch model checkpoint from a .pt file. @@ -32,13 +33,14 @@ def load_checkpoint( if model_name[0:3] == "DER": file_name = ( str(path) - + f"{model_name}_noise_{noise}_loss_{loss}_epoch_{epoch}.pt" + + f"{model_name}_noise_{noise}_loss_{loss}" + + f"_COEFF_{COEFF}_epoch_{epoch}.pt" ) checkpoint = torch.load(file_name, map_location=device) elif model_name[0:2] == "DE": file_name = ( str(path) - + f"{model_name}_noise_{noise}_beta_{beta}_" + + f"{model_name}_noise_{noise}_beta_{BETA}_" f"nmodel_{nmodel}_epoch_{epoch}.pt" ) checkpoint = torch.load(file_name, map_location=device) diff --git a/src/models/models.py b/src/models/models.py index 8140562..f0ccf80 100644 --- a/src/models/models.py +++ b/src/models/models.py @@ -210,7 +210,7 @@ def loss_der(y, y_pred, coeff): u_ep, ) - +# simplified def loss_sder(y, y_pred, coeff): gamma, nu, alpha, beta = y[:, 0], y[:, 1], y[:, 2], y[:, 3] error = gamma - y_pred @@ -218,8 +218,8 @@ def loss_sder(y, y_pred, coeff): # define aleatoric and epistemic uncert u_al = np.sqrt( - beta.detach().numpy() - * (1 + nu.detach().numpy()) + (beta.detach().numpy() + * (1 + nu.detach().numpy())) / (alpha.detach().numpy() * nu.detach().numpy()) ) u_ep = 1 / np.sqrt(nu.detach().numpy()) diff --git a/src/scripts/Aleatoric.py b/src/scripts/Aleatoric.py index e64d735..8adaaec 100644 --- a/src/scripts/Aleatoric.py +++ b/src/scripts/Aleatoric.py @@ -43,6 +43,20 @@ def parse_args(): there are string options: linear_decrease, \ step_decrease_to_0.5, and step_decrease_to_1.0", ) + parser.add_argument( + "--COEFF", + type=float, + required=False, + default=DefaultsAnalysis["model"]["COEFF"], + help="COEFF for DER", + ) + parser.add_argument( + "--loss_type", + type=str, + required=False, + default=DefaultsAnalysis["model"]["loss_type"], + help="loss_type for DER, either SDER or DER", + ) parser.add_argument( "--noise_level_list", type=list, @@ -108,7 +122,9 @@ def parse_args(): "common": {"dir": args.dir}, "model": {"n_models": args.n_models, "n_epochs": args.n_epochs, - "BETA": args.BETA}, + "BETA": args.BETA, + "COEFF": args.COEFF, + "loss_type": args.loss_type}, "analysis": { "noise_level_list": args.noise_level_list, "model_names_list": args.model_names_list, @@ -149,6 +165,8 @@ def beta_type(value): noise_list = config.get_item("analysis", "noise_level_list", "Analysis") color_list = config.get_item("plots", "color_list", "Analysis") BETA = config.get_item("model", "BETA", "Analysis") + COEFF = config.get_item("model", "COEFF", "Analysis") + loss_type = config.get_item("model", "loss_type", "Analysis") sigma_list = [] for noise in noise_list: sigma_list.append(DataPreparation.get_sigma(noise)) @@ -197,9 +215,10 @@ def beta_type(value): model, noise, epoch, - BETA, DEVICE, path=path_to_chk, + COEFF=COEFF, + loss=loss_type ) # path=path_to_chk) # things to grab: 'valid_mse' and 'valid_bnll' @@ -221,10 +240,10 @@ def beta_type(value): model, noise, epoch, - BETA, DEVICE, - nmodel=nmodels, path=path_to_chk, + BETA=BETA, + nmodel=nmodels, ) mu_vals, sig_vals = chk_module.ep_al_checkpoint_DE(chk) list_mus.append(mu_vals) @@ -255,32 +274,33 @@ def beta_type(value): al - al_std, al + al_std, color=color_list[i], - alpha=0.5, + alpha=0.25, + edgecolor=None ) - ax.scatter( + ax.plot( range(n_epochs), np.sqrt(al_dict[model][noise]), color=color_list[i], - edgecolors="black", - label=r"$\sigma = " + str(sigma_list[i]), + #edgecolors="black", + label=r"$\sigma = $" + str(sigma_list[i]), ) - ax.axhline(y=sigma_list[i], color=color_list[i]) + ax.axhline(y=sigma_list[i], color=color_list[i], ls='--') ax.set_ylabel("Aleatoric Uncertainty") ax.set_xlabel("Epoch") if model[0:3] == "DER": ax.set_title("Deep Evidential Regression") elif model[0:2] == "DE": ax.set_title("Deep Ensemble (100 models)") - ax.set_ylim([-1, 15]) + ax.set_ylim([0, 14]) plt.legend() if config.get_item("analysis", "savefig", "Analysis"): plt.savefig( - str(path_to_out) - + "aleatoric_uncertainty_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) + str(path_to_out) + + "aleatoric_uncertainty_n_epochs_" + + str(n_epochs) + + "_n_models_DE_" + + str(n_models) + + ".png" + ) if config.get_item("analysis", "plot", "Analysis"): plt.show() diff --git a/src/scripts/AleatoricandEpistemic.py b/src/scripts/AleatoricandEpistemic.py index c1adde9..9595be8 100644 --- a/src/scripts/AleatoricandEpistemic.py +++ b/src/scripts/AleatoricandEpistemic.py @@ -4,10 +4,8 @@ import numpy as np import torch import matplotlib.pyplot as plt - -from data import DataModules from utils.config import Config -from utils.defaults import DefaultsAnalysis, DefaultsDE +from utils.defaults import DefaultsAnalysis from data.data import DataPreparation from analyze.analyze import AggregateCheckpoints @@ -24,40 +22,41 @@ def parse_args(): # option to pass name of config parser.add_argument("--config", "-c", default=None) - # data info - parser.add_argument( - "--data_path", - "-d", - default=DefaultsAnalysis["data"]["data_path"], - ) - parser.add_argument( - "--data_engine", - "-dl", - default=DefaultsAnalysis["data"]["data_engine"], - choices=DataModules.keys(), - ) - # model + # we need some info about the model to run this analysis # path to save the model results - parser.add_argument("--out_dir", - default=DefaultsAnalysis["common"]["out_dir"]) - + parser.add_argument("--dir", + default=DefaultsAnalysis["common"]["dir"]) # now args for model parser.add_argument( "--n_models", type=int, - default=DefaultsDE["model"]["n_models"], + default=DefaultsAnalysis["model"]["n_models"], help="Number of MVEs in the ensemble", ) parser.add_argument( "--BETA", type=beta_type, required=False, - default=DefaultsDE["model"]["BETA"], + default=DefaultsAnalysis["model"]["BETA"], help="If loss_type is bnn_loss, specify a beta as a float or \ there are string options: linear_decrease, \ step_decrease_to_0.5, and step_decrease_to_1.0", ) + parser.add_argument( + "--COEFF", + type=float, + required=False, + default=DefaultsAnalysis["model"]["COEFF"], + help="COEFF for DER", + ) + parser.add_argument( + "--loss_type", + type=str, + required=False, + default=DefaultsAnalysis["model"]["loss_type"], + help="loss_type for DER", + ) parser.add_argument( "--noise_level_list", type=list, @@ -76,7 +75,7 @@ def parse_args(): "--n_epochs", type=int, required=False, - default=DefaultsAnalysis["analysis"]["n_epochs"], + default=DefaultsAnalysis["model"]["n_epochs"], help="number of epochs", ) parser.add_argument( @@ -85,6 +84,12 @@ def parse_args(): default=DefaultsAnalysis["analysis"]["plot"], help="option to plot in notebook", ) + parser.add_argument( + "--color_list", + type=list, + default=DefaultsAnalysis["plots"]["color_list"], + help="list of named or hexcode colors to use for the noise levels", + ) parser.add_argument( "--savefig", action="store_true", @@ -113,23 +118,21 @@ def parse_args(): os.makedirs(os.path.dirname(temp_config), exist_ok=True) # check if args were specified in cli - # if not, default is from DefaultsDE dictionary input_yaml = { - "common": {"out_dir": args.out_dir}, - "data": { - "data_path": args.data_path, - "data_engine": args.data_engine, - }, - "model": {"n_models": args.n_models, "BETA": args.BETA}, + "common": {"dir": args.dir}, + "model": {"n_models": args.n_models, + "n_epochs": args.n_epochs, + "BETA": args.BETA, + "COEFF": args.COEFF, + "loss_type": args.loss_type}, "analysis": { "noise_level_list": args.noise_level_list, "model_names_list": args.model_names_list, - "n_epochs": args.n_epochs, "plot": args.plot, "savefig": args.savefig, "verbose": args.verbose, }, - # "plots": {key: {} for key in args.plots}, + "plots": {"color_list": args.color_list}, # "metrics": {key: {} for key in args.metrics}, } @@ -160,16 +163,27 @@ def beta_type(value): config = parse_args() DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") noise_list = config.get_item("analysis", "noise_level_list", "Analysis") + color_list = config.get_item("plots", "color_list", "Analysis") + BETA = config.get_item("model", "BETA", "Analysis") + COEFF = config.get_item("model", "COEFF", "Analysis") + loss_type = config.get_item("model", "loss_type", "Analysis") sigma_list = [] for noise in noise_list: sigma_list.append(DataPreparation.get_sigma(noise)) - print("noise list", noise_list) - print("sigma list", sigma_list) - path_to_chk = config.get_item("common", "out_dir", "Analysis") + root_dir = config.get_item("common", "dir", "Analysis") + path_to_chk = root_dir + "checkpoints/" + path_to_out = root_dir + "analysis/" + # check that this exists and if not make it + if not os.path.isdir(path_to_out): + print('does not exist, making dir', path_to_out) + os.mkdir(path_to_out) + else: + print('already exists', path_to_out) model_name_list = config.get_item("analysis", "model_names_list", "Analysis") - print("model list", model_name_list, len(model_name_list)) + print("model list", model_name_list) + print("noise list", noise_list) chk_module = AggregateCheckpoints() # make an empty nested dictionary with keys for # model names followed by noise levels @@ -177,7 +191,7 @@ def beta_type(value): model_name: {noise: [] for noise in noise_list} for model_name in model_name_list } - al_var_dict = { + al_dict = { model_name: {noise: [] for noise in noise_list} for model_name in model_name_list } @@ -186,12 +200,11 @@ def beta_type(value): model_name: {noise: [] for noise in noise_list} for model_name in model_name_list } - al_var_std_dict = { + al_std_dict = { model_name: {noise: [] for noise in noise_list} for model_name in model_name_list } - - n_epochs = config.get_item("analysis", "n_epochs", "Analysis") + n_epochs = config.get_item("model", "n_epochs", "Analysis") for model in model_name_list: for noise in noise_list: # append a noise key @@ -202,8 +215,10 @@ def beta_type(value): model, noise, epoch, - config.get_item("model", "BETA", "DE"), DEVICE, + path=path_to_chk, + COEFF=COEFF, + loss=loss_type, ) # path=path_to_chk) # things to grab: 'valid_mse' and 'valid_bnll' @@ -211,16 +226,12 @@ def beta_type(value): chk_module.ep_al_checkpoint_DER(chk) ) ep_dict[model][noise].append(epistemic_m) - al_var_dict[model][noise].append(aleatoric_m) + al_dict[model][noise].append(aleatoric_m) ep_std_dict[model][noise].append(e_std) - al_var_std_dict[model][noise].append(a_std) + al_std_dict[model][noise].append(a_std) elif model[0:2] == "DE": - n_models = config.get_item("model", "n_models", "DE") - print("n_models", n_models) - print("n_epochs", n_epochs) - for epoch in range(n_epochs): list_mus = [] list_sigs = [] @@ -229,116 +240,70 @@ def beta_type(value): model, noise, epoch, - config.get_item("model", "BETA", "DE"), DEVICE, + path=path_to_chk, + BETA=BETA, nmodel=nmodels, ) mu_vals, sig_vals = chk_module.ep_al_checkpoint_DE(chk) - list_mus.append(mu_vals.detach().numpy()) - list_sigs.append(sig_vals.detach().numpy() ** 2) + list_mus.append(mu_vals) + list_sigs.append(sig_vals) ep_dict[model][noise].append(np.median(np.std(list_mus, axis=0))) - al_var_dict[model][noise].append( - np.median(np.mean(list_sigs, axis=0)) - ) + al_dict[model][noise].append(np.median(np.mean(list_sigs, + axis=0))) ep_std_dict[model][noise].append(np.std(np.std(list_mus, axis=0))) - al_var_std_dict[model][noise].append( - np.std(np.mean(list_sigs, axis=0)) - ) + al_std_dict[model][noise].append(np.std(np.mean(list_sigs, + axis=0))) # make a two-paneled plot for the different noise levels - # make one plot per model - for model in model_name_list: - plt.clf() - fig = plt.figure(figsize=(10, 4)) - ax1 = fig.add_subplot(121) - color_list = ["#8EA8C3", "#406E8E", "#23395B"] - for i, noise in enumerate(noise_list): - ax1.errorbar( - range(n_epochs), - ep_dict[model][noise], - yerr=ep_std_dict[model][noise], - color=color_list[i], - capsize=5, - ls=None, - ) - ax1.scatter(range(n_epochs), - ep_dict[model][noise], - color=color_list[i]) - ax1.axhline(y=sigma_list[i], color=color_list[i]) - ax1.set_ylabel("Epistemic Uncertainty") - ax1.set_xlabel("Epoch") - ax2 = fig.add_subplot(122) - for i, noise in enumerate(noise_list): - ax2.errorbar( - range(n_epochs), - np.sqrt(al_var_dict[model][noise]), - yerr=np.sqrt(al_var_std_dict[model][noise]), - color=color_list[i], - capsize=5, - ls=None, - ) - ax2.scatter( - range(n_epochs), - np.sqrt(al_var_dict[model][noise]), - color=color_list[i], - label=r"$\sigma = $"+str(sigma_list[i]), - ) - ax2.axhline(y=sigma_list[i], color=color_list[i]) - ax2.set_ylabel("Aleatoric Uncertainty") - ax2.set_xlabel("Epoch") - fig.suptitle(model) - plt.legend() - plt.show() - + # make one panel per model + # for the noise levels: + plt.clf() + fig = plt.figure(figsize=(10, 4)) # try this instead with a fill_between method - for model in model_name_list: - plt.clf() - fig = plt.figure(figsize=(10, 4)) - ax1 = fig.add_subplot(121) - color_list = ["#8EA8C3", "#406E8E", "#23395B"] + for i, model in enumerate(model_name_list): + ax = fig.add_subplot(1, len(model_name_list), i + 1) + # Your plotting code for each model here + ax.set_title(model) # Set title for each subplot for i, noise in enumerate(noise_list): + al = np.array(np.sqrt(al_dict[model][noise])) + al_std = np.array(np.sqrt(al_std_dict[model][noise])) ep = np.array(ep_dict[model][noise]) ep_std = np.array(ep_std_dict[model][noise]) - # list_minus = [ep[i] - ep_std[i] for i in range(len(ep))] - # list_plus = [ep[i] + ep_std[i] for i in range(len(ep))] - ax1.fill_between( - range(n_epochs), - ep - ep_std, - ep + ep_std, - color=color_list[i], - alpha=0.5, - ) - ax1.scatter( - range(n_epochs), - ep_dict[model][noise], - color=color_list[i], - edgecolors="black", - ) - ax1.axhline(y=sigma_list[i], color=color_list[i]) - ax1.set_ylabel("Epistemic Uncertainty") - ax1.set_xlabel("Epoch") - ax2 = fig.add_subplot(122) - for i, noise in enumerate(noise_list): - al = np.array(np.sqrt(al_var_dict[model][noise])) - al_std = np.array(np.sqrt(al_var_std_dict[model][noise])) - ax2.fill_between( + total = np.sqrt(al**2 + ep**2) + total_std = np.sqrt(al_std**2 + ep_std**2) + ax.fill_between( range(n_epochs), - al - al_std, - al + al_std, + total - total_std, + total + total_std, color=color_list[i], - alpha=0.5, + alpha=0.25, + edgecolor=None ) - ax2.scatter( + ax.plot( range(n_epochs), - np.sqrt(al_var_dict[model][noise]), + total, color=color_list[i], - edgecolors="black", - label=r"$\sigma = $"+str(sigma_list[i]), + label=r"$\sigma = $" + str(sigma_list[i]), ) - ax2.axhline(y=sigma_list[i], color=color_list[i]) - ax2.set_ylabel("Aleatoric Uncertainty") - ax2.set_xlabel("Epoch") - fig.suptitle(model) - plt.legend() + ax.axhline(y=sigma_list[i], color=color_list[i]) + ax.set_ylabel("Total Uncertainty") + ax.set_xlabel("Epoch") + if model[0:3] == "DER": + ax.set_title("Deep Evidential Regression") + elif model[0:2] == "DE": + ax.set_title("Deep Ensemble (100 models)") + #ax.set_ylim([-1, 15]) + plt.legend() + if config.get_item("analysis", "savefig", "Analysis"): + plt.savefig( + str(path_to_out) + + "aleatoric_and_epistemic_uncertainty_n_epochs_" + + str(n_epochs) + + "_n_models_DE_" + + str(n_models) + + ".png" + ) + if config.get_item("analysis", "plot", "Analysis"): plt.show() diff --git a/src/scripts/Epistemic.py b/src/scripts/Epistemic.py new file mode 100644 index 0000000..1d0348c --- /dev/null +++ b/src/scripts/Epistemic.py @@ -0,0 +1,305 @@ +import os +import yaml +import argparse +import numpy as np +import torch +import matplotlib.pyplot as plt +from utils.config import Config +from utils.defaults import DefaultsAnalysis +from data.data import DataPreparation +from analyze.analyze import AggregateCheckpoints + +# from plots import Plots + + +def parse_args(): + parser = argparse.ArgumentParser(description="data handling module") + # there are three options with the parser: + # 1) Read from a yaml + # 2) Reads from the command line and default file + # and dumps to yaml + + # option to pass name of config + parser.add_argument("--config", "-c", default=None) + + # model + # we need some info about the model to run this analysis + # path to save the model results + parser.add_argument("--dir", + default=DefaultsAnalysis["common"]["dir"]) + # now args for model + parser.add_argument( + "--n_models", + type=int, + default=DefaultsAnalysis["model"]["n_models"], + help="Number of MVEs in the ensemble", + ) + parser.add_argument( + "--BETA", + type=beta_type, + required=False, + default=DefaultsAnalysis["model"]["BETA"], + help="If loss_type is bnn_loss, specify a beta as a float or \ + there are string options: linear_decrease, \ + step_decrease_to_0.5, and step_decrease_to_1.0", + ) + parser.add_argument( + "--COEFF", + type=float, + required=False, + default=DefaultsAnalysis["model"]["COEFF"], + help="COEFF for DER", + ) + parser.add_argument( + "--loss_type", + type=str, + required=False, + default=DefaultsAnalysis["model"]["loss_type"], + help="loss_type for DER, either DER or SDER", + ) + parser.add_argument( + "--noise_level_list", + type=list, + required=False, + default=DefaultsAnalysis["analysis"]["noise_level_list"], + help="Noise levels to compare", + ) + parser.add_argument( + "--model_names_list", + type=list, + required=False, + default=DefaultsAnalysis["analysis"]["model_names_list"], + help="Beginning of name for saved checkpoints and figures", + ) + parser.add_argument( + "--n_epochs", + type=int, + required=False, + default=DefaultsAnalysis["model"]["n_epochs"], + help="number of epochs", + ) + parser.add_argument( + "--plot", + action="store_true", + default=DefaultsAnalysis["analysis"]["plot"], + help="option to plot in notebook", + ) + parser.add_argument( + "--color_list", + type=list, + default=DefaultsAnalysis["plots"]["color_list"], + help="list of named or hexcode colors to use for the noise levels", + ) + parser.add_argument( + "--savefig", + action="store_true", + default=DefaultsAnalysis["analysis"]["savefig"], + help="option to save a figure of the true and predicted values", + ) + parser.add_argument( + "--verbose", + action="store_true", + default=DefaultsAnalysis["analysis"]["verbose"], + help="verbose option for train", + ) + args = parser.parse_args() + args = parser.parse_args() + if args.config is not None: + print("Reading settings from config file", args.config) + config = Config(args.config) + + else: + temp_config = DefaultsAnalysis["common"]["temp_config"] + print( + "Reading settings from cli and default, \ + dumping to temp config: ", + temp_config, + ) + os.makedirs(os.path.dirname(temp_config), exist_ok=True) + + # check if args were specified in cli + input_yaml = { + "common": {"dir": args.dir}, + "model": {"n_models": args.n_models, + "n_epochs": args.n_epochs, + "BETA": args.BETA, + "COEFF": args.COEFF, + "loss_type": args.loss_type}, + "analysis": { + "noise_level_list": args.noise_level_list, + "model_names_list": args.model_names_list, + "plot": args.plot, + "savefig": args.savefig, + "verbose": args.verbose, + }, + "plots": {"color_list": args.color_list}, + # "metrics": {key: {} for key in args.metrics}, + } + + yaml.dump(input_yaml, open(temp_config, "w")) + config = Config(temp_config) + + return config + # return parser.parse_args() + + +def beta_type(value): + if isinstance(value, float): + return value + elif value.lower() == "linear_decrease": + return value + elif value.lower() == "step_decrease_to_0.5": + return value + elif value.lower() == "step_decrease_to_1.0": + return value + else: + raise argparse.ArgumentTypeError( + "BETA must be a float or one of 'linear_decrease', \ + 'step_decrease_to_0.5', 'step_decrease_to_1.0'" + ) + + +if __name__ == "__main__": + config = parse_args() + DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") + noise_list = config.get_item("analysis", "noise_level_list", "Analysis") + color_list = config.get_item("plots", "color_list", "Analysis") + BETA = config.get_item("model", "BETA", "Analysis") + COEFF = config.get_item("model", "COEFF", "Analysis") + loss_type = config.get_item("model", "loss_type", "Analysis") + sigma_list = [] + for noise in noise_list: + sigma_list.append(DataPreparation.get_sigma(noise)) + root_dir = config.get_item("common", "dir", "Analysis") + path_to_chk = root_dir + "checkpoints/" + path_to_out = root_dir + "analysis/" + # check that this exists and if not make it + if not os.path.isdir(path_to_out): + print('does not exist, making dir', path_to_out) + os.mkdir(path_to_out) + else: + print('already exists', path_to_out) + model_name_list = config.get_item("analysis", + "model_names_list", + "Analysis") + print("model list", model_name_list) + print("noise list", noise_list) + chk_module = AggregateCheckpoints() + # make an empty nested dictionary with keys for + # model names followed by noise levels + ep_dict = { + model_name: {noise: [] for noise in noise_list} + for model_name in model_name_list + } + al_dict = { + model_name: {noise: [] for noise in noise_list} + for model_name in model_name_list + } + + ep_std_dict = { + model_name: {noise: [] for noise in noise_list} + for model_name in model_name_list + } + al_std_dict = { + model_name: {noise: [] for noise in noise_list} + for model_name in model_name_list + } + n_epochs = config.get_item("model", "n_epochs", "Analysis") + for model in model_name_list: + for noise in noise_list: + # append a noise key + # now run the analysis on the resulting checkpoints + if model[0:3] == "DER": + for epoch in range(n_epochs): + chk = chk_module.load_checkpoint( + model, + noise, + epoch, + DEVICE, + path=path_to_chk, + COEFF=COEFF, + loss=loss_type + ) + # things to grab: 'valid_mse' and 'valid_bnll' + epistemic_m, aleatoric_m, e_std, a_std = ( + chk_module.ep_al_checkpoint_DER(chk) + ) + ep_dict[model][noise].append(epistemic_m) + al_dict[model][noise].append(aleatoric_m) + ep_std_dict[model][noise].append(e_std) + al_std_dict[model][noise].append(a_std) + + elif model[0:2] == "DE": + n_models = config.get_item("model", "n_models", "DE") + for epoch in range(n_epochs): + list_mus = [] + list_sigs = [] + for nmodels in range(n_models): + chk = chk_module.load_checkpoint( + model, + noise, + epoch, + DEVICE, + path=path_to_chk, + BETA=BETA, + nmodel=nmodels, + ) + mu_vals, sig_vals = chk_module.ep_al_checkpoint_DE(chk) + list_mus.append(mu_vals) + list_sigs.append(sig_vals) + ep_dict[model][noise].append(np.median(np.std(list_mus, + axis=0))) + al_dict[model][noise].append(np.median(np.mean(list_sigs, + axis=0))) + ep_std_dict[model][noise].append(np.std(np.std(list_mus, + axis=0))) + al_std_dict[model][noise].append(np.std(np.mean(list_sigs, + axis=0))) + # make a two-paneled plot for the different noise levels + # make one panel per model + # for the noise levels: + plt.clf() + fig = plt.figure(figsize=(10, 4)) + # try this instead with a fill_between method + for i, model in enumerate(model_name_list): + ax = fig.add_subplot(1, len(model_name_list), i + 1) + # Your plotting code for each model here + ax.set_title(model) # Set title for each subplot + for i, noise in enumerate(noise_list): + ep = np.array(ep_dict[model][noise]) + ep_std = np.array(ep_std_dict[model][noise]) + ax.fill_between( + range(n_epochs), + ep - ep_std, + ep + ep_std, + color=color_list[i], + alpha=0.25, + edgecolor=None + ) + ax.plot( + range(n_epochs), + ep_dict[model][noise], + color=color_list[i], + #edgecolors="black", + label=r"$\sigma = $" + str(sigma_list[i]), + ) + ax.axhline(y=sigma_list[i], color=color_list[i]) + ax.set_ylabel("Epistemic Uncertainty") + ax.set_xlabel("Epoch") + if model[0:3] == "DER": + ax.set_title("Deep Evidential Regression") + elif model[0:2] == "DE": + ax.set_title("Deep Ensemble (100 models)") + #ax.set_ylim([-1, 15]) + plt.legend() + if config.get_item("analysis", "savefig", "Analysis"): + plt.savefig( + str(path_to_out) + + "epistemic_uncertainty_n_epochs_" + + str(n_epochs) + + "_n_models_DE_" + + str(n_models) + + ".png" + ) + if config.get_item("analysis", "plot", "Analysis"): + plt.show() diff --git a/src/train/train.py b/src/train/train.py index 183c08e..f2b59f2 100644 --- a/src/train/train.py +++ b/src/train/train.py @@ -233,6 +233,8 @@ def train_DER( + str(model_name) + "_loss_" + str(loss_type) + + "_COEFF_" + + str(COEFF) + "_epoch_" + str(epoch) + ".png" @@ -260,6 +262,8 @@ def train_DER( + str(model_name) + "_loss_" + str(loss_type) + + "_COEFF_" + + str(COEFF) + "_epoch_" + str(epoch) + ".pt", @@ -284,6 +288,8 @@ def train_DER( + str(model_name) + "_loss_" + str(loss_type) + + "_COEFF_" + + str(COEFF) + "_epoch_" + str(epoch) + ".pt", diff --git a/src/utils/defaults.py b/src/utils/defaults.py index 46ad93e..1732109 100644 --- a/src/utils/defaults.py +++ b/src/utils/defaults.py @@ -42,7 +42,7 @@ } DefaultsDER = { "common": { - "out_dir": "./DeepUQResources/results/", + "out_dir": "./DeepUQResources/", "temp_config": "./DeepUQResources/temp/temp_config_DER.yml", }, "data": { @@ -93,15 +93,20 @@ "n_models": 100, "n_epochs": 100, "BETA": 0.5, + "COEFF": 0.5, + "loss_type": "SDER" }, "analysis": { "noise_level_list": ["low", "medium", "high"], - "model_names_list": ["DER_desiderata_2", "DE_desiderata_2"], + "model_names_list": ["DER", "DE_desiderata_2"], + #["DER_desiderata_2", "DE_desiderata_2"] "plot": True, - "savefig": True, + "savefig": False, "verbose": False, }, - "plots": {"color_list": ["#8EA8C3", "#406E8E", "#23395B"]}, + "plots": {"color_list": ["#F4D58D", "#339989", "#292F36"]}, + # Pinks ["#EC4067", "#A01A7D", "#311847"]}, + # Blues: ["#8EA8C3", "#406E8E", "#23395B"]}, "metrics_common": { "use_progress_bar": False, "samples_per_inference": 1000, From b25fb70d1376b48884c5e85fed4a040c3d6abe22 Mon Sep 17 00:00:00 2001 From: beckynevin Date: Tue, 14 May 2024 20:39:34 -0600 Subject: [PATCH 2/3] flake8 --- src/models/models.py | 5 +++-- src/scripts/Aleatoric.py | 1 - src/scripts/AleatoricandEpistemic.py | 2 +- src/scripts/Epistemic.py | 3 +-- src/utils/defaults.py | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/models/models.py b/src/models/models.py index f0ccf80..50619c3 100644 --- a/src/models/models.py +++ b/src/models/models.py @@ -210,7 +210,8 @@ def loss_der(y, y_pred, coeff): u_ep, ) -# simplified + +# simplified DER loss (from Meinert) def loss_sder(y, y_pred, coeff): gamma, nu, alpha, beta = y[:, 0], y[:, 1], y[:, 2], y[:, 3] error = gamma - y_pred @@ -219,7 +220,7 @@ def loss_sder(y, y_pred, coeff): # define aleatoric and epistemic uncert u_al = np.sqrt( (beta.detach().numpy() - * (1 + nu.detach().numpy())) + * (1 + nu.detach().numpy())) / (alpha.detach().numpy() * nu.detach().numpy()) ) u_ep = 1 / np.sqrt(nu.detach().numpy()) diff --git a/src/scripts/Aleatoric.py b/src/scripts/Aleatoric.py index 8adaaec..74e0b63 100644 --- a/src/scripts/Aleatoric.py +++ b/src/scripts/Aleatoric.py @@ -281,7 +281,6 @@ def beta_type(value): range(n_epochs), np.sqrt(al_dict[model][noise]), color=color_list[i], - #edgecolors="black", label=r"$\sigma = $" + str(sigma_list[i]), ) ax.axhline(y=sigma_list[i], color=color_list[i], ls='--') diff --git a/src/scripts/AleatoricandEpistemic.py b/src/scripts/AleatoricandEpistemic.py index 9595be8..1a6a3ff 100644 --- a/src/scripts/AleatoricandEpistemic.py +++ b/src/scripts/AleatoricandEpistemic.py @@ -294,7 +294,7 @@ def beta_type(value): ax.set_title("Deep Evidential Regression") elif model[0:2] == "DE": ax.set_title("Deep Ensemble (100 models)") - #ax.set_ylim([-1, 15]) + # ax.set_ylim([-1, 15]) plt.legend() if config.get_item("analysis", "savefig", "Analysis"): plt.savefig( diff --git a/src/scripts/Epistemic.py b/src/scripts/Epistemic.py index 1d0348c..8acf311 100644 --- a/src/scripts/Epistemic.py +++ b/src/scripts/Epistemic.py @@ -280,7 +280,6 @@ def beta_type(value): range(n_epochs), ep_dict[model][noise], color=color_list[i], - #edgecolors="black", label=r"$\sigma = $" + str(sigma_list[i]), ) ax.axhline(y=sigma_list[i], color=color_list[i]) @@ -290,7 +289,7 @@ def beta_type(value): ax.set_title("Deep Evidential Regression") elif model[0:2] == "DE": ax.set_title("Deep Ensemble (100 models)") - #ax.set_ylim([-1, 15]) + # ax.set_ylim([-1, 15]) plt.legend() if config.get_item("analysis", "savefig", "Analysis"): plt.savefig( diff --git a/src/utils/defaults.py b/src/utils/defaults.py index 1732109..b52981c 100644 --- a/src/utils/defaults.py +++ b/src/utils/defaults.py @@ -99,7 +99,7 @@ "analysis": { "noise_level_list": ["low", "medium", "high"], "model_names_list": ["DER", "DE_desiderata_2"], - #["DER_desiderata_2", "DE_desiderata_2"] + # ["DER_desiderata_2", "DE_desiderata_2"] "plot": True, "savefig": False, "verbose": False, From 1f9e3daec4a1941f62d729b08ce1f3247ce546f1 Mon Sep 17 00:00:00 2001 From: beckynevin Date: Tue, 14 May 2024 20:43:20 -0600 Subject: [PATCH 3/3] updating config file for testing --- test/test_Aleatoric.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_Aleatoric.py b/test/test_Aleatoric.py index 3b23da9..716e804 100644 --- a/test/test_Aleatoric.py +++ b/test/test_Aleatoric.py @@ -82,6 +82,8 @@ def create_test_config_aleatoric( "n_models": n_models, "n_epochs": n_epochs, "BETA": 0.5, + "COEFF": 0.01, + "loss_type": "DER", }, "plots": {"color_list": ["#8EA8C3", "#406E8E", "#23395B"]}, } @@ -135,9 +137,9 @@ def create_test_config_DER( "model": { "model_engine": "DER", "model_type": "DER", - "loss_type": "SDER", + "loss_type": "DER", "init_lr": 0.001, - "COEFF": 0.5, + "COEFF": 0.01, "n_epochs": n_epochs, "save_all_checkpoints": True, "save_final_checkpoint": True,