From 4ea1ac2523b2f610b81d30f4eb1daba8e8ec1fa9 Mon Sep 17 00:00:00 2001 From: beckynevin Date: Fri, 24 May 2024 07:50:38 -0600 Subject: [PATCH 1/5] modified to have one plot with three lines for ep, al, tot --- src/scripts/AleatoricandEpistemic.py | 67 ++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/src/scripts/AleatoricandEpistemic.py b/src/scripts/AleatoricandEpistemic.py index eb5aae6..5845822 100644 --- a/src/scripts/AleatoricandEpistemic.py +++ b/src/scripts/AleatoricandEpistemic.py @@ -315,3 +315,70 @@ def beta_type(value): ) if config.get_item("analysis", "plot", "Analysis"): plt.show() + + plt.clf() + fig = plt.figure(figsize=(10, 4)) + # try this instead with a fill_between method + for i, model in enumerate(model_name_list): + ax = fig.add_subplot(1, len(model_name_list), i + 1) + # Your plotting code for each model here + ax.set_title(model) # Set title for each subplot + for i, noise in enumerate(noise_list): + if model[0:3] == "DER": + al = np.array(al_dict[model][noise]) + al_std = np.array(al_std_dict[model][noise]) + elif model[0:2] == "DE": + al = np.array(np.sqrt(al_dict[model][noise])) + al_std = np.array(np.sqrt(al_std_dict[model][noise])) + ep = np.array(ep_dict[model][noise]) + ep_std = np.array(ep_std_dict[model][noise]) + total = np.sqrt(al**2 + ep**2) + total_std = np.sqrt(al_std**2 + ep_std**2) + ax.fill_between( + range(n_epochs), + total - total_std, + total + total_std, + color=color_list[i], + alpha=0.25, + edgecolor=None + ) + ax.plot( + range(n_epochs), + al, + color=color_list[i], + ls='dashed', + label=r"Aleatoric, $\sigma = $" + str(sigma_list[i]), + ) + ax.plot( + range(n_epochs), + ep, + color=color_list[i], + ls='dotted', + label=r"Epistemic, $\sigma = $" + str(sigma_list[i]), + ) + ax.plot( + range(n_epochs), + total, + color=color_list[i], + label=r"Total, $\sigma = $" + str(sigma_list[i]), + ) + ax.axhline(y=sigma_list[i], color=color_list[i], ls='--') + ax.set_ylabel("Total Uncertainty") + ax.set_xlabel("Epoch") + if model[0:3] == "DER": + ax.set_title("Deep Evidential Regression") + elif model[0:2] == "DE": + ax.set_title("Deep Ensemble (100 models)") + ax.set_ylim([0, 15]) + plt.legend() + if config.get_item("analysis", "savefig", "Analysis"): + plt.savefig( + str(path_to_out) + + "aleatoric_and_epistemic_and_total_uncertainty_n_epochs_" + + str(n_epochs) + + "_n_models_DE_" + + str(n_models) + + ".png" + ) + if config.get_item("analysis", "plot", "Analysis"): + plt.show() From 4b71d28806077015c25ab87dc4d4e639f818d0b4 Mon Sep 17 00:00:00 2001 From: beckynevin Date: Fri, 24 May 2024 07:51:09 -0600 Subject: [PATCH 2/5] loss functions re-ran and made into a four panel --- src/scripts/LossFunctions.py | 77 ++++++++++++++---------------------- 1 file changed, 29 insertions(+), 48 deletions(-) diff --git a/src/scripts/LossFunctions.py b/src/scripts/LossFunctions.py index 5d49113..465772e 100644 --- a/src/scripts/LossFunctions.py +++ b/src/scripts/LossFunctions.py @@ -253,10 +253,10 @@ def beta_type(value): # make one panel per model # for the noise levels: plt.clf() - fig = plt.figure(figsize=(10, 4)) + fig = plt.figure(figsize=(12, 10)) # try this instead with a fill_between method for i, model in enumerate(model_name_list): - ax = fig.add_subplot(1, len(model_name_list), i + 1) + ax = fig.add_subplot(2, len(model_name_list), i + 1) # Your plotting code for each model here ax.set_title(model) # Set title for each subplot for i, noise in enumerate(noise_list): @@ -275,18 +275,17 @@ def beta_type(value): label=r"Validation; $\sigma = $" + str(sigma_list[i]), ) else: - for n in range(n_models): - ax.plot( - range(n_epochs), - mse_loss_train[model][noise][n], - color=color_list[i], - ls='--' - ) - ax.plot( - range(n_epochs), - mse_loss[model][noise][n], - color=color_list[i] - ) + ax.plot( + range(n_epochs), + mse_loss_train[model][noise][0], + color=color_list[i], + ls='--' + ) + ax.plot( + range(n_epochs), + mse_loss[model][noise][0], + color=color_list[i] + ) ax.set_ylabel("MSE Loss") ax.set_xlabel("Epoch") if model[0:3] == "DER": @@ -295,25 +294,10 @@ def beta_type(value): elif model[0:2] == "DE": ax.set_title("Deep Ensemble") ax.set_ylim([0, 250]) - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "mse_loss_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() - - plt.clf() - fig = plt.figure(figsize=(10, 4)) - # try this instead with a fill_between method + # now make the other loss plots for i, model in enumerate(model_name_list): - ax = fig.add_subplot(1, len(model_name_list), i + 1) + ax = fig.add_subplot(2, len(model_name_list), i + 3) # Your plotting code for each model here - ax.set_title(model) # Set title for each subplot for i, noise in enumerate(noise_list): if model[0:3] == "DER": ax.plot( @@ -330,31 +314,28 @@ def beta_type(value): label=r"Validation; $\sigma = $" + str(sigma_list[i]), ) else: - for n in range(n_models): - ax.plot( - range(n_epochs), - loss_train[model][noise][n], - color=color_list[i], - ls='--' - ) - ax.plot( - range(n_epochs), - loss[model][noise][n], - color=color_list[i], - ) + ax.plot( + range(n_epochs), + loss_train[model][noise][0], + color=color_list[i], + ls='--' + ) + ax.plot( + range(n_epochs), + loss[model][noise][0], + color=color_list[i], + ) + ax.set_xlabel("Epoch") if model[0:3] == "DER": - ax.set_title("Deep Evidential Regression") ax.set_ylabel("NIG Loss") - plt.legend() elif model[0:2] == "DE": - ax.set_title("Deep Ensemble") ax.set_ylabel(r"$\beta-$NLL Loss") - # ax.set_ylim([0, 11]) + # ax.set_ylim([0, 5]) if config.get_item("analysis", "savefig", "Analysis"): plt.savefig( str(path_to_out) - + "loss_n_epochs_" + + "all_loss_n_epochs_" + str(n_epochs) + "_n_models_DE_" + str(n_models) From f5cd4e52ca96e8e5acca10849dcfd865b7341a09 Mon Sep 17 00:00:00 2001 From: beckynevin Date: Fri, 24 May 2024 07:51:22 -0600 Subject: [PATCH 3/5] extended line to 15 --- src/scripts/ParitySigma.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/scripts/ParitySigma.py b/src/scripts/ParitySigma.py index 502e308..2967e14 100644 --- a/src/scripts/ParitySigma.py +++ b/src/scripts/ParitySigma.py @@ -253,14 +253,14 @@ def beta_type(value): ax = fig.add_subplot(1, len(model_name_list), m + 1) # Your plotting code for each model here for i, noise in enumerate(noise_list): - if model[0:3] == "DE_": + if model[0:3] == "DER": + al = np.array(al_dict[model][noise]) + al_std = np.array(al_std_dict[model][noise]) + elif model[0:2] == "DE": # only take the sqrt for the case of DE, # which is the variance al = np.array(np.sqrt(al_dict[model][noise])) al_std = np.array(np.sqrt(al_std_dict[model][noise])) - else: - al = np.array(al_dict[model][noise]) - al_std = np.array(al_std_dict[model][noise]) # summarize the aleatoric ax.errorbar( sigma_list[i], @@ -277,7 +277,7 @@ def beta_type(value): ) ax.set_ylabel("Aleatoric Uncertainty") ax.set_xlabel("True (Injected) Uncertainty") - ax.plot(range(0, 14), range(0, 14), ls="--", color="black") + ax.plot(range(0, 15), range(0, 15), ls="--", color="black") ax.set_ylim([0, 14]) ax.set_xlim([0, 14]) if model[0:3] == "DER": From ed1872f2ab9b22b31b7d2082a7e69bb85259f027 Mon Sep 17 00:00:00 2001 From: beckynevin Date: Fri, 24 May 2024 07:51:33 -0600 Subject: [PATCH 4/5] changed yellow color --- src/utils/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/defaults.py b/src/utils/defaults.py index a466c15..ed170d3 100644 --- a/src/utils/defaults.py +++ b/src/utils/defaults.py @@ -112,7 +112,7 @@ "verbose": False, }, "plots": {"color_list": - ["#F4D58D", "#339989", "#292F36", "#04A777", "#DF928E"]}, + ["#DFA316", "#339989", "#292F36", "#04A777", "#DF928E"]}, # Pinks ["#EC4067", "#A01A7D", "#311847"]}, # Blues: ["#8EA8C3", "#406E8E", "#23395B"]}, "metrics_common": { From 02caaea8a541fc7bd11fcc89065871685393b303 Mon Sep 17 00:00:00 2001 From: beckynevin Date: Fri, 24 May 2024 07:51:55 -0600 Subject: [PATCH 5/5] putting aleatoric and inits ont he same plot --- src/scripts/Aleatoric_and_inits.py | 401 +++++++++++++++++++++++++++++ 1 file changed, 401 insertions(+) create mode 100644 src/scripts/Aleatoric_and_inits.py diff --git a/src/scripts/Aleatoric_and_inits.py b/src/scripts/Aleatoric_and_inits.py new file mode 100644 index 0000000..0b248e2 --- /dev/null +++ b/src/scripts/Aleatoric_and_inits.py @@ -0,0 +1,401 @@ +import os +import yaml +import argparse +import numpy as np +import torch +import matplotlib.pyplot as plt +from utils.config import Config +from utils.defaults import DefaultsAnalysis +from data.data import DataPreparation +from analyze.analyze import AggregateCheckpoints + +# from plots import Plots + + +def parse_args(): + parser = argparse.ArgumentParser(description="data handling module") + # there are three options with the parser: + # 1) Read from a yaml + # 2) Reads from the command line and default file + # and dumps to yaml + + # option to pass name of config + parser.add_argument("--config", "-c", default=None) + + # model + # we need some info about the model to run this analysis + # path to save the model results + parser.add_argument("--dir", + default=DefaultsAnalysis["common"]["dir"]) + # now args for model + parser.add_argument( + "--n_models", + type=int, + default=DefaultsAnalysis["model"]["n_models"], + help="Number of MVEs in the ensemble", + ) + parser.add_argument( + "--BETA", + type=beta_type, + required=False, + default=DefaultsAnalysis["model"]["BETA"], + help="If loss_type is bnn_loss, specify a beta as a float or \ + there are string options: linear_decrease, \ + step_decrease_to_0.5, and step_decrease_to_1.0", + ) + parser.add_argument( + "--COEFF", + type=float, + required=False, + default=DefaultsAnalysis["model"]["COEFF"], + help="COEFF for DER", + ) + parser.add_argument( + "--loss_type", + type=str, + required=False, + default=DefaultsAnalysis["model"]["loss_type"], + help="loss_type for DER, either SDER or DER", + ) + parser.add_argument( + "--noise_level_list", + type=list, + required=False, + default=DefaultsAnalysis["analysis"]["noise_level_list"], + help="Noise levels to compare", + ) + parser.add_argument( + "--model_names_list", + type=list, + required=False, + default=DefaultsAnalysis["analysis"]["model_names_list"], + help="Beginning of name for saved checkpoints and figures", + ) + parser.add_argument( + "--n_epochs", + type=int, + required=False, + default=DefaultsAnalysis["model"]["n_epochs"], + help="number of epochs", + ) + parser.add_argument( + "--plot", + action="store_true", + default=DefaultsAnalysis["analysis"]["plot"], + help="option to plot in notebook", + ) + parser.add_argument( + "--color_list", + type=list, + default=DefaultsAnalysis["plots"]["color_list"], + help="list of named or hexcode colors to use for the noise levels", + ) + parser.add_argument( + "--savefig", + action="store_true", + default=DefaultsAnalysis["analysis"]["savefig"], + help="option to save a figure of the true and predicted values", + ) + parser.add_argument( + "--verbose", + action="store_true", + default=DefaultsAnalysis["analysis"]["verbose"], + help="verbose option for train", + ) + args = parser.parse_args() + args = parser.parse_args() + if args.config is not None: + print("Reading settings from config file", args.config) + config = Config(args.config) + + else: + temp_config = DefaultsAnalysis["common"]["temp_config"] + print( + "Reading settings from cli and default, \ + dumping to temp config: ", + temp_config, + ) + os.makedirs(os.path.dirname(temp_config), exist_ok=True) + + # check if args were specified in cli + input_yaml = { + "common": {"dir": args.dir}, + "model": {"n_models": args.n_models, + "n_epochs": args.n_epochs, + "BETA": args.BETA, + "COEFF": args.COEFF, + "loss_type": args.loss_type}, + "analysis": { + "noise_level_list": args.noise_level_list, + "model_names_list": args.model_names_list, + "plot": args.plot, + "savefig": args.savefig, + "verbose": args.verbose, + }, + "plots": {"color_list": args.color_list}, + # "metrics": {key: {} for key in args.metrics}, + } + + yaml.dump(input_yaml, open(temp_config, "w")) + config = Config(temp_config) + + return config + # return parser.parse_args() + + +def beta_type(value): + if isinstance(value, float): + return value + elif value.lower() == "linear_decrease": + return value + elif value.lower() == "step_decrease_to_0.5": + return value + elif value.lower() == "step_decrease_to_1.0": + return value + else: + raise argparse.ArgumentTypeError( + "BETA must be a float or one of 'linear_decrease', \ + 'step_decrease_to_0.5', 'step_decrease_to_1.0'" + ) + + +if __name__ == "__main__": + config = parse_args() + DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") + noise_list = config.get_item("analysis", "noise_level_list", "Analysis") + color_list = config.get_item("plots", "color_list", "Analysis") + BETA = config.get_item("model", "BETA", "Analysis") + COEFF = config.get_item("model", "COEFF", "Analysis") + loss_type = config.get_item("model", "loss_type", "Analysis") + sigma_list = [] + for noise in noise_list: + sigma_list.append(DataPreparation.get_sigma(noise)) + root_dir = config.get_item("common", "dir", "Analysis") + path_to_chk = root_dir + "checkpoints/" + path_to_out = root_dir + "analysis/" + # check that this exists and if not make it + if not os.path.isdir(path_to_out): + print('does not exist, making dir', path_to_out) + os.mkdir(path_to_out) + else: + print('already exists', path_to_out) + model_name_list = config.get_item("analysis", + "model_names_list", + "Analysis") + print("model list", model_name_list) + print("noise list", noise_list) + chk_module = AggregateCheckpoints() + # make an empty nested dictionary with keys for + # model names followed by noise levels + al_dict = { + model_name: {noise: [] for noise in noise_list} + for model_name in model_name_list + } + al_std_dict = { + model_name: {noise: [] for noise in noise_list} + for model_name in model_name_list + } + n_epochs = config.get_item("model", "n_epochs", "Analysis") + for model in model_name_list: + for noise in noise_list: + # append a noise key + # now run the analysis on the resulting checkpoints + if model[0:3] == "DER": + for epoch in range(n_epochs): + chk = chk_module.load_checkpoint( + model, + noise, + epoch, + DEVICE, + path=path_to_chk, + COEFF=COEFF, + loss=loss_type, + load_nh_chk=False, + ) + # path=path_to_chk) + # things to grab: 'valid_mse' and 'valid_bnll' + epistemic_m, aleatoric_m, e_std, a_std = ( + chk_module.ep_al_checkpoint_DER(chk) + ) + al_dict[model][noise].append(aleatoric_m) + al_std_dict[model][noise].append(a_std) + + elif model[0:2] == "DE": + n_models = config.get_item("model", "n_models", "DE") + for epoch in range(n_epochs): + list_mus = [] + list_vars = [] + for nmodels in range(n_models): + chk = chk_module.load_checkpoint( + model, + noise, + epoch, + DEVICE, + path=path_to_chk, + BETA=BETA, + nmodel=nmodels, + ) + mu_vals, var_vals = chk_module.ep_al_checkpoint_DE(chk) + list_mus.append(mu_vals) + list_vars.append(var_vals) + # first taking the mean across the validation data + # then looking at the mean and standard deviation + # across all of the nmodels + al_dict[model][noise].append( + np.mean(np.mean(list_vars, axis=0)) + ) + al_std_dict[model][noise].append( + np.std(np.mean(list_vars, axis=0)) + ) + # make a two-paneled plot for the different noise levels + # make one panel per model + # for the noise levels: + # this needs to be redone + rs_list = [10, 11, 12, 13, 14] + # make an empty nested dictionary with keys for + # model names followed by noise levels + al_rs_dict = { + model_name: {noise: {rs: [] for rs in rs_list} for noise in noise_list} + for model_name in model_name_list + } + ''' + al_rs_std_dict = { + model_name: {noise: {rs: [] for rs in rs_list} for noise in noise_list} + for model_name in model_name_list + } + ''' + n_epochs = config.get_item("model", "n_epochs", "Analysis") + for model in model_name_list: + for noise in noise_list: + for rs in rs_list: + + # append a noise key + # now run the analysis on the resulting checkpoints + if model[0:3] == "DER": + for epoch in range(n_epochs): + chk = chk_module.load_checkpoint( + model, + noise, + epoch, + DEVICE, + path=path_to_chk, + COEFF=COEFF, + loss=loss_type, + load_rs_chk=True, + rs=rs, + load_nh_chk=False, + ) + # path=path_to_chk) + # things to grab: 'valid_mse' and 'valid_bnll' + _, aleatoric_m, _, a_std = \ + chk_module.ep_al_checkpoint_DER(chk) + al_rs_dict[model][noise][rs].append(aleatoric_m) + # al_std_dict[model][noise][rs].append(a_std) + if model[0:2] == "DE" and model[0:3] != "DER": + n_models = config.get_item("model", "n_models", "DE") + for epoch in range(n_epochs): + list_mus = [] + list_vars = [] + for nmodels in range(n_models): + chk = chk_module.load_checkpoint( + model, + noise, + epoch, + DEVICE, + path=path_to_chk, + BETA=BETA, + nmodel=nmodels, + ) + mu_vals, var_vals = chk_module.ep_al_checkpoint_DE(chk) + list_mus.append(mu_vals) + list_vars.append(var_vals) + try: + al_rs_dict[model][noise][nmodels + 1].append( + np.mean(list_vars) + ) + except KeyError: + continue + # make a two-paneled plot for the different noise levels + # make one panel per model + # for the noise levels: + plt.clf() + fig = plt.figure(figsize=(10, 4)) + # try this instead with a fill_between method + for i, model in enumerate(model_name_list): + ax = fig.add_subplot(1, len(model_name_list), i + 1) + # Your plotting code for each model here + ax.set_title(model) # Set title for each subplot + for n, noise in enumerate(noise_list): + if model[0:3] == "DER": + al = np.array(al_dict[model][noise]) + al_std = np.array(al_std_dict[model][noise]) + elif model[0:2] == "DE": + # only take the sqrt for the case of DE, + # which is the variance + al = np.array(np.sqrt(al_dict[model][noise])) + al_std = np.array(np.sqrt(al_std_dict[model][noise])) + ax.fill_between( + range(n_epochs), + al - al_std, + al + al_std, + color=color_list[n], + alpha=0.25, + edgecolor=None + ) + ax.plot( + range(n_epochs), + al, + color=color_list[n], + label=r"$\sigma = $" + str(sigma_list[n]), + lw=3 + ) + for r, rs in enumerate(rs_list): + if model[0:3] == "DER": + al = np.array(al_rs_dict[model][noise][rs]) + elif model[0:2] == "DE": + al = np.array(np.sqrt(al_rs_dict[model][noise][rs])) + """ + # it doesn't really make sense to plot the std for the + # case of the DE because each individual model + # makes up one in the ensemble + """ + ''' + if model[0:3] == "DER": + al_std = np.array(al_std_dict[model][noise][rs]) + ax.fill_between( + range(n_epochs), + al - al_std, + al + al_std, + color=color_list[n], + alpha=0.1, + edgecolor=None, + ) + ''' + if r == 0: + ax.plot( + range(n_epochs), + al, + color=color_list[n], + ) + else: + ax.plot(range(n_epochs), al, color=color_list[n]) + ax.axhline(y=sigma_list[n], color=color_list[n], ls="--") + ax.set_ylabel("Aleatoric Uncertainty") + ax.set_xlabel("Epoch") + if model[0:3] == "DER": + ax.set_title("Deep Evidential Regression") + elif model[0:2] == "DE": + ax.set_title("Deep Ensemble (100 models)") + ax.set_ylim([0, 15]) + plt.legend() + if config.get_item("analysis", "savefig", "Analysis"): + plt.savefig( + str(path_to_out) + + "aleatoric_uncertainty_and_inits_n_epochs_" + + str(n_epochs) + + "_n_models_DE_" + + str(n_models) + + ".png" + ) + if config.get_item("analysis", "plot", "Analysis"): + plt.show()