diff --git a/CHANGELOG.md b/CHANGELOG.md index c92035e..b8374f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [0.1.2] - 2024-10-25 +### Fixed +- Decided to rename src/ to deepuq/ in order for easier imports + ## [0.1.1] - 2024-10-23 ### Fixed - Fixed packaging to enable use of commands for running scripts. diff --git a/src/__init__.py b/deepuq/__init__.py similarity index 100% rename from src/__init__.py rename to deepuq/__init__.py diff --git a/src/analyze/__init__.py b/deepuq/analyze/__init__.py similarity index 100% rename from src/analyze/__init__.py rename to deepuq/analyze/__init__.py diff --git a/src/analyze/analyze.py b/deepuq/analyze/analyze.py similarity index 100% rename from src/analyze/analyze.py rename to deepuq/analyze/analyze.py diff --git a/src/data/__init__.py b/deepuq/data/__init__.py similarity index 61% rename from src/data/__init__.py rename to deepuq/data/__init__.py index 69b0fe6..bf2324a 100644 --- a/src/data/__init__.py +++ b/deepuq/data/__init__.py @@ -1,4 +1,4 @@ -from src.data.data import MyDataLoader, DataPreparation +from deepuq.data.data import MyDataLoader, DataPreparation DataModules = { "MyDataLoader": MyDataLoader, diff --git a/src/data/data.py b/deepuq/data/data.py similarity index 100% rename from src/data/data.py rename to deepuq/data/data.py diff --git a/src/models/__init__.py b/deepuq/models/__init__.py similarity index 50% rename from src/models/__init__.py rename to deepuq/models/__init__.py index 808d5ca..dbfd249 100644 --- a/src/models/__init__.py +++ b/deepuq/models/__init__.py @@ -1,3 +1,3 @@ -from src.models.models import ModelLoader +from deepuq.models.models import ModelLoader ModelModules = {"ModelLoader": ModelLoader} diff --git a/src/models/models.py b/deepuq/models/models.py similarity index 100% rename from src/models/models.py rename to deepuq/models/models.py diff --git a/src/scripts/DeepEnsemble.py b/deepuq/scripts/DeepEnsemble.py similarity index 98% rename from src/scripts/DeepEnsemble.py rename to deepuq/scripts/DeepEnsemble.py index 1963e13..24096e4 100644 --- a/src/scripts/DeepEnsemble.py +++ b/deepuq/scripts/DeepEnsemble.py @@ -7,19 +7,13 @@ import torch from torch.utils.data import TensorDataset, DataLoader -# from scripts import train, models, io -from src.train import train -from src.models import models -from src.data import DataModules -from src.models import ModelModules -from src.utils.config import Config -from src.utils.defaults import DefaultsDE -from src.data.data import DataPreparation, MyDataLoader - -# from analyze.analyze import AggregateCheckpoints - -# from plots import Plots - +from deepuq.train import train +from deepuq.models import models +from deepuq.data import DataModules +from deepuq.models import ModelModules +from deepuq.utils.config import Config +from deepuq.utils.defaults import DefaultsDE +from deepuq.data.data import DataPreparation, MyDataLoader def parse_args(): """Parses command-line (or config) arguments for the DeepEnsemble script. diff --git a/src/scripts/DeepEvidentialRegression.py b/deepuq/scripts/DeepEvidentialRegression.py similarity index 98% rename from src/scripts/DeepEvidentialRegression.py rename to deepuq/scripts/DeepEvidentialRegression.py index b230157..867d0f5 100644 --- a/src/scripts/DeepEvidentialRegression.py +++ b/deepuq/scripts/DeepEvidentialRegression.py @@ -8,13 +8,13 @@ from torch.utils.data import TensorDataset, DataLoader # from scripts import train, models, io -from src.train import train -from src.models import models -from src.data import DataModules -from src.models import ModelModules -from src.utils.config import Config -from src.utils.defaults import DefaultsDER -from src.data.data import DataPreparation, MyDataLoader +from deepuq.train import train +from deepuq.models import models +from deepuq.data import DataModules +from deepuq.models import ModelModules +from deepuq.utils.config import Config +from deepuq.utils.defaults import DefaultsDER +from deepuq.data.data import DataPreparation, MyDataLoader # from plots import Plots diff --git a/src/scripts/__init__.py b/deepuq/scripts/__init__.py similarity index 100% rename from src/scripts/__init__.py rename to deepuq/scripts/__init__.py diff --git a/src/train/__init__.py b/deepuq/train/__init__.py similarity index 100% rename from src/train/__init__.py rename to deepuq/train/__init__.py diff --git a/src/train/train.py b/deepuq/train/train.py similarity index 99% rename from src/train/train.py rename to deepuq/train/train.py index d66bab0..3fc79e1 100644 --- a/src/train/train.py +++ b/deepuq/train/train.py @@ -4,7 +4,7 @@ import glob import numpy as np import matplotlib.pyplot as plt -from src.models import models +from deepuq.models import models def set_random_seeds(seed_value=42): diff --git a/src/utils/__init__.py b/deepuq/utils/__init__.py similarity index 100% rename from src/utils/__init__.py rename to deepuq/utils/__init__.py diff --git a/src/utils/config.py b/deepuq/utils/config.py similarity index 99% rename from src/utils/config.py rename to deepuq/utils/config.py index 7e58a69..8e0f87f 100644 --- a/src/utils/config.py +++ b/deepuq/utils/config.py @@ -1,7 +1,7 @@ from typing import Optional import os import yaml -from src.utils.defaults import ( +from deepuq.utils.defaults import ( DefaultsDE, DefaultsDER, DefaultsAnalysis, diff --git a/src/utils/defaults.py b/deepuq/utils/defaults.py similarity index 100% rename from src/utils/defaults.py rename to deepuq/utils/defaults.py diff --git a/pyproject.toml b/pyproject.toml index 47beef1..6dea21d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "DeepUQ" -packages = [{include = "src"}] -version = "0.1.1" +packages = [{include = "deepuq"}] +version = "0.1.2" description = "a package for investigating and comparing the predictive uncertainties from deep learning models" authors = ["beckynevin "] readme = "README.md" diff --git a/src/scripts/LossFunctions.py b/src/scripts/LossFunctions.py deleted file mode 100644 index 83c493f..0000000 --- a/src/scripts/LossFunctions.py +++ /dev/null @@ -1,379 +0,0 @@ -import os -import yaml -import argparse -import torch -import matplotlib.pyplot as plt -from data.data import DataPreparation -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from analyze.analyze import AggregateCheckpoints - -# from plots import Plots - - -def parse_args(): - parser = argparse.ArgumentParser(description="data handling module") - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--data_prescription", - "-dp", - default=DefaultsAnalysis["model"]["data_prescription"], - ) - parser.add_argument( - "--data_dimension", - "-dd", - default=DefaultsAnalysis["model"]["data_dimension"], - ) - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER, either SDER or DER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--inject_type_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["inject_type_list"], - help="Feature and predictive", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "data_prescription": args.data_prescription, - "data_dimension": args.data_dimension, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "inject_type_list": args.inject_type_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - n_models = config.get_item("model", "n_models", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - prescription = config.get_item("model", "data_prescription", "Analysis") - inject_type_list = config.get_item( - "analysis", "inject_type_list", "Analysis" - ) - dim = config.get_item("model", "data_dimension", "Analysis") - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - mse_loss = { - typei: {noise: [] for noise in noise_list} - for typei in inject_type_list - } - loss = { - typei: {noise: [] for noise in noise_list} - for typei in inject_type_list - } - mse_loss_train = { - typei: {noise: [] for noise in noise_list} - for typei in inject_type_list - } - loss_train = { - typei: {noise: [] for noise in noise_list} - for typei in inject_type_list - } - n_epochs = config.get_item("model", "n_epochs", "Analysis") - n_models = config.get_item("model", "n_models", "Analysis") - model = model_name_list[0] - for typei in inject_type_list: - for noise in noise_list: - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - prescription, - typei, - dim, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - ) - # path=path_to_chk) - # things to grab: 'valid_mse' and 'valid_bnll' - mse_loss[typei][noise].append(chk["valid_mse"]) - loss[typei][noise].append(chk["valid_loss"]) - mse_loss_train[typei][noise].append(chk["train_mse"]) - loss_train[typei][noise].append(chk["train_loss"]) - elif model[0:2] == "DE": - for nmodel in range(n_models): - mse_loss_one_model = [] - loss_one_model = [] - train_mse_loss_one_model = [] - train_loss_one_model = [] - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - prescription, - typei, - dim, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodel, - ) - mse_loss_one_model.append(chk["valid_mse"]) - loss_one_model.append(chk["valid_loss"]) - train_mse_loss_one_model.append(chk["train_mse"]) - train_loss_one_model.append(chk["train_loss"]) - - mse_loss[typei][noise].append(mse_loss_one_model) - loss[typei][noise].append(loss_one_model) - mse_loss_train[typei][noise].append( - train_mse_loss_one_model - ) - loss_train[typei][noise].append(train_loss_one_model) - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - plt.clf() - fig = plt.figure(figsize=(12, 10)) - # try this instead with a fill_between method - for i, typei in enumerate(inject_type_list): - ax = fig.add_subplot(2, len(inject_type_list), i + 1) - # Your plotting code for each model here - ax.set_title(typei) # Set title for each subplot - for i, noise in enumerate(noise_list): - sigma = DataPreparation.get_sigma( - noise, inject_type=typei, data_dimension=dim - ) - if model[0:3] == "DER": - ax.plot( - range(n_epochs), - mse_loss_train[typei][noise], - color=color_list[i], - label=r"Train; $\sigma = $" + str(sigma), - ls="--", - ) - ax.plot( - range(n_epochs), - mse_loss[typei][noise], - color=color_list[i], - label=r"Validation; $\sigma = $" + str(sigma), - ) - else: - ax.plot( - range(n_epochs), - mse_loss_train[typei][noise][0], - color=color_list[i], - ls="--", - ) - ax.plot( - range(n_epochs), - mse_loss[typei][noise][0], - color=color_list[i], - ) - ax.set_ylabel("MSE Loss") - ax.set_xlabel("Epoch") - ax.set_title(typei) - plt.legend() - # ax.set_ylim([0, 250]) - # now make the other loss plots - for i, typei in enumerate(inject_type_list): - ax = fig.add_subplot(2, len(inject_type_list), i + 3) - # Your plotting code for each model here - for i, noise in enumerate(noise_list): - sigma = DataPreparation.get_sigma( - noise, inject_type=typei, data_dimension=dim - ) - if model[0:3] == "DER": - ax.plot( - range(n_epochs), - loss_train[typei][noise], - color=color_list[i], - label=r"Train; $\sigma = $" + str(sigma), - ls="--", - ) - ax.plot( - range(n_epochs), - loss[typei][noise], - color=color_list[i], - label=r"Validation; $\sigma = $" + str(sigma), - ) - else: - ax.plot( - range(n_epochs), - loss_train[typei][noise][0], - color=color_list[i], - ls="--", - ) - ax.plot( - range(n_epochs), - loss[typei][noise][0], - color=color_list[i], - ) - - ax.set_xlabel("Epoch") - if model[0:3] == "DER": - ax.set_ylabel("NIG Loss") - elif model[0:2] == "DE": - ax.set_ylabel(r"$\beta-$NLL Loss") - # ax.set_ylim([0, 5]) - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "all_loss_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() diff --git a/src/scripts/__version__.py b/src/scripts/__version__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/scripts/old_analysis/Aleatoric.py b/src/scripts/old_analysis/Aleatoric.py deleted file mode 100644 index ac883b7..0000000 --- a/src/scripts/old_analysis/Aleatoric.py +++ /dev/null @@ -1,343 +0,0 @@ -import os -import yaml -import argparse -import numpy as np -import torch -import matplotlib.pyplot as plt -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from data.data import DataPreparation -from analyze.analyze import AggregateCheckpoints - -# from plots import Plots - - -def parse_args(): - parser = argparse.ArgumentParser(description="data handling module") - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--data_prescription", - "-dp", - default=DefaultsAnalysis["model"]["data_prescription"], - ) - parser.add_argument( - "--data_dimension", - "-dd", - default=DefaultsAnalysis["model"]["data_dimension"], - ) - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER, either SDER or DER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--inject_type_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["inject_type_list"], - help="Feature and predictive", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "data_prescription": args.data_prescription, - "data_dimension": args.data_dimension, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "inject_type_list": args.inject_type_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - n_models = config.get_item("model", "n_models", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - prescription = config.get_item("model", "data_prescription", "Analysis") - inject_type_list = config.get_item( - "analysis", "inject_type_list", "Analysis" - ) - dim = config.get_item("model", "data_dimension", "Analysis") - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - if len(model_name_list) > 1: - assert "model_name_list should only be one item" - print("inject type list", inject_type_list) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - # make an empty nested dictionary with keys for - # model names followed by noise levels - al_dict = { - typei: {noise: [] for noise in noise_list} - for typei in inject_type_list - } - al_std_dict = { - typei: {noise: [] for noise in noise_list} - for typei in inject_type_list - } - n_epochs = config.get_item("model", "n_epochs", "Analysis") - # for model in model_name_list: - model = model_name_list[0] - for typei in inject_type_list: - for noise in noise_list: - # append a noise key - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - prescription, - typei, - dim, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - ) - # path=path_to_chk) - # things to grab: 'valid_mse' and 'valid_bnll' - ( - epistemic_m, - aleatoric_m, - e_std, - a_std, - ) = chk_module.ep_al_checkpoint_DER(chk) - al_dict[typei][noise].append(aleatoric_m) - al_std_dict[typei][noise].append(a_std) - - else: - n_models = config.get_item("model", "n_models", "DE") - for epoch in range(n_epochs): - list_mus = [] - list_vars = [] - for nmodels in range(n_models): - chk = chk_module.load_checkpoint( - model, - prescription, - typei, - dim, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodels, - ) - mu_vals, var_vals = chk_module.ep_al_checkpoint_DE(chk) - list_mus.append(mu_vals) - list_vars.append(var_vals) - # first taking the mean across the validation data - # then looking at the mean and standard deviation - # across all of the nmodels - al_dict[typei][noise].append( - np.mean(np.mean(list_vars, axis=0)) - ) - al_std_dict[typei][noise].append( - np.std(np.mean(list_vars, axis=0)) - ) - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - plt.clf() - fig = plt.figure(figsize=(10, 4)) - # try this instead with a fill_between method - for i, typei in enumerate(inject_type_list): - ax = fig.add_subplot(1, len(inject_type_list), i + 1) - # Your plotting code for each model here - ax.set_title(typei) # Set title for each subplot - for i, noise in enumerate(noise_list): - if model[0:3] == "DER": - al = np.array(al_dict[typei][noise]) - al_std = np.array(al_std_dict[typei][noise]) - elif model[0:2] == "DE": - # only take the sqrt for the case of DE, - # which is the variance - al = np.array(np.sqrt(al_dict[typei][noise])) - al_std = np.array(np.sqrt(al_std_dict[typei][noise])) - ax.fill_between( - range(n_epochs), - al - al_std, - al + al_std, - color=color_list[i], - alpha=0.25, - edgecolor=None, - ) - sigma = DataPreparation.get_sigma( - noise, inject_type=typei, data_dimension=dim - ) - ax.plot( - range(n_epochs), - al, - color=color_list[i], - label=r"$\sigma = $" + str(sigma), - ) - ax.axhline(y=sigma, color=color_list[i], ls="--") - ax.set_ylabel("Aleatoric Uncertainty") - ax.set_xlabel("Epoch") - # if model[0:3] == "DER": - # ax.set_title("Deep Evidential Regression") - # elif model[0:2] == "DE": - # ax.set_title("Deep Ensemble (100 models)") - ax.set_title(typei) - ax.set_ylim([0, 15]) - plt.legend() - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "aleatoric_uncertainty_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() diff --git a/src/scripts/old_analysis/AleatoricNHidden.py b/src/scripts/old_analysis/AleatoricNHidden.py deleted file mode 100644 index 93e387a..0000000 --- a/src/scripts/old_analysis/AleatoricNHidden.py +++ /dev/null @@ -1,327 +0,0 @@ -import os -import yaml -import argparse -import numpy as np -import torch -import matplotlib.pyplot as plt -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from data.data import DataPreparation -from analyze.analyze import AggregateCheckpoints - - -def parse_args(): - parser = argparse.ArgumentParser( - description="Analyzes the aleatoric uncertainty when the model \ - architecture is jittered" - ) - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER, either SDER or DER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - sigma_list = [] - for noise in noise_list: - sigma_list.append(DataPreparation.get_sigma(noise)) - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # this needs to be redone - n_hidden_list = [64, 54, 44, 34, 24] - rs = 1 - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - # make an empty nested dictionary with keys for - # model names followed by noise levels - al_dict = { - model_name: { - noise: {nh: [] for nh in n_hidden_list} for noise in noise_list - } - for model_name in model_name_list - } - al_std_dict = { - model_name: { - noise: {nh: [] for nh in n_hidden_list} for noise in noise_list - } - for model_name in model_name_list - } - n_epochs = config.get_item("model", "n_epochs", "Analysis") - for model in model_name_list: - for noise in noise_list: - for nh in n_hidden_list: - - # append a noise key - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - load_rs_chk=True, - rs=rs, - load_nh_chk=True, - nh=nh, - ) - # path=path_to_chk) - # things to grab: 'valid_mse' and 'valid_bnll' - ( - epistemic_m, - aleatoric_m, - e_std, - a_std, - ) = chk_module.ep_al_checkpoint_DER(chk) - al_dict[model][noise][nh].append(aleatoric_m) - al_std_dict[model][noise][nh].append(a_std) - - if model[0:3] == "DE_": - n_models = config.get_item("model", "n_models", "DE") - for epoch in range(n_epochs): - list_mus = [] - list_vars = [] - for nmodels in range(n_models): - chk = chk_module.load_checkpoint( - model, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodels, - ) - mu_vals, var_vals = chk_module.ep_al_checkpoint_DE(chk) - list_mus.append(mu_vals) - list_vars.append(var_vals) - try: - al_dict[model][noise][nmodels + 1].append( - np.mean(list_vars) - ) - except KeyError: - continue - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - plt.clf() - fig = plt.figure(figsize=(10, 4)) - # try this instead with a fill_between method - for i, model in enumerate(model_name_list): - ax = fig.add_subplot(1, len(model_name_list), i + 1) - # Your plotting code for each model here - ax.set_title(model) # Set title for each subplot - for n, noise in enumerate(noise_list): - for h, nh in enumerate(n_hidden_list): - if model[0:3] == "DE_": - al = np.array(np.sqrt(al_dict[model][noise][nh])) - al_std = np.array(np.sqrt(al_std_dict[model][noise][nh])) - else: - al = np.array(al_dict[model][noise][nh]) - al_std = np.array(al_std_dict[model][noise][nh]) - ax.fill_between( - range(n_epochs), - al - al_std, - al + al_std, - color=color_list[n], - alpha=0.1, - edgecolor=None, - ) - - if h == 0: - ax.plot( - range(n_epochs), - al, - color=color_list[n], - label=r"$\sigma = $" + str(sigma_list[n]), - ) - else: - ax.plot( - range(n_epochs), - al, - color=color_list[n], - ) - ax.axhline( - y=sigma_list[n], - color=color_list[n], - ls="--", - ) - ax.set_ylabel("Aleatoric Uncertainty") - ax.set_xlabel("Epoch") - if model[0:3] == "DER": - ax.set_title("Deep Evidential Regression") - elif model[0:2] == "DE": - ax.set_title("Deep Ensemble (100 models)") - ax.set_ylim([0, 6]) - plt.legend() - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "aleatoric_uncertainty_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() diff --git a/src/scripts/old_analysis/Aleatoric_and_inits.py b/src/scripts/old_analysis/Aleatoric_and_inits.py deleted file mode 100644 index 3c82ba7..0000000 --- a/src/scripts/old_analysis/Aleatoric_and_inits.py +++ /dev/null @@ -1,460 +0,0 @@ -import os -import yaml -import argparse -import numpy as np -import torch -import matplotlib.pyplot as plt -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from data.data import DataPreparation -from analyze.analyze import AggregateCheckpoints - -# from plots import Plots - - -def parse_args(): - parser = argparse.ArgumentParser(description="data handling module") - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--data_prescription", - type=str, - default=DefaultsAnalysis["model"]["data_prescription"], - help="Current only case is linear homoskedastic", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER, either SDER or DER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--inject_type_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["inject_type_list"], - help="Options are predictive and feature", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "data_prescription": args.data_prescription, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "inject_type_list": args.inject_type_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - prescription = config.get_item("model", "data_prescription", "Analysis") - inject_type_list = config.get_item( - "analysis", "inject_type_list", "Analysis" - ) - dim = config.get_item("model", "data_dimension", "Analysis") - sigma_list = [] - for noise in noise_list: - sigma_list.append(DataPreparation.get_sigma(noise)) - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - model = model_name_list[0] - print("one model at a time", model) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - # make an empty nested dictionary with keys for - # model names followed by noise levels - al_dict = { - typei: { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - for typei in inject_type_list - } - al_std_dict = { - typei: { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - for typei in inject_type_list - } - n_epochs = config.get_item("model", "n_epochs", "Analysis") - # switching from two panels for different models to - # two panels for different injection types - # could eventually make this into a four panel plot - # for model in model_name_list: - for inject_type in inject_type_list: - for model_name in model_name_list: - for noise in noise_list: - # append a noise key - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model_name, - prescription, - inject_type, - dim, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - load_nh_chk=False, - ) - # path=path_to_chk) - # things to grab: 'valid_mse' and 'valid_bnll' - ( - epistemic_m, - aleatoric_m, - e_std, - a_std, - ) = chk_module.ep_al_checkpoint_DER(chk) - al_dict[inject_type][model][noise].append(aleatoric_m) - al_std_dict[inject_type][model][noise].append(a_std) - - elif model[0:2] == "DE": - n_models = config.get_item("model", "n_models", "DE") - for epoch in range(n_epochs): - list_mus = [] - list_vars = [] - for nmodels in range(n_models): - chk = chk_module.load_checkpoint( - model, - prescription, - inject_type, - dim, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodels, - ) - mu_vals, var_vals = chk_module.ep_al_checkpoint_DE( - chk - ) - list_mus.append(mu_vals) - list_vars.append(var_vals) - # first taking the mean across the validation data - # then looking at the mean and standard deviation - # across all of the nmodels - al_dict[inject_type][model][noise].append( - np.mean(np.mean(list_vars, axis=0)) - ) - al_std_dict[inject_type][model][noise].append( - np.std(np.mean(list_vars, axis=0)) - ) - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - # this needs to be redone - rs_list = [10, 11, 12, 13, 14] - # make an empty nested dictionary with keys for - # model names followed by noise levels - al_rs_dict = { - model_name: {noise: {rs: [] for rs in rs_list} for noise in noise_list} - for model_name in model_name_list - } - """ - al_rs_std_dict = { - model_name: {noise: {rs: [] for rs in rs_list} for noise in noise_list} - for model_name in model_name_list - } - """ - n_epochs = config.get_item("model", "n_epochs", "Analysis") - # for model in model_name_list: - for inject_type in inject_type_list: - for noise in noise_list: - for rs in rs_list: - - # append a noise key - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - prescription, - inject_type, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - load_rs_chk=True, - rs=rs, - load_nh_chk=False, - ) - # path=path_to_chk) - # things to grab: 'valid_mse' and 'valid_bnll' - _, aleatoric_m, _, a_std = ( - chk_module.ep_al_checkpoint_DER(chk) - ) - al_rs_dict[model][noise][rs].append(aleatoric_m) - # al_std_dict[model][noise][rs].append(a_std) - if model[0:2] == "DE" and model[0:3] != "DER": - n_models = config.get_item("model", "n_models", "DE") - for epoch in range(n_epochs): - list_mus = [] - list_vars = [] - for nmodels in range(n_models): - chk = chk_module.load_checkpoint( - model, - prescription, - inject_type, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodels, - ) - mu_vals, var_vals = chk_module.ep_al_checkpoint_DE(chk) - list_mus.append(mu_vals) - list_vars.append(var_vals) - try: - al_rs_dict[model][noise][nmodels + 1].append( - np.mean(list_vars) - ) - except KeyError: - continue - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - plt.clf() - fig = plt.figure(figsize=(10, 4)) - # try this instead with a fill_between method - for i, model in enumerate(model_name_list): - ax = fig.add_subplot(1, len(model_name_list), i + 1) - # Your plotting code for each model here - ax.set_title(model) # Set title for each subplot - for n, noise in enumerate(noise_list): - if model[0:3] == "DER": - al = np.array(al_dict[model][noise]) - al_std = np.array(al_std_dict[model][noise]) - elif model[0:2] == "DE": - # only take the sqrt for the case of DE, - # which is the variance - al = np.array(np.sqrt(al_dict[model][noise])) - al_std = np.array(np.sqrt(al_std_dict[model][noise])) - ax.fill_between( - range(n_epochs), - al - al_std, - al + al_std, - color=color_list[n], - alpha=0.25, - edgecolor=None, - ) - ax.plot( - range(n_epochs), - al, - color=color_list[n], - label=r"$\sigma = $" + str(sigma_list[n]), - lw=3, - ) - for r, rs in enumerate(rs_list): - if model[0:3] == "DER": - al = np.array(al_rs_dict[model][noise][rs]) - elif model[0:2] == "DE": - al = np.array(np.sqrt(al_rs_dict[model][noise][rs])) - """ - # it doesn't really make sense to plot the std for the - # case of the DE because each individual model - # makes up one in the ensemble - """ - """ - if model[0:3] == "DER": - al_std = np.array(al_std_dict[model][noise][rs]) - ax.fill_between( - range(n_epochs), - al - al_std, - al + al_std, - color=color_list[n], - alpha=0.1, - edgecolor=None, - ) - """ - if r == 0: - ax.plot( - range(n_epochs), - al, - color=color_list[n], - ) - else: - ax.plot( - range(n_epochs), - al, - color=color_list[n], - ) - ax.axhline( - y=sigma_list[n], - color=color_list[n], - ls="--", - ) - ax.set_ylabel("Aleatoric Uncertainty") - ax.set_xlabel("Epoch") - if model[0:3] == "DER": - ax.set_title("Deep Evidential Regression") - elif model[0:2] == "DE": - ax.set_title("Deep Ensemble (100 models)") - ax.set_ylim([0, 15]) - plt.legend() - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "aleatoric_uncertainty_and_inits_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() diff --git a/src/scripts/old_analysis/AleatoricandEpistemic.py b/src/scripts/old_analysis/AleatoricandEpistemic.py deleted file mode 100644 index f68d53c..0000000 --- a/src/scripts/old_analysis/AleatoricandEpistemic.py +++ /dev/null @@ -1,396 +0,0 @@ -import os -import yaml -import argparse -import numpy as np -import torch -import matplotlib.pyplot as plt -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from data.data import DataPreparation -from analyze.analyze import AggregateCheckpoints - -# from plots import Plots - - -def parse_args(): - parser = argparse.ArgumentParser(description="data handling module") - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - sigma_list = [] - for noise in noise_list: - sigma_list.append(DataPreparation.get_sigma(noise)) - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - # make an empty nested dictionary with keys for - # model names followed by noise levels - ep_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - al_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - - ep_std_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - al_std_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - n_epochs = config.get_item("model", "n_epochs", "Analysis") - for model in model_name_list: - for noise in noise_list: - # append a noise key - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - ) - # path=path_to_chk) - # things to grab: 'valid_mse' and 'valid_bnll' - ( - epistemic_m, - aleatoric_m, - e_std, - a_std, - ) = chk_module.ep_al_checkpoint_DER(chk) - ep_dict[model][noise].append(epistemic_m) - al_dict[model][noise].append(aleatoric_m) - ep_std_dict[model][noise].append(e_std) - al_std_dict[model][noise].append(a_std) - - elif model[0:2] == "DE": - n_models = config.get_item("model", "n_models", "DE") - for epoch in range(n_epochs): - list_mus = [] - list_vars = [] - for nmodels in range(n_models): - chk = chk_module.load_checkpoint( - model, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodels, - ) - mu_vals, var_vals = chk_module.ep_al_checkpoint_DE(chk) - list_mus.append(mu_vals) - list_vars.append(var_vals) - ep_dict[model][noise].append( - np.mean(np.std(list_mus, axis=0)) - ) - ep_std_dict[model][noise].append( - np.std(np.std(list_mus, axis=0)) - ) - al_dict[model][noise].append( - np.mean(np.mean(list_vars, axis=0)) - ) - al_std_dict[model][noise].append( - np.std(np.mean(list_vars, axis=0)) - ) - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - plt.clf() - fig = plt.figure(figsize=(10, 4)) - # try this instead with a fill_between method - for i, model in enumerate(model_name_list): - ax = fig.add_subplot(1, len(model_name_list), i + 1) - # Your plotting code for each model here - ax.set_title(model) # Set title for each subplot - for i, noise in enumerate(noise_list): - if model[0:3] == "DER": - al = np.array(al_dict[model][noise]) - al_std = np.array(al_std_dict[model][noise]) - elif model[0:2] == "DE": - al = np.array(np.sqrt(al_dict[model][noise])) - al_std = np.array(np.sqrt(al_std_dict[model][noise])) - ep = np.array(ep_dict[model][noise]) - ep_std = np.array(ep_std_dict[model][noise]) - total = np.sqrt(al**2 + ep**2) - total_std = np.sqrt(al_std**2 + ep_std**2) - ax.fill_between( - range(n_epochs), - total - total_std, - total + total_std, - color=color_list[i], - alpha=0.25, - edgecolor=None, - ) - ax.plot( - range(n_epochs), - total, - color=color_list[i], - label=r"$\sigma = $" + str(sigma_list[i]), - ) - ax.axhline( - y=sigma_list[i], - color=color_list[i], - ls="--", - ) - ax.set_ylabel("Total Uncertainty") - ax.set_xlabel("Epoch") - if model[0:3] == "DER": - ax.set_title("Deep Evidential Regression") - elif model[0:2] == "DE": - ax.set_title("Deep Ensemble (100 models)") - ax.set_ylim([0, 15]) - plt.legend() - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "aleatoric_and_epistemic_uncertainty_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() - - plt.clf() - fig = plt.figure(figsize=(10, 4)) - # try this instead with a fill_between method - for i, model in enumerate(model_name_list): - ax = fig.add_subplot(1, len(model_name_list), i + 1) - # Your plotting code for each model here - ax.set_title(model) # Set title for each subplot - for i, noise in enumerate(noise_list): - if model[0:3] == "DER": - al = np.array(al_dict[model][noise]) - al_std = np.array(al_std_dict[model][noise]) - elif model[0:2] == "DE": - al = np.array(np.sqrt(al_dict[model][noise])) - al_std = np.array(np.sqrt(al_std_dict[model][noise])) - ep = np.array(ep_dict[model][noise]) - ep_std = np.array(ep_std_dict[model][noise]) - total = np.sqrt(al**2 + ep**2) - total_std = np.sqrt(al_std**2 + ep_std**2) - ax.fill_between( - range(n_epochs), - total - total_std, - total + total_std, - color=color_list[i], - alpha=0.25, - edgecolor=None, - ) - ax.plot( - range(n_epochs), - al, - color=color_list[i], - ls="dashed", - label=r"Aleatoric, $\sigma = $" + str(sigma_list[i]), - ) - ax.plot( - range(n_epochs), - ep, - color=color_list[i], - ls="dotted", - label=r"Epistemic, $\sigma = $" + str(sigma_list[i]), - ) - ax.plot( - range(n_epochs), - total, - color=color_list[i], - label=r"Total, $\sigma = $" + str(sigma_list[i]), - ) - ax.axhline( - y=sigma_list[i], - color=color_list[i], - ls="--", - ) - ax.set_ylabel("Total Uncertainty") - ax.set_xlabel("Epoch") - if model[0:3] == "DER": - ax.set_title("Deep Evidential Regression") - elif model[0:2] == "DE": - ax.set_title("Deep Ensemble (100 models)") - ax.set_ylim([0, 15]) - plt.legend() - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "aleatoric_and_epistemic_and_total_uncertainty_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() diff --git a/src/scripts/old_analysis/Epistemic.py b/src/scripts/old_analysis/Epistemic.py deleted file mode 100644 index 50d5a39..0000000 --- a/src/scripts/old_analysis/Epistemic.py +++ /dev/null @@ -1,312 +0,0 @@ -import os -import yaml -import argparse -import numpy as np -import torch -import matplotlib.pyplot as plt -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from data.data import DataPreparation -from analyze.analyze import AggregateCheckpoints - -# from plots import Plots - - -def parse_args(): - parser = argparse.ArgumentParser(description="data handling module") - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER, either DER or SDER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - sigma_list = [] - for noise in noise_list: - sigma_list.append(DataPreparation.get_sigma(noise)) - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - # make an empty nested dictionary with keys for - # model names followed by noise levels - ep_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - al_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - - ep_std_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - al_std_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - n_epochs = config.get_item("model", "n_epochs", "Analysis") - for model in model_name_list: - for noise in noise_list: - # append a noise key - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - ) - # things to grab: 'valid_mse' and 'valid_bnll' - ( - epistemic_m, - aleatoric_m, - e_std, - a_std, - ) = chk_module.ep_al_checkpoint_DER(chk) - ep_dict[model][noise].append(epistemic_m) - al_dict[model][noise].append(aleatoric_m) - ep_std_dict[model][noise].append(e_std) - al_std_dict[model][noise].append(a_std) - - elif model[0:2] == "DE": - n_models = config.get_item("model", "n_models", "DE") - for epoch in range(n_epochs): - list_mus = [] - list_sigs = [] - for nmodels in range(n_models): - chk = chk_module.load_checkpoint( - model, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodels, - ) - mu_vals, sig_vals = chk_module.ep_al_checkpoint_DE(chk) - list_mus.append(mu_vals) - list_sigs.append(sig_vals) - ep_dict[model][noise].append( - np.median(np.std(list_mus, axis=0)) - ) - al_dict[model][noise].append( - np.median(np.mean(list_sigs, axis=0)) - ) - ep_std_dict[model][noise].append( - np.std(np.std(list_mus, axis=0)) - ) - al_std_dict[model][noise].append( - np.std(np.mean(list_sigs, axis=0)) - ) - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - plt.clf() - fig = plt.figure(figsize=(10, 4)) - # try this instead with a fill_between method - for i, model in enumerate(model_name_list): - ax = fig.add_subplot(1, len(model_name_list), i + 1) - # Your plotting code for each model here - ax.set_title(model) # Set title for each subplot - for i, noise in enumerate(noise_list): - ep = np.array(ep_dict[model][noise]) - ep_std = np.array(ep_std_dict[model][noise]) - ax.fill_between( - range(n_epochs), - ep - ep_std, - ep + ep_std, - color=color_list[i], - alpha=0.25, - edgecolor=None, - ) - ax.plot( - range(n_epochs), - ep_dict[model][noise], - color=color_list[i], - label=r"$\sigma = $" + str(sigma_list[i]), - ) - ax.axhline(y=sigma_list[i], color=color_list[i]) - ax.set_ylabel("Epistemic Uncertainty") - ax.set_xlabel("Epoch") - if model[0:3] == "DER": - ax.set_title("Deep Evidential Regression") - elif model[0:2] == "DE": - ax.set_title("Deep Ensemble (100 models)") - # ax.set_ylim([-1, 15]) - plt.legend() - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "epistemic_uncertainty_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() diff --git a/src/scripts/old_analysis/ParityPlotTest.py b/src/scripts/old_analysis/ParityPlotTest.py deleted file mode 100644 index 42f332e..0000000 --- a/src/scripts/old_analysis/ParityPlotTest.py +++ /dev/null @@ -1,279 +0,0 @@ -import os -import yaml -import argparse -import numpy as np -import torch -import matplotlib.pyplot as plt -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from data.data import DataPreparation -from analyze.analyze import AggregateCheckpoints -from torch.utils.data import TensorDataset -from models import models - -# from plots import Plots - - -def parse_args(): - parser = argparse.ArgumentParser(description="data handling module") - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER, either SDER or DER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - sigma_list = [] - for noise in noise_list: - sigma_list.append(DataPreparation.get_sigma(noise)) - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - - # now create a test set - data = DataPreparation() - data.sample_params_from_prior(1000) - data.simulate_data(data.params, 1, "linear_homogeneous", seed=41) - df_array = data.get_dict() - # Convert non-tensor entries to tensors - df = {} - for key, value in df_array.items(): - - if isinstance(value, TensorDataset): - # Keep tensors as they are - df[key] = value - else: - # Convert lists to tensors - df[key] = torch.tensor(value) - len_df = len(df["params"][:, 0].numpy()) - len_x = len(df["inputs"].numpy()) - ms_array = np.repeat(df["params"][:, 0].numpy(), len_x) - bs_array = np.repeat(df["params"][:, 1].numpy(), len_x) - xs_array = np.tile(df["inputs"].numpy(), len_df) - ys_array = np.reshape(df["output"].numpy(), (len_df * len_x)) - - inputs = np.array([xs_array, ms_array, bs_array]).T - model_inputs, model_outputs = DataPreparation.normalize( - inputs, ys_array, False - ) - _, x_test, _, y_test = DataPreparation.train_val_split( - model_inputs, - model_outputs, - val_proportion=0.1, - random_state=41, - ) - - # load up the checkpoints for DER - # and run it on the test data, make a parity plot - DERmodel, lossFn = models.model_setup_DER("DER", DEVICE, 64) - plt.clf() - fig = plt.figure() - ax = fig.add_subplot(211) - axr = fig.add_subplot(212) - - for i, noise in enumerate(noise_list): - chk = chk_module.load_checkpoint( - "DER", - noise, - 99, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - ) - # first, define the model at this epoch - DERmodel.load_state_dict(chk.get("model_state_dict")) - # checkpoint['model_state_dict']) - DERmodel.eval() - # now run on the x_test - y_pred = DERmodel(torch.Tensor(x_test)).detach().numpy() - - ax.scatter( - y_test, - y_pred[:, 0], - color=color_list[i], - label=r"$\sigma = $" + str(sigma_list[i]), - s=3, - ) - # ax.set_xlabel('True y') - ax.set_ylabel(r"Predicted $y$") - ax.plot( - range(-100, 1100), - range(-100, 1100), - ls="--", - color="grey", - ) - - axr.scatter( - y_test, - y_pred[:, 0] - y_test, - color=color_list[i], - label=r"$\sigma = $" + str(sigma_list[i]), - s=3, - zorder=-100, - ) - - axr.set_xlabel(r"True $y^*$") - axr.set_ylabel("Residual (predicted - true)") - axr.axhline(y=0, ls="--", color="grey") - plt.legend() - plt.show()