diff --git a/.cz.toml b/.cz.toml deleted file mode 100644 index 1566585..0000000 --- a/.cz.toml +++ /dev/null @@ -1,3 +0,0 @@ -[tool.commitizen] -name = "cz_conventional_commits" -version = "0.1.0" diff --git a/.pre-commit-config.yml b/.pre-commit-config.yml deleted file mode 100644 index 7a05a42..0000000 --- a/.pre-commit-config.yml +++ /dev/null @@ -1,17 +0,0 @@ -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 - hooks: - - id: check-toml - - id: name-tests-test - - id: requirements-txt-fixer - - id: trailing-whitespace - - id: mixed-line-ending - - repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black - - repo: https://github.com/asottile/pyupgrade - rev: v2.31.0 - hooks: - - id: pyupgrade diff --git a/CHANGELOG.md b/CHANGELOG.md index c92035e..b8374f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [0.1.2] - 2024-10-25 +### Fixed +- Decided to rename src/ to deepuq/ in order for easier imports + ## [0.1.1] - 2024-10-23 ### Fixed - Fixed packaging to enable use of commands for running scripts. diff --git a/README.md b/README.md index 1ace3ba..abb21b7 100644 --- a/README.md +++ b/README.md @@ -59,11 +59,10 @@ DeepUQ/ ├── README.md ├── DeepUQResources/ ├── data/ -├── environment.yml ├── notebooks/ ├── poetry.lock ├── pyproject.toml -├── src/ +├── deepuq/ │ ├── __init__.py │ ├── analyze/ │ │ ├── __init__.py @@ -91,7 +90,7 @@ DeepUQ/ │ ├── test_DeepEnsemble.py │ └── test_DeepEvidentialRegression.py ``` -The `src/` folder contains the relevant modules for config settings, data generation, model parameters, training, and the two scripts for training the Deep Ensemble and the Deep Evidential Regression models. It also includes tools for loading and analyzing the saved checkpoints in `analysis/`. +The `deepuq/` folder contains the relevant modules for config settings, data generation, model parameters, training, and the two scripts for training the Deep Ensemble and the Deep Evidential Regression models. It also includes tools for loading and analyzing the saved checkpoints in `analysis/`. Example notebooks for how to train and analyze the results of the models can be found in the `notebooks/` folder. @@ -100,7 +99,7 @@ The `DeepUQResources/` folder is the default location for saving checkpoints fro ## How to run the workflow The scripts can be accessed via the ipython example notebooks or via the model modules (ie `DeepEnsemble.py`). For example, to ingest data and train a Deep Ensemble from the DeepUQ/ directory: -> python src/scripts/DeepEnsemble.py +> python deepuq/scripts/DeepEnsemble.py The equivalent shortcut command: > UQensemble @@ -109,21 +108,21 @@ With no config file specified, this command will pull settings from the `default Another option is to specify your own config file: -> python src/scripts/DeepEnsemble.py --config "path/to/config/myconfig.yaml" +> python deepuq/scripts/DeepEnsemble.py --config "path/to/config/myconfig.yaml" Where you would modify the "path/to/config/myconfig.yaml" to specify where your own yaml lives. The third option is to input settings on the command line. These choices are then combined with the default settings and output in a temporary yaml. -> python src/scripts/DeepEnsemble.py --noise_level "low" --n_models 10 --out_dir ./DeepUQResources/results/ --save_final_checkpoint True --savefig True --n_epochs 10 +> python deepuq/scripts/DeepEnsemble.py --noise_level "low" --n_models 10 --out_dir ./DeepUQResources/results/ --save_final_checkpoint True --savefig True --n_epochs 10 This command will train a 10 network, 10 epoch ensemble on the low noise data and will save figures and final checkpoints to the specified directory. Required arguments are the noise setting (low/medium/high), the number of ensembles, and the working directory. For more information on the arguments: -> python src/scripts/DeepEnsemble.py --help +> python deepuq/scripts/DeepEnsemble.py --help The other available script is the `DeepEvidentialRegression.py` script: -> python src/scripts/DeepEvidentialRegression.py --help +> python deepuq/scripts/DeepEvidentialRegression.py --help The shortcut: > UQder diff --git a/src/__init__.py b/deepuq/__init__.py similarity index 100% rename from src/__init__.py rename to deepuq/__init__.py diff --git a/src/analyze/__init__.py b/deepuq/analyze/__init__.py similarity index 100% rename from src/analyze/__init__.py rename to deepuq/analyze/__init__.py diff --git a/src/analyze/analyze.py b/deepuq/analyze/analyze.py similarity index 100% rename from src/analyze/analyze.py rename to deepuq/analyze/analyze.py diff --git a/src/data/__init__.py b/deepuq/data/__init__.py similarity index 61% rename from src/data/__init__.py rename to deepuq/data/__init__.py index 69b0fe6..bf2324a 100644 --- a/src/data/__init__.py +++ b/deepuq/data/__init__.py @@ -1,4 +1,4 @@ -from src.data.data import MyDataLoader, DataPreparation +from deepuq.data.data import MyDataLoader, DataPreparation DataModules = { "MyDataLoader": MyDataLoader, diff --git a/src/data/data.py b/deepuq/data/data.py similarity index 100% rename from src/data/data.py rename to deepuq/data/data.py diff --git a/src/models/__init__.py b/deepuq/models/__init__.py similarity index 50% rename from src/models/__init__.py rename to deepuq/models/__init__.py index 808d5ca..dbfd249 100644 --- a/src/models/__init__.py +++ b/deepuq/models/__init__.py @@ -1,3 +1,3 @@ -from src.models.models import ModelLoader +from deepuq.models.models import ModelLoader ModelModules = {"ModelLoader": ModelLoader} diff --git a/src/models/models.py b/deepuq/models/models.py similarity index 100% rename from src/models/models.py rename to deepuq/models/models.py diff --git a/src/scripts/DeepEnsemble.py b/deepuq/scripts/DeepEnsemble.py similarity index 98% rename from src/scripts/DeepEnsemble.py rename to deepuq/scripts/DeepEnsemble.py index 1963e13..212a6ff 100644 --- a/src/scripts/DeepEnsemble.py +++ b/deepuq/scripts/DeepEnsemble.py @@ -7,18 +7,13 @@ import torch from torch.utils.data import TensorDataset, DataLoader -# from scripts import train, models, io -from src.train import train -from src.models import models -from src.data import DataModules -from src.models import ModelModules -from src.utils.config import Config -from src.utils.defaults import DefaultsDE -from src.data.data import DataPreparation, MyDataLoader - -# from analyze.analyze import AggregateCheckpoints - -# from plots import Plots +from deepuq.train import train +from deepuq.models import models +from deepuq.data import DataModules +from deepuq.models import ModelModules +from deepuq.utils.config import Config +from deepuq.utils.defaults import DefaultsDE +from deepuq.data.data import DataPreparation, MyDataLoader def parse_args(): diff --git a/src/scripts/DeepEvidentialRegression.py b/deepuq/scripts/DeepEvidentialRegression.py similarity index 98% rename from src/scripts/DeepEvidentialRegression.py rename to deepuq/scripts/DeepEvidentialRegression.py index b230157..867d0f5 100644 --- a/src/scripts/DeepEvidentialRegression.py +++ b/deepuq/scripts/DeepEvidentialRegression.py @@ -8,13 +8,13 @@ from torch.utils.data import TensorDataset, DataLoader # from scripts import train, models, io -from src.train import train -from src.models import models -from src.data import DataModules -from src.models import ModelModules -from src.utils.config import Config -from src.utils.defaults import DefaultsDER -from src.data.data import DataPreparation, MyDataLoader +from deepuq.train import train +from deepuq.models import models +from deepuq.data import DataModules +from deepuq.models import ModelModules +from deepuq.utils.config import Config +from deepuq.utils.defaults import DefaultsDER +from deepuq.data.data import DataPreparation, MyDataLoader # from plots import Plots diff --git a/src/scripts/__init__.py b/deepuq/scripts/__init__.py similarity index 100% rename from src/scripts/__init__.py rename to deepuq/scripts/__init__.py diff --git a/src/train/__init__.py b/deepuq/train/__init__.py similarity index 100% rename from src/train/__init__.py rename to deepuq/train/__init__.py diff --git a/src/train/train.py b/deepuq/train/train.py similarity index 99% rename from src/train/train.py rename to deepuq/train/train.py index d66bab0..3fc79e1 100644 --- a/src/train/train.py +++ b/deepuq/train/train.py @@ -4,7 +4,7 @@ import glob import numpy as np import matplotlib.pyplot as plt -from src.models import models +from deepuq.models import models def set_random_seeds(seed_value=42): diff --git a/src/utils/__init__.py b/deepuq/utils/__init__.py similarity index 100% rename from src/utils/__init__.py rename to deepuq/utils/__init__.py diff --git a/src/utils/config.py b/deepuq/utils/config.py similarity index 99% rename from src/utils/config.py rename to deepuq/utils/config.py index 7e58a69..8e0f87f 100644 --- a/src/utils/config.py +++ b/deepuq/utils/config.py @@ -1,7 +1,7 @@ from typing import Optional import os import yaml -from src.utils.defaults import ( +from deepuq.utils.defaults import ( DefaultsDE, DefaultsDER, DefaultsAnalysis, diff --git a/src/utils/defaults.py b/deepuq/utils/defaults.py similarity index 100% rename from src/utils/defaults.py rename to deepuq/utils/defaults.py diff --git a/pyproject.toml b/pyproject.toml index 47beef1..9441d8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,15 @@ [tool.poetry] name = "DeepUQ" -packages = [{include = "src"}] -version = "0.1.1" +packages = [{include = "deepuq"}] +version = "0.1.2" description = "a package for investigating and comparing the predictive uncertainties from deep learning models" authors = ["beckynevin "] readme = "README.md" license = "MIT" [tool.poetry.scripts] -UQensemble = "src.scripts.DeepEnsemble:main" -UQder = "src.scripts.DeepEvidentialRegression:main" +UQensemble = "deepuq.scripts.DeepEnsemble:main" +UQder = "deepuq.scripts.DeepEvidentialRegression:main" [tool.poetry.dependencies] python = ">=3.10,<3.11" diff --git a/src/scripts/LossFunctions.py b/src/scripts/LossFunctions.py deleted file mode 100644 index 83c493f..0000000 --- a/src/scripts/LossFunctions.py +++ /dev/null @@ -1,379 +0,0 @@ -import os -import yaml -import argparse -import torch -import matplotlib.pyplot as plt -from data.data import DataPreparation -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from analyze.analyze import AggregateCheckpoints - -# from plots import Plots - - -def parse_args(): - parser = argparse.ArgumentParser(description="data handling module") - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--data_prescription", - "-dp", - default=DefaultsAnalysis["model"]["data_prescription"], - ) - parser.add_argument( - "--data_dimension", - "-dd", - default=DefaultsAnalysis["model"]["data_dimension"], - ) - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER, either SDER or DER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--inject_type_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["inject_type_list"], - help="Feature and predictive", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "data_prescription": args.data_prescription, - "data_dimension": args.data_dimension, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "inject_type_list": args.inject_type_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - n_models = config.get_item("model", "n_models", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - prescription = config.get_item("model", "data_prescription", "Analysis") - inject_type_list = config.get_item( - "analysis", "inject_type_list", "Analysis" - ) - dim = config.get_item("model", "data_dimension", "Analysis") - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - mse_loss = { - typei: {noise: [] for noise in noise_list} - for typei in inject_type_list - } - loss = { - typei: {noise: [] for noise in noise_list} - for typei in inject_type_list - } - mse_loss_train = { - typei: {noise: [] for noise in noise_list} - for typei in inject_type_list - } - loss_train = { - typei: {noise: [] for noise in noise_list} - for typei in inject_type_list - } - n_epochs = config.get_item("model", "n_epochs", "Analysis") - n_models = config.get_item("model", "n_models", "Analysis") - model = model_name_list[0] - for typei in inject_type_list: - for noise in noise_list: - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - prescription, - typei, - dim, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - ) - # path=path_to_chk) - # things to grab: 'valid_mse' and 'valid_bnll' - mse_loss[typei][noise].append(chk["valid_mse"]) - loss[typei][noise].append(chk["valid_loss"]) - mse_loss_train[typei][noise].append(chk["train_mse"]) - loss_train[typei][noise].append(chk["train_loss"]) - elif model[0:2] == "DE": - for nmodel in range(n_models): - mse_loss_one_model = [] - loss_one_model = [] - train_mse_loss_one_model = [] - train_loss_one_model = [] - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - prescription, - typei, - dim, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodel, - ) - mse_loss_one_model.append(chk["valid_mse"]) - loss_one_model.append(chk["valid_loss"]) - train_mse_loss_one_model.append(chk["train_mse"]) - train_loss_one_model.append(chk["train_loss"]) - - mse_loss[typei][noise].append(mse_loss_one_model) - loss[typei][noise].append(loss_one_model) - mse_loss_train[typei][noise].append( - train_mse_loss_one_model - ) - loss_train[typei][noise].append(train_loss_one_model) - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - plt.clf() - fig = plt.figure(figsize=(12, 10)) - # try this instead with a fill_between method - for i, typei in enumerate(inject_type_list): - ax = fig.add_subplot(2, len(inject_type_list), i + 1) - # Your plotting code for each model here - ax.set_title(typei) # Set title for each subplot - for i, noise in enumerate(noise_list): - sigma = DataPreparation.get_sigma( - noise, inject_type=typei, data_dimension=dim - ) - if model[0:3] == "DER": - ax.plot( - range(n_epochs), - mse_loss_train[typei][noise], - color=color_list[i], - label=r"Train; $\sigma = $" + str(sigma), - ls="--", - ) - ax.plot( - range(n_epochs), - mse_loss[typei][noise], - color=color_list[i], - label=r"Validation; $\sigma = $" + str(sigma), - ) - else: - ax.plot( - range(n_epochs), - mse_loss_train[typei][noise][0], - color=color_list[i], - ls="--", - ) - ax.plot( - range(n_epochs), - mse_loss[typei][noise][0], - color=color_list[i], - ) - ax.set_ylabel("MSE Loss") - ax.set_xlabel("Epoch") - ax.set_title(typei) - plt.legend() - # ax.set_ylim([0, 250]) - # now make the other loss plots - for i, typei in enumerate(inject_type_list): - ax = fig.add_subplot(2, len(inject_type_list), i + 3) - # Your plotting code for each model here - for i, noise in enumerate(noise_list): - sigma = DataPreparation.get_sigma( - noise, inject_type=typei, data_dimension=dim - ) - if model[0:3] == "DER": - ax.plot( - range(n_epochs), - loss_train[typei][noise], - color=color_list[i], - label=r"Train; $\sigma = $" + str(sigma), - ls="--", - ) - ax.plot( - range(n_epochs), - loss[typei][noise], - color=color_list[i], - label=r"Validation; $\sigma = $" + str(sigma), - ) - else: - ax.plot( - range(n_epochs), - loss_train[typei][noise][0], - color=color_list[i], - ls="--", - ) - ax.plot( - range(n_epochs), - loss[typei][noise][0], - color=color_list[i], - ) - - ax.set_xlabel("Epoch") - if model[0:3] == "DER": - ax.set_ylabel("NIG Loss") - elif model[0:2] == "DE": - ax.set_ylabel(r"$\beta-$NLL Loss") - # ax.set_ylim([0, 5]) - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "all_loss_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() diff --git a/src/scripts/__version__.py b/src/scripts/__version__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/scripts/old_analysis/Aleatoric.py b/src/scripts/old_analysis/Aleatoric.py deleted file mode 100644 index ac883b7..0000000 --- a/src/scripts/old_analysis/Aleatoric.py +++ /dev/null @@ -1,343 +0,0 @@ -import os -import yaml -import argparse -import numpy as np -import torch -import matplotlib.pyplot as plt -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from data.data import DataPreparation -from analyze.analyze import AggregateCheckpoints - -# from plots import Plots - - -def parse_args(): - parser = argparse.ArgumentParser(description="data handling module") - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--data_prescription", - "-dp", - default=DefaultsAnalysis["model"]["data_prescription"], - ) - parser.add_argument( - "--data_dimension", - "-dd", - default=DefaultsAnalysis["model"]["data_dimension"], - ) - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER, either SDER or DER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--inject_type_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["inject_type_list"], - help="Feature and predictive", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "data_prescription": args.data_prescription, - "data_dimension": args.data_dimension, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "inject_type_list": args.inject_type_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - n_models = config.get_item("model", "n_models", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - prescription = config.get_item("model", "data_prescription", "Analysis") - inject_type_list = config.get_item( - "analysis", "inject_type_list", "Analysis" - ) - dim = config.get_item("model", "data_dimension", "Analysis") - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - if len(model_name_list) > 1: - assert "model_name_list should only be one item" - print("inject type list", inject_type_list) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - # make an empty nested dictionary with keys for - # model names followed by noise levels - al_dict = { - typei: {noise: [] for noise in noise_list} - for typei in inject_type_list - } - al_std_dict = { - typei: {noise: [] for noise in noise_list} - for typei in inject_type_list - } - n_epochs = config.get_item("model", "n_epochs", "Analysis") - # for model in model_name_list: - model = model_name_list[0] - for typei in inject_type_list: - for noise in noise_list: - # append a noise key - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - prescription, - typei, - dim, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - ) - # path=path_to_chk) - # things to grab: 'valid_mse' and 'valid_bnll' - ( - epistemic_m, - aleatoric_m, - e_std, - a_std, - ) = chk_module.ep_al_checkpoint_DER(chk) - al_dict[typei][noise].append(aleatoric_m) - al_std_dict[typei][noise].append(a_std) - - else: - n_models = config.get_item("model", "n_models", "DE") - for epoch in range(n_epochs): - list_mus = [] - list_vars = [] - for nmodels in range(n_models): - chk = chk_module.load_checkpoint( - model, - prescription, - typei, - dim, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodels, - ) - mu_vals, var_vals = chk_module.ep_al_checkpoint_DE(chk) - list_mus.append(mu_vals) - list_vars.append(var_vals) - # first taking the mean across the validation data - # then looking at the mean and standard deviation - # across all of the nmodels - al_dict[typei][noise].append( - np.mean(np.mean(list_vars, axis=0)) - ) - al_std_dict[typei][noise].append( - np.std(np.mean(list_vars, axis=0)) - ) - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - plt.clf() - fig = plt.figure(figsize=(10, 4)) - # try this instead with a fill_between method - for i, typei in enumerate(inject_type_list): - ax = fig.add_subplot(1, len(inject_type_list), i + 1) - # Your plotting code for each model here - ax.set_title(typei) # Set title for each subplot - for i, noise in enumerate(noise_list): - if model[0:3] == "DER": - al = np.array(al_dict[typei][noise]) - al_std = np.array(al_std_dict[typei][noise]) - elif model[0:2] == "DE": - # only take the sqrt for the case of DE, - # which is the variance - al = np.array(np.sqrt(al_dict[typei][noise])) - al_std = np.array(np.sqrt(al_std_dict[typei][noise])) - ax.fill_between( - range(n_epochs), - al - al_std, - al + al_std, - color=color_list[i], - alpha=0.25, - edgecolor=None, - ) - sigma = DataPreparation.get_sigma( - noise, inject_type=typei, data_dimension=dim - ) - ax.plot( - range(n_epochs), - al, - color=color_list[i], - label=r"$\sigma = $" + str(sigma), - ) - ax.axhline(y=sigma, color=color_list[i], ls="--") - ax.set_ylabel("Aleatoric Uncertainty") - ax.set_xlabel("Epoch") - # if model[0:3] == "DER": - # ax.set_title("Deep Evidential Regression") - # elif model[0:2] == "DE": - # ax.set_title("Deep Ensemble (100 models)") - ax.set_title(typei) - ax.set_ylim([0, 15]) - plt.legend() - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "aleatoric_uncertainty_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() diff --git a/src/scripts/old_analysis/AleatoricNHidden.py b/src/scripts/old_analysis/AleatoricNHidden.py deleted file mode 100644 index 93e387a..0000000 --- a/src/scripts/old_analysis/AleatoricNHidden.py +++ /dev/null @@ -1,327 +0,0 @@ -import os -import yaml -import argparse -import numpy as np -import torch -import matplotlib.pyplot as plt -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from data.data import DataPreparation -from analyze.analyze import AggregateCheckpoints - - -def parse_args(): - parser = argparse.ArgumentParser( - description="Analyzes the aleatoric uncertainty when the model \ - architecture is jittered" - ) - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER, either SDER or DER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - sigma_list = [] - for noise in noise_list: - sigma_list.append(DataPreparation.get_sigma(noise)) - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # this needs to be redone - n_hidden_list = [64, 54, 44, 34, 24] - rs = 1 - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - # make an empty nested dictionary with keys for - # model names followed by noise levels - al_dict = { - model_name: { - noise: {nh: [] for nh in n_hidden_list} for noise in noise_list - } - for model_name in model_name_list - } - al_std_dict = { - model_name: { - noise: {nh: [] for nh in n_hidden_list} for noise in noise_list - } - for model_name in model_name_list - } - n_epochs = config.get_item("model", "n_epochs", "Analysis") - for model in model_name_list: - for noise in noise_list: - for nh in n_hidden_list: - - # append a noise key - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - load_rs_chk=True, - rs=rs, - load_nh_chk=True, - nh=nh, - ) - # path=path_to_chk) - # things to grab: 'valid_mse' and 'valid_bnll' - ( - epistemic_m, - aleatoric_m, - e_std, - a_std, - ) = chk_module.ep_al_checkpoint_DER(chk) - al_dict[model][noise][nh].append(aleatoric_m) - al_std_dict[model][noise][nh].append(a_std) - - if model[0:3] == "DE_": - n_models = config.get_item("model", "n_models", "DE") - for epoch in range(n_epochs): - list_mus = [] - list_vars = [] - for nmodels in range(n_models): - chk = chk_module.load_checkpoint( - model, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodels, - ) - mu_vals, var_vals = chk_module.ep_al_checkpoint_DE(chk) - list_mus.append(mu_vals) - list_vars.append(var_vals) - try: - al_dict[model][noise][nmodels + 1].append( - np.mean(list_vars) - ) - except KeyError: - continue - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - plt.clf() - fig = plt.figure(figsize=(10, 4)) - # try this instead with a fill_between method - for i, model in enumerate(model_name_list): - ax = fig.add_subplot(1, len(model_name_list), i + 1) - # Your plotting code for each model here - ax.set_title(model) # Set title for each subplot - for n, noise in enumerate(noise_list): - for h, nh in enumerate(n_hidden_list): - if model[0:3] == "DE_": - al = np.array(np.sqrt(al_dict[model][noise][nh])) - al_std = np.array(np.sqrt(al_std_dict[model][noise][nh])) - else: - al = np.array(al_dict[model][noise][nh]) - al_std = np.array(al_std_dict[model][noise][nh]) - ax.fill_between( - range(n_epochs), - al - al_std, - al + al_std, - color=color_list[n], - alpha=0.1, - edgecolor=None, - ) - - if h == 0: - ax.plot( - range(n_epochs), - al, - color=color_list[n], - label=r"$\sigma = $" + str(sigma_list[n]), - ) - else: - ax.plot( - range(n_epochs), - al, - color=color_list[n], - ) - ax.axhline( - y=sigma_list[n], - color=color_list[n], - ls="--", - ) - ax.set_ylabel("Aleatoric Uncertainty") - ax.set_xlabel("Epoch") - if model[0:3] == "DER": - ax.set_title("Deep Evidential Regression") - elif model[0:2] == "DE": - ax.set_title("Deep Ensemble (100 models)") - ax.set_ylim([0, 6]) - plt.legend() - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "aleatoric_uncertainty_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() diff --git a/src/scripts/old_analysis/Aleatoric_and_inits.py b/src/scripts/old_analysis/Aleatoric_and_inits.py deleted file mode 100644 index 3c82ba7..0000000 --- a/src/scripts/old_analysis/Aleatoric_and_inits.py +++ /dev/null @@ -1,460 +0,0 @@ -import os -import yaml -import argparse -import numpy as np -import torch -import matplotlib.pyplot as plt -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from data.data import DataPreparation -from analyze.analyze import AggregateCheckpoints - -# from plots import Plots - - -def parse_args(): - parser = argparse.ArgumentParser(description="data handling module") - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--data_prescription", - type=str, - default=DefaultsAnalysis["model"]["data_prescription"], - help="Current only case is linear homoskedastic", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER, either SDER or DER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--inject_type_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["inject_type_list"], - help="Options are predictive and feature", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "data_prescription": args.data_prescription, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "inject_type_list": args.inject_type_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - prescription = config.get_item("model", "data_prescription", "Analysis") - inject_type_list = config.get_item( - "analysis", "inject_type_list", "Analysis" - ) - dim = config.get_item("model", "data_dimension", "Analysis") - sigma_list = [] - for noise in noise_list: - sigma_list.append(DataPreparation.get_sigma(noise)) - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - model = model_name_list[0] - print("one model at a time", model) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - # make an empty nested dictionary with keys for - # model names followed by noise levels - al_dict = { - typei: { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - for typei in inject_type_list - } - al_std_dict = { - typei: { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - for typei in inject_type_list - } - n_epochs = config.get_item("model", "n_epochs", "Analysis") - # switching from two panels for different models to - # two panels for different injection types - # could eventually make this into a four panel plot - # for model in model_name_list: - for inject_type in inject_type_list: - for model_name in model_name_list: - for noise in noise_list: - # append a noise key - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model_name, - prescription, - inject_type, - dim, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - load_nh_chk=False, - ) - # path=path_to_chk) - # things to grab: 'valid_mse' and 'valid_bnll' - ( - epistemic_m, - aleatoric_m, - e_std, - a_std, - ) = chk_module.ep_al_checkpoint_DER(chk) - al_dict[inject_type][model][noise].append(aleatoric_m) - al_std_dict[inject_type][model][noise].append(a_std) - - elif model[0:2] == "DE": - n_models = config.get_item("model", "n_models", "DE") - for epoch in range(n_epochs): - list_mus = [] - list_vars = [] - for nmodels in range(n_models): - chk = chk_module.load_checkpoint( - model, - prescription, - inject_type, - dim, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodels, - ) - mu_vals, var_vals = chk_module.ep_al_checkpoint_DE( - chk - ) - list_mus.append(mu_vals) - list_vars.append(var_vals) - # first taking the mean across the validation data - # then looking at the mean and standard deviation - # across all of the nmodels - al_dict[inject_type][model][noise].append( - np.mean(np.mean(list_vars, axis=0)) - ) - al_std_dict[inject_type][model][noise].append( - np.std(np.mean(list_vars, axis=0)) - ) - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - # this needs to be redone - rs_list = [10, 11, 12, 13, 14] - # make an empty nested dictionary with keys for - # model names followed by noise levels - al_rs_dict = { - model_name: {noise: {rs: [] for rs in rs_list} for noise in noise_list} - for model_name in model_name_list - } - """ - al_rs_std_dict = { - model_name: {noise: {rs: [] for rs in rs_list} for noise in noise_list} - for model_name in model_name_list - } - """ - n_epochs = config.get_item("model", "n_epochs", "Analysis") - # for model in model_name_list: - for inject_type in inject_type_list: - for noise in noise_list: - for rs in rs_list: - - # append a noise key - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - prescription, - inject_type, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - load_rs_chk=True, - rs=rs, - load_nh_chk=False, - ) - # path=path_to_chk) - # things to grab: 'valid_mse' and 'valid_bnll' - _, aleatoric_m, _, a_std = ( - chk_module.ep_al_checkpoint_DER(chk) - ) - al_rs_dict[model][noise][rs].append(aleatoric_m) - # al_std_dict[model][noise][rs].append(a_std) - if model[0:2] == "DE" and model[0:3] != "DER": - n_models = config.get_item("model", "n_models", "DE") - for epoch in range(n_epochs): - list_mus = [] - list_vars = [] - for nmodels in range(n_models): - chk = chk_module.load_checkpoint( - model, - prescription, - inject_type, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodels, - ) - mu_vals, var_vals = chk_module.ep_al_checkpoint_DE(chk) - list_mus.append(mu_vals) - list_vars.append(var_vals) - try: - al_rs_dict[model][noise][nmodels + 1].append( - np.mean(list_vars) - ) - except KeyError: - continue - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - plt.clf() - fig = plt.figure(figsize=(10, 4)) - # try this instead with a fill_between method - for i, model in enumerate(model_name_list): - ax = fig.add_subplot(1, len(model_name_list), i + 1) - # Your plotting code for each model here - ax.set_title(model) # Set title for each subplot - for n, noise in enumerate(noise_list): - if model[0:3] == "DER": - al = np.array(al_dict[model][noise]) - al_std = np.array(al_std_dict[model][noise]) - elif model[0:2] == "DE": - # only take the sqrt for the case of DE, - # which is the variance - al = np.array(np.sqrt(al_dict[model][noise])) - al_std = np.array(np.sqrt(al_std_dict[model][noise])) - ax.fill_between( - range(n_epochs), - al - al_std, - al + al_std, - color=color_list[n], - alpha=0.25, - edgecolor=None, - ) - ax.plot( - range(n_epochs), - al, - color=color_list[n], - label=r"$\sigma = $" + str(sigma_list[n]), - lw=3, - ) - for r, rs in enumerate(rs_list): - if model[0:3] == "DER": - al = np.array(al_rs_dict[model][noise][rs]) - elif model[0:2] == "DE": - al = np.array(np.sqrt(al_rs_dict[model][noise][rs])) - """ - # it doesn't really make sense to plot the std for the - # case of the DE because each individual model - # makes up one in the ensemble - """ - """ - if model[0:3] == "DER": - al_std = np.array(al_std_dict[model][noise][rs]) - ax.fill_between( - range(n_epochs), - al - al_std, - al + al_std, - color=color_list[n], - alpha=0.1, - edgecolor=None, - ) - """ - if r == 0: - ax.plot( - range(n_epochs), - al, - color=color_list[n], - ) - else: - ax.plot( - range(n_epochs), - al, - color=color_list[n], - ) - ax.axhline( - y=sigma_list[n], - color=color_list[n], - ls="--", - ) - ax.set_ylabel("Aleatoric Uncertainty") - ax.set_xlabel("Epoch") - if model[0:3] == "DER": - ax.set_title("Deep Evidential Regression") - elif model[0:2] == "DE": - ax.set_title("Deep Ensemble (100 models)") - ax.set_ylim([0, 15]) - plt.legend() - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "aleatoric_uncertainty_and_inits_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() diff --git a/src/scripts/old_analysis/AleatoricandEpistemic.py b/src/scripts/old_analysis/AleatoricandEpistemic.py deleted file mode 100644 index f68d53c..0000000 --- a/src/scripts/old_analysis/AleatoricandEpistemic.py +++ /dev/null @@ -1,396 +0,0 @@ -import os -import yaml -import argparse -import numpy as np -import torch -import matplotlib.pyplot as plt -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from data.data import DataPreparation -from analyze.analyze import AggregateCheckpoints - -# from plots import Plots - - -def parse_args(): - parser = argparse.ArgumentParser(description="data handling module") - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - sigma_list = [] - for noise in noise_list: - sigma_list.append(DataPreparation.get_sigma(noise)) - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - # make an empty nested dictionary with keys for - # model names followed by noise levels - ep_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - al_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - - ep_std_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - al_std_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - n_epochs = config.get_item("model", "n_epochs", "Analysis") - for model in model_name_list: - for noise in noise_list: - # append a noise key - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - ) - # path=path_to_chk) - # things to grab: 'valid_mse' and 'valid_bnll' - ( - epistemic_m, - aleatoric_m, - e_std, - a_std, - ) = chk_module.ep_al_checkpoint_DER(chk) - ep_dict[model][noise].append(epistemic_m) - al_dict[model][noise].append(aleatoric_m) - ep_std_dict[model][noise].append(e_std) - al_std_dict[model][noise].append(a_std) - - elif model[0:2] == "DE": - n_models = config.get_item("model", "n_models", "DE") - for epoch in range(n_epochs): - list_mus = [] - list_vars = [] - for nmodels in range(n_models): - chk = chk_module.load_checkpoint( - model, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodels, - ) - mu_vals, var_vals = chk_module.ep_al_checkpoint_DE(chk) - list_mus.append(mu_vals) - list_vars.append(var_vals) - ep_dict[model][noise].append( - np.mean(np.std(list_mus, axis=0)) - ) - ep_std_dict[model][noise].append( - np.std(np.std(list_mus, axis=0)) - ) - al_dict[model][noise].append( - np.mean(np.mean(list_vars, axis=0)) - ) - al_std_dict[model][noise].append( - np.std(np.mean(list_vars, axis=0)) - ) - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - plt.clf() - fig = plt.figure(figsize=(10, 4)) - # try this instead with a fill_between method - for i, model in enumerate(model_name_list): - ax = fig.add_subplot(1, len(model_name_list), i + 1) - # Your plotting code for each model here - ax.set_title(model) # Set title for each subplot - for i, noise in enumerate(noise_list): - if model[0:3] == "DER": - al = np.array(al_dict[model][noise]) - al_std = np.array(al_std_dict[model][noise]) - elif model[0:2] == "DE": - al = np.array(np.sqrt(al_dict[model][noise])) - al_std = np.array(np.sqrt(al_std_dict[model][noise])) - ep = np.array(ep_dict[model][noise]) - ep_std = np.array(ep_std_dict[model][noise]) - total = np.sqrt(al**2 + ep**2) - total_std = np.sqrt(al_std**2 + ep_std**2) - ax.fill_between( - range(n_epochs), - total - total_std, - total + total_std, - color=color_list[i], - alpha=0.25, - edgecolor=None, - ) - ax.plot( - range(n_epochs), - total, - color=color_list[i], - label=r"$\sigma = $" + str(sigma_list[i]), - ) - ax.axhline( - y=sigma_list[i], - color=color_list[i], - ls="--", - ) - ax.set_ylabel("Total Uncertainty") - ax.set_xlabel("Epoch") - if model[0:3] == "DER": - ax.set_title("Deep Evidential Regression") - elif model[0:2] == "DE": - ax.set_title("Deep Ensemble (100 models)") - ax.set_ylim([0, 15]) - plt.legend() - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "aleatoric_and_epistemic_uncertainty_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() - - plt.clf() - fig = plt.figure(figsize=(10, 4)) - # try this instead with a fill_between method - for i, model in enumerate(model_name_list): - ax = fig.add_subplot(1, len(model_name_list), i + 1) - # Your plotting code for each model here - ax.set_title(model) # Set title for each subplot - for i, noise in enumerate(noise_list): - if model[0:3] == "DER": - al = np.array(al_dict[model][noise]) - al_std = np.array(al_std_dict[model][noise]) - elif model[0:2] == "DE": - al = np.array(np.sqrt(al_dict[model][noise])) - al_std = np.array(np.sqrt(al_std_dict[model][noise])) - ep = np.array(ep_dict[model][noise]) - ep_std = np.array(ep_std_dict[model][noise]) - total = np.sqrt(al**2 + ep**2) - total_std = np.sqrt(al_std**2 + ep_std**2) - ax.fill_between( - range(n_epochs), - total - total_std, - total + total_std, - color=color_list[i], - alpha=0.25, - edgecolor=None, - ) - ax.plot( - range(n_epochs), - al, - color=color_list[i], - ls="dashed", - label=r"Aleatoric, $\sigma = $" + str(sigma_list[i]), - ) - ax.plot( - range(n_epochs), - ep, - color=color_list[i], - ls="dotted", - label=r"Epistemic, $\sigma = $" + str(sigma_list[i]), - ) - ax.plot( - range(n_epochs), - total, - color=color_list[i], - label=r"Total, $\sigma = $" + str(sigma_list[i]), - ) - ax.axhline( - y=sigma_list[i], - color=color_list[i], - ls="--", - ) - ax.set_ylabel("Total Uncertainty") - ax.set_xlabel("Epoch") - if model[0:3] == "DER": - ax.set_title("Deep Evidential Regression") - elif model[0:2] == "DE": - ax.set_title("Deep Ensemble (100 models)") - ax.set_ylim([0, 15]) - plt.legend() - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "aleatoric_and_epistemic_and_total_uncertainty_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() diff --git a/src/scripts/old_analysis/Epistemic.py b/src/scripts/old_analysis/Epistemic.py deleted file mode 100644 index 50d5a39..0000000 --- a/src/scripts/old_analysis/Epistemic.py +++ /dev/null @@ -1,312 +0,0 @@ -import os -import yaml -import argparse -import numpy as np -import torch -import matplotlib.pyplot as plt -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from data.data import DataPreparation -from analyze.analyze import AggregateCheckpoints - -# from plots import Plots - - -def parse_args(): - parser = argparse.ArgumentParser(description="data handling module") - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER, either DER or SDER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - sigma_list = [] - for noise in noise_list: - sigma_list.append(DataPreparation.get_sigma(noise)) - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - # make an empty nested dictionary with keys for - # model names followed by noise levels - ep_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - al_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - - ep_std_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - al_std_dict = { - model_name: {noise: [] for noise in noise_list} - for model_name in model_name_list - } - n_epochs = config.get_item("model", "n_epochs", "Analysis") - for model in model_name_list: - for noise in noise_list: - # append a noise key - # now run the analysis on the resulting checkpoints - if model[0:3] == "DER": - for epoch in range(n_epochs): - chk = chk_module.load_checkpoint( - model, - noise, - epoch, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - ) - # things to grab: 'valid_mse' and 'valid_bnll' - ( - epistemic_m, - aleatoric_m, - e_std, - a_std, - ) = chk_module.ep_al_checkpoint_DER(chk) - ep_dict[model][noise].append(epistemic_m) - al_dict[model][noise].append(aleatoric_m) - ep_std_dict[model][noise].append(e_std) - al_std_dict[model][noise].append(a_std) - - elif model[0:2] == "DE": - n_models = config.get_item("model", "n_models", "DE") - for epoch in range(n_epochs): - list_mus = [] - list_sigs = [] - for nmodels in range(n_models): - chk = chk_module.load_checkpoint( - model, - noise, - epoch, - DEVICE, - path=path_to_chk, - BETA=BETA, - nmodel=nmodels, - ) - mu_vals, sig_vals = chk_module.ep_al_checkpoint_DE(chk) - list_mus.append(mu_vals) - list_sigs.append(sig_vals) - ep_dict[model][noise].append( - np.median(np.std(list_mus, axis=0)) - ) - al_dict[model][noise].append( - np.median(np.mean(list_sigs, axis=0)) - ) - ep_std_dict[model][noise].append( - np.std(np.std(list_mus, axis=0)) - ) - al_std_dict[model][noise].append( - np.std(np.mean(list_sigs, axis=0)) - ) - # make a two-paneled plot for the different noise levels - # make one panel per model - # for the noise levels: - plt.clf() - fig = plt.figure(figsize=(10, 4)) - # try this instead with a fill_between method - for i, model in enumerate(model_name_list): - ax = fig.add_subplot(1, len(model_name_list), i + 1) - # Your plotting code for each model here - ax.set_title(model) # Set title for each subplot - for i, noise in enumerate(noise_list): - ep = np.array(ep_dict[model][noise]) - ep_std = np.array(ep_std_dict[model][noise]) - ax.fill_between( - range(n_epochs), - ep - ep_std, - ep + ep_std, - color=color_list[i], - alpha=0.25, - edgecolor=None, - ) - ax.plot( - range(n_epochs), - ep_dict[model][noise], - color=color_list[i], - label=r"$\sigma = $" + str(sigma_list[i]), - ) - ax.axhline(y=sigma_list[i], color=color_list[i]) - ax.set_ylabel("Epistemic Uncertainty") - ax.set_xlabel("Epoch") - if model[0:3] == "DER": - ax.set_title("Deep Evidential Regression") - elif model[0:2] == "DE": - ax.set_title("Deep Ensemble (100 models)") - # ax.set_ylim([-1, 15]) - plt.legend() - if config.get_item("analysis", "savefig", "Analysis"): - plt.savefig( - str(path_to_out) - + "epistemic_uncertainty_n_epochs_" - + str(n_epochs) - + "_n_models_DE_" - + str(n_models) - + ".png" - ) - if config.get_item("analysis", "plot", "Analysis"): - plt.show() diff --git a/src/scripts/old_analysis/ParityPlotTest.py b/src/scripts/old_analysis/ParityPlotTest.py deleted file mode 100644 index 42f332e..0000000 --- a/src/scripts/old_analysis/ParityPlotTest.py +++ /dev/null @@ -1,279 +0,0 @@ -import os -import yaml -import argparse -import numpy as np -import torch -import matplotlib.pyplot as plt -from utils.config import Config -from utils.defaults import DefaultsAnalysis -from data.data import DataPreparation -from analyze.analyze import AggregateCheckpoints -from torch.utils.data import TensorDataset -from models import models - -# from plots import Plots - - -def parse_args(): - parser = argparse.ArgumentParser(description="data handling module") - # there are three options with the parser: - # 1) Read from a yaml - # 2) Reads from the command line and default file - # and dumps to yaml - - # option to pass name of config - parser.add_argument("--config", "-c", default=None) - - # model - # we need some info about the model to run this analysis - # path to save the model results - parser.add_argument("--dir", default=DefaultsAnalysis["common"]["dir"]) - # now args for model - parser.add_argument( - "--n_models", - type=int, - default=DefaultsAnalysis["model"]["n_models"], - help="Number of MVEs in the ensemble", - ) - parser.add_argument( - "--BETA", - type=beta_type, - required=False, - default=DefaultsAnalysis["model"]["BETA"], - help="If loss_type is bnn_loss, specify a beta as a float or \ - there are string options: linear_decrease, \ - step_decrease_to_0.5, and step_decrease_to_1.0", - ) - parser.add_argument( - "--COEFF", - type=float, - required=False, - default=DefaultsAnalysis["model"]["COEFF"], - help="COEFF for DER", - ) - parser.add_argument( - "--loss_type", - type=str, - required=False, - default=DefaultsAnalysis["model"]["loss_type"], - help="loss_type for DER, either SDER or DER", - ) - parser.add_argument( - "--noise_level_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["noise_level_list"], - help="Noise levels to compare", - ) - parser.add_argument( - "--model_names_list", - type=list, - required=False, - default=DefaultsAnalysis["analysis"]["model_names_list"], - help="Beginning of name for saved checkpoints and figures", - ) - parser.add_argument( - "--n_epochs", - type=int, - required=False, - default=DefaultsAnalysis["model"]["n_epochs"], - help="number of epochs", - ) - parser.add_argument( - "--plot", - action="store_true", - default=DefaultsAnalysis["analysis"]["plot"], - help="option to plot in notebook", - ) - parser.add_argument( - "--color_list", - type=list, - default=DefaultsAnalysis["plots"]["color_list"], - help="list of named or hexcode colors to use for the noise levels", - ) - parser.add_argument( - "--savefig", - action="store_true", - default=DefaultsAnalysis["analysis"]["savefig"], - help="option to save a figure of the true and predicted values", - ) - parser.add_argument( - "--verbose", - action="store_true", - default=DefaultsAnalysis["analysis"]["verbose"], - help="verbose option for train", - ) - args = parser.parse_args() - args = parser.parse_args() - if args.config is not None: - print("Reading settings from config file", args.config) - config = Config(args.config) - - else: - temp_config = DefaultsAnalysis["common"]["temp_config"] - print( - "Reading settings from cli and default, \ - dumping to temp config: ", - temp_config, - ) - os.makedirs(os.path.dirname(temp_config), exist_ok=True) - - # check if args were specified in cli - input_yaml = { - "common": {"dir": args.dir}, - "model": { - "n_models": args.n_models, - "n_epochs": args.n_epochs, - "BETA": args.BETA, - "COEFF": args.COEFF, - "loss_type": args.loss_type, - }, - "analysis": { - "noise_level_list": args.noise_level_list, - "model_names_list": args.model_names_list, - "plot": args.plot, - "savefig": args.savefig, - "verbose": args.verbose, - }, - "plots": {"color_list": args.color_list}, - # "metrics": {key: {} for key in args.metrics}, - } - - yaml.dump(input_yaml, open(temp_config, "w")) - config = Config(temp_config) - - return config - # return parser.parse_args() - - -def beta_type(value): - if isinstance(value, float): - return value - elif value.lower() == "linear_decrease": - return value - elif value.lower() == "step_decrease_to_0.5": - return value - elif value.lower() == "step_decrease_to_1.0": - return value - else: - raise argparse.ArgumentTypeError( - "BETA must be a float or one of 'linear_decrease', \ - 'step_decrease_to_0.5', 'step_decrease_to_1.0'" - ) - - -if __name__ == "__main__": - config = parse_args() - DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") - noise_list = config.get_item("analysis", "noise_level_list", "Analysis") - color_list = config.get_item("plots", "color_list", "Analysis") - BETA = config.get_item("model", "BETA", "Analysis") - COEFF = config.get_item("model", "COEFF", "Analysis") - loss_type = config.get_item("model", "loss_type", "Analysis") - sigma_list = [] - for noise in noise_list: - sigma_list.append(DataPreparation.get_sigma(noise)) - root_dir = config.get_item("common", "dir", "Analysis") - path_to_chk = root_dir + "checkpoints/" - path_to_out = root_dir + "analysis/" - # check that this exists and if not make it - if not os.path.isdir(path_to_out): - print("does not exist, making dir", path_to_out) - os.mkdir(path_to_out) - else: - print("already exists", path_to_out) - model_name_list = config.get_item( - "analysis", "model_names_list", "Analysis" - ) - print("model list", model_name_list) - print("noise list", noise_list) - chk_module = AggregateCheckpoints() - - # now create a test set - data = DataPreparation() - data.sample_params_from_prior(1000) - data.simulate_data(data.params, 1, "linear_homogeneous", seed=41) - df_array = data.get_dict() - # Convert non-tensor entries to tensors - df = {} - for key, value in df_array.items(): - - if isinstance(value, TensorDataset): - # Keep tensors as they are - df[key] = value - else: - # Convert lists to tensors - df[key] = torch.tensor(value) - len_df = len(df["params"][:, 0].numpy()) - len_x = len(df["inputs"].numpy()) - ms_array = np.repeat(df["params"][:, 0].numpy(), len_x) - bs_array = np.repeat(df["params"][:, 1].numpy(), len_x) - xs_array = np.tile(df["inputs"].numpy(), len_df) - ys_array = np.reshape(df["output"].numpy(), (len_df * len_x)) - - inputs = np.array([xs_array, ms_array, bs_array]).T - model_inputs, model_outputs = DataPreparation.normalize( - inputs, ys_array, False - ) - _, x_test, _, y_test = DataPreparation.train_val_split( - model_inputs, - model_outputs, - val_proportion=0.1, - random_state=41, - ) - - # load up the checkpoints for DER - # and run it on the test data, make a parity plot - DERmodel, lossFn = models.model_setup_DER("DER", DEVICE, 64) - plt.clf() - fig = plt.figure() - ax = fig.add_subplot(211) - axr = fig.add_subplot(212) - - for i, noise in enumerate(noise_list): - chk = chk_module.load_checkpoint( - "DER", - noise, - 99, - DEVICE, - path=path_to_chk, - COEFF=COEFF, - loss=loss_type, - ) - # first, define the model at this epoch - DERmodel.load_state_dict(chk.get("model_state_dict")) - # checkpoint['model_state_dict']) - DERmodel.eval() - # now run on the x_test - y_pred = DERmodel(torch.Tensor(x_test)).detach().numpy() - - ax.scatter( - y_test, - y_pred[:, 0], - color=color_list[i], - label=r"$\sigma = $" + str(sigma_list[i]), - s=3, - ) - # ax.set_xlabel('True y') - ax.set_ylabel(r"Predicted $y$") - ax.plot( - range(-100, 1100), - range(-100, 1100), - ls="--", - color="grey", - ) - - axr.scatter( - y_test, - y_pred[:, 0] - y_test, - color=color_list[i], - label=r"$\sigma = $" + str(sigma_list[i]), - s=3, - zorder=-100, - ) - - axr.set_xlabel(r"True $y^*$") - axr.set_ylabel("Residual (predicted - true)") - axr.axhline(y=0, ls="--", color="grey") - plt.legend() - plt.show() diff --git a/test/test_DeepEnsemble.py b/test/test_DeepEnsemble.py index ad50912..d77d4f8 100644 --- a/test/test_DeepEnsemble.py +++ b/test/test_DeepEnsemble.py @@ -4,7 +4,7 @@ import tempfile import shutil import yaml -from src.data.data import MyDataLoader, DataPreparation +from deepuq.data.data import MyDataLoader, DataPreparation @pytest.fixture() @@ -231,7 +231,7 @@ def test_DE_from_saved_data( n_epochs = 2 subprocess_args = [ "python", - "src/scripts/DeepEnsemble.py", + "deepuq/scripts/DeepEnsemble.py", "--data_path", str(temp_data), "--size_df", @@ -303,7 +303,7 @@ def test_DE_chkpt_saved( n_epochs = 2 subprocess_args = [ "python", - "src/scripts/DeepEnsemble.py", + "deepuq/scripts/DeepEnsemble.py", "--data_path", str(temp_data), "--size_df", @@ -377,7 +377,7 @@ def test_DE_from_config( create_test_config(temp_directory + "/", temp_data, n_epochs) subprocess_args = [ "python", - "src/scripts/DeepEnsemble.py", + "deepuq/scripts/DeepEnsemble.py", "--config", str(temp_directory) + "/yamls/DE.yaml", ] @@ -437,7 +437,7 @@ def test_DE_no_chkpt_saved_xfail( n_epochs = 2 subprocess_args = [ "python", - "src/scripts/DeepEnsemble.py", + "deepuq/scripts/DeepEnsemble.py", "--data_path", str(temp_data), "--noise_level", @@ -484,7 +484,7 @@ def test_DE_only_final_chkpt_saved( n_epochs = 10 subprocess_args = [ "python", - "src/scripts/DeepEnsemble.py", + "deepuq/scripts/DeepEnsemble.py", "--data_path", temp_data, "--noise_level", @@ -533,7 +533,7 @@ def test_DE_all_epochs_chkpt_saved( n_epochs = 10 subprocess_args = [ "python", - "src/scripts/DeepEnsemble.py", + "deepuq/scripts/DeepEnsemble.py", "--data_path", temp_data, "--noise_level", @@ -582,7 +582,7 @@ def test_DE_run_simple_ensemble( n_models = 2 subprocess_args = [ "python", - "src/scripts/DeepEnsemble.py", + "deepuq/scripts/DeepEnsemble.py", "--data_path", temp_data, "--noise_level", diff --git a/test/test_DeepEvidentialRegression.py b/test/test_DeepEvidentialRegression.py index df1fde0..c41c84c 100644 --- a/test/test_DeepEvidentialRegression.py +++ b/test/test_DeepEvidentialRegression.py @@ -4,7 +4,7 @@ import tempfile import shutil import yaml -from src.data.data import MyDataLoader, DataPreparation +from deepuq.data.data import MyDataLoader, DataPreparation @pytest.fixture() @@ -230,7 +230,7 @@ def test_DER_all_chkpts_saved( n_epochs = 10 subprocess_args = [ "python", - "src/scripts/DeepEvidentialRegression.py", + "deepuq/scripts/DeepEvidentialRegression.py", "--data_path", str(temp_data), "--noise_level", @@ -287,7 +287,7 @@ def test_DER_one_chkpt_saved( n_epochs = 2 subprocess_args = [ "python", - "src/scripts/DeepEvidentialRegression.py", + "deepuq/scripts/DeepEvidentialRegression.py", "--data_path", str(temp_data), "--noise_level", @@ -367,7 +367,7 @@ def test_DER_from_config( create_test_config(temp_directory + "/", temp_data, n_epochs) subprocess_args = [ "python", - "src/scripts/DeepEvidentialRegression.py", + "deepuq/scripts/DeepEvidentialRegression.py", "--config", str(temp_directory) + "/yamls/DER.yaml", ]