From 09a474d78c69e1d882b669abae3fb2fa9f85f95f Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sat, 10 Oct 2020 10:26:39 +0200 Subject: [PATCH 001/211] Update nn.py --- traja/models/nn.py | 275 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 274 insertions(+), 1 deletion(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 6c1487d3..b89b8858 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -14,13 +14,286 @@ ) import torch.nn as nn import torch.optim as optim - +<<<<<<< Updated upstream + +======= +import os +import pandas as pd +from time import time +from sklearn.preprocessing import MinMaxScaler +from datetime import datetime +from sklearn.model_selection import train_test_split +from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler +>>>>>>> Stashed changes nb_steps = 10 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +<<<<<<< Updated upstream +======= +class TimeseriesDataset(Dataset): + # Loads the dataset and splits it into equally sized chunks. + # Whereas this can lead to uneven training data, + # with sufficiently long sequence lengths the + # bias should even out. + + def __init__(self, data_frame, sequence_length): + self.data = data_frame + self.sequence_length = sequence_length + + def __len__(self): + return int((self.data.shape[0]) / self.sequence_length) + + def __getitem__(self, index): + data = self.data[index * self.sequence_length: (index + 1) * self.sequence_length] + return data + +def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int): + """ Scale the timeseries dataset and generate train and test dataloaders + + Args: + data_frame (pd.DataFrame): Dataset + sequence_length (int): Sequence length of time series for a single gradient step + train_fraction (float): train data vs test data ratio + batch_size (int): Batch size of single gradient measure + + Returns: + train_loader (Dataloader) + validation_loader(Dataloader) + scaler (instance): Data scaler instance + """ + # Dataset transformation + scaler = MinMaxScaler(copy=False) + # scaler.fit(data_frame.values) + # scaled_dataset = scaler.fit_transform(data_frame.values) + dataset_length = int(data_frame.values.shape[0] / sequence_length) + indices = list(range(dataset_length)) + split = int(np.floor(train_fraction * dataset_length)) + train_indices, val_indices = indices[split:], indices[:split] + + # Creating PT data samplers and loaders: + train_sampler = SubsetRandomSampler(train_indices) + valid_sampler = SubsetRandomSampler(val_indices) + + dataset = TimeseriesDataset(data_frame.values, sequence_length) + + train_loader = DataLoader(dataset, batch_size=batch_size, + sampler=train_sampler) + validation_loader = DataLoader(dataset, batch_size=batch_size, + sampler=valid_sampler) + train_loader.name = "time_series" + return train_loader, validation_loader, scaler + +def get_transformed_timeseries_dataloaders_(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int): + """ Scale the timeseries dataset and generate train and test dataloaders + + Args: + data_frame (pd.DataFrame): Dataset + sequence_length (int): Sequence length of time series for a single gradient step + train_fraction (float): train data vs test data ratio + batch_size (int): Batch size of single gradient measure + + Returns: + train_loader (Dataloader) + validation_loader(Dataloader) + scaler (instance): Data scaler instance + """ + # Split the data into train and test: + train_dataset, test_dataset = train_test_split(data_frame.values, train_size=train_fraction) + dataset_length = int(data_frame.values.shape[0] / sequence_length) + indices = list(range(dataset_length)) + split = int(np.floor(train_fraction * dataset_length)) + train_indices, val_indices = indices[split:], indices[:split] + + # Creating PT data samplers and loaders: + train_sampler = SubsetRandomSampler(train_indices) + valid_sampler = SubsetRandomSampler(val_indices) + + # Dataset transformation; Train and test should have different scaler instances; + train_dataset_scaler = MinMaxScaler(copy=False) + scaled_train_dataset = train_dataset_scaler.fit_transform(train_dataset) + test_dataset_scaler = MinMaxScaler(copy=False) + scaled_test_dataset = test_dataset_scaler.fit_transform(test_dataset) + + # Convert transformed data into 3D tensor (batch_size,sequence_length, num_features) + train_dataset_3d = TimeseriesDataset(scaled_train_dataset, sequence_length) + test_dataset_3d = TimeseriesDataset(scaled_test_dataset, sequence_length) + + train_loader = DataLoader(train_dataset_3d, batch_size=batch_size, + sampler=train_sampler, drop_last=False ) + validation_loader = DataLoader(test_dataset_3d, batch_size=batch_size, + sampler=valid_sampler, drop_last=False ) + train_loader.name = "time_series" + return train_loader, validation_loader, train_dataset_scaler, test_dataset_scaler +class LossMseWarmup: + """ + Calculate the Mean Squared Error between y_true and y_pred, + but ignore the beginning "warmup" part of the sequences. + + y_true is the desired output. + y_pred is the model's output. + """ + def __init__(self, warmup_steps=50): + self.warmup_steps = warmup_steps + + def __call__(self, y_pred, y_true): + + y_true_slice = y_true[:, self.warmup_steps:, :] + y_pred_slice = y_pred[:, self.warmup_steps:, :] + + # Calculate the Mean Squared Error and use it as loss. + mse = torch.mean(torch.square(y_true_slice - y_pred_slice)) + + return mse + + +class Trainer: + def __init__(self, model, + train_loader, + test_loader, + epochs=200, + batch_size=60, + run_id=0, + logs_dir='logs', + device='cpu', + optimizer='None', + plot=True, + downsampling=None, + warmup_steps=50): + self.device = device + self.model = model + self.epochs = epochs + self.plot = plot + + self.train_loader = train_loader + self.test_loader = test_loader + + self.warmup_steps = warmup_steps + + self.criterion = LossMseWarmup(self.warmup_steps) + print('Checking for optimizer for {}'.format(optimizer)) + if optimizer == "adam": + print('Using adam') + self.optimizer = optim.Adam(model.parameters()) + elif optimizer == "adam_lr": + print("Using adam with higher learning rate") + self.optimizer = optim.Adam(model.parameters(), lr=0.01) + elif optimizer == 'adam_lr2': + print('Using adam with to large learning rate') + self.optimizer = optim.Adam(model.parameters(), lr=0.0001) + elif optimizer == "SGD": + print('Using SGD') + self.optimizer = optim.SGD(model.parameters(), momentum=0.9, weight_decay=5e-4) + elif optimizer == "LRS": + print('Using LRS') + self.optimizer = optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4) + self.lr_scheduler = optim.lr_scheduler.StepLR(self.optimizer, self.epochs // 3) + elif optimizer == "radam": + print('Using radam') + self.optimizer = RAdam(model.parameters()) + elif optimizer == "RMSprop": + print('Using RMSprop') + self.optimizer = optim.RMSprop(model.parameters()) + else: + raise ValueError('Unknown optimizer {}'.format(optimizer)) + self.opt_name = optimizer + save_dir = os.path.join(logs_dir, model.name, train_loader.name) + if not os.path.exists(save_dir): + os.makedirs(save_dir) + + self.savepath = os.path.join(save_dir, f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') + self.experiment_done = False + if os.path.exists(self.savepath): + trained_epochs = len(pd.read_csv(self.savepath, sep=';')) + + if trained_epochs >= epochs: + self.experiment_done = True + print(f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') + if os.path.exists((self.savepath.replace('.csv', '.pt'))): + self.model.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['model_state_dict']) + self.model = self.model.to(self.device) + + self.optimizer.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['optimizer']) + self.start_epoch = torch.load(self.savepath.replace('.csv', '.pt'))['epoch'] + 1 + else: + + self.start_epoch = 0 + self.model = self.model.to(self.device) + + + def _infer_initial_epoch(self, savepath): + if not os.path.exists(savepath): + return 0 + else: + df = pd.read_csv(savepath, sep=';', index_col=0) + print(len(df)+1) + return len(df) + + def train(self): + if self.experiment_done: + return + for epoch in range(self.start_epoch, self.epochs): + + print('Start training epoch', epoch) + print("{} Epoch {}, training loss: {}".format(datetime.now(), epoch, self.train_epoch())) + self.test(epoch=epoch) + if self.opt_name == "LRS": + print('LRS step') + self.lr_scheduler.step() + return self.savepath+'.csv' + + def train_epoch(self): + self.model.train() + total = 0 + running_loss = 0 + old_time = time() + for batch, data in enumerate(self.train_loader): + if batch % 10 == 0 and batch != 0: + print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'loss:', running_loss/total) + old_time = time() + inputs= data.to(self.device).float() + + self.optimizer.zero_grad() + outputs = self.model(inputs) + # For time series step prediction, targets are inputs + loss = self.criterion(outputs, inputs) + loss.backward() + self.optimizer.step() + + running_loss += loss.item() + # Increment number of batches + total += 1 + return running_loss/total + + def test(self, epoch, save=True): + self.model.eval() + total = 0 + test_loss = 0 + with torch.no_grad(): + for batch, data in enumerate(self.test_loader): + if batch % 10 == 0: + print('Processing eval batch', batch,'of', len(self.test_loader)) + inputs = data.to(self.device).float() + + outputs = self.model(inputs) + # For time series step prediction, targets are inputs + loss = self.criterion(outputs, inputs) + total += 1 + test_loss += loss.item() + + if save: + torch.save({ + 'model_state_dict': self.model.state_dict(), + 'optimizer': self.optimizer.state_dict(), + 'epoch': epoch, + 'test_loss': test_loss / total + }, self.savepath.replace('.csv', '.pt')) + return test_loss / total + + +>>>>>>> Stashed changes class LSTM(nn.Module): def __init__(self): super(LSTM, self).__init__() From 20a7ea1ece8411906c293d2455c377e623ceae74 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sat, 10 Oct 2020 10:30:34 +0200 Subject: [PATCH 002/211] updated stashed changes --- traja/models/nn.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index b89b8858..0c4d35df 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -14,9 +14,7 @@ ) import torch.nn as nn import torch.optim as optim -<<<<<<< Updated upstream -======= import os import pandas as pd from time import time @@ -24,15 +22,12 @@ from datetime import datetime from sklearn.model_selection import train_test_split from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler ->>>>>>> Stashed changes nb_steps = 10 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -<<<<<<< Updated upstream -======= class TimeseriesDataset(Dataset): # Loads the dataset and splits it into equally sized chunks. # Whereas this can lead to uneven training data, @@ -293,7 +288,6 @@ def test(self, epoch, save=True): return test_loss / total ->>>>>>> Stashed changes class LSTM(nn.Module): def __init__(self): super(LSTM, self).__init__() From 612f35818dd2c686f3715b02d67d208be075ffca Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sat, 10 Oct 2020 12:18:11 +0200 Subject: [PATCH 003/211] Update nn.py --- traja/models/nn.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 0c4d35df..89f79d64 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -20,8 +20,12 @@ from time import time from sklearn.preprocessing import MinMaxScaler from datetime import datetime +<<<<<<< Updated upstream from sklearn.model_selection import train_test_split +======= +>>>>>>> Stashed changes from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler +from sklearn.model_selection import train_test_split nb_steps = 10 @@ -122,6 +126,10 @@ def get_transformed_timeseries_dataloaders_(data_frame: pd.DataFrame, sequence_l sampler=valid_sampler, drop_last=False ) train_loader.name = "time_series" return train_loader, validation_loader, train_dataset_scaler, test_dataset_scaler +<<<<<<< Updated upstream +======= + +>>>>>>> Stashed changes class LossMseWarmup: """ Calculate the Mean Squared Error between y_true and y_pred, @@ -1031,4 +1039,4 @@ def forward(self, traj, traj_rel, seq_start_end=None): else: classifier_input = self.pool_net(final_h.squeeze(), seq_start_end, traj[0]) scores = self.real_classifier(classifier_input) - return scores + return scores \ No newline at end of file From 1607f5d9f6bb75e61dd1edf2aebe03ecd1726e34 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sat, 10 Oct 2020 12:18:29 +0200 Subject: [PATCH 004/211] Update nn.py --- traja/models/nn.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 89f79d64..41697a05 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -20,10 +20,7 @@ from time import time from sklearn.preprocessing import MinMaxScaler from datetime import datetime -<<<<<<< Updated upstream from sklearn.model_selection import train_test_split -======= ->>>>>>> Stashed changes from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler from sklearn.model_selection import train_test_split From c8dc55fa208550abd452b7bdd8d01c7a057952c2 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sat, 10 Oct 2020 12:19:49 +0200 Subject: [PATCH 005/211] Update __init__.py --- traja/models/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/traja/models/__init__.py b/traja/models/__init__.py index e69de29b..4fb6be76 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -0,0 +1 @@ +from .nn import LSTM, TimeseriesDataset, get_transformed_timeseries_dataloaders, get_transformed_timeseries_dataloaders_, Trainer \ No newline at end of file From 3c75d9e11b4547ddc6f96769795129bccbc72883 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sat, 10 Oct 2020 12:21:40 +0200 Subject: [PATCH 006/211] Update nn.py --- traja/models/nn.py | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 41697a05..b6509c13 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -123,10 +123,7 @@ def get_transformed_timeseries_dataloaders_(data_frame: pd.DataFrame, sequence_l sampler=valid_sampler, drop_last=False ) train_loader.name = "time_series" return train_loader, validation_loader, train_dataset_scaler, test_dataset_scaler -<<<<<<< Updated upstream -======= ->>>>>>> Stashed changes class LossMseWarmup: """ Calculate the Mean Squared Error between y_true and y_pred, @@ -292,20 +289,41 @@ def test(self, epoch, save=True): }, self.savepath.replace('.csv', '.pt')) return test_loss / total - class LSTM(nn.Module): - def __init__(self): + """ Deep LSTM network. This implementation + returns output_size outputs. + Args: + input_size: The number of expected features in the input `x` + hidden_size: The number of features in the hidden state `h` + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` + would mean stacking two LSTMs together to form a `stacked LSTM`, + with the second LSTM taking in outputs of the first LSTM and + computing the final results. Default: 1 + output_size: The number of output dimensions + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + LSTM layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + """ + + name = "LSTM" + + def __init__(self, input_size: int, hidden_size: int, num_layers: int, + output_size: int, dropout: float, bidirectional: bool): super(LSTM, self).__init__() - self.lstm = nn.LSTM(2, 100) - self.head = nn.Linear(100, 2) - def forward(self, x): - outputs, states = self.lstm(x) - outputs = outputs.reshape(x.shape[0] * x.shape[1], -1) - pred = self.head(outputs) + self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size, + num_layers=num_layers, dropout=dropout, + bidirectional=bidirectional) - return pred + self.head = nn.Linear(hidden_size, output_size) + def forward(self, x): + x, states = self.lstm(x) + #x = x.permute([1, 0, 2]) + #x = x.reshape(x.shape[0], x.shape[1] * x.shape[2]) + x = self.head(x) + return x class TrajectoryLSTM: def __init__( From 11cdaf7b182c17a24580fba9de5525798ac930b0 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sun, 11 Oct 2020 00:28:32 +0200 Subject: [PATCH 007/211] Update nn.py --- traja/models/nn.py | 172 ++++++++++++++++++++++++++++++--------------- 1 file changed, 115 insertions(+), 57 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index b6509c13..36e86eef 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -14,7 +14,6 @@ ) import torch.nn as nn import torch.optim as optim - import os import pandas as pd from time import time @@ -22,31 +21,74 @@ from datetime import datetime from sklearn.model_selection import train_test_split from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler -from sklearn.model_selection import train_test_split +import torchvision.transforms as transforms nb_steps = 10 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -class TimeseriesDataset(Dataset): - # Loads the dataset and splits it into equally sized chunks. - # Whereas this can lead to uneven training data, - # with sufficiently long sequence lengths the - # bias should even out. +class DataSequenceGenerator(torch.utils.data.Dataset): + + """ + Support class for the loading and batching of sequences of samples + + Args: + dataset (Tensor): Tensor containing all the samples + sequence_length (int): length of the analyzed sequence by the LSTM + transforms (object torchvision.transform): Pytorch's transforms used to process the data + """ - def __init__(self, data_frame, sequence_length): - self.data = data_frame - self.sequence_length = sequence_length + ## Constructor + def __init__(self, dataset, batch_size, sequence_length, transforms=None): + self.dataset = dataset + self.seq_length = sequence_length + self.transforms = transforms # List to tensors + self.batch_size = batch_size + ## Override total dataset's length getter def __len__(self): - return int((self.data.shape[0]) / self.sequence_length) + return self.dataset.__len__() + + ## Override single items' getter + def __getitem__(self, idx): + # if idx + self.seq_length > self.__len__(): + # if self.transforms is not None: + # item = torch.zeros(self.seq_length, self.dataset[0].__len__()) + # item[:self.__len__()-idx] = self.transforms(self.dataset[idx:]) + # return item, item + # else: + # item = [] + # item[:self.__len__()-idx] = self.dataset[idx:] + # return item, item + # else: + if self.transforms is not None: + return self.transforms(self.dataset[idx:idx+self.seq_length]), self.transforms(self.dataset[idx+self.seq_length:idx+self.seq_length+1]) + else: + return self.dataset[idx:idx+self.seq_length], self.dataset[idx+self.seq_length:idx+self.seq_length+1] - def __getitem__(self, index): - data = self.data[index * self.sequence_length: (index + 1) * self.sequence_length] - return data - -def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int): +class DataLoaderLSTM: + def __init__(self,dataset,batch_size, seq_length, num_workers=6): + self.dataset = dataset + self.batch_size = batch_size + self.seq_length = seq_length + self.num_workers = num_workers + self.name = None + def listToTensor(list): + tensor = torch.empty(list.__len__(), list[0].__len__()) + for i in range(list.__len__()): + tensor[i, :] = torch.FloatTensor(list[i]) + return tensor + + def load(self): + + data_transform = transforms.Lambda(lambda x: self.__class__.listToTensor(x)) + dataset = DataSequenceGenerator(self.dataset, self.batch_size, self.seq_length, transforms=data_transform) + data_loader = torch.utils.data.DataLoader(dataset, self.batch_size, shuffle=False, num_workers=self.num_workers) + setattr( data_loader, 'name', 'time_series' ) + return data_loader + +def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int, num_workers:int): """ Scale the timeseries dataset and generate train and test dataloaders Args: @@ -60,27 +102,26 @@ def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_le validation_loader(Dataloader) scaler (instance): Data scaler instance """ - # Dataset transformation - scaler = MinMaxScaler(copy=False) - # scaler.fit(data_frame.values) - # scaled_dataset = scaler.fit_transform(data_frame.values) - dataset_length = int(data_frame.values.shape[0] / sequence_length) - indices = list(range(dataset_length)) - split = int(np.floor(train_fraction * dataset_length)) - train_indices, val_indices = indices[split:], indices[:split] + + # Split the dataset into train and val + train,test = train_test_split(data_frame.values,train_size=train_fraction) - # Creating PT data samplers and loaders: - train_sampler = SubsetRandomSampler(train_indices) - valid_sampler = SubsetRandomSampler(val_indices) + # Train Dataset transformation + train_scaler = MinMaxScaler(copy=False) + train_scaler_fit = train_scaler.fit(train) + train_scaled_dataset = train_scaler_fit.transform(train) - dataset = TimeseriesDataset(data_frame.values, sequence_length) - - train_loader = DataLoader(dataset, batch_size=batch_size, - sampler=train_sampler) - validation_loader = DataLoader(dataset, batch_size=batch_size, - sampler=valid_sampler) - train_loader.name = "time_series" - return train_loader, validation_loader, scaler + # Test Dataset transformation + val_scaler = MinMaxScaler(copy=False) + val_scaler_fit = val_scaler.fit(test) + val_scaled_dataset = val_scaler_fit.transform(test) + + # Dataset and Dataloader for training and validation + train_loader = DataLoaderLSTM(train_scaled_dataset,batch_size,sequence_length,num_workers=num_workers) + validation_loader = DataLoaderLSTM(val_scaled_dataset,batch_size,sequence_length,num_workers=num_workers) + # train_loader.name = "time_series" + + return train_loader.load(), validation_loader.load(), train_scaler_fit, val_scaler_fit def get_transformed_timeseries_dataloaders_(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int): """ Scale the timeseries dataset and generate train and test dataloaders @@ -145,6 +186,22 @@ def __call__(self, y_pred, y_true): return mse +class LossMse: + """ + Calculate the Mean Squared Error between y_true and y_pred + + y_true is the desired output. + y_pred is the model's output. + """ + def __init__(self,warmup_steps=None): + self.warmup_steps = warmup_steps + + def __call__(self, y_pred, y_true): + + # Calculate the Mean Squared Error and use it as loss. + mse = torch.mean(torch.square(y_true - y_pred)) + + return mse class Trainer: def __init__(self, model, @@ -166,10 +223,11 @@ def __init__(self, model, self.train_loader = train_loader self.test_loader = test_loader - self.warmup_steps = warmup_steps - - self.criterion = LossMseWarmup(self.warmup_steps) + if self.warmup_steps!=None: + self.criterion = LossMseWarmup(self.warmup_steps) + else: + self.criterion = LossMse(self.warmup_steps) # warmup_steps == None print('Checking for optimizer for {}'.format(optimizer)) if optimizer == "adam": print('Using adam') @@ -247,19 +305,19 @@ def train_epoch(self): running_loss = 0 old_time = time() for batch, data in enumerate(self.train_loader): - if batch % 10 == 0 and batch != 0: - print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'loss:', running_loss/total) - old_time = time() - inputs= data.to(self.device).float() - + + inputs, targets= data[0].to(self.device).float(), data[1].to(self.device).float() self.optimizer.zero_grad() outputs = self.model(inputs) - # For time series step prediction, targets are inputs - loss = self.criterion(outputs, inputs) + loss = self.criterion(outputs, targets) loss.backward() self.optimizer.step() - running_loss += loss.item() + + if batch % 10 == 0 and batch != 0: + print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'loss:', running_loss/total) + old_time = time() + # Increment number of batches total += 1 return running_loss/total @@ -289,9 +347,12 @@ def test(self, epoch, save=True): }, self.savepath.replace('.csv', '.pt')) return test_loss / total + class LSTM(nn.Module): """ Deep LSTM network. This implementation returns output_size outputs. + + Args: input_size: The number of expected features in the input `x` hidden_size: The number of features in the hidden state `h` @@ -314,17 +375,20 @@ def __init__(self, input_size: int, hidden_size: int, num_layers: int, self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers, dropout=dropout, - bidirectional=bidirectional) + bidirectional=bidirectional, ) self.head = nn.Linear(hidden_size, output_size) def forward(self, x): - x, states = self.lstm(x) - #x = x.permute([1, 0, 2]) - #x = x.reshape(x.shape[0], x.shape[1] * x.shape[2]) + x, state = self.lstm(x) + # x = x.permute([1, 0, 2]) + # x = x.reshape(x.shape[0], x.shape[1] * x.shape[2]) + # Use the last hidden state of last layer + x = state[0][-1] # (batch_size,hidden_dim) x = self.head(x) return x + class TrajectoryLSTM: def __init__( self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() @@ -409,11 +473,6 @@ def plot(self, interactive=True): self._plot() return self.fig - -import torch -import torch.nn as nn - - def make_mlp(dim_list, activation="relu", batch_norm=True, dropout=0): layers = [] for dim_in, dim_out in zip(dim_list[:-1], dim_list[1:]): @@ -985,7 +1044,6 @@ def forward(self, obs_traj, obs_traj_rel, seq_start_end, user_noise=None): return pred_traj_fake_rel - class TrajectoryDiscriminator(nn.Module): def __init__( self, @@ -1054,4 +1112,4 @@ def forward(self, traj, traj_rel, seq_start_end=None): else: classifier_input = self.pool_net(final_h.squeeze(), seq_start_end, traj[0]) scores = self.real_classifier(classifier_input) - return scores \ No newline at end of file + return scores From e2d76bb3ec48182a9a74e665907f82835bf8b59b Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sun, 11 Oct 2020 13:19:29 +0200 Subject: [PATCH 008/211] Single step pred dataloader update Using Sequential batch sampler and random batch sampler --- traja/models/__init__.py | 3 +- traja/models/nn.py | 120 ++++++++++++++++++++++++++++++--------- 2 files changed, 94 insertions(+), 29 deletions(-) diff --git a/traja/models/__init__.py b/traja/models/__init__.py index 4fb6be76..6f734754 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1 +1,2 @@ -from .nn import LSTM, TimeseriesDataset, get_transformed_timeseries_dataloaders, get_transformed_timeseries_dataloaders_, Trainer \ No newline at end of file +from .nn import LSTM, TimeseriesDataset, get_transformed_timeseries_dataloaders, get_transformed_timeseries_dataloaders_, Trainer +from .nn import LSTM, TimeseriesDataset \ No newline at end of file diff --git a/traja/models/nn.py b/traja/models/nn.py index 36e86eef..72c30622 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -27,68 +27,105 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +class TimeseriesDataset(Dataset): + """ + Support class for the loading and Random batching of sequences of samples, + Args: + dataset (Tensor): Tensor containing all the samples + sequence_length (int): length of the analyzed sequence by the LSTM + """ + # Constructor + def __init__(self, dataset: np.array, sequence_length: int): + self.data = dataset + self.sequence_length = sequence_length + + # Override total dataset's length getter + def __len__(self): + return int((self.data.shape[0]) / self.sequence_length) + + # Override single items' getter + def __getitem__(self, index): + data = (self.data[index * self.sequence_length: (index + 1) * self.sequence_length], + self.data[index * self.sequence_length : (index + 1) * self.sequence_length]) + return data + class DataSequenceGenerator(torch.utils.data.Dataset): - """ - Support class for the loading and batching of sequences of samples + Support class for the loading and Sequential batching of sequences of samples, Args: dataset (Tensor): Tensor containing all the samples sequence_length (int): length of the analyzed sequence by the LSTM transforms (object torchvision.transform): Pytorch's transforms used to process the data + batch_size (int): Batch size """ - - ## Constructor - def __init__(self, dataset, batch_size, sequence_length, transforms=None): + # Constructor + def __init__(self, dataset: np.array, batch_size: int, sequence_length: int, transforms=None): + self.dataset = dataset self.seq_length = sequence_length self.transforms = transforms # List to tensors self.batch_size = batch_size - ## Override total dataset's length getter + # Override total dataset's length getter def __len__(self): return self.dataset.__len__() - ## Override single items' getter + # Override single items' getter def __getitem__(self, idx): - # if idx + self.seq_length > self.__len__(): - # if self.transforms is not None: - # item = torch.zeros(self.seq_length, self.dataset[0].__len__()) - # item[:self.__len__()-idx] = self.transforms(self.dataset[idx:]) - # return item, item - # else: - # item = [] - # item[:self.__len__()-idx] = self.dataset[idx:] - # return item, item - # else: + if self.transforms is not None: return self.transforms(self.dataset[idx:idx+self.seq_length]), self.transforms(self.dataset[idx+self.seq_length:idx+self.seq_length+1]) else: return self.dataset[idx:idx+self.seq_length], self.dataset[idx+self.seq_length:idx+self.seq_length+1] class DataLoaderLSTM: - def __init__(self,dataset,batch_size, seq_length, num_workers=6): + + """Dataloader object for both Random and sequential samplers. + + Args: + dataset (np.array): Dataset + batch_size (int): Batch size + seq_length (int): Sequence length at each batch + num_workers (int, optional): Number of subprocesses to deploy for dataloading. Defaults to 6. + random_sampler (Sampler object): + """ + + def __init__(self,dataset: np.array, batch_size: int, seq_length: int, random_sampler: str = None, num_workers: int=6): + self.dataset = dataset self.batch_size = batch_size self.seq_length = seq_length self.num_workers = num_workers self.name = None + def listToTensor(list): tensor = torch.empty(list.__len__(), list[0].__len__()) for i in range(list.__len__()): tensor[i, :] = torch.FloatTensor(list[i]) return tensor - def load(self): - + def load(self, random_sampler: str=None): + """Load the dataset using corresponding sampler + + Args: + random_sampler (Sampler instance name): If !=None, Random sampling of batches, otherwise Sequential. Defaults to None. + + Returns: + [torch.utils.data.Dataloader]: Dataloader + """ data_transform = transforms.Lambda(lambda x: self.__class__.listToTensor(x)) dataset = DataSequenceGenerator(self.dataset, self.batch_size, self.seq_length, transforms=data_transform) - data_loader = torch.utils.data.DataLoader(dataset, self.batch_size, shuffle=False, num_workers=self.num_workers) + if random_sampler!=None: + data_loader = torch.utils.data.DataLoader(dataset, self.batch_size, shuffle=False, sampler = random_sampler, num_workers=self.num_workers) + else: + data_loader = torch.utils.data.DataLoader(dataset, self.batch_size, shuffle=False, num_workers=self.num_workers) + setattr( data_loader, 'name', 'time_series' ) return data_loader -def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int, num_workers:int): +def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int, random_sampler: bool, num_workers:int): """ Scale the timeseries dataset and generate train and test dataloaders Args: @@ -116,12 +153,39 @@ def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_le val_scaler_fit = val_scaler.fit(test) val_scaled_dataset = val_scaler_fit.transform(test) - # Dataset and Dataloader for training and validation - train_loader = DataLoaderLSTM(train_scaled_dataset,batch_size,sequence_length,num_workers=num_workers) - validation_loader = DataLoaderLSTM(val_scaled_dataset,batch_size,sequence_length,num_workers=num_workers) - # train_loader.name = "time_series" - - return train_loader.load(), validation_loader.load(), train_scaler_fit, val_scaler_fit + if random_sampler: + # Concatenate train and val data and slice them again using their indices and SubsetRandomSampler + concat_dataset = np.concatenate((train_scaled_dataset,val_scaled_dataset),axis=0) + dataset_length = int(concat_dataset.shape[0] / sequence_length) + indices = list(range(dataset_length)) + split = int(np.floor(train_fraction * dataset_length)) + train_indices, val_indices = indices[split:], indices[:split] + + # Creating PT data samplers and loaders: + train_sampler = SubsetRandomSampler(train_indices) + valid_sampler = SubsetRandomSampler(val_indices) + # dataset = TimeseriesDataset(concat_dataset, sequence_length) + + # train_loader = DataLoader(dataset, batch_size=batch_size, + # sampler=train_sampler) + # validation_loader = DataLoader(dataset, batch_size=batch_size, + # sampler=valid_sampler) + # train_loader.name = "time_series" + # Sequential Dataloader for training and validation + train_loader = DataLoaderLSTM(train_scaled_dataset,batch_size,sequence_length, + random_sampler= train_sampler, num_workers=num_workers) + validation_loader = DataLoaderLSTM(val_scaled_dataset,batch_size,sequence_length, + random_sampler = valid_sampler, num_workers=num_workers) + + return train_loader.load(), validation_loader.load(), train_scaler_fit, val_scaler_fit + + else: + # Sequential Dataloader for training and validation + train_loader = DataLoaderLSTM(train_scaled_dataset,batch_size,sequence_length, + random_sampler= None, num_workers=num_workers) + validation_loader = DataLoaderLSTM(val_scaled_dataset,batch_size,sequence_length, + random_sampler = None, num_workers=num_workers) + return train_loader.load(), validation_loader.load(), train_scaler_fit, val_scaler_fit def get_transformed_timeseries_dataloaders_(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int): """ Scale the timeseries dataset and generate train and test dataloaders From 2aaf115ffa7f61f4ccb1c18cb4cd3ebf6438aae3 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sun, 11 Oct 2020 13:35:32 +0200 Subject: [PATCH 009/211] TODO: Remove warmup steps while train and test Use warmup steps feature defined in Optimizers --- traja/models/nn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 72c30622..06345e2c 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -82,7 +82,7 @@ def __getitem__(self, idx): class DataLoaderLSTM: - """Dataloader object for both Random and sequential samplers. + """Dataloader object for Single Step-prediction problems using Random and sequential samplers. Args: dataset (np.array): Dataset From 5cbaf2d1b2111ac0e6a0cb18ab050ac3a3ca131d Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sun, 11 Oct 2020 15:11:28 +0200 Subject: [PATCH 010/211] idx err resolved in dataloader --- traja/models/nn.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 06345e2c..8730d800 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -70,7 +70,7 @@ def __init__(self, dataset: np.array, batch_size: int, sequence_length: int, tra # Override total dataset's length getter def __len__(self): - return self.dataset.__len__() + return self.dataset.__len__()-self.seq_length # Override single items' getter def __getitem__(self, idx): @@ -117,6 +117,7 @@ def load(self, random_sampler: str=None): """ data_transform = transforms.Lambda(lambda x: self.__class__.listToTensor(x)) dataset = DataSequenceGenerator(self.dataset, self.batch_size, self.seq_length, transforms=data_transform) + # dataset = DataSequenceGenerator(self.dataset, self.batch_size, self.seq_length, transforms=None) if random_sampler!=None: data_loader = torch.utils.data.DataLoader(dataset, self.batch_size, shuffle=False, sampler = random_sampler, num_workers=self.num_workers) else: @@ -164,18 +165,12 @@ def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_le # Creating PT data samplers and loaders: train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) - # dataset = TimeseriesDataset(concat_dataset, sequence_length) - # train_loader = DataLoader(dataset, batch_size=batch_size, - # sampler=train_sampler) - # validation_loader = DataLoader(dataset, batch_size=batch_size, - # sampler=valid_sampler) - # train_loader.name = "time_series" # Sequential Dataloader for training and validation train_loader = DataLoaderLSTM(train_scaled_dataset,batch_size,sequence_length, random_sampler= train_sampler, num_workers=num_workers) validation_loader = DataLoaderLSTM(val_scaled_dataset,batch_size,sequence_length, - random_sampler = valid_sampler, num_workers=num_workers) + random_sampler= valid_sampler, num_workers=num_workers) return train_loader.load(), validation_loader.load(), train_scaler_fit, val_scaler_fit @@ -184,7 +179,7 @@ def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_le train_loader = DataLoaderLSTM(train_scaled_dataset,batch_size,sequence_length, random_sampler= None, num_workers=num_workers) validation_loader = DataLoaderLSTM(val_scaled_dataset,batch_size,sequence_length, - random_sampler = None, num_workers=num_workers) + random_sampler= None, num_workers=num_workers) return train_loader.load(), validation_loader.load(), train_scaler_fit, val_scaler_fit def get_transformed_timeseries_dataloaders_(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int): @@ -394,11 +389,9 @@ def test(self, epoch, save=True): for batch, data in enumerate(self.test_loader): if batch % 10 == 0: print('Processing eval batch', batch,'of', len(self.test_loader)) - inputs = data.to(self.device).float() - + inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() outputs = self.model(inputs) - # For time series step prediction, targets are inputs - loss = self.criterion(outputs, inputs) + loss = self.criterion(outputs, targets) total += 1 test_loss += loss.item() From 9f09d0e5e1df85c4ec51d7b2637b3344ab4d55dc Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sun, 11 Oct 2020 15:42:46 +0200 Subject: [PATCH 011/211] rmv ununsed __init__ imports --- traja/models/__init__.py | 3 +-- traja/models/nn.py | 50 +++------------------------------------- 2 files changed, 4 insertions(+), 49 deletions(-) diff --git a/traja/models/__init__.py b/traja/models/__init__.py index 6f734754..9dc49961 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1,2 +1 @@ -from .nn import LSTM, TimeseriesDataset, get_transformed_timeseries_dataloaders, get_transformed_timeseries_dataloaders_, Trainer -from .nn import LSTM, TimeseriesDataset \ No newline at end of file +from .nn import LSTM, TimeseriesDataset, get_transformed_timeseries_dataloaders, Trainer diff --git a/traja/models/nn.py b/traja/models/nn.py index 8730d800..be2bd0a2 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -182,48 +182,6 @@ def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_le random_sampler= None, num_workers=num_workers) return train_loader.load(), validation_loader.load(), train_scaler_fit, val_scaler_fit -def get_transformed_timeseries_dataloaders_(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int): - """ Scale the timeseries dataset and generate train and test dataloaders - - Args: - data_frame (pd.DataFrame): Dataset - sequence_length (int): Sequence length of time series for a single gradient step - train_fraction (float): train data vs test data ratio - batch_size (int): Batch size of single gradient measure - - Returns: - train_loader (Dataloader) - validation_loader(Dataloader) - scaler (instance): Data scaler instance - """ - # Split the data into train and test: - train_dataset, test_dataset = train_test_split(data_frame.values, train_size=train_fraction) - dataset_length = int(data_frame.values.shape[0] / sequence_length) - indices = list(range(dataset_length)) - split = int(np.floor(train_fraction * dataset_length)) - train_indices, val_indices = indices[split:], indices[:split] - - # Creating PT data samplers and loaders: - train_sampler = SubsetRandomSampler(train_indices) - valid_sampler = SubsetRandomSampler(val_indices) - - # Dataset transformation; Train and test should have different scaler instances; - train_dataset_scaler = MinMaxScaler(copy=False) - scaled_train_dataset = train_dataset_scaler.fit_transform(train_dataset) - test_dataset_scaler = MinMaxScaler(copy=False) - scaled_test_dataset = test_dataset_scaler.fit_transform(test_dataset) - - # Convert transformed data into 3D tensor (batch_size,sequence_length, num_features) - train_dataset_3d = TimeseriesDataset(scaled_train_dataset, sequence_length) - test_dataset_3d = TimeseriesDataset(scaled_test_dataset, sequence_length) - - train_loader = DataLoader(train_dataset_3d, batch_size=batch_size, - sampler=train_sampler, drop_last=False ) - validation_loader = DataLoader(test_dataset_3d, batch_size=batch_size, - sampler=valid_sampler, drop_last=False ) - train_loader.name = "time_series" - return train_loader, validation_loader, train_dataset_scaler, test_dataset_scaler - class LossMseWarmup: """ Calculate the Mean Squared Error between y_true and y_pred, @@ -436,16 +394,14 @@ def __init__(self, input_size: int, hidden_size: int, num_layers: int, self.head = nn.Linear(hidden_size, output_size) - def forward(self, x): + def forward(self, x): + # lstm(batch_size,seq_length,hidden_dim) ---> fc(hidden_dim,output_dim) x, state = self.lstm(x) - # x = x.permute([1, 0, 2]) - # x = x.reshape(x.shape[0], x.shape[1] * x.shape[2]) # Use the last hidden state of last layer - x = state[0][-1] # (batch_size,hidden_dim) + x = state[0][-1] x = self.head(x) return x - class TrajectoryLSTM: def __init__( self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() From 6cbbfb105b60ef2906616bfdfbf01aaa7a9c3a26 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sat, 3 Oct 2020 17:11:15 +0100 Subject: [PATCH 012/211] Add proper neural network support --- traja/__init__.py | 4 + traja/accessor.py | 14 ++- traja/models/__init__.py | 2 +- traja/models/nn.py | 229 ++++++++++++++++++++++++++++++++++++++- traja/tests/.gitignore | 1 + traja/tests/test_nn.py | 45 ++++++++ 6 files changed, 286 insertions(+), 9 deletions(-) create mode 100644 traja/tests/.gitignore create mode 100644 traja/tests/test_nn.py diff --git a/traja/__init__.py b/traja/__init__.py index 658c8b61..2e884412 100644 --- a/traja/__init__.py +++ b/traja/__init__.py @@ -13,3 +13,7 @@ __version__ = "0.2.3" logging.basicConfig(level=logging.INFO) + + +def set_traja_axes(axes: list): + TrajaAccessor._set_axes(axes) diff --git a/traja/accessor.py b/traja/accessor.py index 4002410c..0d9be885 100644 --- a/traja/accessor.py +++ b/traja/accessor.py @@ -16,6 +16,14 @@ def __init__(self, pandas_obj): self._validate(pandas_obj) self._obj = pandas_obj + __axes = ["x", "y"] + + @staticmethod + def _set_axes(axes): + if len(axes) != 2: + raise ValueError("TrajaAccessor requires precisely two axes, got {}".format(len(axes))) + TrajaAccessor.__axes = axes + def _strip(self, text): try: return text.strip() @@ -24,15 +32,15 @@ def _strip(self, text): @staticmethod def _validate(obj): - if "x" not in obj.columns or "y" not in obj.columns: - raise AttributeError("Must have 'x' and 'y'.") + if TrajaAccessor.__axes[0] not in obj.columns or TrajaAccessor.__axes[1] not in obj.columns: + raise AttributeError("Must have '{}' and '{}'.".format(*TrajaAccessor.__axes)) @property def center(self): """Return the center point of this trajectory.""" x = self._obj.x y = self._obj.y - return (float(x.mean()), float(y.mean())) + return float(x.mean()), float(y.mean()) @property def bounds(self): diff --git a/traja/models/__init__.py b/traja/models/__init__.py index 9dc49961..844f5020 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1 +1 @@ -from .nn import LSTM, TimeseriesDataset, get_transformed_timeseries_dataloaders, Trainer +from .nn import LSTM, TimeseriesDataset, get_timeseries_data_loaders, Trainer \ No newline at end of file diff --git a/traja/models/nn.py b/traja/models/nn.py index be2bd0a2..d711ff85 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -17,11 +17,10 @@ import os import pandas as pd from time import time -from sklearn.preprocessing import MinMaxScaler + from datetime import datetime -from sklearn.model_selection import train_test_split + from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler -import torchvision.transforms as transforms nb_steps = 10 @@ -363,6 +362,219 @@ def test(self, epoch, save=True): return test_loss / total +class TimeseriesDataset(Dataset): + # Loads the dataset and splits it into equally sized chunks. + # Whereas this can lead to uneven training data, + # with sufficiently long sequence lengths the + # bias should even out. + + def __init__(self, data_frame, sequence_length): + self.data = data_frame.filter(items=['x', 'y']) + self.sequence_length = sequence_length + + def __len__(self): + return int(self.data.shape[0] / self.sequence_length) + + def __getitem__(self, index): + data = (self.data.values[index * self.sequence_length: (index + 1) * self.sequence_length], + self.data.values[index * self.sequence_length: (index + 1) * self.sequence_length]) + return data + + +def get_timeseries_data_loaders(data_frame, sequence_length, train_fraction, batch_size): + + dataset_length = int(data_frame.shape[0] / sequence_length) + indices = list(range(dataset_length)) + split = int(np.floor(train_fraction * dataset_length)) + train_indices, val_indices = indices[split:], indices[:split] + + # Creating PT data samplers and loaders: + train_sampler = SubsetRandomSampler(train_indices) + valid_sampler = SubsetRandomSampler(val_indices) + + dataset = TimeseriesDataset(data_frame, sequence_length) + + train_loader = DataLoader(dataset, batch_size=batch_size, + sampler=train_sampler) + validation_loader = DataLoader(dataset, batch_size=batch_size, + sampler=valid_sampler) + + train_loader.name = "time_series" + + return train_loader, validation_loader + + +class LossMseWarmup: + """ + Calculate the Mean Squared Error between y_true and y_pred, + but ignore the beginning "warmup" part of the sequences. + + y_true is the desired output. + y_pred is the model's output. + """ + def __init__(self, warmup_steps=50): + self.warmup_steps = warmup_steps + + def __call__(self): + + y_true_slice = y_true[:, self.warmup_steps:, :] + y_pred_slice = y_pred[:, self.warmup_steps:, :] + + # Calculate the Mean Squared Error and use it as loss. + mse = torch.mean(torch.square(y_true_slice - y_pred_slice)) + + return mse + + +class Trainer: + def __init__(self, model, + train_loader, + test_loader, + epochs=200, + batch_size=60, + run_id=0, + logs_dir='logs', + device='cpu', + optimizer='None', + plot=True, + downsampling=None, + warmup_steps=50): + self.device = device + self.model = model + self.epochs = epochs + self.plot = plot + + self.train_loader = train_loader + self.test_loader = test_loader + + self.warmup_steps = warmup_steps + + self.criterion = LossMseWarmup(self.warmup_steps) + print('Checking for optimizer for {}'.format(optimizer)) + if optimizer == "adam": + print('Using adam') + self.optimizer = optim.Adam(model.parameters()) + elif optimizer == "adam_lr": + print("Using adam with higher learning rate") + self.optimizer = optim.Adam(model.parameters(), lr=0.01) + elif optimizer == 'adam_lr2': + print('Using adam with to large learning rate') + self.optimizer = optim.Adam(model.parameters(), lr=0.0001) + elif optimizer == "SGD": + print('Using SGD') + self.optimizer = optim.SGD(model.parameters(), momentum=0.9, weight_decay=5e-4) + elif optimizer == "LRS": + print('Using LRS') + self.optimizer = optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4) + self.lr_scheduler = optim.lr_scheduler.StepLR(self.optimizer, self.epochs // 3) + elif optimizer == "radam": + print('Using radam') + self.optimizer = RAdam(model.parameters()) + else: + raise ValueError('Unknown optimizer {}'.format(optimizer)) + self.opt_name = optimizer + save_dir = os.path.join(logs_dir, model.name, train_loader.name) + if not os.path.exists(save_dir): + os.makedirs(save_dir) + + self.savepath = os.path.join(save_dir, f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') + self.experiment_done = False + if os.path.exists(self.savepath): + trained_epochs = len(pd.read_csv(self.savepath, sep=';')) + + if trained_epochs >= epochs: + self.experiment_done = True + print(f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') + if os.path.exists((self.savepath.replace('.csv', '.pt'))): + self.model.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['model_state_dict']) + self.model = self.model.to(self.device) + + self.optimizer.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['optimizer']) + self.start_epoch = torch.load(self.savepath.replace('.csv', '.pt'))['epoch'] + 1 + else: + + self.start_epoch = 0 + self.model = self.model.to(self.device) + + + def _infer_initial_epoch(self, savepath): + if not os.path.exists(savepath): + return 0 + else: + df = pd.read_csv(savepath, sep=';', index_col=0) + print(len(df)+1) + return len(df) + + def train(self): + if self.experiment_done: + return + for epoch in range(self.start_epoch, self.epochs): + + print('Start training epoch', epoch) + print("{} Epoch {}, training loss: {}, training accuracy: {}".format(datetime.now(), epoch, *self.train_epoch())) + self.test(epoch=epoch) + if self.opt_name == "LRS": + print('LRS step') + self.lr_scheduler.step() + return self.savepath+'.csv' + + def train_epoch(self): + self.model.train() + correct = 0 + total = 0 + running_loss = 0 + old_time = time() + top5_accumulator = 0 + for batch, data in enumerate(self.train_loader): + if batch % 10 == 0 and batch != 0: + print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, "top5_acc:" if self.compute_top_k else 'acc:', round(top5_accumulator/(batch),3) if self.compute_top_k else correct/total) + old_time = time() + inputs, labels = data + inputs, labels = inputs.to(self.device).float(), labels.to(self.device).float() + + self.optimizer.zero_grad() + outputs = self.model(inputs) + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + + loss = self.criterion(outputs, labels) + loss.backward() + self.optimizer.step() + correct += (predicted == labels.long()).sum().item() + + running_loss += loss.item() + + return running_loss/total, correct/total + + def test(self, epoch, save=True): + self.model.eval() + correct = 0 + total = 0 + test_loss = 0 + with torch.no_grad(): + for batch, data in enumerate(self.test_loader): + if batch % 10 == 0: + print('Processing eval batch', batch,'of', len(self.test_loader)) + inputs, labels = data + inputs, labels = inputs.to(self.device).float(), labels.to(self.device).float() + + outputs = self.model(inputs) + loss = self.criterion(outputs, labels) + _, predicted = torch.max(outputs.data, 1) + total += labels.size(0) + correct += (predicted == labels.long()).sum().item() + test_loss += loss.item() + + if save: + torch.save({ + 'model_state_dict': self.model.state_dict(), + 'optimizer': self.optimizer.state_dict(), + 'epoch': epoch, + 'test_loss': test_loss / total + }, self.savepath.replace('.csv', '.pt')) + return correct / total, test_loss / total + + class LSTM(nn.Module): """ Deep LSTM network. This implementation returns output_size outputs. @@ -390,10 +602,17 @@ def __init__(self, input_size: int, hidden_size: int, num_layers: int, self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers, dropout=dropout, - bidirectional=bidirectional, ) + bidirectional=bidirectional) self.head = nn.Linear(hidden_size, output_size) + def forward(self, x): + x, states = self.lstm(x) + #x = x.permute([1, 0, 2]) + #x = x.reshape(x.shape[0], x.shape[1] * x.shape[2]) + x = self.head(x) + return x + def forward(self, x): # lstm(batch_size,seq_length,hidden_dim) ---> fc(hidden_dim,output_dim) x, state = self.lstm(x) @@ -409,7 +628,7 @@ def __init__( fig, ax = plt.subplots(2, 1) self.fig = fig self.ax = ax - assert xy.shape[1] is 2, f"xy should be an N x 2 array, but is {xy.shape}" + assert xy.shape[1] == 2, f"xy should be an N x 2 array, but is {xy.shape}" self.xy = xy self.nb_steps = nb_steps self.epochs = epochs diff --git a/traja/tests/.gitignore b/traja/tests/.gitignore new file mode 100644 index 00000000..333c1e91 --- /dev/null +++ b/traja/tests/.gitignore @@ -0,0 +1 @@ +logs/ diff --git a/traja/tests/test_nn.py b/traja/tests/test_nn.py new file mode 100644 index 00000000..64150c41 --- /dev/null +++ b/traja/tests/test_nn.py @@ -0,0 +1,45 @@ + +import sys +sys.path.append('../../delve') + +import delve + + +import pandas as pd + +import traja + + + + + + +def test_from_df(): + df = traja.generate(n=20) + save_path = 'temp/test' + + warmup_steps = 1 + + # Run 1 + timeseries_method = 'last_timestep' + + model = traja.models.LSTM(input_size=2, hidden_size=2, num_layers=3, output_size=2, dropout=0, bidirectional=False) + writer = delve.writers.CSVandPlottingWriter(save_path, fontsize=16, primary_metric='test_accuracy') + saturation = delve.CheckLayerSat(save_path, + [writer], model, + stats=['embed'], + timeseries_method=timeseries_method) + + sequence_length = 5 + train_fraction = .25 + batch_size = 1 + + train_loader, test_loader = traja.models.get_timeseries_data_loaders(df, sequence_length, + train_fraction, batch_size) + + trainer = traja.models.Trainer(model, train_loader, test_loader, epochs=6, optimizer="adam", + warmup_steps=warmup_steps) + + trainer.train() + + pass \ No newline at end of file From cb4f6bfa1a30473800b6397ecab6a1f4a475d523 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sat, 3 Oct 2020 23:18:49 +0100 Subject: [PATCH 013/211] Add time shift to neural network data sets --- traja/models/nn.py | 11 ++++++----- traja/tests/test_nn.py | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index d711ff85..30bb53a0 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -368,20 +368,21 @@ class TimeseriesDataset(Dataset): # with sufficiently long sequence lengths the # bias should even out. - def __init__(self, data_frame, sequence_length): + def __init__(self, data_frame, sequence_length, shift): self.data = data_frame.filter(items=['x', 'y']) self.sequence_length = sequence_length + self.shift = shift def __len__(self): - return int(self.data.shape[0] / self.sequence_length) + return int((self.data.shape[0] - self.shift) / self.sequence_length) def __getitem__(self, index): data = (self.data.values[index * self.sequence_length: (index + 1) * self.sequence_length], - self.data.values[index * self.sequence_length: (index + 1) * self.sequence_length]) + self.data.values[index * self.sequence_length: (index + 1) * self.sequence_length + self.shift]) return data -def get_timeseries_data_loaders(data_frame, sequence_length, train_fraction, batch_size): +def get_timeseries_data_loaders(data_frame, sequence_length, train_fraction, batch_size, shift): dataset_length = int(data_frame.shape[0] / sequence_length) indices = list(range(dataset_length)) @@ -392,7 +393,7 @@ def get_timeseries_data_loaders(data_frame, sequence_length, train_fraction, bat train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) - dataset = TimeseriesDataset(data_frame, sequence_length) + dataset = TimeseriesDataset(data_frame, sequence_length, shift) train_loader = DataLoader(dataset, batch_size=batch_size, sampler=train_sampler) diff --git a/traja/tests/test_nn.py b/traja/tests/test_nn.py index 64150c41..2e73bc11 100644 --- a/traja/tests/test_nn.py +++ b/traja/tests/test_nn.py @@ -33,9 +33,10 @@ def test_from_df(): sequence_length = 5 train_fraction = .25 batch_size = 1 + shift = 2 train_loader, test_loader = traja.models.get_timeseries_data_loaders(df, sequence_length, - train_fraction, batch_size) + train_fraction, batch_size, shift) trainer = traja.models.Trainer(model, train_loader, test_loader, epochs=6, optimizer="adam", warmup_steps=warmup_steps) From ef8df986f80ebc68c177c5e000cd62fa184b45be Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sun, 4 Oct 2020 11:53:09 +0100 Subject: [PATCH 014/211] Fix mse call function error --- traja/models/nn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 30bb53a0..58907f20 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -416,7 +416,7 @@ class LossMseWarmup: def __init__(self, warmup_steps=50): self.warmup_steps = warmup_steps - def __call__(self): + def __call__(self, y_pred, y_true): y_true_slice = y_true[:, self.warmup_steps:, :] y_pred_slice = y_pred[:, self.warmup_steps:, :] From b04cfe4811064b71f0e5395898d60f225246cb5b Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sun, 4 Oct 2020 11:57:49 +0100 Subject: [PATCH 015/211] Remove top 5 references from regressor --- traja/models/nn.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 58907f20..99810b10 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -525,10 +525,9 @@ def train_epoch(self): total = 0 running_loss = 0 old_time = time() - top5_accumulator = 0 for batch, data in enumerate(self.train_loader): if batch % 10 == 0 and batch != 0: - print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, "top5_acc:" if self.compute_top_k else 'acc:', round(top5_accumulator/(batch),3) if self.compute_top_k else correct/total) + print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'acc:', correct/total) old_time = time() inputs, labels = data inputs, labels = inputs.to(self.device).float(), labels.to(self.device).float() From cdb1e5e22fd94749003b9e7f6a455d43c9dfce32 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sun, 4 Oct 2020 12:06:30 +0100 Subject: [PATCH 016/211] Move df filtering to test --- traja/models/nn.py | 1 - traja/tests/test_nn.py | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 99810b10..b792ec30 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -369,7 +369,6 @@ class TimeseriesDataset(Dataset): # bias should even out. def __init__(self, data_frame, sequence_length, shift): - self.data = data_frame.filter(items=['x', 'y']) self.sequence_length = sequence_length self.shift = shift diff --git a/traja/tests/test_nn.py b/traja/tests/test_nn.py index 2e73bc11..359b3c03 100644 --- a/traja/tests/test_nn.py +++ b/traja/tests/test_nn.py @@ -16,6 +16,9 @@ def test_from_df(): df = traja.generate(n=20) + + df = df.filter(items=['x', 'y']) + save_path = 'temp/test' warmup_steps = 1 From 454591f9512643a8047acd255c7002ddfc99e2ad Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sun, 4 Oct 2020 12:09:23 +0100 Subject: [PATCH 017/211] Keep dataframe in dataset --- traja/models/nn.py | 1 + 1 file changed, 1 insertion(+) diff --git a/traja/models/nn.py b/traja/models/nn.py index b792ec30..983276f4 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -369,6 +369,7 @@ class TimeseriesDataset(Dataset): # bias should even out. def __init__(self, data_frame, sequence_length, shift): + self.data = data_frame self.sequence_length = sequence_length self.shift = shift From e570c1525001f33eaf424df1679b58554da4ea2a Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sun, 4 Oct 2020 12:17:28 +0100 Subject: [PATCH 018/211] Debugging the shape of mse predictions --- traja/models/nn.py | 1 + 1 file changed, 1 insertion(+) diff --git a/traja/models/nn.py b/traja/models/nn.py index 983276f4..32395dbc 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -422,6 +422,7 @@ def __call__(self, y_pred, y_true): y_pred_slice = y_pred[:, self.warmup_steps:, :] # Calculate the Mean Squared Error and use it as loss. + print("Shapes", y_true_slice.shape, y_pred_slice.shape) mse = torch.mean(torch.square(y_true_slice - y_pred_slice)) return mse From 4dcf0ff407aa97ddccb912d174435d4ef51d8588 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sun, 4 Oct 2020 12:25:58 +0100 Subject: [PATCH 019/211] Additional debugging statements --- traja/models/nn.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 32395dbc..06a44d24 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -417,12 +417,13 @@ def __init__(self, warmup_steps=50): self.warmup_steps = warmup_steps def __call__(self, y_pred, y_true): + print("Shapes", y_true, y_pred) y_true_slice = y_true[:, self.warmup_steps:, :] y_pred_slice = y_pred[:, self.warmup_steps:, :] # Calculate the Mean Squared Error and use it as loss. - print("Shapes", y_true_slice.shape, y_pred_slice.shape) + print("Shapes truncated", y_true_slice.shape, y_pred_slice.shape) mse = torch.mean(torch.square(y_true_slice - y_pred_slice)) return mse @@ -532,6 +533,7 @@ def train_epoch(self): old_time = time() inputs, labels = data inputs, labels = inputs.to(self.device).float(), labels.to(self.device).float() + print("Input, labels", inputs.shape, labels.shape) self.optimizer.zero_grad() outputs = self.model(inputs) From f7e4a9a27cb1afaf49d021651c2ff57b8016a69c Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sun, 4 Oct 2020 12:31:20 +0100 Subject: [PATCH 020/211] Shift both start and end of ground truth --- traja/models/nn.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 06a44d24..79a7a3e2 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -378,7 +378,7 @@ def __len__(self): def __getitem__(self, index): data = (self.data.values[index * self.sequence_length: (index + 1) * self.sequence_length], - self.data.values[index * self.sequence_length: (index + 1) * self.sequence_length + self.shift]) + self.data.values[index * self.sequence_length + self.shift: (index + 1) * self.sequence_length + self.shift]) return data @@ -417,13 +417,11 @@ def __init__(self, warmup_steps=50): self.warmup_steps = warmup_steps def __call__(self, y_pred, y_true): - print("Shapes", y_true, y_pred) y_true_slice = y_true[:, self.warmup_steps:, :] y_pred_slice = y_pred[:, self.warmup_steps:, :] # Calculate the Mean Squared Error and use it as loss. - print("Shapes truncated", y_true_slice.shape, y_pred_slice.shape) mse = torch.mean(torch.square(y_true_slice - y_pred_slice)) return mse @@ -533,7 +531,6 @@ def train_epoch(self): old_time = time() inputs, labels = data inputs, labels = inputs.to(self.device).float(), labels.to(self.device).float() - print("Input, labels", inputs.shape, labels.shape) self.optimizer.zero_grad() outputs = self.model(inputs) From c2788905c3a64c74fc79ecf03c4683357cdbcd6d Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sun, 4 Oct 2020 14:45:11 +0100 Subject: [PATCH 021/211] Remove label 'correct' count --- traja/models/nn.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 79a7a3e2..47673bc9 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -512,7 +512,7 @@ def train(self): for epoch in range(self.start_epoch, self.epochs): print('Start training epoch', epoch) - print("{} Epoch {}, training loss: {}, training accuracy: {}".format(datetime.now(), epoch, *self.train_epoch())) + print("{} Epoch {}, training loss: {}".format(datetime.now(), epoch, self.train_epoch())) self.test(epoch=epoch) if self.opt_name == "LRS": print('LRS step') @@ -521,13 +521,12 @@ def train(self): def train_epoch(self): self.model.train() - correct = 0 total = 0 running_loss = 0 old_time = time() for batch, data in enumerate(self.train_loader): if batch % 10 == 0 and batch != 0: - print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'acc:', correct/total) + print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'loss:', running_loss/total) old_time = time() inputs, labels = data inputs, labels = inputs.to(self.device).float(), labels.to(self.device).float() @@ -540,11 +539,10 @@ def train_epoch(self): loss = self.criterion(outputs, labels) loss.backward() self.optimizer.step() - correct += (predicted == labels.long()).sum().item() running_loss += loss.item() - return running_loss/total, correct/total + return running_loss/total def test(self, epoch, save=True): self.model.eval() From 8becd5c8fae9ab683398d1a821fd3c37da4d5f4a Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sun, 4 Oct 2020 14:48:19 +0100 Subject: [PATCH 022/211] Remove the same from test --- traja/models/nn.py | 4 +--- traja/tests/test_nn.py | 8 ++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 47673bc9..84af0629 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -546,7 +546,6 @@ def train_epoch(self): def test(self, epoch, save=True): self.model.eval() - correct = 0 total = 0 test_loss = 0 with torch.no_grad(): @@ -560,7 +559,6 @@ def test(self, epoch, save=True): loss = self.criterion(outputs, labels) _, predicted = torch.max(outputs.data, 1) total += labels.size(0) - correct += (predicted == labels.long()).sum().item() test_loss += loss.item() if save: @@ -570,7 +568,7 @@ def test(self, epoch, save=True): 'epoch': epoch, 'test_loss': test_loss / total }, self.savepath.replace('.csv', '.pt')) - return correct / total, test_loss / total + return test_loss / total class LSTM(nn.Module): diff --git a/traja/tests/test_nn.py b/traja/tests/test_nn.py index 359b3c03..3ddb4570 100644 --- a/traja/tests/test_nn.py +++ b/traja/tests/test_nn.py @@ -15,13 +15,13 @@ def test_from_df(): - df = traja.generate(n=20) + df = traja.generate(n=599581) df = df.filter(items=['x', 'y']) save_path = 'temp/test' - warmup_steps = 1 + warmup_steps = 50 # Run 1 timeseries_method = 'last_timestep' @@ -33,9 +33,9 @@ def test_from_df(): stats=['embed'], timeseries_method=timeseries_method) - sequence_length = 5 + sequence_length = 2000 train_fraction = .25 - batch_size = 1 + batch_size = 50 shift = 2 train_loader, test_loader = traja.models.get_timeseries_data_loaders(df, sequence_length, From 48d467ae5889a6606bb13109bae007512fc6dcce Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sun, 4 Oct 2020 22:32:02 +0100 Subject: [PATCH 023/211] Add RMSprop to neural network --- traja/models/nn.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/traja/models/nn.py b/traja/models/nn.py index 84af0629..ef33ae6a 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -471,6 +471,9 @@ def __init__(self, model, elif optimizer == "radam": print('Using radam') self.optimizer = RAdam(model.parameters()) + elif optimizer == "RMSprop": + print('Using RMSprop') + self.optimizer = optim.RMSprop(model.parameters()) else: raise ValueError('Unknown optimizer {}'.format(optimizer)) self.opt_name = optimizer From 887cf7ab171630231b370498521dc47b6ea3e444 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 7 Oct 2020 11:32:41 +0200 Subject: [PATCH 024/211] dataset transformer func --- traja/models/nn.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index ef33ae6a..9ca6d513 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -17,7 +17,7 @@ import os import pandas as pd from time import time - +from sklearn.preprocessing import MinMaxScaler from datetime import datetime from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler @@ -380,8 +380,7 @@ def __getitem__(self, index): data = (self.data.values[index * self.sequence_length: (index + 1) * self.sequence_length], self.data.values[index * self.sequence_length + self.shift: (index + 1) * self.sequence_length + self.shift]) return data - - + def get_timeseries_data_loaders(data_frame, sequence_length, train_fraction, batch_size, shift): dataset_length = int(data_frame.shape[0] / sequence_length) @@ -404,6 +403,29 @@ def get_timeseries_data_loaders(data_frame, sequence_length, train_fraction, bat return train_loader, validation_loader +def get_transformed_timeseries_dataloaders(data_frame, sequence_length, train_fraction, batch_size, shift): + + dataset_length = int(data_frame.shape[0] / sequence_length) + indices = list(range(dataset_length)) + split = int(np.floor(train_fraction * dataset_length)) + train_indices, val_indices = indices[split:], indices[:split] + + # Creating PT data samplers and loaders: + train_sampler = SubsetRandomSampler(train_indices) + valid_sampler = SubsetRandomSampler(val_indices) + + dataset = TimeseriesDataset(data_frame, sequence_length, shift) + + # Dataset transformation + scaler = MinMaxScaler() + scaled_dataset = scaler.fit_transform(dataset) + + train_loader = DataLoader(scaled_dataset, batch_size=batch_size, + sampler=train_sampler) + validation_loader = DataLoader(scaled_dataset, batch_size=batch_size, + sampler=valid_sampler) + train_loader.name = "time_series" + return train_loader, validation_loader, scaler class LossMseWarmup: """ From 5765537b734104cdf7bcece5ec67e6e453c23dcb Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 7 Oct 2020 20:34:29 +0200 Subject: [PATCH 025/211] get_timeseries_data_loaders deprecated get_timeseries_data_loaders replaced by get_transformed_timeseries_data_loaders --- traja/models/nn.py | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 9ca6d513..fb3e2925 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -381,29 +381,21 @@ def __getitem__(self, index): self.data.values[index * self.sequence_length + self.shift: (index + 1) * self.sequence_length + self.shift]) return data -def get_timeseries_data_loaders(data_frame, sequence_length, train_fraction, batch_size, shift): +def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int, shift:int): + """ Scale the timeseries dataset and return train and test dataloaders - dataset_length = int(data_frame.shape[0] / sequence_length) - indices = list(range(dataset_length)) - split = int(np.floor(train_fraction * dataset_length)) - train_indices, val_indices = indices[split:], indices[:split] - - # Creating PT data samplers and loaders: - train_sampler = SubsetRandomSampler(train_indices) - valid_sampler = SubsetRandomSampler(val_indices) - - dataset = TimeseriesDataset(data_frame, sequence_length, shift) - - train_loader = DataLoader(dataset, batch_size=batch_size, - sampler=train_sampler) - validation_loader = DataLoader(dataset, batch_size=batch_size, - sampler=valid_sampler) - - train_loader.name = "time_series" - - return train_loader, validation_loader + Args: + data_frame (pd.DataFrame): Dataset + sequence_length (int): Sequence length of time series for a single gradient step + train_fraction (float): train data vs test data ratio + batch_size (int): Batch size of single gradient measure + shift (int): [description] -def get_transformed_timeseries_dataloaders(data_frame, sequence_length, train_fraction, batch_size, shift): + Returns: + train_loader (Dataloader) + validation_loader(Dataloader) + scaler (instance): Data scaler instance + """ dataset_length = int(data_frame.shape[0] / sequence_length) indices = list(range(dataset_length)) From fbd62797dea17d998f7fd37444282ad49ed4bddb Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 7 Oct 2020 20:42:11 +0200 Subject: [PATCH 026/211] remoevd shift var from TimeSeriesDataset class --- traja/models/nn.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index fb3e2925..237e0bce 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -368,28 +368,26 @@ class TimeseriesDataset(Dataset): # with sufficiently long sequence lengths the # bias should even out. - def __init__(self, data_frame, sequence_length, shift): + def __init__(self, data_frame, sequence_length): self.data = data_frame self.sequence_length = sequence_length - self.shift = shift def __len__(self): - return int((self.data.shape[0] - self.shift) / self.sequence_length) + return int((self.data.shape[0]) / self.sequence_length) def __getitem__(self, index): data = (self.data.values[index * self.sequence_length: (index + 1) * self.sequence_length], - self.data.values[index * self.sequence_length + self.shift: (index + 1) * self.sequence_length + self.shift]) + self.data.values[index * self.sequence_length : (index + 1) * self.sequence_length]) return data -def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int, shift:int): - """ Scale the timeseries dataset and return train and test dataloaders +def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int): + """ Scale the timeseries dataset and generate train and test dataloaders Args: data_frame (pd.DataFrame): Dataset sequence_length (int): Sequence length of time series for a single gradient step train_fraction (float): train data vs test data ratio batch_size (int): Batch size of single gradient measure - shift (int): [description] Returns: train_loader (Dataloader) @@ -406,7 +404,7 @@ def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_le train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) - dataset = TimeseriesDataset(data_frame, sequence_length, shift) + dataset = TimeseriesDataset(data_frame, sequence_length) # Dataset transformation scaler = MinMaxScaler() From 55033031268fbaa8491fe77b114b67646ba8482a Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 8 Oct 2020 19:22:50 +0200 Subject: [PATCH 027/211] Update __init__.py --- traja/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/__init__.py b/traja/models/__init__.py index 844f5020..e01389e0 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1 +1 @@ -from .nn import LSTM, TimeseriesDataset, get_timeseries_data_loaders, Trainer \ No newline at end of file +from .nn import LSTM, TimeseriesDataset, get_transformed_timeseries_data_loaders, Trainer \ No newline at end of file From d4462ffe242cf3e0fbc079a436104907431ad315 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 8 Oct 2020 22:10:25 +0200 Subject: [PATCH 028/211] TimeSeriesDataset type(data): pd.Dataframe --> np array --- traja/models/__init__.py | 2 +- traja/models/nn.py | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/traja/models/__init__.py b/traja/models/__init__.py index e01389e0..399397d7 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1 +1 @@ -from .nn import LSTM, TimeseriesDataset, get_transformed_timeseries_data_loaders, Trainer \ No newline at end of file +from .nn import LSTM, TimeseriesDataset, get_transformed_timeseries_dataloaders, Trainer \ No newline at end of file diff --git a/traja/models/nn.py b/traja/models/nn.py index 237e0bce..8b18922a 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -376,8 +376,8 @@ def __len__(self): return int((self.data.shape[0]) / self.sequence_length) def __getitem__(self, index): - data = (self.data.values[index * self.sequence_length: (index + 1) * self.sequence_length], - self.data.values[index * self.sequence_length : (index + 1) * self.sequence_length]) + data = (self.data[index * self.sequence_length: (index + 1) * self.sequence_length], + self.data[index * self.sequence_length : (index + 1) * self.sequence_length]) return data def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int): @@ -394,8 +394,10 @@ def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_le validation_loader(Dataloader) scaler (instance): Data scaler instance """ - - dataset_length = int(data_frame.shape[0] / sequence_length) + # Dataset transformation + scaler = MinMaxScaler(copy=False) + scaled_dataset = scaler.fit_transform(data_frame.values) + dataset_length = int(scaled_dataset.shape[0] / sequence_length) indices = list(range(dataset_length)) split = int(np.floor(train_fraction * dataset_length)) train_indices, val_indices = indices[split:], indices[:split] @@ -404,15 +406,11 @@ def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_le train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) - dataset = TimeseriesDataset(data_frame, sequence_length) - - # Dataset transformation - scaler = MinMaxScaler() - scaled_dataset = scaler.fit_transform(dataset) + dataset = TimeseriesDataset(scaled_dataset, sequence_length) - train_loader = DataLoader(scaled_dataset, batch_size=batch_size, + train_loader = DataLoader(dataset, batch_size=batch_size, sampler=train_sampler) - validation_loader = DataLoader(scaled_dataset, batch_size=batch_size, + validation_loader = DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler) train_loader.name = "time_series" return train_loader, validation_loader, scaler From 72dec6a49d8b549fb08d6c1724600b6bdf9475a2 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sun, 11 Oct 2020 23:39:03 +0200 Subject: [PATCH 029/211] Update nn.py user defined trial dir name, removed warmup_step arg --- traja/models/nn.py | 93 ++++++++++++++++------------------------------ 1 file changed, 33 insertions(+), 60 deletions(-) diff --git a/traja/models/nn.py b/traja/models/nn.py index 8b18922a..9d9b43da 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -5,6 +5,7 @@ import matplotlib.pyplot as plt import numpy as np +from torch.utils.data import sampler try: import torch @@ -88,15 +89,18 @@ class DataLoaderLSTM: batch_size (int): Batch size seq_length (int): Sequence length at each batch num_workers (int, optional): Number of subprocesses to deploy for dataloading. Defaults to 6. - random_sampler (Sampler object): + random_sampler (Sampler instance name): If !=None, Random sampling of batches, otherwise Sequential. Defaults to None. + dir_name (str): Target folder name where the model will be saved after training: './logs//' """ - def __init__(self,dataset: np.array, batch_size: int, seq_length: int, random_sampler: str = None, num_workers: int=6): + def __init__(self,dataset: np.array, batch_size: int, seq_length: int, random_sampler: str=None, dir_name: str=None, num_workers: int=6): self.dataset = dataset self.batch_size = batch_size self.seq_length = seq_length self.num_workers = num_workers + self.random_sampler = random_sampler + self.dir_name = dir_name self.name = None def listToTensor(list): @@ -105,27 +109,24 @@ def listToTensor(list): tensor[i, :] = torch.FloatTensor(list[i]) return tensor - def load(self, random_sampler: str=None): + def load(self): """Load the dataset using corresponding sampler - - Args: - random_sampler (Sampler instance name): If !=None, Random sampling of batches, otherwise Sequential. Defaults to None. - Returns: [torch.utils.data.Dataloader]: Dataloader """ data_transform = transforms.Lambda(lambda x: self.__class__.listToTensor(x)) dataset = DataSequenceGenerator(self.dataset, self.batch_size, self.seq_length, transforms=data_transform) - # dataset = DataSequenceGenerator(self.dataset, self.batch_size, self.seq_length, transforms=None) - if random_sampler!=None: - data_loader = torch.utils.data.DataLoader(dataset, self.batch_size, shuffle=False, sampler = random_sampler, num_workers=self.num_workers) + + if self.random_sampler!=None: + data_loader = torch.utils.data.DataLoader(dataset, self.batch_size, shuffle=False, sampler = self.random_sampler, num_workers=self.num_workers) else: data_loader = torch.utils.data.DataLoader(dataset, self.batch_size, shuffle=False, num_workers=self.num_workers) - - setattr( data_loader, 'name', 'time_series' ) + + # Directory where the model will be saved + setattr( data_loader, 'name', self.dir_name ) return data_loader -def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int, random_sampler: bool, num_workers:int): +def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int, random_sampler: bool, dir_name: str, num_workers:int): """ Scale the timeseries dataset and generate train and test dataloaders Args: @@ -133,7 +134,8 @@ def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_le sequence_length (int): Sequence length of time series for a single gradient step train_fraction (float): train data vs test data ratio batch_size (int): Batch size of single gradient measure - + dir_name (str: optional): Directory name that corresponds to type of running task, where the trained model will be saved' + Returns: train_loader (Dataloader) validation_loader(Dataloader) @@ -155,53 +157,30 @@ def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_le if random_sampler: # Concatenate train and val data and slice them again using their indices and SubsetRandomSampler - concat_dataset = np.concatenate((train_scaled_dataset,val_scaled_dataset),axis=0) - dataset_length = int(concat_dataset.shape[0] / sequence_length) + concat_dataset = np.concatenate((train_scaled_dataset, val_scaled_dataset),axis=0) + dataset_length = int(concat_dataset.shape[0]- sequence_length) indices = list(range(dataset_length)) split = int(np.floor(train_fraction * dataset_length)) - train_indices, val_indices = indices[split:], indices[:split] + train_indices, val_indices = indices[:split], indices[split:] # Creating PT data samplers and loaders: train_sampler = SubsetRandomSampler(train_indices) valid_sampler = SubsetRandomSampler(val_indices) - # Sequential Dataloader for training and validation - train_loader = DataLoaderLSTM(train_scaled_dataset,batch_size,sequence_length, - random_sampler= train_sampler, num_workers=num_workers) - validation_loader = DataLoaderLSTM(val_scaled_dataset,batch_size,sequence_length, - random_sampler= valid_sampler, num_workers=num_workers) - - return train_loader.load(), validation_loader.load(), train_scaler_fit, val_scaler_fit + # Random Dataloader for training and validation + train_loader = DataLoaderLSTM(concat_dataset, batch_size, sequence_length, + random_sampler= train_sampler, dir_name = dir_name, num_workers=num_workers) + validation_loader = DataLoaderLSTM(concat_dataset,batch_size,sequence_length, + random_sampler= valid_sampler, dir_name = dir_name, num_workers=num_workers) + + return train_loader.load() , validation_loader.load(), train_scaler_fit, val_scaler_fit else: # Sequential Dataloader for training and validation - train_loader = DataLoaderLSTM(train_scaled_dataset,batch_size,sequence_length, - random_sampler= None, num_workers=num_workers) - validation_loader = DataLoaderLSTM(val_scaled_dataset,batch_size,sequence_length, - random_sampler= None, num_workers=num_workers) + train_loader = DataLoaderLSTM(train_scaled_dataset, batch_size, sequence_length, random_sampler= None, dir_name = dir_name, num_workers=num_workers) + validation_loader = DataLoaderLSTM(val_scaled_dataset,batch_size,sequence_length, random_sampler= None, dir_name = dir_name, num_workers=num_workers) return train_loader.load(), validation_loader.load(), train_scaler_fit, val_scaler_fit -class LossMseWarmup: - """ - Calculate the Mean Squared Error between y_true and y_pred, - but ignore the beginning "warmup" part of the sequences. - - y_true is the desired output. - y_pred is the model's output. - """ - def __init__(self, warmup_steps=50): - self.warmup_steps = warmup_steps - - def __call__(self, y_pred, y_true): - - y_true_slice = y_true[:, self.warmup_steps:, :] - y_pred_slice = y_pred[:, self.warmup_steps:, :] - - # Calculate the Mean Squared Error and use it as loss. - mse = torch.mean(torch.square(y_true_slice - y_pred_slice)) - - return mse - class LossMse: """ Calculate the Mean Squared Error between y_true and y_pred @@ -209,9 +188,8 @@ class LossMse: y_true is the desired output. y_pred is the model's output. """ - def __init__(self,warmup_steps=None): - self.warmup_steps = warmup_steps - + def __init__(self) -> None: + pass def __call__(self, y_pred, y_true): # Calculate the Mean Squared Error and use it as loss. @@ -230,8 +208,7 @@ def __init__(self, model, device='cpu', optimizer='None', plot=True, - downsampling=None, - warmup_steps=50): + downsampling=None): self.device = device self.model = model self.epochs = epochs @@ -239,11 +216,8 @@ def __init__(self, model, self.train_loader = train_loader self.test_loader = test_loader - self.warmup_steps = warmup_steps - if self.warmup_steps!=None: - self.criterion = LossMseWarmup(self.warmup_steps) - else: - self.criterion = LossMse(self.warmup_steps) # warmup_steps == None + + self.criterion = LossMse() print('Checking for optimizer for {}'.format(optimizer)) if optimizer == "adam": print('Using adam') @@ -623,7 +597,6 @@ def forward(self, x): return x def forward(self, x): - # lstm(batch_size,seq_length,hidden_dim) ---> fc(hidden_dim,output_dim) x, state = self.lstm(x) # Use the last hidden state of last layer x = state[0][-1] From a2d8d53711fb41b2af487640223a57c70ed99ef7 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sat, 28 Nov 2020 17:46:33 +0100 Subject: [PATCH 030/211] check ae, vae todo vaegan,irl,lstm --- traja/datasets/dataset.py | 89 +++- traja/datasets/utils.py | 120 +++++ traja/models/ae.py | 257 ++++++++++ traja/models/experiment.py | 929 ++++++++++++++++++++++++++++++++++++ traja/models/generator.py | 62 +++ traja/models/interpretor.py | 26 + traja/models/irl.py | 20 + traja/models/losses.py | 69 +++ traja/models/lstm.py | 22 + traja/models/nn.py | 156 ------ traja/models/train.py | 295 ++++++++++++ traja/models/utils.py | 114 +++++ traja/models/vae.py | 7 + traja/models/vaegan.py | 27 ++ 14 files changed, 2035 insertions(+), 158 deletions(-) create mode 100644 traja/datasets/utils.py create mode 100644 traja/models/ae.py create mode 100644 traja/models/experiment.py create mode 100644 traja/models/generator.py create mode 100644 traja/models/interpretor.py create mode 100644 traja/models/irl.py create mode 100644 traja/models/losses.py create mode 100644 traja/models/lstm.py create mode 100644 traja/models/train.py create mode 100644 traja/models/utils.py create mode 100644 traja/models/vae.py create mode 100644 traja/models/vaegan.py diff --git a/traja/datasets/dataset.py b/traja/datasets/dataset.py index b30ff986..55b51138 100644 --- a/traja/datasets/dataset.py +++ b/traja/datasets/dataset.py @@ -1,15 +1,27 @@ """ Modified from https://github.com/agrimgupta92/sgan/blob/master/sgan/data/trajectories.py. +This module contains: + +Classes: +1. Pytorch Time series dataset class instance +2. Weighted train and test dataset loader with respect to class distribution + +Helpers: +1. Class distribution in the dataset + """ import logging import os import math - import numpy as np - import torch from torch.utils.data import Dataset +from collections import Counter +from torch.utils.data.sampler import WeightedRandomSampler +import pandas as pd +from sklearn.utils import shuffle +from datasets import utils logger = logging.getLogger(__name__) @@ -210,3 +222,76 @@ def __getitem__(self, index): self.loss_mask[start:end, :], ] return out + +class TimeSeriesDataset(Dataset): + r"""Pytorch Dataset object + + Args: + Dataset (torch.utils.data.Dataset): Pyptorch dataset object + """ + def __init__(self,data, target, category): + r""" + Args: + data (array): Data + target (array): Target + category (array): Category + """ + self.data = data + self.target = target + self.category = category + + def __getitem__(self, index): + x = self.data[index] + y = self.target[index] + z = self.category[index] + return x, y,z + + def __len__(self): + return len(self.data) + +class MultiModalDataLoader(object): + r""" Data loader object. Wrap Data preprocessors, dataset, sampler and return the data loader object""" + + def __init__(self, df:pd.DataFrame, batch_size:int, n_past:int, n_future:int, num_workers: int): + + # Extract/generate data from the pandas df + train_data, target_data, target_category = utils.generate_dataset(df, n_past, n_future) + + # Shuffle and split the data + [train_x,train_y,train_z],[test_x,test_y,test_z] = utils.shuffle_split(train_data, target_data, target_category, train_ratio = 0.75) + + # Scale data + (train_x, train_x_scaler),(train_y,train_y_scaler) = utils.scale_data(train_x,sequence_length=n_past),utils.scale_data(train_y,sequence_length=n_future) + (test_x,test_x_scaler),(test_y,test_y_scaler) = utils.scale_data(test_x,sequence_length=n_past),utils.scale_data(test_y,sequence_length=n_future) + + # Weighted Random Sampler + train_weighted_sampler, test_weighted_sampler = utils.weighted_random_samplers(train_z,test_z) + + # Dataset + train_dataset = TimeSeriesDataset(train_x,train_y,train_z) + test_dataset = TimeSeriesDataset(test_x,test_y,test_z) + + # Dataloader with weighted samplers + self.train_loader = torch.utils.data.DataLoader(dataset=train_dataset, shuffle=False, + batch_size=batch_size, sampler=train_weighted_sampler, + drop_last=True, num_workers = num_workers) + self.test_loader = torch.utils.data.DataLoader(dataset=test_dataset, shuffle=False, + batch_size=batch_size, sampler=test_weighted_sampler, + drop_last=True, num_workers=num_workers) + + def __new__(cls, df:pd.DataFrame, batch_size:int, n_past:int, n_future:int, num_workers:int): + """Constructor of MultiModalDataLoader""" + # Loader instance + loader_instance = super(MultiModalDataLoader, cls).__new__(cls) + loader_instance.__init__(df, batch_size, n_past, n_future, num_workers) + # Return train and test loader attributes + return loader_instance.train_loader, loader_instance.test_loader + + + + + + + + + diff --git a/traja/datasets/utils.py b/traja/datasets/utils.py new file mode 100644 index 00000000..767b87d6 --- /dev/null +++ b/traja/datasets/utils.py @@ -0,0 +1,120 @@ +import logging +import os +import math +import numpy as np +import torch +from torch.utils.data import Dataset +from collections import Counter +from torch.utils.data.sampler import WeightedRandomSampler +import pandas as pd +from sklearn.utils import shuffle +from sklearn.preprocessing import MinMaxScaler +import torch + +logger = logging.getLogger(__name__) + +def get_class_distribution(targets): + """Compute class distribution, returns number of classes and their count in the targets""" + targets_ = np.unique(targets, return_counts=True) + return targets_[0],targets_[1] + +def generate_dataset(df, n_past, n_future): + """ + df : Dataframe + n_past: Number of past observations + n_future: Number of future observations + Returns: + X: Past steps + Y: Future steps (Sequence target) + Z: Sequence category""" + + # Split the dataframe with respect to IDs + series_ids = dict(tuple(df.groupby('ID'))) # Dict of ids as keys and x,y,id as values + train_data, target_data, target_category = list(), list(), list() + + for id in series_ids.keys(): + X, Y, Z= list(), list(), list() + # Drop the column ids and convert the pandas into arrays + series = series_ids[id].drop(columns = ['ID']).to_numpy() + for window_start in range(len(series)): + past_end = window_start + n_past + future_end = past_end + n_future + if not future_end > len(series): + # slicing the past and future parts of the window + past, future = series[window_start:past_end, :], series[past_end:future_end, :] + X.append(past) + Y.append(future) + # For each sequence length set target category + Z.append(int(id)) + + train_data.extend(np.array(X)) + target_data.extend(np.array(Y)) + target_category.extend(np.array(Z)) + + return train_data, target_data, target_category + +def shuffle_split(train_data:np.array, target_data:np.array,target_category:np.array, train_ratio:float): + + # Shuffle the IDs and the corresponding sequence , preserving the order + train_data, target_data, target_category = shuffle(train_data, target_data, target_category) + + assert train_ratio<=1.0,"Train data ratio should be less than or equal to 1 " + + # Train test split + split = int(train_ratio*len(train_data)) + + train_x = train_data[:split] + train_y = target_data[:split] + train_z = target_category[:split] + + test_x = train_data[split:] + test_y = target_data[split :] + test_z = target_category[split:] + + return [train_x,train_y,train_z],[test_x,test_y,test_z] + +def scale_data(data, sequence_length): + assert len(data[0].shape)==2 + scalers={} + data = np.vstack(data) + + for i in range(data.shape[1]): + scaler = MinMaxScaler(feature_range=(-1,1)) + s_s = scaler.fit_transform(data[:,i].reshape(-1,1)) + s_s=np.reshape(s_s,len(s_s)) + scalers['scaler_'+ str(i)] = scaler + data[:,i]=s_s + # Slice the data into batches + data = [data[i:i + sequence_length] for i in range(0, len(data), sequence_length)] + return data, scalers + +def weighted_random_samplers(train_z,test_z): + + # Prepare weighted random sampler: + train_target_list = torch.tensor(train_z) + test_target_list = torch.tensor(test_z) + + # Number of classes and their frequencies + train_targets_, train_class_count = get_class_distribution(train_target_list) + test_targets_, test_class_count = get_class_distribution(test_target_list) + + # Compute class weights + train_class_weights = 1./torch.tensor(train_class_count, dtype=torch.float) + test_class_weights = 1./torch.tensor(test_class_count, dtype=torch.float) + + # Assign weights to original target list + train_class_weights_all = train_class_weights[train_target_list-1] # Note the targets start from 1, to python idx to apply,-1 + test_class_weights_all = test_class_weights[test_target_list-1] + + # Weighted samplers + train_weighted_sampler = WeightedRandomSampler( + weights=train_class_weights_all, + num_samples=len(train_class_weights_all), + replacement=True + ) + test_weighted_sampler = WeightedRandomSampler( + weights=test_class_weights_all, + num_samples=len(test_class_weights_all), + replacement=True + ) + return train_weighted_sampler, test_weighted_sampler \ No newline at end of file diff --git a/traja/models/ae.py b/traja/models/ae.py new file mode 100644 index 00000000..b5481828 --- /dev/null +++ b/traja/models/ae.py @@ -0,0 +1,257 @@ +"""This module contains the autoencoders and its variants +1. classic autoencoder +2. wasserstein autoencoder""" + +import torch +from .utils import TimeDistributed +from .utils import load_model + +# Encoder +class LSTMEncoder(torch.nn.Module): + """ Deep LSTM network. This implementation + returns output_size hidden size. + Args: + input_size: The number of expected features in the input `x` + batch_size: + sequence_length: The number of in each sample + hidden_size: The number of features in the hidden state `h` + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` + would mean stacking two LSTMs together to form a `stacked LSTM`, + with the second LSTM taking in outputs of the first LSTM and + computing the final results. Default: 1 + output_size: The number of output dimensions + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + LSTM layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + """ + + def __init__(self, input_size: int, sequence_length:int, batch_size:int, + hidden_size: int, num_layers: int, + batch_first: bool, dropout: float, + reset_state: bool, bidirectional: bool): + + super(LSTMEncoder, self).__init__() + + self.input_size = input_size + self.sequence_length = sequence_length + self.batch_size = batch_size + self.hidden_size = hidden_size + self.num_layers = num_layers + self.batch_first = batch_first + self.dropout = dropout + self.reset_state = reset_state + self.bidirectional = bidirectional + + # RNN Encoder + self.lstm_encoder = torch.nn.LSTM(input_size=input_size, hidden_size=self.hidden_size, + num_layers=num_layers, dropout=dropout, + bidirectional=self.bidirectional, batch_first=True) + + def _init_hidden(self): + return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size), torch.zeros(self.num_layers, self.batch_size, self.hidden_size)) + + def forward(self, x): + + # Encoder + enc_init_hidden = self._init_hidden() + enc_output,enc_states = self.lstm_encoder(x,enc_init_hidden) + # RNNs obeys, Markovian. Consider the last state of the hidden is the markovian of the entire sequence in that batch. + enc_output = enc_output[ : , -1, : ] # Shape(batch_size,hidden_dim) + return enc_output +# Latent +class DisentangledAELatent(torch.nn.Module): + """Dense Latent Layer """ + + def __init__(self, hidden_size: int, latent_size:int, dropout: float): + super(DisentangledAELatent, self).__init__() + + self.latent_size = latent_size + self.hidden_size = hidden_size + self.dropout = dropout + self.latent = torch.nn.Linear(self.hidden_size,self.latent_size) + + def forward(self, x, training=True): + # Feed it into the disentanled latent layer + z = self.latent(x) # Shape(batch_size, latent_size*2) + return z + +# Decoder +class LSTMDecoder(torch.nn.Module): + """ Deep LSTM network. This implementation + returns output_size outputs. + Args: + input_size: The number of expected features in the input `x` + batch_size: + sequence_length: The number of in each sample + hidden_size: The number of features in the hidden state `h` + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` + would mean stacking two LSTMs together to form a `stacked LSTM`, + with the second LSTM taking in outputs of the first LSTM and + computing the final results. Default: 1 + output_size: The number of output dimensions + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + LSTM layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + """ + + def __init__(self, batch_size:int, num_future:int, hidden_size: int, num_layers: int, + output_size: int, latent_size:int, batch_first: bool, dropout: float, + reset_state: bool, bidirectional: bool): + super(LSTMDecoder, self).__init__() + + self.batch_size = batch_size + self.latent_size = latent_size + self.num_future = num_future + self.hidden_size = hidden_size + self.num_layers = num_layers + self.output_size = output_size + self.batch_first = batch_first + self.dropout = dropout + self.reset_state = reset_state + self.bidirectional = bidirectional + + # RNN decoder + self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, hidden_size=self.hidden_size, + num_layers=self.num_layers, dropout=self.dropout, + bidirectional=self.bidirectional, batch_first=True) + self.output = TimeDistributed(torch.nn.Linear(self.hidden_size , self.output_size)) + + def _init_hidden(self): + return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device), torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) + + def forward(self, x, num_future = None): + + # To feed the latent states into lstm decoder, repeat the tensor n_future times at second dim + _init_hidden = self._init_hidden() + decoder_inputs = x.unsqueeze(1) + + if num_future==None: + decoder_inputs = decoder_inputs.repeat(1, self.num_future, 1) + else: # For multistep a prediction after training + decoder_inputs = decoder_inputs.repeat(1, num_future, 1) + + # Decoder input Shape(batch_size, num_futures, latent_size) + dec,(dec_hidden,dec_cell) = self.lstm_decoder(decoder_inputs,_init_hidden) + + # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) to Time Dsitributed Linear Layer + output = self.output(dec) + return output + +class MLPClassifier(torch.nn.Module): + """ MLP classifier + """ + def __init__(self, hidden_size: int, num_classes:int, latent_size:int, dropout:float ): + + super(MLPClassifier, self).__init__() + + self.latent_size = latent_size + self.hidden_size = hidden_size + self.num_classes = num_classes + self.dropout = dropout + + # Classifier + self.classifier1 = torch.nn.Linear(self.latent_size , self.hidden_size) + self.classifier2 = torch.nn.Linear(self.hidden_size , self.hidden_size) + self.classifier3 = torch.nn.Linear(self.hidden_size , self.hidden_size) + self.classifier4 = torch.nn.Linear(self.hidden_size , self.num_classes) + self.dropout = torch.nn.Dropout(p=dropout) + + def forward(self, x): + + classifier1 = self.dropout(self.classifier1(x)) + classifier2 = self.dropout(self.classifier2(classifier1)) + classifier3 = self.dropout(self.classifier3(classifier2)) + classifier4 = self.classifier4(classifier3) + # classifier_out = F.softmax(classifier4) + + return classifier4 + + +class MultiModelAE(torch.nn.Module): + + def __init__(self, *model_hyperparameters, **kwargs): + + super(MultiModelAE, self).__init__() + # self.input_size = input_size + # self.sequence_length = sequence_length + # self.batch_size = batch_size + # self.latent_size = latent_size + # self.num_future = num_future + # self.hidden_size = hidden_size + # self.num_layers = num_layers + # self.output_size = output_size + # self.num_classes = num_classes + # self.batch_first = batch_first + # self.dropout = dropout + # self.reset_state = reset_state + # self.bidirectional = bidirectional + for dictionary in model_hyperparameters: + for key in dictionary: + setattr(self, key, dictionary[key]) + for key in kwargs: + setattr(self, key, kwargs[key]) + + # Network instances in the model + self.encoder = LSTMEncoder(input_size=self.input_size, sequence_length=self.sequence_length, batch_size=self.batch_size, + hidden_size=self.hidden_size, num_layers=self.num_layers, + batch_first=self.batch_first, dropout=self.dropout, + reset_state=True, bidirectional=self.bidirectional) + + self.latent = DisentangledAELatent(hidden_size=self.hidden_size, latent_size=self.latent_size, dropout=self.dropout) + + self.decoder = LSTMDecoder(batch_size=self.batch_size, num_future=self.num_future, + hidden_size=self.hidden_size, num_layers=self.num_layers, output_size=self.output_size, + latent_size=self.latent_size, batch_first=self.batch_first, dropout=self.dropout, + reset_state=True, bidirectional=self.bidirectional) + + self.classifier = MLPClassifier(hidden_size=self.hidden_size,num_classes=self.num_classes, latent_size=self.latent_size, dropout=self.dropout) + + def forward(self, data, training=True, is_classification=False): + + if not is_classification: + # Set the classifier grad off + for param in self.classifier.parameters(): + param.requires_grad = False + + for param in self.encoder.parameters(): + param.requires_grad = True + + for param in self.decoder.parameters(): + param.requires_grad = True + + for param in self.latent.parameters(): + param.requires_grad = True + + enc_out = self.encoder(data) + # Latent + latent_out= self.latent(enc_out,training=training) + # Decoder + decoder_out = self.decoder(latent_out) + + return decoder_out,latent_out + + else: + # training_mode = 'classification' + # Freeze decoder parameters; + for param in self.classifier.parameters(): + param.requires_grad = True + + for param in self.encoder.parameters(): + param.requires_grad = False + + for param in self.decoder.parameters(): + param.requires_grad = False + + for param in self.latent.parameters(): + param.requires_grad = False + + # Encoder + enc_out = self.encoder(data) + # Latent + latent_out= self.latent(enc_out,training=training) + #Classifier + classifier_out = self.classifier(latent_out) # Deterministic + return classifier_out + diff --git a/traja/models/experiment.py b/traja/models/experiment.py new file mode 100644 index 00000000..8e800790 --- /dev/null +++ b/traja/models/experiment.py @@ -0,0 +1,929 @@ +#! /usr/local/env python3 +"""Pytorch visualization code modified from Chad Jensen's implementation +(https://discuss.pytorch.org/t/lstm-for-sequence-prediction/22021/3).""" +import logging + +import matplotlib.pyplot as plt +import numpy as np +from torch.utils.data import sampler + +try: + import torch +except ImportError: + raise ImportError( + "Missing optional dependency 'pytorch'. Install it via pytorch.org" + ) +import torch.nn as nn +import torch.optim as optim +import os +import pandas as pd +from time import time +from sklearn.preprocessing import MinMaxScaler +from datetime import datetime +from sklearn.model_selection import train_test_split +from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler +import torchvision.transforms as transforms + + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +class Trainer: + def __init__(self, model, + train_loader, + test_loader, + epochs=200, + batch_size=60, + run_id=0, + logs_dir='logs', + device='cpu', + optimizer='None', + plot=True, + downsampling=None): + self.device = device + self.model = model + self.epochs = epochs + self.plot = plot + + self.train_loader = train_loader + self.test_loader = test_loader + + self.criterion =torch.nn.MSELoss() + print('Checking for optimizer for {}'.format(optimizer)) + if optimizer == "adam": + print('Using adam') + self.optimizer = optim.Adam(model.parameters()) + elif optimizer == "adam_lr": + print("Using adam with higher learning rate") + self.optimizer = optim.Adam(model.parameters(), lr=0.01) + elif optimizer == 'adam_lr2': + print('Using adam with to large learning rate') + self.optimizer = optim.Adam(model.parameters(), lr=0.0001) + elif optimizer == "SGD": + print('Using SGD') + self.optimizer = optim.SGD(model.parameters(), momentum=0.9, weight_decay=5e-4) + elif optimizer == "LRS": + print('Using LRS') + self.optimizer = optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4) + self.lr_scheduler = optim.lr_scheduler.StepLR(self.optimizer, self.epochs // 3) + elif optimizer == "radam": + print('Using radam') + self.optimizer = RAdam(model.parameters()) + elif optimizer == "RMSprop": + print('Using RMSprop') + self.optimizer = optim.RMSprop(model.parameters()) + else: + raise ValueError('Unknown optimizer {}'.format(optimizer)) + self.opt_name = optimizer + save_dir = os.path.join(logs_dir, model.name, train_loader.name) + if not os.path.exists(save_dir): + os.makedirs(save_dir) + + self.savepath = os.path.join(save_dir, f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') + self.experiment_done = False + if os.path.exists(self.savepath): + trained_epochs = len(pd.read_csv(self.savepath, sep=';')) + + if trained_epochs >= epochs: + self.experiment_done = True + print(f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') + if os.path.exists((self.savepath.replace('.csv', '.pt'))): + self.model.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['model_state_dict']) + self.model = self.model.to(self.device) + + self.optimizer.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['optimizer']) + self.start_epoch = torch.load(self.savepath.replace('.csv', '.pt'))['epoch'] + 1 + else: + + self.start_epoch = 0 + self.model = self.model.to(self.device) + + + def _infer_initial_epoch(self, savepath): + if not os.path.exists(savepath): + return 0 + else: + df = pd.read_csv(savepath, sep=';', index_col=0) + print(len(df)+1) + return len(df) + + def train(self): + if self.experiment_done: + return + for epoch in range(self.start_epoch, self.epochs): + + print('Start training epoch', epoch) + print("{} Epoch {}, training loss: {}".format(datetime.now(), epoch, self.train_epoch())) + self.test(epoch=epoch) + if self.opt_name == "LRS": + print('LRS step') + self.lr_scheduler.step() + return self.savepath+'.csv' + + def train_epoch(self): + self.model.train() + total = 0 + running_loss = 0 + old_time = time() + for batch, data in enumerate(self.train_loader): + inputs, targets= data[0].to(self.device).float(), data[1].to(self.device).float() + self.optimizer.zero_grad() + outputs = self.model(inputs) + loss = self.criterion(outputs, targets) + loss.backward() + self.optimizer.step() + running_loss += loss.item() + + if batch % 10 == 0 and batch != 0: + print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'loss:', running_loss/total) + old_time = time() + + # Increment number of batches + total += 1 + return running_loss/total + + def test(self, epoch, save=True): + self.model.eval() + total = 0 + test_loss = 0 + with torch.no_grad(): + for batch, data in enumerate(self.test_loader): + if batch % 10 == 0: + print('Processing eval batch', batch,'of', len(self.test_loader)) + inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() + outputs = self.model(inputs) + loss = self.criterion(outputs, targets) + total += 1 + test_loss += loss.item() + + if save: + torch.save({ + 'model_state_dict': self.model.state_dict(), + 'optimizer': self.optimizer.state_dict(), + 'epoch': epoch, + 'test_loss': test_loss / total + }, self.savepath.replace('.csv', '.pt')) + return test_loss / total + + +class LSTM(nn.Module): + """ Deep LSTM network. This implementation + returns output_size outputs. + + + Args: + input_size: The number of expected features in the input `x` + hidden_size: The number of features in the hidden state `h` + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` + would mean stacking two LSTMs together to form a `stacked LSTM`, + with the second LSTM taking in outputs of the first LSTM and + computing the final results. Default: 1 + output_size: The number of output dimensions + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + LSTM layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + """ + + name = "LSTM" + + def __init__(self, input_size: int, hidden_size: int, num_layers: int, + output_size: int, dropout: float, bidirectional: bool): + super(LSTM, self).__init__() + + self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size, + num_layers=num_layers, dropout=dropout, + bidirectional=bidirectional, ) + + self.head = nn.Linear(hidden_size, output_size) + + def forward(self, x): + x, state = self.lstm(x) + # Use the last hidden state of last layer + x = state[0][-1] + x = self.head(x) + return x + +class TrajectoryLSTM: + def __init__( + self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() + ): + fig, ax = plt.subplots(2, 1) + self.fig = fig + self.ax = ax + assert xy.shape[1] is 2, f"xy should be an N x 2 array, but is {xy.shape}" + self.xy = xy + self.nb_steps = nb_steps + self.epochs = epochs + self.batch_size = batch_size + self.criterion = criterion + self.rnn = LSTM() + + def load_batch(self, batch_size=32): + t_1_b = np.zeros((self.nb_steps, self.batch_size, 2)) + t_b = np.zeros((self.nb_steps * self.batch_size, 2)) + + inds = np.random.randint(0, len(self.xy) - self.nb_steps, (self.batch_size)) + for i, ind in enumerate(inds): + t_1_b[:, i] = self.xy[ind : ind + self.nb_steps] + t_b[i * nb_steps : (i + 1) * self.nb_steps] = self.xy[ + ind + 1 : ind + nb_steps + 1 + ] + return torch.from_numpy(t_1_b).float(), torch.from_numpy(t_b).float() + + def train(self): + self.mean_loss = 0.0 + for epoch in range(1, self.epochs + 1): + t_1_b, t_b = self.load_batch(self.batch_size) + + def closure(): + global loss + optimizer.zero_grad() + pred = self.rnn(t_1_b) + shaped_pred = pred.reshape(-1, 2) + loss = self.criterion(abs(shaped_pred), abs(t_b)) + loss.backward() + + return loss + + optimizer = optim.Adam(self.rnn.parameters(), 1e-3) + optimizer.step(closure) + self.mean_loss += loss.item() + + if epoch % 100 == 0: + print("Epoch: {} | Loss: {:.6f}".format(epoch, self.mean_loss)) + self.mean_loss = 0 + + def savefig(self, filepath): + self.fig.savefig(filepath) + + def _plot(self): + t_1_b, t_b = self.load_batch(1) + pred = self.rnn(t_1_b).detach().numpy().reshape(-1, 2) + + real = t_1_b.numpy().reshape(-1, 2) + x, y = self.xy.T + self.ax[0].plot(x, y, label="Real") + self.ax[0].plot(real[:, 0], real[:, 1], label="Real batch") + self.ax[0].plot(pred[:, 0], pred[:, 1], label="Pred") + + self.ax[1].scatter(real[:, 0], real[:, 1], label="Real") + self.ax[1].scatter(pred[:, 0], pred[:, 1], label="Pred") + + for a in self.ax: + a.legend() + + def plot(self, interactive=True): + if interactive and (plt.get_backend() == "agg"): + logging.ERROR("Not able to use interactive plotting in mpl `agg` mode.") + # interactive = False + elif interactive: + while True: + for a in self.ax: + a.clear() + self._plot() + plt.pause(1) + plt.show(block=False) + else: + self._plot() + return self.fig + +def make_mlp(dim_list, activation="relu", batch_norm=True, dropout=0): + layers = [] + for dim_in, dim_out in zip(dim_list[:-1], dim_list[1:]): + layers.append(nn.Linear(dim_in, dim_out)) + if batch_norm: + layers.append(nn.BatchNorm1d(dim_out)) + if activation == "relu": + layers.append(nn.ReLU()) + elif activation == "leakyrelu": + layers.append(nn.LeakyReLU()) + if dropout > 0: + layers.append(nn.Dropout(p=dropout)) + return nn.Sequential(*layers) + + +def get_noise(shape, noise_type): + if noise_type == "gaussian": + return torch.randn(*shape).cuda() + elif noise_type == "uniform": + return torch.rand(*shape).sub_(0.5).mul_(2.0).cuda() + raise ValueError('Unrecognized noise type "%s"' % noise_type) + + +class Encoder(nn.Module): + """Encoder is part of both TrajectoryGenerator and + TrajectoryDiscriminator""" + + def __init__( + self, embedding_dim=64, h_dim=64, mlp_dim=1024, num_layers=1, dropout=0.0 + ): + super(Encoder, self).__init__() + + self.mlp_dim = 1024 + self.h_dim = h_dim + self.embedding_dim = embedding_dim + self.num_layers = num_layers + + self.encoder = nn.LSTM(embedding_dim, h_dim, num_layers, dropout=dropout) + + self.spatial_embedding = nn.Linear(2, embedding_dim) + + def init_hidden(self, batch): + return ( + torch.zeros(self.num_layers, batch, self.h_dim).cuda(), + torch.zeros(self.num_layers, batch, self.h_dim).cuda(), + ) + + def forward(self, obs_traj): + """ + Inputs: + - obs_traj: Tensor of shape (obs_len, batch, 2) + Output: + - final_h: Tensor of shape (self.num_layers, batch, self.h_dim) + """ + # Encode observed Trajectory + batch = obs_traj.size(1) + obs_traj_embedding = self.spatial_embedding(obs_traj.view(-1, 2)) + obs_traj_embedding = obs_traj_embedding.view(-1, batch, self.embedding_dim) + state_tuple = self.init_hidden(batch) + output, state = self.encoder(obs_traj_embedding, state_tuple) + final_h = state[0] + return final_h + + +class Decoder(nn.Module): + """Decoder is part of TrajectoryGenerator""" + + def __init__( + self, + seq_len, + embedding_dim=64, + h_dim=128, + mlp_dim=1024, + num_layers=1, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + pooling_type="pool_net", + neighborhood_size=2.0, + grid_size=8, + ): + super(Decoder, self).__init__() + + self.seq_len = seq_len + self.mlp_dim = mlp_dim + self.h_dim = h_dim + self.embedding_dim = embedding_dim + self.pool_every_timestep = pool_every_timestep + + self.decoder = nn.LSTM(embedding_dim, h_dim, num_layers, dropout=dropout) + + if pool_every_timestep: + if pooling_type == "pool_net": + self.pool_net = PoolHiddenNet( + embedding_dim=self.embedding_dim, + h_dim=self.h_dim, + mlp_dim=mlp_dim, + bottleneck_dim=bottleneck_dim, + activation=activation, + batch_norm=batch_norm, + dropout=dropout, + ) + elif pooling_type == "spool": + self.pool_net = SocialPooling( + h_dim=self.h_dim, + activation=activation, + batch_norm=batch_norm, + dropout=dropout, + neighborhood_size=neighborhood_size, + grid_size=grid_size, + ) + + mlp_dims = [h_dim + bottleneck_dim, mlp_dim, h_dim] + self.mlp = make_mlp( + mlp_dims, activation=activation, batch_norm=batch_norm, dropout=dropout + ) + + self.spatial_embedding = nn.Linear(2, embedding_dim) + self.hidden2pos = nn.Linear(h_dim, 2) + + def forward(self, last_pos, last_pos_rel, state_tuple, seq_start_end): + """ + Inputs: + - last_pos: Tensor of shape (batch, 2) + - last_pos_rel: Tensor of shape (batch, 2) + - state_tuple: (hh, ch) each tensor of shape (num_layers, batch, h_dim) + - seq_start_end: A list of tuples which delimit sequences within batch + Output: + - pred_traj: tensor of shape (self.seq_len, batch, 2) + """ + batch = last_pos.size(0) + pred_traj_fake_rel = [] + decoder_input = self.spatial_embedding(last_pos_rel) + decoder_input = decoder_input.view(1, batch, self.embedding_dim) + + for _ in range(self.seq_len): + output, state_tuple = self.decoder(decoder_input, state_tuple) + rel_pos = self.hidden2pos(output.view(-1, self.h_dim)) + curr_pos = rel_pos + last_pos + + if self.pool_every_timestep: + decoder_h = state_tuple[0] + pool_h = self.pool_net(decoder_h, seq_start_end, curr_pos) + decoder_h = torch.cat([decoder_h.view(-1, self.h_dim), pool_h], dim=1) + decoder_h = self.mlp(decoder_h) + decoder_h = torch.unsqueeze(decoder_h, 0) + state_tuple = (decoder_h, state_tuple[1]) + + embedding_input = rel_pos + + decoder_input = self.spatial_embedding(embedding_input) + decoder_input = decoder_input.view(1, batch, self.embedding_dim) + pred_traj_fake_rel.append(rel_pos.view(batch, -1)) + last_pos = curr_pos + + pred_traj_fake_rel = torch.stack(pred_traj_fake_rel, dim=0) + return pred_traj_fake_rel, state_tuple[0] + + +class PoolHiddenNet(nn.Module): + """Pooling module as proposed in our paper""" + + def __init__( + self, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + dropout=0.0, + ): + super(PoolHiddenNet, self).__init__() + + self.mlp_dim = 1024 + self.h_dim = h_dim + self.bottleneck_dim = bottleneck_dim + self.embedding_dim = embedding_dim + + mlp_pre_dim = embedding_dim + h_dim + mlp_pre_pool_dims = [mlp_pre_dim, 512, bottleneck_dim] + + self.spatial_embedding = nn.Linear(2, embedding_dim) + self.mlp_pre_pool = make_mlp( + mlp_pre_pool_dims, + activation=activation, + batch_norm=batch_norm, + dropout=dropout, + ) + + def repeat(self, tensor, num_reps): + """ + Inputs: + -tensor: 2D tensor of any shape + -num_reps: Number of times to repeat each row + Outpus: + -repeat_tensor: Repeat each row such that: R1, R1, R2, R2 + """ + col_len = tensor.size(1) + tensor = tensor.unsqueeze(dim=1).repeat(1, num_reps, 1) + tensor = tensor.view(-1, col_len) + return tensor + + def forward(self, h_states, seq_start_end, end_pos): + """ + Inputs: + - h_states: Tensor of shape (num_layers, batch, h_dim) + - seq_start_end: A list of tuples which delimit sequences within batch + - end_pos: Tensor of shape (batch, 2) + Output: + - pool_h: Tensor of shape (batch, bottleneck_dim) + """ + pool_h = [] + for _, (start, end) in enumerate(seq_start_end): + start = start.item() + end = end.item() + num_ped = end - start + curr_hidden = h_states.view(-1, self.h_dim)[start:end] + curr_end_pos = end_pos[start:end] + # Repeat -> H1, H2, H1, H2 + curr_hidden_1 = curr_hidden.repeat(num_ped, 1) + # Repeat position -> P1, P2, P1, P2 + curr_end_pos_1 = curr_end_pos.repeat(num_ped, 1) + # Repeat position -> P1, P1, P2, P2 + curr_end_pos_2 = self.repeat(curr_end_pos, num_ped) + curr_rel_pos = curr_end_pos_1 - curr_end_pos_2 + curr_rel_embedding = self.spatial_embedding(curr_rel_pos) + mlp_h_input = torch.cat([curr_rel_embedding, curr_hidden_1], dim=1) + curr_pool_h = self.mlp_pre_pool(mlp_h_input) + curr_pool_h = curr_pool_h.view(num_ped, num_ped, -1).max(1)[0] + pool_h.append(curr_pool_h) + pool_h = torch.cat(pool_h, dim=0) + return pool_h + + +class SocialPooling(nn.Module): + """Current state of the art pooling mechanism: + http://cvgl.stanford.edu/papers/CVPR16_Social_LSTM.pdf""" + + def __init__( + self, + h_dim=64, + activation="relu", + batch_norm=True, + dropout=0.0, + neighborhood_size=2.0, + grid_size=8, + pool_dim=None, + ): + super(SocialPooling, self).__init__() + self.h_dim = h_dim + self.grid_size = grid_size + self.neighborhood_size = neighborhood_size + if pool_dim: + mlp_pool_dims = [grid_size * grid_size * h_dim, pool_dim] + else: + mlp_pool_dims = [grid_size * grid_size * h_dim, h_dim] + + self.mlp_pool = make_mlp( + mlp_pool_dims, activation=activation, batch_norm=batch_norm, dropout=dropout + ) + + def get_bounds(self, ped_pos): + top_left_x = ped_pos[:, 0] - self.neighborhood_size / 2 + top_left_y = ped_pos[:, 1] + self.neighborhood_size / 2 + bottom_right_x = ped_pos[:, 0] + self.neighborhood_size / 2 + bottom_right_y = ped_pos[:, 1] - self.neighborhood_size / 2 + top_left = torch.stack([top_left_x, top_left_y], dim=1) + bottom_right = torch.stack([bottom_right_x, bottom_right_y], dim=1) + return top_left, bottom_right + + def get_grid_locations(self, top_left, other_pos): + cell_x = torch.floor( + ((other_pos[:, 0] - top_left[:, 0]) / self.neighborhood_size) + * self.grid_size + ) + cell_y = torch.floor( + ((top_left[:, 1] - other_pos[:, 1]) / self.neighborhood_size) + * self.grid_size + ) + grid_pos = cell_x + cell_y * self.grid_size + return grid_pos + + def repeat(self, tensor, num_reps): + """ + Inputs: + -tensor: 2D tensor of any shape + -num_reps: Number of times to repeat each row + Outpus: + -repeat_tensor: Repeat each row such that: R1, R1, R2, R2 + """ + col_len = tensor.size(1) + tensor = tensor.unsqueeze(dim=1).repeat(1, num_reps, 1) + tensor = tensor.view(-1, col_len) + return tensor + + def forward(self, h_states, seq_start_end, end_pos): + """ + Inputs: + - h_states: Tesnsor of shape (num_layers, batch, h_dim) + - seq_start_end: A list of tuples which delimit sequences within batch. + - end_pos: Absolute end position of obs_traj (batch, 2) + Output: + - pool_h: Tensor of shape (batch, h_dim) + """ + pool_h = [] + for _, (start, end) in enumerate(seq_start_end): + start = start.item() + end = end.item() + num_ped = end - start + grid_size = self.grid_size * self.grid_size + curr_hidden = h_states.view(-1, self.h_dim)[start:end] + curr_hidden_repeat = curr_hidden.repeat(num_ped, 1) + curr_end_pos = end_pos[start:end] + curr_pool_h_size = (num_ped * grid_size) + 1 + curr_pool_h = curr_hidden.new_zeros((curr_pool_h_size, self.h_dim)) + # curr_end_pos = curr_end_pos.data + top_left, bottom_right = self.get_bounds(curr_end_pos) + + # Repeat position -> P1, P2, P1, P2 + curr_end_pos = curr_end_pos.repeat(num_ped, 1) + # Repeat bounds -> B1, B1, B2, B2 + top_left = self.repeat(top_left, num_ped) + bottom_right = self.repeat(bottom_right, num_ped) + + grid_pos = self.get_grid_locations(top_left, curr_end_pos).type_as( + seq_start_end + ) + # Make all positions to exclude as non-zero + # Find which peds to exclude + x_bound = (curr_end_pos[:, 0] >= bottom_right[:, 0]) + ( + curr_end_pos[:, 0] <= top_left[:, 0] + ) + y_bound = (curr_end_pos[:, 1] >= top_left[:, 1]) + ( + curr_end_pos[:, 1] <= bottom_right[:, 1] + ) + + within_bound = x_bound + y_bound + within_bound[0 :: num_ped + 1] = 1 # Don't include the ped itself + within_bound = within_bound.view(-1) + + # This is a tricky way to get scatter add to work. Helps me avoid a + # for loop. Offset everything by 1. Use the initial 0 position to + # dump all uncessary adds. + grid_pos += 1 + total_grid_size = self.grid_size * self.grid_size + offset = torch.arange( + 0, total_grid_size * num_ped, total_grid_size + ).type_as(seq_start_end) + + offset = self.repeat(offset.view(-1, 1), num_ped).view(-1) + grid_pos += offset + grid_pos[within_bound != 0] = 0 + grid_pos = grid_pos.view(-1, 1).expand_as(curr_hidden_repeat) + + curr_pool_h = curr_pool_h.scatter_add(0, grid_pos, curr_hidden_repeat) + curr_pool_h = curr_pool_h[1:] + pool_h.append(curr_pool_h.view(num_ped, -1)) + + pool_h = torch.cat(pool_h, dim=0) + pool_h = self.mlp_pool(pool_h) + return pool_h + + +class TrajectoryGenerator(nn.Module): + """Modified from @agrimgupta92's https://github.com/agrimgupta92/sgan/blob/master/sgan/models.py.""" + + def __init__( + self, + obs_len, + pred_len, + embedding_dim=64, + encoder_h_dim=64, + decoder_h_dim=128, + mlp_dim=1024, + num_layers=1, + noise_dim=(0,), + noise_type="gaussian", + noise_mix_type="ped", + pooling_type=None, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + neighborhood_size=2.0, + grid_size=8, + ): + super(TrajectoryGenerator, self).__init__() + + if pooling_type and pooling_type.lower() == "none": + pooling_type = None + + self.obs_len = obs_len + self.pred_len = pred_len + self.mlp_dim = mlp_dim + self.encoder_h_dim = encoder_h_dim + self.decoder_h_dim = decoder_h_dim + self.embedding_dim = embedding_dim + self.noise_dim = noise_dim + self.num_layers = num_layers + self.noise_type = noise_type + self.noise_mix_type = noise_mix_type + self.pooling_type = pooling_type + self.noise_first_dim = 0 + self.pool_every_timestep = pool_every_timestep + self.bottleneck_dim = 1024 + + self.encoder = Encoder( + embedding_dim=embedding_dim, + h_dim=encoder_h_dim, + mlp_dim=mlp_dim, + num_layers=num_layers, + dropout=dropout, + ) + + self.decoder = Decoder( + pred_len, + embedding_dim=embedding_dim, + h_dim=decoder_h_dim, + mlp_dim=mlp_dim, + num_layers=num_layers, + pool_every_timestep=pool_every_timestep, + dropout=dropout, + bottleneck_dim=bottleneck_dim, + activation=activation, + batch_norm=batch_norm, + pooling_type=pooling_type, + grid_size=grid_size, + neighborhood_size=neighborhood_size, + ) + + if pooling_type == "pool_net": + self.pool_net = PoolHiddenNet( + embedding_dim=self.embedding_dim, + h_dim=encoder_h_dim, + mlp_dim=mlp_dim, + bottleneck_dim=bottleneck_dim, + activation=activation, + batch_norm=batch_norm, + ) + elif pooling_type == "spool": + self.pool_net = SocialPooling( + h_dim=encoder_h_dim, + activation=activation, + batch_norm=batch_norm, + dropout=dropout, + neighborhood_size=neighborhood_size, + grid_size=grid_size, + ) + + if self.noise_dim[0] == 0: + self.noise_dim = None + else: + self.noise_first_dim = noise_dim[0] + + # Decoder Hidden + if pooling_type: + input_dim = encoder_h_dim + bottleneck_dim + else: + input_dim = encoder_h_dim + + if self.mlp_decoder_needed(): + mlp_decoder_context_dims = [ + input_dim, + mlp_dim, + decoder_h_dim - self.noise_first_dim, + ] + + self.mlp_decoder_context = make_mlp( + mlp_decoder_context_dims, + activation=activation, + batch_norm=batch_norm, + dropout=dropout, + ) + + def add_noise(self, _input, seq_start_end, user_noise=None): + """ + Inputs: + - _input: Tensor of shape (_, decoder_h_dim - noise_first_dim) + - seq_start_end: A list of tuples which delimit sequences within batch. + - user_noise: Generally used for inference when you want to see + relation between different types of noise and outputs. + Outputs: + - decoder_h: Tensor of shape (_, decoder_h_dim) + """ + if not self.noise_dim: + return _input + + if self.noise_mix_type == "global": + noise_shape = (seq_start_end.size(0),) + self.noise_dim + else: + noise_shape = (_input.size(0),) + self.noise_dim + + if user_noise is not None: + z_decoder = user_noise + else: + z_decoder = get_noise(noise_shape, self.noise_type) + + if self.noise_mix_type == "global": + _list = [] + for idx, (start, end) in enumerate(seq_start_end): + start = start.item() + end = end.item() + _vec = z_decoder[idx].view(1, -1) + _to_cat = _vec.repeat(end - start, 1) + _list.append(torch.cat([_input[start:end], _to_cat], dim=1)) + decoder_h = torch.cat(_list, dim=0) + return decoder_h + + decoder_h = torch.cat([_input, z_decoder], dim=1) + + return decoder_h + + def mlp_decoder_needed(self): + if ( + self.noise_dim + or self.pooling_type + or self.encoder_h_dim != self.decoder_h_dim + ): + return True + else: + return False + + def forward(self, obs_traj, obs_traj_rel, seq_start_end, user_noise=None): + """ + Inputs: + - obs_traj: Tensor of shape (obs_len, batch, 2) + - obs_traj_rel: Tensor of shape (obs_len, batch, 2) + - seq_start_end: A list of tuples which delimit sequences within batch. + - user_noise: Generally used for inference when you want to see + relation between different types of noise and outputs. + Output: + - pred_traj_rel: Tensor of shape (self.pred_len, batch, 2) + """ + batch = obs_traj_rel.size(1) + # Encode seq + final_encoder_h = self.encoder(obs_traj_rel) + # Pool States + if self.pooling_type: + end_pos = obs_traj[-1, :, :] + pool_h = self.pool_net(final_encoder_h, seq_start_end, end_pos) + # Construct input hidden states for decoder + mlp_decoder_context_input = torch.cat( + [final_encoder_h.view(-1, self.encoder_h_dim), pool_h], dim=1 + ) + else: + mlp_decoder_context_input = final_encoder_h.view(-1, self.encoder_h_dim) + + # Add Noise + if self.mlp_decoder_needed(): + noise_input = self.mlp_decoder_context(mlp_decoder_context_input) + else: + noise_input = mlp_decoder_context_input + decoder_h = self.add_noise(noise_input, seq_start_end, user_noise=user_noise) + decoder_h = torch.unsqueeze(decoder_h, 0) + + decoder_c = torch.zeros(self.num_layers, batch, self.decoder_h_dim).cuda() + + state_tuple = (decoder_h, decoder_c) + last_pos = obs_traj[-1] + last_pos_rel = obs_traj_rel[-1] + # Predict Trajectory + + decoder_out = self.decoder(last_pos, last_pos_rel, state_tuple, seq_start_end) + pred_traj_fake_rel, final_decoder_h = decoder_out + + return pred_traj_fake_rel + +class TrajectoryDiscriminator(nn.Module): + def __init__( + self, + obs_len, + pred_len, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + num_layers=1, + activation="relu", + batch_norm=True, + dropout=0.0, + d_type="local", + ): + super(TrajectoryDiscriminator, self).__init__() + + self.obs_len = obs_len + self.pred_len = pred_len + self.seq_len = obs_len + pred_len + self.mlp_dim = mlp_dim + self.h_dim = h_dim + self.d_type = d_type + + self.encoder = Encoder( + embedding_dim=embedding_dim, + h_dim=h_dim, + mlp_dim=mlp_dim, + num_layers=num_layers, + dropout=dropout, + ) + + real_classifier_dims = [h_dim, mlp_dim, 1] + self.real_classifier = make_mlp( + real_classifier_dims, + activation=activation, + batch_norm=batch_norm, + dropout=dropout, + ) + if d_type == "global": + mlp_pool_dims = [h_dim + embedding_dim, mlp_dim, h_dim] + self.pool_net = PoolHiddenNet( + embedding_dim=embedding_dim, + h_dim=h_dim, + mlp_dim=mlp_pool_dims, + bottleneck_dim=h_dim, + activation=activation, + batch_norm=batch_norm, + ) + + def forward(self, traj, traj_rel, seq_start_end=None): + """ + Inputs: + - traj: Tensor of shape (obs_len + pred_len, batch, 2) + - traj_rel: Tensor of shape (obs_len + pred_len, batch, 2) + - seq_start_end: A list of tuples which delimit sequences within batch + Output: + - scores: Tensor of shape (batch,) with real/fake scores + """ + final_h = self.encoder(traj_rel) + # Note: In case of 'global' option we are using start_pos as opposed to + # end_pos. The intution being that hidden state has the whole + # trajectory and relative postion at the start when combined with + # trajectory information should help in discriminative behavior. + if self.d_type == "local": + classifier_input = final_h.squeeze() + else: + classifier_input = self.pool_net(final_h.squeeze(), seq_start_end, traj[0]) + scores = self.real_classifier(classifier_input) + return scores diff --git a/traja/models/generator.py b/traja/models/generator.py new file mode 100644 index 00000000..6df87234 --- /dev/null +++ b/traja/models/generator.py @@ -0,0 +1,62 @@ +"""Generate time series from model""" + +import plotly.express as px +import torch +from .ae import MultiModelAE +from .vae import MultiModelVAE +from .vaegan import MultiModelVAEGAN +from .irl import MultiModelIRL +from .utils import load_model +import matplotlib.pyplot as plt + +device = 'cuda' if torch.cuda.is_available() else 'cpu' + +def timeseries(model_type:str, model_hyperparameters:dict, model_path:str, batch_size:int, num_future:int, ): + # Generating few samples + batch_size = model_hyperparameters.batch_size # Number of samples + num_future = model_hyperparameters.num_future # Number of time steps in each sample + if model_type == 'ae': + model = MultiModelAE(**model_hyperparameters) + + if model_type == 'vae': + model = MultiModelVAE(**model_hyperparameters) + + if model_type == 'vaegan': + model = MultiModelVAEGAN(**model_hyperparameters) + return NotImplementedError + + if model_type == 'irl': + model = MultiModelIRL(**model_hyperparameters) + return NotImplementedError + + # Load the model from model path: + model = load_model(model, model_hyperparameters, model_path) + # z = torch.randn((batch_size, latent_size)).to(device) + z = torch.empty(10, model_hyperparameters.latent_size).normal_(mean=0,std=.1).to(device) + # Category from the noise + cat = model.classifier(z) + # Generate trajectories from the noise + out = model.decoder(z,num_future).cpu().detach().numpy() + out = out.reshape(out.shape[0]*out.shape[1],out.shape[2]) + + # for index, i in enumerate(train_df.columns): + # scaler = scalers['scaler_'+i] + # out[:,index] = scaler.inverse_transform(out[:,index].reshape(1, -1)) + print('IDs in this batch of synthetic data',torch.max(cat,1).indices+1) + plt.figure(figsize=(12,4)) + plt.plot(out[:,0], label='Generated x: Longitude') + plt.plot(out[:,1], label='Generated y: Latitude') + plt.legend() + + fig, ax = plt.subplots(nrows=2, ncols= 5, figsize=(16, 5), sharey=True) + # plt.ticklabel_format(useOffset=False) + fig.set_size_inches(20,5) + for i in range(2): + for j in range(5): + ax[i,j].plot(out[:,0][(i+j)*num_future:(i+j)*num_future + num_future],out[:,1][(i+j)*num_future:(i+j)*num_future+ num_future],label = 'Animal ID {}'.format((torch.max(cat,1).indices+1).detach()[i+j]),color='g') + ax[i,j].legend() + plt.show() + + return out + + diff --git a/traja/models/interpretor.py b/traja/models/interpretor.py new file mode 100644 index 00000000..7ac0eb91 --- /dev/null +++ b/traja/models/interpretor.py @@ -0,0 +1,26 @@ +"""Model interpretion and Visualization""" +import plotly.express as px + +from .ae import MultiModelAE +from .vae import MultiModelVAE +from .vaegan import MultiModelVAEGAN +from .irl import MultiModelIRL + +def DisplayLatentDynamics(latent): + r"""Visualize the dynamics of combination of latents + Args: + latent(tensor): Each point in the list is latent's state at the end of a sequence of each batch. + Latent shape (batch_size, latent_dim) + Usage: + DisplayLatentDynamics(latent)""" + + latents = {} + latents.fromkeys(list(range(latent.shape[1]))) + for i in range(latent.shape[1]): + latents[f'{i}']=latent[:,i].cpu().detach().numpy() + fig= px.scatter_matrix(latents) + fig.update_layout( + autosize=False, + width=1600, + height=1000,) + return fig.show() \ No newline at end of file diff --git a/traja/models/irl.py b/traja/models/irl.py new file mode 100644 index 00000000..2fac5848 --- /dev/null +++ b/traja/models/irl.py @@ -0,0 +1,20 @@ +""" Implementation of Inverse Reinforcement Learning algorithm for Time series""" +import torch + +class MultiModelIRL(torch.nn.Module): + def __init__(self,*model_hyperparameters, **kwargs): + super(MultiModelIRL,self).__init__() + + for dictionary in model_hyperparameters: + for key in dictionary: + setattr(self, key, dictionary[key]) + for key in kwargs: + setattr(self, key, kwargs[key]) + + def __new__(cls): + pass + + def forward(self, *input:None, **kwargs: None): + return NotImplementedError + + \ No newline at end of file diff --git a/traja/models/losses.py b/traja/models/losses.py new file mode 100644 index 00000000..f1019c1e --- /dev/null +++ b/traja/models/losses.py @@ -0,0 +1,69 @@ +import torch + +class Criterion(object): + + def __init__(self, model_type): + self.model_type = model_type + + @staticmethod + def ae_criterion(recon_x, x, loss_type='huber'): + """[summary] + + Args: + recon_x ([type]): [description] + x ([type]): [description] + loss_type(str): Type of Loss; huber or RMSE + + Returns: + [type]: [description] + """ + if loss_type == 'huber': + + huber_loss = torch.nn.SmoothL1Loss(reduction='sum') + dist_x = huber_loss(recon_x,x) + return dist_x + else: # RMSE + return torch.sqrt(torch.mean((recon_x-x)**2)) + + @staticmethod + def vae_criterion(recon_x, x, mu, logvar, loss_type='huber'): + r"""Time series generative model loss function + + Args: + recon_x ([type]): [description] + x ([type]): [description] + mu ([type]): [description] + logvar ([type]): [description] + + Returns: + [type]: [description] + """ + if loss_type=='huber': + huber_loss = torch.nn.SmoothL1Loss(reduction='sum') + dist_x = huber_loss(recon_x,x) + else: + dist_x = torch.sqrt(torch.mean((recon_x-x)**2)) + + KLD = -0.5 * torch.sum(1 + logvar - mu**2 - logvar.exp()) + + return dist_x + KLD + + @staticmethod + def classifier_criterion(): + """Classifier loss function""" + classifier_criterion = torch.nn.CrossEntropyLoss() + return classifier_criterion + + def vaegan_criterion(): + return NotImplementedError + + def lstm_criterion(): + return NotImplementedError + + + +# VAE loss + +# VAE-GAN loss + +# LSTM \ No newline at end of file diff --git a/traja/models/lstm.py b/traja/models/lstm.py new file mode 100644 index 00000000..7ecdd287 --- /dev/null +++ b/traja/models/lstm.py @@ -0,0 +1,22 @@ +"""Implementation of Multimodel LSTM""" + +import torch + +class MultiModelLSTM(torch.nn.Module): + + def __init__(self,*model_hyperparameters, **kwargs): + super(MultiModelLSTM,self).__init__() + + for dictionary in model_hyperparameters: + for key in dictionary: + setattr(self, key, dictionary[key]) + for key in kwargs: + setattr(self, key, kwargs[key]) + + def __new__(cls): + pass + + def forward(self, *input:None, **kwargs: None): + return NotImplementedError + + diff --git a/traja/models/nn.py b/traja/models/nn.py index 9d9b43da..3e8a1d44 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -2,7 +2,6 @@ """Pytorch visualization code modified from Chad Jensen's implementation (https://discuss.pytorch.org/t/lstm-for-sequence-prediction/22021/3).""" import logging - import matplotlib.pyplot as plt import numpy as np from torch.utils.data import sampler @@ -24,163 +23,8 @@ from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler nb_steps = 10 - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - -class TimeseriesDataset(Dataset): - """ - Support class for the loading and Random batching of sequences of samples, - - Args: - dataset (Tensor): Tensor containing all the samples - sequence_length (int): length of the analyzed sequence by the LSTM - """ - # Constructor - def __init__(self, dataset: np.array, sequence_length: int): - self.data = dataset - self.sequence_length = sequence_length - - # Override total dataset's length getter - def __len__(self): - return int((self.data.shape[0]) / self.sequence_length) - - # Override single items' getter - def __getitem__(self, index): - data = (self.data[index * self.sequence_length: (index + 1) * self.sequence_length], - self.data[index * self.sequence_length : (index + 1) * self.sequence_length]) - return data - -class DataSequenceGenerator(torch.utils.data.Dataset): - """ - Support class for the loading and Sequential batching of sequences of samples, - - Args: - dataset (Tensor): Tensor containing all the samples - sequence_length (int): length of the analyzed sequence by the LSTM - transforms (object torchvision.transform): Pytorch's transforms used to process the data - batch_size (int): Batch size - """ - # Constructor - def __init__(self, dataset: np.array, batch_size: int, sequence_length: int, transforms=None): - - self.dataset = dataset - self.seq_length = sequence_length - self.transforms = transforms # List to tensors - self.batch_size = batch_size - - # Override total dataset's length getter - def __len__(self): - return self.dataset.__len__()-self.seq_length - - # Override single items' getter - def __getitem__(self, idx): - - if self.transforms is not None: - return self.transforms(self.dataset[idx:idx+self.seq_length]), self.transforms(self.dataset[idx+self.seq_length:idx+self.seq_length+1]) - else: - return self.dataset[idx:idx+self.seq_length], self.dataset[idx+self.seq_length:idx+self.seq_length+1] - -class DataLoaderLSTM: - """Dataloader object for Single Step-prediction problems using Random and sequential samplers. - - Args: - dataset (np.array): Dataset - batch_size (int): Batch size - seq_length (int): Sequence length at each batch - num_workers (int, optional): Number of subprocesses to deploy for dataloading. Defaults to 6. - random_sampler (Sampler instance name): If !=None, Random sampling of batches, otherwise Sequential. Defaults to None. - dir_name (str): Target folder name where the model will be saved after training: './logs//' - """ - - def __init__(self,dataset: np.array, batch_size: int, seq_length: int, random_sampler: str=None, dir_name: str=None, num_workers: int=6): - - self.dataset = dataset - self.batch_size = batch_size - self.seq_length = seq_length - self.num_workers = num_workers - self.random_sampler = random_sampler - self.dir_name = dir_name - self.name = None - - def listToTensor(list): - tensor = torch.empty(list.__len__(), list[0].__len__()) - for i in range(list.__len__()): - tensor[i, :] = torch.FloatTensor(list[i]) - return tensor - - def load(self): - """Load the dataset using corresponding sampler - Returns: - [torch.utils.data.Dataloader]: Dataloader - """ - data_transform = transforms.Lambda(lambda x: self.__class__.listToTensor(x)) - dataset = DataSequenceGenerator(self.dataset, self.batch_size, self.seq_length, transforms=data_transform) - - if self.random_sampler!=None: - data_loader = torch.utils.data.DataLoader(dataset, self.batch_size, shuffle=False, sampler = self.random_sampler, num_workers=self.num_workers) - else: - data_loader = torch.utils.data.DataLoader(dataset, self.batch_size, shuffle=False, num_workers=self.num_workers) - - # Directory where the model will be saved - setattr( data_loader, 'name', self.dir_name ) - return data_loader - -def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int, random_sampler: bool, dir_name: str, num_workers:int): - """ Scale the timeseries dataset and generate train and test dataloaders - - Args: - data_frame (pd.DataFrame): Dataset - sequence_length (int): Sequence length of time series for a single gradient step - train_fraction (float): train data vs test data ratio - batch_size (int): Batch size of single gradient measure - dir_name (str: optional): Directory name that corresponds to type of running task, where the trained model will be saved' - - Returns: - train_loader (Dataloader) - validation_loader(Dataloader) - scaler (instance): Data scaler instance - """ - - # Split the dataset into train and val - train,test = train_test_split(data_frame.values,train_size=train_fraction) - - # Train Dataset transformation - train_scaler = MinMaxScaler(copy=False) - train_scaler_fit = train_scaler.fit(train) - train_scaled_dataset = train_scaler_fit.transform(train) - - # Test Dataset transformation - val_scaler = MinMaxScaler(copy=False) - val_scaler_fit = val_scaler.fit(test) - val_scaled_dataset = val_scaler_fit.transform(test) - - if random_sampler: - # Concatenate train and val data and slice them again using their indices and SubsetRandomSampler - concat_dataset = np.concatenate((train_scaled_dataset, val_scaled_dataset),axis=0) - dataset_length = int(concat_dataset.shape[0]- sequence_length) - indices = list(range(dataset_length)) - split = int(np.floor(train_fraction * dataset_length)) - train_indices, val_indices = indices[:split], indices[split:] - - # Creating PT data samplers and loaders: - train_sampler = SubsetRandomSampler(train_indices) - valid_sampler = SubsetRandomSampler(val_indices) - - # Random Dataloader for training and validation - train_loader = DataLoaderLSTM(concat_dataset, batch_size, sequence_length, - random_sampler= train_sampler, dir_name = dir_name, num_workers=num_workers) - validation_loader = DataLoaderLSTM(concat_dataset,batch_size,sequence_length, - random_sampler= valid_sampler, dir_name = dir_name, num_workers=num_workers) - - return train_loader.load() , validation_loader.load(), train_scaler_fit, val_scaler_fit - - else: - # Sequential Dataloader for training and validation - train_loader = DataLoaderLSTM(train_scaled_dataset, batch_size, sequence_length, random_sampler= None, dir_name = dir_name, num_workers=num_workers) - validation_loader = DataLoaderLSTM(val_scaled_dataset,batch_size,sequence_length, random_sampler= None, dir_name = dir_name, num_workers=num_workers) - return train_loader.load(), validation_loader.load(), train_scaler_fit, val_scaler_fit - class LossMse: """ Calculate the Mean Squared Error between y_true and y_pred diff --git a/traja/models/train.py b/traja/models/train.py new file mode 100644 index 00000000..7e7e2ab9 --- /dev/null +++ b/traja/models/train.py @@ -0,0 +1,295 @@ +from .ae import MultiModelAE +from .vae import MultiModelVAE +from .vaegan import MultiModelVAEGAN +from .lstm import LSTM +import torch +from functools import wraps +import inspect +from . import utils +from .losses import Criterion + +device = 'cuda' if torch.cuda.is_available() else 'cpu' + +def initializer(func): + """ + Automatically assigns the parameters. + https://stackoverflow.com/questions/109087/how-to-get-instance-variables-in-python + + >>> class process: + ... @initializer + ... def __init__(self, cmd, reachable=False, user='root'): + ... pass + >>> p = process('halt', True) + >>> p.cmd, p.reachable, p.user + ('halt', True, 'root') + """ + names, varargs, keywords, defaults = inspect.getargspec(func) + + @wraps(func) + def wrapper(self, *args, **kargs): + for name, arg in list(zip(names[1:], args)) + list(kargs.items()): + setattr(self, name, arg) + + for name, default in zip(reversed(names), reversed(defaults)): + if not hasattr(self, name): + setattr(self, name, default) + + func(self, *args, **kargs) + + return wrapper + +class Trainer(object): + # @initializer + def __init__(self, model_type:str, device:str, + input_size:int, output_size:int, + lstm_hidden_size:int, lstm_num_layers:int, + reset_state:bool, num_classes:int, + latent_size:int, dropout:float, + num_layers:int, epochs:int, batch_size:int, + num_future:int, sequence_length:int, + bidirectional:bool =False, + batch_first:bool =True, + loss_type:str = 'huber'): + + white_keys = ['ae','vae','lstm','vaegan', 'irl'] + assert model_type in white_keys, "Valid models are {}".format(white_keys) + self.model_type = model_type + self.device = device + self.input_size = input_size + self.lstm_hidden_size = lstm_hidden_size + self.lstm_num_layers = lstm_num_layers + self.num_layers = lstm_num_layers + self.hidden_size = lstm_hidden_size # For classifiers too + self.batch_first = batch_first + self.reset_state = reset_state + self.output_size = output_size + self.num_classes = num_classes + self.latent_size = latent_size + self.num_layers = num_layers + self.num_future = num_future + self.epochs = epochs + self.batch_size = batch_size + self.sequence_length = sequence_length + self.dropout = dropout + self.bidirectional= bidirectional + self.loss_type = loss_type + + self.model_hyperparameters = {'input_size':self.input_size, + 'sequence_length':self.sequence_length, + 'epochs':self.epochs, + 'batch_size':self.batch_size, + 'batch_first':self.batch_first, + 'lstm_hidden_size':self.lstm_hidden_size, + 'hidden_size':self.lstm_hidden_size, + 'num_future':self.num_future, + 'lstm_num_layers':self.lstm_num_layers, + 'num_layers':self.lstm_num_layers, + 'latent_size':self.latent_size, + 'output_size':self.output_size, + 'num_classes':self.num_classes, + 'batch_first':self.batch_first, + 'dropout':self.dropout, + 'reset_state':self.reset_state, + 'bidirectional':self.bidirectional, + 'dropout':self.dropout, + 'loss_type':self.loss_type + } + + if self.model_type == 'lstm': + self.model = LSTM(self.model_hyperparameters) + + if self.model_type == 'ae': + self.model = MultiModelAE(self.model_hyperparameters) + + if self.model_type == 'vae': + self.model = MultiModelVAE(self.model_hyperparameters) + + if self.model_type == 'vaegan': + self.model = MultiModelVAEGAN(self.model_hyperparameters) + + if self.model_type == 'irl': + return NotImplementedError + + # Get the optimizers for each network in the model + [self.encoder_optimizer, self.latent_optimizer, self.decoder_optimizer, self.classifier_optimizer] = utils.set_optimizers(self.model_type, self.model) + + # Learning rate schedulers for the models + [self.encoder_scheduler, self.latent_scheduler, self.decoder_scheduler, self.classifier_scheduler] = utils.get_lrschedulers(self.model_type, self.model, factor=0.1, patience=10) + + + def __str__(self): + return "Training model type {}".format(self.model_type) + + # TRAIN AUTOENCODERS + def train_ae(self, train_loader, test_loader, model_save_path): + + assert self.model_type == 'ae' + # Move the model to target device + self.model.to(device) + + # Training mode: Switch from Generative to classifier training mode + training_mode = 'forecasting' + + # Training + for epoch in range(self.epochs*2): # First half for generative model and next for classifier + if epoch>0: # Initial step is to test and set LR schduler + # Training + self.model.train() + total_loss = 0 + for idx, (data, target,category) in enumerate(train_loader): + # Reset optimizer states + self.encoder_optimizer.zero_grad() + self.latent_optimizer.zero_grad() + self.decoder_optimizer.zero_grad() + self.classifier_optimizer.zero_grad() + + data, target,category = data.float().to(device), target.float().to(device), category.to(device) + + if training_mode =='forecasting': + + decoder_out,latent_out= self.model(data,training=True, is_classification=False) + loss = Criterion.ae_criterion(decoder_out, target) + loss.backward() + + self.encoder_optimizer.step() + self.decoder_optimizer.step() + self.latent_optimizer.step() + + else: + + classifier_out = self.model(data,training=True, is_classification=True) + loss = Criterion.classifier_criterion(classifier_out, category-1) + loss.backward() + + self.classifier_optimizer.step() + + total_loss+=loss + + print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss/(idx+1))) + + if epoch+1 == self.epochs: # + training_mode = 'classification' + + # Testing + if epoch%10==0: + with torch.no_grad(): + self.model.eval() + test_loss_forecasting = 0 + test_loss_classification = 0 + for idx, (data, target,category) in enumerate(list(test_loader)): + data, target, category = data.float().to(device), target.float().to(device), category.to(device) + out, latent = self.model(data, training=False, is_classification=False) + test_loss_forecasting += Criterion.ae_criterion(out,target).item() + + classifier_out= self.model(data,training=False, is_classification=True) + test_loss_classification += Criterion.classifier_criterion(classifier_out, category-1).item() + + test_loss_forecasting /= len(test_loader.dataset) + print(f'====> Mean test set generator loss: {test_loss_forecasting:.4f}') + test_loss_classification /= len(test_loader.dataset) + print(f'====> Mean test set classifier loss: {test_loss_classification:.4f}') + + # Scheduler metric is test set loss + if training_mode =='forecasting': + self.encoder_scheduler.step(self.test_loss_forecasting) + self.decoder_scheduler.step(self.test_loss_forecasting) + self.latent_scheduler.step(self.test_loss_forecasting) + else: + self.classifier_scheduler.step(self.test_loss_classification) + + # Save the model at target path + utils.save_model(self.model,PATH = model_save_path) + + # TRAIN VARIATIONAL AUTOENCODERS + def train_vae(self, train_loader, test_loader, model_save_path): + assert self.model_type == 'vae' + # Move the model to target device + self.model.to(device) + + # Training mode: Switch from Generative to classifier training mode + training_mode = 'forecasting' + + # Training + for epoch in range(self.epochs*2): # First half for generative model and next for classifier + if epoch>0: # Initial step is to test and set LR schduler + # Training + self.model.train() + total_loss = 0 + for idx, (data, target,category) in enumerate(train_loader): + # Reset optimizer states + self.encoder_optimizer.zero_grad() + self.latent_optimizer.zero_grad() + self.decoder_optimizer.zero_grad() + self.classifier_optimizer.zero_grad() + + data, target,category = data.float().to(device), target.float().to(device), category.to(device) + + if training_mode =='forecasting': + + decoder_out,latent_out,mu,logvar= self.model(data,training=True, is_classification=False) + loss = Criterion.vae_criterion(decoder_out, target,mu,logvar) + loss.backward() + + self.encoder_optimizer.step() + self.decoder_optimizer.step() + self.latent_optimizer.step() + + else: + + classifier_out = self.model(data,training=True, is_classification=True) + loss = Criterion.classifier_criterion(classifier_out, category-1) + loss.backward() + + self.classifier_optimizer.step() + + total_loss+=loss + + print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss/(idx+1))) + + if epoch+1 == self.epochs: # + training_mode = 'classification' + + # Testing + if epoch%10==0: + with torch.no_grad(): + self.model.eval() + test_loss_forecasting = 0 + test_loss_classification = 0 + for idx, (data, target,category) in enumerate(list(test_loader)): + data, target, category = data.float().to(device), target.float().to(device), category.to(device) + out, latent, mu, logvar = self.model(data, training=False, is_classification=False) + test_loss_forecasting += Criterion.vae_criterion(out,target, mu, logvar).item() + + classifier_out= self.model(data,training=False, is_classification=True) + test_loss_classification += Criterion.classifier_criterion(classifier_out, category-1).item() + + test_loss_forecasting /= len(test_loader.dataset) + print(f'====> Mean test set generator loss: {test_loss_forecasting:.4f}') + test_loss_classification /= len(test_loader.dataset) + print(f'====> Mean test set classifier loss: {test_loss_classification:.4f}') + + # Scheduler metric is test set loss + if training_mode =='forecasting': + self.encoder_scheduler.step(self.test_loss_forecasting) + self.decoder_scheduler.step(self.test_loss_forecasting) + self.latent_scheduler.step(self.test_loss_forecasting) + else: + self.classifier_scheduler.step(self.test_loss_classification) + + # Save the model at target path + utils.save_model(self.model,PATH = model_save_path) + + # TRAIN VARIATIONAL AUTOENCODERS-GAN + def train_vaegan(self): + assert self.model_type == 'vaegan' + return NotImplementedError + + # TRAIN INVERSE RL + def train_irl(self): + assert self.model_type == 'irl' + return NotImplementedError + + # TRAIN LSTM + def train_lstm(self): + assert self.model_type == 'lstm' + return NotImplementedError \ No newline at end of file diff --git a/traja/models/utils.py b/traja/models/utils.py new file mode 100644 index 00000000..5425977b --- /dev/null +++ b/traja/models/utils.py @@ -0,0 +1,114 @@ +import torch +import matplotlib.pyplot as plt +from torch.optim.lr_scheduler import ReduceLROnPlateau + +class TimeDistributed(torch.nn.Module): + """ Time distributed wrapper compatible with linear/dense pytorch layer modules""" + def __init__(self, module, batch_first=True): + super(TimeDistributed, self).__init__() + self.module = module + self.batch_first = batch_first + + def forward(self, x): + + # Linear layer accept 2D input + if len(x.size()) <= 2: + return self.module(x) + + # Squash samples and timesteps into a single axis + x_reshape = x.contiguous().view(-1, x.size(-1)) # (samples * timesteps, input_size) + out = self.module(x_reshape) + + # We have to reshape Y back to the target shape + if self.batch_first: + out = out.contiguous().view(x.size(0), -1, out.size(-1)) # (samples, timesteps, output_size) + else: + out = out.view(-1, x.size(1), out.size(-1)) # (timesteps, samples, output_size) + + return out + +def get_optimizers(model_type, model, lr=0.0001): + r"""Optimizers for each network in the model + + Args: + model_type ([type]): [description] + model ([type]): [description] + lr (float, optional): [description]. Defaults to 0.0001. + + Returns: + [type]: [description] + """ + + if model_type == 'ae' or 'vae': + # Optimizers for each network in the model + encoder_optimizer = torch.optim.Adam(model.encoder.parameters(), lr=lr) + latent_optimizer = torch.optim.Adam(model.latent.parameters(), lr=lr) + decoder_optimizer = torch.optim.Adam(model.decoder.parameters(), lr=lr) + classifier_optimizer = torch.optim.Adam(model.classifier.parameters(), lr=lr) + return [encoder_optimizer, latent_optimizer, decoder_optimizer, classifier_optimizer] + + elif model_type == 'vaegan': + return NotImplementedError + + else: # LSTM + return NotImplementedError + +def get_lrschedulers(model_type, model, factor=0.1, patience=10 ): + r"""Learning rate scheduler for each network in the model + NOTE: Scheduler metric should be test set loss + + Args: + model_type ([type]): [description] + model ([type]): [description] + factor (float, optional): [description]. Defaults to 0.1. + patience (int, optional): [description]. Defaults to 10. + + Returns: + [type]: [description] + """ + if model_type == 'ae' or 'vae': + encoder_scheduler = ReduceLROnPlateau(model.encoder_optimizer, mode='max', factor=factor, patience=patience, verbose=True) + decoder_scheduler = ReduceLROnPlateau(model.decoder_optimizer, mode='max', factor=factor, patience=patience, verbose=True) + latent_scheduler = ReduceLROnPlateau(model.latent_optimizer, mode='max', factor=factor, patience=patience, verbose=True) + classifier_scheduler = ReduceLROnPlateau(model.classifier_optimizer, mode='max', factor=factor, patience=patience, verbose=True) + return [encoder_scheduler, decoder_scheduler, latent_scheduler, classifier_scheduler] + + elif model_type == 'vaegan': + return NotImplementedError + + else: # LSTM + return NotImplementedError + + + +def save_model(model, PATH): + r"""[summary] + + Args: + model ([type]): [description] + PATH ([type]): [description] + """ + + # PATH = "state_dict_model.pt" + # Save + torch.save(model.state_dict(), PATH) + print('Model saved at {}'.format(PATH)) + +def load_model(model,model_hyperparameters, PATH): + r"""[summary] + + Args: + model ([type]): [description] + model_hyperparameters ([type]): [description] + PATH ([type]): [description] + + Returns: + [type]: [description] + """ + # Load + model = model(model_hyperparameters) + model.load_state_dict(torch.load(PATH)) + + return model + + \ No newline at end of file diff --git a/traja/models/vae.py b/traja/models/vae.py new file mode 100644 index 00000000..b29312b1 --- /dev/null +++ b/traja/models/vae.py @@ -0,0 +1,7 @@ +"""This module contains the variational autoencoders and its variants +1. classic variational autoencoder + + +Loss functions: +1. MSE +2. Huber Loss""" \ No newline at end of file diff --git a/traja/models/vaegan.py b/traja/models/vaegan.py new file mode 100644 index 00000000..ce47a12f --- /dev/null +++ b/traja/models/vaegan.py @@ -0,0 +1,27 @@ +"""This module contains the variational autoencoders - GAN and its variants +1. classic VAE-GAN +2. ***** + +Loss functions: +1. MSE +2. Huber Loss""" + + +import torch + +class MultiModelVAEGAN(torch.nn.Module): + + def __init__(self,*model_hyperparameters, **kwargs): + super(MultiModelVAEGAN,self).__init__() + + for dictionary in model_hyperparameters: + for key in dictionary: + setattr(self, key, dictionary[key]) + for key in kwargs: + setattr(self, key, kwargs[key]) + + def __new__(cls): + pass + + def forward(self, *input:None, **kwargs: None): + return NotImplementedError \ No newline at end of file From 3a2b700d62b20a7eac4f0a1e217ec1f1d9ebc1c9 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 30 Nov 2020 12:22:13 +0100 Subject: [PATCH 031/211] addressing issues as per #25 --- .vscode/settings.json | 6 + traja/datasets/dataset.py | 26 ++- traja/models/ae.py | 329 +++++++++++++++++++++----------------- traja/models/train.py | 143 +++-------------- traja/models/vae.py | 312 +++++++++++++++++++++++++++++++++++- 5 files changed, 546 insertions(+), 270 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..18837826 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "python.pythonPath":"C:\\Users\\saran\\Anaconda3\\envs\\tfgpu\\python.exe", + "python.linting.pycodestyleEnabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.provider": "autopep8" +} \ No newline at end of file diff --git a/traja/datasets/dataset.py b/traja/datasets/dataset.py index 55b51138..45cd5610 100644 --- a/traja/datasets/dataset.py +++ b/traja/datasets/dataset.py @@ -250,7 +250,31 @@ def __len__(self): return len(self.data) class MultiModalDataLoader(object): - r""" Data loader object. Wrap Data preprocessors, dataset, sampler and return the data loader object""" + """ + MultiModalDataLoader wraps the following data preparation steps, + + 1. Data generator: Extract x and y time series and corresponding ID (category) in the dataset. This process split the dataset into + i) Train samples with sequence length equals n_past + ii) Target samples with sequence length equals n_future + iii) Target category(ID) of both train and target data + 2. Data scalling: Scale the train and target data columns between the range (-1,1) using MinMaxScalers; TODO: It is more optimal to scale data for each ID(category) + 3. Data shuffling: Shuffle the order of samples in the dataset without loosing the train<->target<->category combination + 4. Create train test split: Split the shuffled batches into train (data, target, category) and test(data, target, category) + 5. Weighted Random sampling: Apply weights with respect to category counts in the dataset: category_sample_weight = 1/num_category_samples; This avoid model overfit to category appear often in the dataset + 6. Create pytorch Dataset instances + 7. Returns the train and test data loader instances given the dataset instances and batch size + + Args: + df (pd.DataFrame): Dataset + batch_size (int): Number of samples per batch of data + n_past (int): Input sequence length. Number of time steps from the past. + n_future (int): Target sequence length. Number of time steps to the future. + num_workers (int): Number of cpu subprocess to be occupied during data loading process + + Usage: + train_dataloader, test_dataloader = MultiModalDataLoader(df = data_frame, batch_size=32, + n_past = 20, n_future = 10, num_workers=4) + """ def __init__(self, df:pd.DataFrame, batch_size:int, n_past:int, n_future:int, num_workers: int): diff --git a/traja/models/ae.py b/traja/models/ae.py index b5481828..4677336a 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -1,12 +1,35 @@ -"""This module contains the autoencoders and its variants -1. classic autoencoder -2. wasserstein autoencoder""" +""" This module implement the Auto encoder model for both forecasting +and classification of time series data. + +```USAGE``` to train AE model: +trainer = Trainer(model_type='ae', + device=device, + input_size=input_size, + output_size=output_size, + lstm_hidden_size=lstm_hidden_size, + lstm_num_layers=lstm_num_layers, + reset_state=True, + num_classes=num_classes, + latent_size=latent_size, + dropout=0.1, + num_layers=num_layers, + epochs=epochs, + batch_size=batch_size, + num_future=num_future, + sequence_length=sequence_length, + bidirectional =False, + batch_first =True, + loss_type = 'huber') + +trainer.train_latent_model(train_dataloader, test_dataloader, model_save_path=PATH)""" import torch from .utils import TimeDistributed from .utils import load_model -# Encoder +device = 'cuda' if torch.cuda.is_available() else 'cpu' + + class LSTMEncoder(torch.nn.Module): """ Deep LSTM network. This implementation returns output_size hidden size. @@ -26,13 +49,13 @@ class LSTMEncoder(torch.nn.Module): bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` """ - def __init__(self, input_size: int, sequence_length:int, batch_size:int, + def __init__(self, input_size: int, sequence_length: int, batch_size: int, hidden_size: int, num_layers: int, - batch_first: bool, dropout: float, + batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): - + super(LSTMEncoder, self).__init__() - + self.input_size = input_size self.sequence_length = sequence_length self.batch_size = batch_size @@ -43,64 +66,60 @@ def __init__(self, input_size: int, sequence_length:int, batch_size:int, self.reset_state = reset_state self.bidirectional = bidirectional - # RNN Encoder self.lstm_encoder = torch.nn.LSTM(input_size=input_size, hidden_size=self.hidden_size, - num_layers=num_layers, dropout=dropout, - bidirectional=self.bidirectional, batch_first=True) + num_layers=num_layers, dropout=dropout, + bidirectional=self.bidirectional, batch_first=True) def _init_hidden(self): return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size), torch.zeros(self.num_layers, self.batch_size, self.hidden_size)) def forward(self, x): - - # Encoder - enc_init_hidden = self._init_hidden() - enc_output,enc_states = self.lstm_encoder(x,enc_init_hidden) + + enc_init_hidden = self._init_hidden() + enc_output, _ = self.lstm_encoder(x, enc_init_hidden) # RNNs obeys, Markovian. Consider the last state of the hidden is the markovian of the entire sequence in that batch. - enc_output = enc_output[ : , -1, : ] # Shape(batch_size,hidden_dim) + enc_output = enc_output[:, -1, :] # Shape(batch_size,hidden_dim) return enc_output -# Latent -class DisentangledAELatent(torch.nn.Module): - """Dense Latent Layer """ - def __init__(self, hidden_size: int, latent_size:int, dropout: float): + +class DisentangledAELatent(torch.nn.Module): + """Dense Dientangled Latent Layer between encoder and decoder""" + def __init__(self, hidden_size: int, latent_size: int, dropout: float): super(DisentangledAELatent, self).__init__() - self.latent_size = latent_size self.hidden_size = hidden_size self.dropout = dropout - self.latent = torch.nn.Linear(self.hidden_size,self.latent_size) - - def forward(self, x, training=True): - # Feed it into the disentanled latent layer - z = self.latent(x) # Shape(batch_size, latent_size*2) + self.latent = torch.nn.Linear(self.hidden_size, self.latent_size) + def forward(self, x): + z = self.latent(x) # Shape(batch_size, latent_size*2) return z -# Decoder + class LSTMDecoder(torch.nn.Module): """ Deep LSTM network. This implementation returns output_size outputs. Args: - input_size: The number of expected features in the input `x` - batch_size: - sequence_length: The number of in each sample + latent_size: The number of dimensions of the latent layer + batch_size: Number of samples in each batch of training data hidden_size: The number of features in the hidden state `h` num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would mean stacking two LSTMs together to form a `stacked LSTM`, with the second LSTM taking in outputs of the first LSTM and computing the final results. Default: 1 - output_size: The number of output dimensions + output_size: The number of output/input dimensions + num_future: The number of time steps in future predictions dropout: If non-zero, introduces a `Dropout` layer on the outputs of each LSTM layer except the last layer, with dropout probability equal to :attr:`dropout`. Default: 0 bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + reset_state: If ``True``, the hidden and cell states of the LSTM will + be reset at the beginning of each batch of input """ - - def __init__(self, batch_size:int, num_future:int, hidden_size: int, num_layers: int, - output_size: int, latent_size:int, batch_first: bool, dropout: float, + def __init__(self, batch_size: int, num_future: int, hidden_size: int, + num_layers: int, output_size: int, latent_size: int, + batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMDecoder, self).__init__() - self.batch_size = batch_size self.latent_size = latent_size self.num_future = num_future @@ -113,145 +132,167 @@ def __init__(self, batch_size:int, num_future:int, hidden_size: int, num_layers: self.bidirectional = bidirectional # RNN decoder - self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, hidden_size=self.hidden_size, - num_layers=self.num_layers, dropout=self.dropout, - bidirectional=self.bidirectional, batch_first=True) - self.output = TimeDistributed(torch.nn.Linear(self.hidden_size , self.output_size)) + self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + dropout=self.dropout, + bidirectional=self.bidirectional, + batch_first=True) + self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, + self.output_size)) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device), torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) + return (torch.zeros(self.num_layers, self.batch_size, + self.hidden_size).to(device), + torch.zeros(self.num_layers, self.batch_size, + self.hidden_size).to(device)) - def forward(self, x, num_future = None): - - # To feed the latent states into lstm decoder, repeat the tensor n_future times at second dim + def forward(self, x, num_future=None): + + # To feed the latent states into lstm decoder, + # repeat the tensor n_future times at second dim _init_hidden = self._init_hidden() decoder_inputs = x.unsqueeze(1) - if num_future==None: + if num_future is None: decoder_inputs = decoder_inputs.repeat(1, self.num_future, 1) - else: # For multistep a prediction after training + else: # For multistep a prediction after training decoder_inputs = decoder_inputs.repeat(1, num_future, 1) - + # Decoder input Shape(batch_size, num_futures, latent_size) - dec,(dec_hidden,dec_cell) = self.lstm_decoder(decoder_inputs,_init_hidden) - - # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) to Time Dsitributed Linear Layer + dec, _ = self.lstm_decoder(decoder_inputs, _init_hidden) + + # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) + # to Time Dsitributed Linear Layer output = self.output(dec) - return output + return output + class MLPClassifier(torch.nn.Module): """ MLP classifier """ - def __init__(self, hidden_size: int, num_classes:int, latent_size:int, dropout:float ): - + def __init__(self, hidden_size: int, num_classes: int, latent_size: int, + dropout: float): super(MLPClassifier, self).__init__() - self.latent_size = latent_size self.hidden_size = hidden_size self.num_classes = num_classes self.dropout = dropout - - # Classifier - self.classifier1 = torch.nn.Linear(self.latent_size , self.hidden_size) - self.classifier2 = torch.nn.Linear(self.hidden_size , self.hidden_size) - self.classifier3 = torch.nn.Linear(self.hidden_size , self.hidden_size) - self.classifier4 = torch.nn.Linear(self.hidden_size , self.num_classes) + + # Classifier layers + self.classifier1 = torch.nn.Linear(self.latent_size, self.hidden_size) + self.classifier2 = torch.nn.Linear(self.hidden_size, self.hidden_size) + self.classifier3 = torch.nn.Linear(self.hidden_size, self.hidden_size) + self.classifier4 = torch.nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) def forward(self, x): - + classifier1 = self.dropout(self.classifier1(x)) classifier2 = self.dropout(self.classifier2(classifier1)) classifier3 = self.dropout(self.classifier3(classifier2)) classifier4 = self.classifier4(classifier3) - # classifier_out = F.softmax(classifier4) - return classifier4 - - + + class MultiModelAE(torch.nn.Module): - - def __init__(self, *model_hyperparameters, **kwargs): - + + def __init__(self, input_size: int, + sequence_length: int, + batch_size: int, + num_future: int, + hidden_size: int, + num_layers: int, + output_size: int, + num_classes: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool ): + super(MultiModelAE, self).__init__() - # self.input_size = input_size - # self.sequence_length = sequence_length - # self.batch_size = batch_size - # self.latent_size = latent_size - # self.num_future = num_future - # self.hidden_size = hidden_size - # self.num_layers = num_layers - # self.output_size = output_size - # self.num_classes = num_classes - # self.batch_first = batch_first - # self.dropout = dropout - # self.reset_state = reset_state - # self.bidirectional = bidirectional - for dictionary in model_hyperparameters: - for key in dictionary: - setattr(self, key, dictionary[key]) - for key in kwargs: - setattr(self, key, kwargs[key]) - - # Network instances in the model - self.encoder = LSTMEncoder(input_size=self.input_size, sequence_length=self.sequence_length, batch_size=self.batch_size, - hidden_size=self.hidden_size, num_layers=self.num_layers, - batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, bidirectional=self.bidirectional) - - self.latent = DisentangledAELatent(hidden_size=self.hidden_size, latent_size=self.latent_size, dropout=self.dropout) - - self.decoder = LSTMDecoder(batch_size=self.batch_size, num_future=self.num_future, - hidden_size=self.hidden_size, num_layers=self.num_layers, output_size=self.output_size, - latent_size=self.latent_size, batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, bidirectional=self.bidirectional) - - self.classifier = MLPClassifier(hidden_size=self.hidden_size,num_classes=self.num_classes, latent_size=self.latent_size, dropout=self.dropout) + self.input_size = input_size + self.sequence_length = sequence_length + self.batch_size = batch_size + self.latent_size = latent_size + self.num_future = num_future + self.hidden_size = hidden_size + self.num_layers = num_layers + self.output_size = output_size + self.num_classes = num_classes + self.batch_first = batch_first + self.dropout = dropout + self.reset_state = reset_state + self.bidirectional = bidirectional + + self.encoder = LSTMEncoder(input_size=self.input_size, + sequence_length=self.sequence_length, + batch_size=self.batch_size, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + batch_first=self.batch_first, + dropout=self.dropout, + reset_state=True, + bidirectional=self.bidirectional) + + self.latent = DisentangledAELatent(hidden_size=self.hidden_size, + latent_size=self.latent_size, + dropout=self.dropout) + + self.decoder = LSTMDecoder(batch_size=self.batch_size, + num_future=self.num_future, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + output_size=self.output_size, + latent_size=self.latent_size, + batch_first=self.batch_first, + dropout=self.dropout, + reset_state=True, + bidirectional=self.bidirectional) + + self.classifier = MLPClassifier(hidden_size=self.hidden_size, + num_classes=self.num_classes, + latent_size=self.latent_size, + dropout=self.dropout) def forward(self, data, training=True, is_classification=False): + if not is_classification: + # Set the classifier grad off + for param in self.classifier.parameters(): + param.requires_grad = False + for param in self.encoder.parameters(): + param.requires_grad = True + for param in self.decoder.parameters(): + param.requires_grad = True + for param in self.latent.parameters(): + param.requires_grad = True - if not is_classification: - # Set the classifier grad off - for param in self.classifier.parameters(): - param.requires_grad = False - - for param in self.encoder.parameters(): - param.requires_grad = True - - for param in self.decoder.parameters(): - param.requires_grad = True - - for param in self.latent.parameters(): - param.requires_grad = True - - enc_out = self.encoder(data) - # Latent - latent_out= self.latent(enc_out,training=training) - # Decoder - decoder_out = self.decoder(latent_out) - - return decoder_out,latent_out - - else: - # training_mode = 'classification' - # Freeze decoder parameters; - for param in self.classifier.parameters(): - param.requires_grad = True - - for param in self.encoder.parameters(): - param.requires_grad = False - - for param in self.decoder.parameters(): - param.requires_grad = False - - for param in self.latent.parameters(): - param.requires_grad = False - - # Encoder - enc_out = self.encoder(data) - # Latent - latent_out= self.latent(enc_out,training=training) - #Classifier - classifier_out = self.classifier(latent_out) # Deterministic - return classifier_out + # Encoder + enc_out = self.encoder(data) + # Latent + latent_out = self.latent(enc_out) + # Decoder + decoder_out = self.decoder(latent_out) + + return decoder_out, latent_out + else: # training_mode = 'classification' + # Unfreeze classifier parameters and freeze all other + # network parameters + for param in self.classifier.parameters(): + param.requires_grad = True + for param in self.encoder.parameters(): + param.requires_grad = False + for param in self.decoder.parameters(): + param.requires_grad = False + for param in self.latent.parameters(): + param.requires_grad = False + + # Encoder + enc_out = self.encoder(data) + # Latent + latent_out = self.latent(enc_out) + # Classifier + classifier_out = self.classifier(latent_out) # Deterministic + return classifier_out diff --git a/traja/models/train.py b/traja/models/train.py index 7e7e2ab9..6fd4bdc5 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -10,36 +10,9 @@ device = 'cuda' if torch.cuda.is_available() else 'cpu' -def initializer(func): - """ - Automatically assigns the parameters. - https://stackoverflow.com/questions/109087/how-to-get-instance-variables-in-python - - >>> class process: - ... @initializer - ... def __init__(self, cmd, reachable=False, user='root'): - ... pass - >>> p = process('halt', True) - >>> p.cmd, p.reachable, p.user - ('halt', True, 'root') - """ - names, varargs, keywords, defaults = inspect.getargspec(func) - - @wraps(func) - def wrapper(self, *args, **kargs): - for name, arg in list(zip(names[1:], args)) + list(kargs.items()): - setattr(self, name, arg) - - for name, default in zip(reversed(names), reversed(defaults)): - if not hasattr(self, name): - setattr(self, name, default) - - func(self, *args, **kargs) - - return wrapper class Trainer(object): - # @initializer + def __init__(self, model_type:str, device:str, input_size:int, output_size:int, lstm_hidden_size:int, lstm_num_layers:int, @@ -51,7 +24,7 @@ def __init__(self, model_type:str, device:str, batch_first:bool =True, loss_type:str = 'huber'): - white_keys = ['ae','vae','lstm','vaegan', 'irl'] + white_keys = ['ae','vae','lstm','vaeg an', 'irl'] assert model_type in white_keys, "Valid models are {}".format(white_keys) self.model_type = model_type self.device = device @@ -120,10 +93,10 @@ def __init__(self, model_type:str, device:str, def __str__(self): return "Training model type {}".format(self.model_type) - # TRAIN AUTOENCODERS - def train_ae(self, train_loader, test_loader, model_save_path): + # TRAIN AUTOENCODERS/VARIATIONAL AUTOENCODERS + def train_latent_model(self, train_loader, test_loader, model_save_path): - assert self.model_type == 'ae' + assert self.model_type == 'ae' or 'vae' # Move the model to target device self.model.to(device) @@ -146,102 +119,26 @@ def train_ae(self, train_loader, test_loader, model_save_path): data, target,category = data.float().to(device), target.float().to(device), category.to(device) if training_mode =='forecasting': - - decoder_out,latent_out= self.model(data,training=True, is_classification=False) - loss = Criterion.ae_criterion(decoder_out, target) - loss.backward() - - self.encoder_optimizer.step() - self.decoder_optimizer.step() - self.latent_optimizer.step() - - else: - - classifier_out = self.model(data,training=True, is_classification=True) - loss = Criterion.classifier_criterion(classifier_out, category-1) - loss.backward() - - self.classifier_optimizer.step() - - total_loss+=loss - - print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss/(idx+1))) - - if epoch+1 == self.epochs: # - training_mode = 'classification' - - # Testing - if epoch%10==0: - with torch.no_grad(): - self.model.eval() - test_loss_forecasting = 0 - test_loss_classification = 0 - for idx, (data, target,category) in enumerate(list(test_loader)): - data, target, category = data.float().to(device), target.float().to(device), category.to(device) - out, latent = self.model(data, training=False, is_classification=False) - test_loss_forecasting += Criterion.ae_criterion(out,target).item() + if self.model_type == 'ae': + decoder_out, latent_out = self.model(data, training=True, is_classification=False) + loss = Criterion.ae_criterion(decoder_out, target) - classifier_out= self.model(data,training=False, is_classification=True) - test_loss_classification += Criterion.classifier_criterion(classifier_out, category-1).item() - - test_loss_forecasting /= len(test_loader.dataset) - print(f'====> Mean test set generator loss: {test_loss_forecasting:.4f}') - test_loss_classification /= len(test_loader.dataset) - print(f'====> Mean test set classifier loss: {test_loss_classification:.4f}') - - # Scheduler metric is test set loss - if training_mode =='forecasting': - self.encoder_scheduler.step(self.test_loss_forecasting) - self.decoder_scheduler.step(self.test_loss_forecasting) - self.latent_scheduler.step(self.test_loss_forecasting) - else: - self.classifier_scheduler.step(self.test_loss_classification) - - # Save the model at target path - utils.save_model(self.model,PATH = model_save_path) - - # TRAIN VARIATIONAL AUTOENCODERS - def train_vae(self, train_loader, test_loader, model_save_path): - assert self.model_type == 'vae' - # Move the model to target device - self.model.to(device) - - # Training mode: Switch from Generative to classifier training mode - training_mode = 'forecasting' - - # Training - for epoch in range(self.epochs*2): # First half for generative model and next for classifier - if epoch>0: # Initial step is to test and set LR schduler - # Training - self.model.train() - total_loss = 0 - for idx, (data, target,category) in enumerate(train_loader): - # Reset optimizer states - self.encoder_optimizer.zero_grad() - self.latent_optimizer.zero_grad() - self.decoder_optimizer.zero_grad() - self.classifier_optimizer.zero_grad() - - data, target,category = data.float().to(device), target.float().to(device), category.to(device) - - if training_mode =='forecasting': - - decoder_out,latent_out,mu,logvar= self.model(data,training=True, is_classification=False) - loss = Criterion.vae_criterion(decoder_out, target,mu,logvar) + else: # vae + decoder_out, latent_out, mu, logvar= self.model(data, training=True, is_classification=False) + loss = Criterion.vae_criterion(decoder_out, target, mu, logvar) + loss.backward() self.encoder_optimizer.step() self.decoder_optimizer.step() self.latent_optimizer.step() - else: + else: # training_mode == 'classification' - classifier_out = self.model(data,training=True, is_classification=True) + classifier_out = self.model(data, training=True, is_classification=True) loss = Criterion.classifier_criterion(classifier_out, category-1) loss.backward() - self.classifier_optimizer.step() - total_loss+=loss print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss/(idx+1))) @@ -257,9 +154,14 @@ def train_vae(self, train_loader, test_loader, model_save_path): test_loss_classification = 0 for idx, (data, target,category) in enumerate(list(test_loader)): data, target, category = data.float().to(device), target.float().to(device), category.to(device) - out, latent, mu, logvar = self.model(data, training=False, is_classification=False) - test_loss_forecasting += Criterion.vae_criterion(out,target, mu, logvar).item() - + # Time seriesforecasting test + if self.model_type=='ae': + out, latent = self.model(data, training=False, is_classification=False) + test_loss_forecasting += Criterion.ae_criterion(out,target).item() + else: + decoder_out,latent_out,mu,logvar= self.model(data,training=False, is_classification=False) + test_loss_forecasting += Criterion.vae_criterion(decoder_out, target,mu,logvar) + # Classification test classifier_out= self.model(data,training=False, is_classification=True) test_loss_classification += Criterion.classifier_criterion(classifier_out, category-1).item() @@ -279,6 +181,7 @@ def train_vae(self, train_loader, test_loader, model_save_path): # Save the model at target path utils.save_model(self.model,PATH = model_save_path) + # TRAIN VARIATIONAL AUTOENCODERS-GAN def train_vaegan(self): assert self.model_type == 'vaegan' diff --git a/traja/models/vae.py b/traja/models/vae.py index b29312b1..a0126267 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -1,7 +1,309 @@ -"""This module contains the variational autoencoders and its variants -1. classic variational autoencoder +""" This module implement the Variational Autoencoder model for +both forecasting and classification of time series data. +```USAGE``` to train AE model: +trainer = Trainer(model_type='vae', + device=device, + input_size=input_size, + output_size=output_size, + lstm_hidden_size=lstm_hidden_size, + lstm_num_layers=lstm_num_layers, + reset_state=True, + num_classes=num_classes, + latent_size=latent_size, + dropout=0.1, + num_layers=num_layers, + epochs=epochs, + batch_size=batch_size, + num_future=num_future, + sequence_length=sequence_length, + bidirectional =False, + batch_first =True, + loss_type = 'huber') -Loss functions: -1. MSE -2. Huber Loss""" \ No newline at end of file +trainer.train_latent_model(train_dataloader, test_dataloader, model_save_path=PATH)""" + +import torch +from .utils import TimeDistributed +from .utils import load_model + +device = 'cuda' if torch.cuda.is_available() else 'cpu' + + +class LSTMEncoder(torch.nn.Module): + """ Deep LSTM network. This implementation + returns output_size hidden size. + Args: + input_size: The number of expected features in the input `x` + batch_size: + sequence_length: The number of in each sample + hidden_size: The number of features in the hidden state `h` + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` + would mean stacking two LSTMs together to form a `stacked LSTM`, + with the second LSTM taking in outputs of the first LSTM and + computing the final results. Default: 1 + output_size: The number of output dimensions + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + LSTM layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + """ + + def __init__(self, input_size: int, sequence_length: int, batch_size: int, + hidden_size: int, num_layers: int, + batch_first: bool, dropout: float, + reset_state: bool, bidirectional: bool): + + super(LSTMEncoder, self).__init__() + + self.input_size = input_size + self.sequence_length = sequence_length + self.batch_size = batch_size + self.hidden_size = hidden_size + self.num_layers = num_layers + self.batch_first = batch_first + self.dropout = dropout + self.reset_state = reset_state + self.bidirectional = bidirectional + + self.lstm_encoder = torch.nn.LSTM(input_size=input_size, hidden_size=self.hidden_size, + num_layers=num_layers, dropout=dropout, + bidirectional=self.bidirectional, batch_first=True) + + def _init_hidden(self): + return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size), torch.zeros(self.num_layers, self.batch_size, self.hidden_size)) + + def forward(self, x): + + enc_init_hidden = self._init_hidden() + enc_output, _ = self.lstm_encoder(x, enc_init_hidden) + # RNNs obeys, Markovian. Consider the last state of the hidden is the markovian of the entire sequence in that batch. + enc_output = enc_output[:, -1, :] # Shape(batch_size,hidden_dim) + return enc_output + + +class DisentangledAELatent(torch.nn.Module): + """Dense Dientangled Latent Layer between encoder and decoder""" + def __init__(self, hidden_size: int, latent_size: int, dropout: float): + super(DisentangledAELatent, self).__init__() + self.latent_size = latent_size + self.hidden_size = hidden_size + self.dropout = dropout + self.latent = torch.nn.Linear(self.hidden_size, self.latent_size) + + def reparameterize(self, mu, logvar, training= True): + if training: + std = logvar.mul(0.5).exp_() + eps = std.data.new(std.size()).normal_() + return eps.mul(std).add_(mu) + return mu + + def forward(self, x, training=True): + z_variables = self.latent(x) # [batch_size, latent_size*2] + mu, logvar = torch.chunk(z_variables, 2, dim=1) # [batch_size,latent_size] + # Reparameterize + z = self.reparameterize(mu, logvar, training=training) # [batch_size,latent_size] + return z, mu, logvar + + +class LSTMDecoder(torch.nn.Module): + """ Deep LSTM network. This implementation + returns output_size outputs. + Args: + latent_size: The number of dimensions of the latent layer + batch_size: Number of samples in each batch of training data + hidden_size: The number of features in the hidden state `h` + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` + would mean stacking two LSTMs together to form a `stacked LSTM`, + with the second LSTM taking in outputs of the first LSTM and + computing the final results. Default: 1 + output_size: The number of output/input dimensions + num_future: The number of time steps in future predictions + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + LSTM layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + reset_state: If ``True``, the hidden and cell states of the LSTM will + be reset at the beginning of each batch of input + """ + def __init__(self, batch_size: int, num_future: int, hidden_size: int, + num_layers: int, output_size: int, latent_size: int, + batch_first: bool, dropout: float, + reset_state: bool, bidirectional: bool): + super(LSTMDecoder, self).__init__() + self.batch_size = batch_size + self.latent_size = latent_size + self.num_future = num_future + self.hidden_size = hidden_size + self.num_layers = num_layers + self.output_size = output_size + self.batch_first = batch_first + self.dropout = dropout + self.reset_state = reset_state + self.bidirectional = bidirectional + + # RNN decoder + self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + dropout=self.dropout, + bidirectional=self.bidirectional, + batch_first=True) + self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, + self.output_size)) + + def _init_hidden(self): + return (torch.zeros(self.num_layers, self.batch_size, + self.hidden_size).to(device), + torch.zeros(self.num_layers, self.batch_size, + self.hidden_size).to(device)) + + def forward(self, x, num_future=None): + + # To feed the latent states into lstm decoder, repeat the + # tensor n_future times at second dim + _init_hidden = self._init_hidden() + decoder_inputs = x.unsqueeze(1) + + if num_future is None: + decoder_inputs = decoder_inputs.repeat(1, self.num_future, 1) + else: # For multistep a prediction after training + decoder_inputs = decoder_inputs.repeat(1, num_future, 1) + + # Decoder input Shape(batch_size, num_futures, latent_size) + dec, _ = self.lstm_decoder(decoder_inputs, _init_hidden) + + # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) + # to Time Dsitributed Linear Layer + output = self.output(dec) + return output + + +class MLPClassifier(torch.nn.Module): + """ MLP classifier + """ + def __init__(self, hidden_size: int, num_classes: int, latent_size: int, + dropout: float): + super(MLPClassifier, self).__init__() + self.latent_size = latent_size + self.hidden_size = hidden_size + self.num_classes = num_classes + self.dropout = dropout + + # Classifier layers + self.classifier1 = torch.nn.Linear(self.latent_size, self.hidden_size) + self.classifier2 = torch.nn.Linear(self.hidden_size, self.hidden_size) + self.classifier3 = torch.nn.Linear(self.hidden_size, self.hidden_size) + self.classifier4 = torch.nn.Linear(self.hidden_size, self.num_classes) + self.dropout = torch.nn.Dropout(p=dropout) + + def forward(self, x): + + classifier1 = self.dropout(self.classifier1(x)) + classifier2 = self.dropout(self.classifier2(classifier1)) + classifier3 = self.dropout(self.classifier3(classifier2)) + classifier4 = self.classifier4(classifier3) + return classifier4 + +class MultiModelVAE(torch.nn.Module): + """Implementation of Multimodel Variational autoencoders; + """ + def __init__(self, input_size: int, + sequence_length: int, + batch_size: int, + num_future: int, + hidden_size: int, + num_layers: int, + output_size: int, + num_classes: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool ): + + super(MultiModelVAE, self).__init__() + self.input_size = input_size + self.sequence_length = sequence_length + self.batch_size = batch_size + self.latent_size = latent_size + self.num_future = num_future + self.hidden_size = hidden_size + self.num_layers = num_layers + self.output_size = output_size + self.num_classes = num_classes + self.batch_first = batch_first + self.dropout = dropout + self.reset_state = reset_state + self.bidirectional = bidirectional + + self.encoder = LSTMEncoder(input_size=self.input_size, + sequence_length=self.sequence_length, + batch_size=self.batch_size, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + batch_first=self.batch_first, + dropout=self.dropout, + reset_state=True, + bidirectional=self.bidirectional) + + self.latent = DisentangledAELatent(hidden_size=self.hidden_size, + latent_size=self.latent_size, + dropout=self.dropout) + + self.decoder = LSTMDecoder(batch_size=self.batch_size, + num_future=self.num_future, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + output_size=self.output_size, + latent_size=self.latent_size, + batch_first=self.batch_first, + dropout=self.dropout, + reset_state=True, + bidirectional=self.bidirectional) + + self.classifier = MLPClassifier(hidden_size=self.hidden_size, + num_classes=self.num_classes, + latent_size=self.latent_size, + dropout=self.dropout) + + def forward(self, data, training=True, is_classification=False): + + if not is_classification: + # Set the classifier grad off + for param in self.classifier.parameters(): + param.requires_grad = False + for param in self.encoder.parameters(): + param.requires_grad = True + for param in self.decoder.parameters(): + param.requires_grad = True + for param in self.latent.parameters(): + param.requires_grad = True + + # Encoder + enc_out = self.encoder(data) + # Latent + latent_out = self.latent(enc_out) + # Decoder + decoder_out = self.decoder(latent_out) + return decoder_out, latent_out + + else: # training_mode = 'classification' + # Unfreeze classifier parameters and freeze all other + # network parameters + for param in self.classifier.parameters(): + param.requires_grad = True + for param in self.encoder.parameters(): + param.requires_grad = False + for param in self.decoder.parameters(): + param.requires_grad = False + for param in self.latent.parameters(): + param.requires_grad = False + + # Encoder + enc_out = self.encoder(data) + # Latent + latent_out, mu, logvar = self.latent(enc_out, training=training) + # Classifier + classifier_out = self.classifier(latent_out) # Deterministic + return classifier_out, latent_out, mu, logvar From 132ffefc92404349773ce5da865b9b124b7ed783 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sat, 19 Dec 2020 14:31:54 +0100 Subject: [PATCH 032/211] ae,vae,lstm,doc,trainer,hybrid trainer,optimizers, model tests update --- traja/__init__.py | 5 +- traja/datasets/__init__.py | 3 +- traja/datasets/dataset.py | 21 +- traja/datasets/utils.py | 49 +++- traja/models/__init__.py | 3 +- traja/models/ae.py | 186 +++++++-------- traja/models/experiment.py | 175 +++++++------- traja/models/generator.py | 45 ++-- traja/models/interpretor.py | 17 +- traja/models/irl.py | 17 +- traja/models/losses.py | 109 +++++---- traja/models/lstm.py | 80 +++++-- traja/models/nn.py | 185 +++++++-------- traja/models/optimizers.py | 89 ++++++++ traja/models/test.py | 44 ++++ traja/models/train.py | 442 +++++++++++++++++++++++++----------- traja/models/utils.py | 74 +----- traja/models/vae.py | 191 ++++++++-------- traja/models/vaegan.py | 20 +- traja/models/visualizer.py | 266 ++++++++++++++++++++++ traja/tests/test_models.py | 92 ++++++++ 21 files changed, 1387 insertions(+), 726 deletions(-) create mode 100644 traja/models/optimizers.py create mode 100644 traja/models/test.py create mode 100644 traja/models/visualizer.py create mode 100644 traja/tests/test_models.py diff --git a/traja/__init__.py b/traja/__init__.py index 2e884412..5a3f5355 100644 --- a/traja/__init__.py +++ b/traja/__init__.py @@ -1,11 +1,10 @@ -from . import models -from . import datasets - from .accessor import TrajaAccessor from .frame import TrajaDataFrame, TrajaCollection from .parsers import read_file, from_df from .plotting import * from .trajectory import * +from traja import models +from traja import datasets import logging diff --git a/traja/datasets/__init__.py b/traja/datasets/__init__.py index d195115f..665235a1 100644 --- a/traja/datasets/__init__.py +++ b/traja/datasets/__init__.py @@ -2,9 +2,8 @@ import glob import os from typing import List - import pandas as pd - +from traja.datasets import dataset import traja diff --git a/traja/datasets/dataset.py b/traja/datasets/dataset.py index 45cd5610..8742da2f 100644 --- a/traja/datasets/dataset.py +++ b/traja/datasets/dataset.py @@ -1,16 +1,3 @@ -""" -Modified from https://github.com/agrimgupta92/sgan/blob/master/sgan/data/trajectories.py. - -This module contains: - -Classes: -1. Pytorch Time series dataset class instance -2. Weighted train and test dataset loader with respect to class distribution - -Helpers: -1. Class distribution in the dataset - -""" import logging import os import math @@ -21,7 +8,7 @@ from torch.utils.data.sampler import WeightedRandomSampler import pandas as pd from sklearn.utils import shuffle -from datasets import utils +from traja.datasets import utils logger = logging.getLogger(__name__) @@ -249,7 +236,7 @@ def __getitem__(self, index): def __len__(self): return len(self.data) -class MultiModalDataLoader(object): +class MultiModalDataLoader: """ MultiModalDataLoader wraps the following data preparation steps, @@ -269,7 +256,7 @@ class MultiModalDataLoader(object): batch_size (int): Number of samples per batch of data n_past (int): Input sequence length. Number of time steps from the past. n_future (int): Target sequence length. Number of time steps to the future. - num_workers (int): Number of cpu subprocess to be occupied during data loading process + num_workers (int): Number of cpu subprocess occupied during data loading process Usage: train_dataloader, test_dataloader = MultiModalDataLoader(df = data_frame, batch_size=32, @@ -311,7 +298,7 @@ def __new__(cls, df:pd.DataFrame, batch_size:int, n_past:int, n_future:int, num_ # Return train and test loader attributes return loader_instance.train_loader, loader_instance.test_loader - + diff --git a/traja/datasets/utils.py b/traja/datasets/utils.py index 767b87d6..b95ce362 100644 --- a/traja/datasets/utils.py +++ b/traja/datasets/utils.py @@ -3,6 +3,7 @@ import math import numpy as np import torch +from torch import long from torch.utils.data import Dataset from collections import Counter from torch.utils.data.sampler import WeightedRandomSampler @@ -14,10 +15,17 @@ logger = logging.getLogger(__name__) def get_class_distribution(targets): - """Compute class distribution, returns number of classes and their count in the targets""" + """Compute class distribution, returns number of classes and their count in the targets + + Args: + targets ([type]): [description] + + Returns: + [type]: [description] + """ targets_ = np.unique(targets, return_counts=True) return targets_[0],targets_[1] - + def generate_dataset(df, n_past, n_future): """ df : Dataframe @@ -54,6 +62,17 @@ def generate_dataset(df, n_past, n_future): return train_data, target_data, target_category def shuffle_split(train_data:np.array, target_data:np.array,target_category:np.array, train_ratio:float): + """[summary] + + Args: + train_data (np.array): [description] + target_data (np.array): [description] + target_category (np.array): [description] + train_ratio (float): [description] + + Returns: + [type]: [description] + """ # Shuffle the IDs and the corresponding sequence , preserving the order train_data, target_data, target_category = shuffle(train_data, target_data, target_category) @@ -74,6 +93,15 @@ def shuffle_split(train_data:np.array, target_data:np.array,target_category:np.a return [train_x,train_y,train_z],[test_x,test_y,test_z] def scale_data(data, sequence_length): + """[summary] + + Args: + data ([type]): [description] + sequence_length ([type]): [description] + + Returns: + [type]: [description] + """ assert len(data[0].shape)==2 scalers={} data = np.vstack(data) @@ -89,11 +117,20 @@ def scale_data(data, sequence_length): return data, scalers def weighted_random_samplers(train_z,test_z): + """[summary] - # Prepare weighted random sampler: - train_target_list = torch.tensor(train_z) - test_target_list = torch.tensor(test_z) + Args: + train_z ([type]): [description] + test_z ([type]): [description] + Returns: + [type]: [description] + """ + + # Prepare weighted random sampler: + train_target_list = torch.tensor(train_z).type(torch.LongTensor) + test_target_list = torch.tensor(test_z).type(torch.LongTensor) + # Number of classes and their frequencies train_targets_, train_class_count = get_class_distribution(train_target_list) test_targets_, test_class_count = get_class_distribution(test_target_list) @@ -101,7 +138,7 @@ def weighted_random_samplers(train_z,test_z): # Compute class weights train_class_weights = 1./torch.tensor(train_class_count, dtype=torch.float) test_class_weights = 1./torch.tensor(test_class_count, dtype=torch.float) - + # Assign weights to original target list train_class_weights_all = train_class_weights[train_target_list-1] # Note the targets start from 1, to python idx to apply,-1 test_class_weights_all = test_class_weights[test_target_list-1] diff --git a/traja/models/__init__.py b/traja/models/__init__.py index 399397d7..63328ec4 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1 +1,2 @@ -from .nn import LSTM, TimeseriesDataset, get_transformed_timeseries_dataloaders, Trainer \ No newline at end of file +from .nn import LSTM +from .vae import MultiModelVAE diff --git a/traja/models/ae.py b/traja/models/ae.py index 4677336a..648e6b82 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -1,95 +1,69 @@ -""" This module implement the Auto encoder model for both forecasting -and classification of time series data. - -```USAGE``` to train AE model: -trainer = Trainer(model_type='ae', - device=device, - input_size=input_size, - output_size=output_size, - lstm_hidden_size=lstm_hidden_size, - lstm_num_layers=lstm_num_layers, - reset_state=True, - num_classes=num_classes, - latent_size=latent_size, - dropout=0.1, - num_layers=num_layers, - epochs=epochs, - batch_size=batch_size, - num_future=num_future, - sequence_length=sequence_length, - bidirectional =False, - batch_first =True, - loss_type = 'huber') - -trainer.train_latent_model(train_dataloader, test_dataloader, model_save_path=PATH)""" - import torch -from .utils import TimeDistributed -from .utils import load_model - +from traja.models.utils import TimeDistributed +from torch import nn device = 'cuda' if torch.cuda.is_available() else 'cpu' class LSTMEncoder(torch.nn.Module): - """ Deep LSTM network. This implementation - returns output_size hidden size. - Args: - input_size: The number of expected features in the input `x` - batch_size: - sequence_length: The number of in each sample - hidden_size: The number of features in the hidden state `h` - num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` - would mean stacking two LSTMs together to form a `stacked LSTM`, - with the second LSTM taking in outputs of the first LSTM and - computing the final results. Default: 1 - output_size: The number of output dimensions - dropout: If non-zero, introduces a `Dropout` layer on the outputs of each - LSTM layer except the last layer, with dropout probability equal to - :attr:`dropout`. Default: 0 - bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + + """ Implementation of Encoder network using LSTM layers + :param input_size: The number of expected features in the input x + :param num_past: Number of time steps to look backwards to predict num_future steps forward + :param batch_size: Number of samples in a batch + :param hidden_size: The number of features in the hidden state h + :param num_lstm_layers: Number of layers in the LSTM model + + :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param reset_state: If True, will reset the hidden and cell state for each batch of data + :param bidirectional: If True, becomes a bidirectional LSTM """ - def __init__(self, input_size: int, sequence_length: int, batch_size: int, - hidden_size: int, num_layers: int, + def __init__(self, input_size: int, num_past: int, batch_size: int, + hidden_size: int, num_lstm_layers: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMEncoder, self).__init__() self.input_size = input_size - self.sequence_length = sequence_length + self.num_past = num_past self.batch_size = batch_size self.hidden_size = hidden_size - self.num_layers = num_layers + self.num_lstm_layers = num_lstm_layers self.batch_first = batch_first self.dropout = dropout self.reset_state = reset_state self.bidirectional = bidirectional self.lstm_encoder = torch.nn.LSTM(input_size=input_size, hidden_size=self.hidden_size, - num_layers=num_layers, dropout=dropout, + num_layers=num_lstm_layers, dropout=dropout, bidirectional=self.bidirectional, batch_first=True) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size), torch.zeros(self.num_layers, self.batch_size, self.hidden_size)) + return (torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size), + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size)) def forward(self, x): - enc_init_hidden = self._init_hidden() enc_output, _ = self.lstm_encoder(x, enc_init_hidden) - # RNNs obeys, Markovian. Consider the last state of the hidden is the markovian of the entire sequence in that batch. + # RNNs obeys, Markovian. So, the last state of the hidden is the markovian state for the entire + # sequence in that batch. enc_output = enc_output[:, -1, :] # Shape(batch_size,hidden_dim) return enc_output class DisentangledAELatent(torch.nn.Module): """Dense Dientangled Latent Layer between encoder and decoder""" - def __init__(self, hidden_size: int, latent_size: int, dropout: float): + + def __init__(self, hidden_size: int, latent_size: int, dropout: float): super(DisentangledAELatent, self).__init__() self.latent_size = latent_size self.hidden_size = hidden_size self.dropout = dropout self.latent = torch.nn.Linear(self.hidden_size, self.latent_size) + def forward(self, x): z = self.latent(x) # Shape(batch_size, latent_size*2) return z @@ -115,8 +89,9 @@ class LSTMDecoder(torch.nn.Module): reset_state: If ``True``, the hidden and cell states of the LSTM will be reset at the beginning of each batch of input """ - def __init__(self, batch_size: int, num_future: int, hidden_size: int, - num_layers: int, output_size: int, latent_size: int, + + def __init__(self, batch_size: int, num_future: int, hidden_size: int, + num_lstm_layers: int, output_size: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMDecoder, self).__init__() @@ -124,7 +99,7 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, self.latent_size = latent_size self.num_future = num_future self.hidden_size = hidden_size - self.num_layers = num_layers + self.num_lstm_layers = num_lstm_layers self.output_size = output_size self.batch_first = batch_first self.dropout = dropout @@ -134,17 +109,17 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, # RNN decoder self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, hidden_size=self.hidden_size, - num_layers=self.num_layers, + num_layers=self.num_lstm_layers, dropout=self.dropout, bidirectional=self.bidirectional, batch_first=True) - self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, + self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, self.output_size)) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, - self.hidden_size).to(device), - torch.zeros(self.num_layers, self.batch_size, + return (torch.zeros(self.num_lstm_layers, self.batch_size, + self.hidden_size).to(device), + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device)) def forward(self, x, num_future=None): @@ -171,89 +146,84 @@ def forward(self, x, num_future=None): class MLPClassifier(torch.nn.Module): """ MLP classifier """ - def __init__(self, hidden_size: int, num_classes: int, latent_size: int, + + def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_size: int, num_classifier_layers: int, dropout: float): super(MLPClassifier, self).__init__() self.latent_size = latent_size + self.input_size = input_size self.hidden_size = hidden_size self.num_classes = num_classes + self.num_classifier_layers = num_classifier_layers self.dropout = dropout # Classifier layers - self.classifier1 = torch.nn.Linear(self.latent_size, self.hidden_size) - self.classifier2 = torch.nn.Linear(self.hidden_size, self.hidden_size) - self.classifier3 = torch.nn.Linear(self.hidden_size, self.hidden_size) - self.classifier4 = torch.nn.Linear(self.hidden_size, self.num_classes) + self.hidden = nn.ModuleList([nn.Linear(self.input_size, self.hidden_size)]) + self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_classifier_layers - 1)]) + self.hidden = nn.Sequential(*self.hidden) + self.out = nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) def forward(self, x): - - classifier1 = self.dropout(self.classifier1(x)) - classifier2 = self.dropout(self.classifier2(classifier1)) - classifier3 = self.dropout(self.classifier3(classifier2)) - classifier4 = self.classifier4(classifier3) - return classifier4 + x = self.dropout(self.hidden(x)) + out = self.out(x) + return out class MultiModelAE(torch.nn.Module): - def __init__(self, input_size: int, - sequence_length: int, - batch_size: int, - num_future: int, - hidden_size: int, - num_layers: int, - output_size: int, - num_classes: int, - latent_size: int, - batch_first: bool, - dropout: float, - reset_state: bool, - bidirectional: bool ): + def __init__(self, input_size: int, num_past: int, batch_size: int, num_future: int, lstm_hidden_size: int, + num_lstm_layers: int, classifier_hidden_size: int, num_classifier_layers: int, output_size: int, + num_classes: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, + bidirectional: bool): super(MultiModelAE, self).__init__() self.input_size = input_size - self.sequence_length = sequence_length + self.num_past = num_past self.batch_size = batch_size self.latent_size = latent_size self.num_future = num_future - self.hidden_size = hidden_size - self.num_layers = num_layers + self.lstm_hidden_size = lstm_hidden_size + self.num_lstm_layers = num_lstm_layers + self.num_classifier_layers = num_classifier_layers + self.classifier_hidden_size = classifier_hidden_size self.output_size = output_size self.num_classes = num_classes self.batch_first = batch_first self.dropout = dropout self.reset_state = reset_state self.bidirectional = bidirectional - - self.encoder = LSTMEncoder(input_size=self.input_size, - sequence_length=self.sequence_length, + + self.encoder = LSTMEncoder(input_size=self.input_size, + num_past=self.num_past, batch_size=self.batch_size, - hidden_size=self.hidden_size, - num_layers=self.num_layers, - batch_first=self.batch_first, + hidden_size=self.lstm_hidden_size, + num_lstm_layers=self.num_lstm_layers, + batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, + reset_state=True, bidirectional=self.bidirectional) - self.latent = DisentangledAELatent(hidden_size=self.hidden_size, - latent_size=self.latent_size, + self.latent = DisentangledAELatent(hidden_size=self.lstm_hidden_size, + latent_size=self.latent_size, dropout=self.dropout) - self.decoder = LSTMDecoder(batch_size=self.batch_size, + self.decoder = LSTMDecoder(batch_size=self.batch_size, num_future=self.num_future, - hidden_size=self.hidden_size, - num_layers=self.num_layers, + hidden_size=self.lstm_hidden_size, + num_lstm_layers=self.num_lstm_layers, output_size=self.output_size, - latent_size=self.latent_size, - batch_first=self.batch_first, + latent_size=self.latent_size, + batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, + reset_state=True, bidirectional=self.bidirectional) - self.classifier = MLPClassifier(hidden_size=self.hidden_size, - num_classes=self.num_classes, - latent_size=self.latent_size, + self.classifier = MLPClassifier(input_size=self.latent_size, + hidden_size=self.classifier_hidden_size, + num_classes=self.num_classes, + latent_size=self.latent_size, + num_classifier_layers=self.num_classifier_layers, dropout=self.dropout) def forward(self, data, training=True, is_classification=False): @@ -267,7 +237,7 @@ def forward(self, data, training=True, is_classification=False): param.requires_grad = True for param in self.latent.parameters(): param.requires_grad = True - + # Encoder enc_out = self.encoder(data) # Latent @@ -288,7 +258,7 @@ def forward(self, data, training=True, is_classification=False): param.requires_grad = False for param in self.latent.parameters(): param.requires_grad = False - + # Encoder enc_out = self.encoder(data) # Latent diff --git a/traja/models/experiment.py b/traja/models/experiment.py index 8e800790..5e805571 100644 --- a/traja/models/experiment.py +++ b/traja/models/experiment.py @@ -24,9 +24,9 @@ from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler import torchvision.transforms as transforms - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + class Trainer: def __init__(self, model, train_loader, @@ -47,7 +47,7 @@ def __init__(self, model, self.train_loader = train_loader self.test_loader = test_loader - self.criterion =torch.nn.MSELoss() + self.criterion = torch.nn.MSELoss() print('Checking for optimizer for {}'.format(optimizer)) if optimizer == "adam": print('Using adam') @@ -78,14 +78,16 @@ def __init__(self, model, if not os.path.exists(save_dir): os.makedirs(save_dir) - self.savepath = os.path.join(save_dir, f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') + self.savepath = os.path.join(save_dir, + f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') self.experiment_done = False if os.path.exists(self.savepath): trained_epochs = len(pd.read_csv(self.savepath, sep=';')) if trained_epochs >= epochs: self.experiment_done = True - print(f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') + print( + f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') if os.path.exists((self.savepath.replace('.csv', '.pt'))): self.model.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['model_state_dict']) self.model = self.model.to(self.device) @@ -97,13 +99,12 @@ def __init__(self, model, self.start_epoch = 0 self.model = self.model.to(self.device) - def _infer_initial_epoch(self, savepath): if not os.path.exists(savepath): return 0 else: df = pd.read_csv(savepath, sep=';', index_col=0) - print(len(df)+1) + print(len(df) + 1) return len(df) def train(self): @@ -117,7 +118,7 @@ def train(self): if self.opt_name == "LRS": print('LRS step') self.lr_scheduler.step() - return self.savepath+'.csv' + return self.savepath + '.csv' def train_epoch(self): self.model.train() @@ -125,7 +126,7 @@ def train_epoch(self): running_loss = 0 old_time = time() for batch, data in enumerate(self.train_loader): - inputs, targets= data[0].to(self.device).float(), data[1].to(self.device).float() + inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() self.optimizer.zero_grad() outputs = self.model(inputs) loss = self.criterion(outputs, targets) @@ -134,12 +135,13 @@ def train_epoch(self): running_loss += loss.item() if batch % 10 == 0 and batch != 0: - print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'loss:', running_loss/total) + print(batch, 'of', len(self.train_loader), 'processing time', time() - old_time, 'loss:', + running_loss / total) old_time = time() # Increment number of batches total += 1 - return running_loss/total + return running_loss / total def test(self, epoch, save=True): self.model.eval() @@ -148,7 +150,7 @@ def test(self, epoch, save=True): with torch.no_grad(): for batch, data in enumerate(self.test_loader): if batch % 10 == 0: - print('Processing eval batch', batch,'of', len(self.test_loader)) + print('Processing eval batch', batch, 'of', len(self.test_loader)) inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() outputs = self.model(inputs) loss = self.criterion(outputs, targets) @@ -196,16 +198,17 @@ def __init__(self, input_size: int, hidden_size: int, num_layers: int, self.head = nn.Linear(hidden_size, output_size) - def forward(self, x): + def forward(self, x): x, state = self.lstm(x) # Use the last hidden state of last layer - x = state[0][-1] + x = state[0][-1] x = self.head(x) return x + class TrajectoryLSTM: def __init__( - self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() + self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() ): fig, ax = plt.subplots(2, 1) self.fig = fig @@ -224,10 +227,10 @@ def load_batch(self, batch_size=32): inds = np.random.randint(0, len(self.xy) - self.nb_steps, (self.batch_size)) for i, ind in enumerate(inds): - t_1_b[:, i] = self.xy[ind : ind + self.nb_steps] - t_b[i * nb_steps : (i + 1) * self.nb_steps] = self.xy[ - ind + 1 : ind + nb_steps + 1 - ] + t_1_b[:, i] = self.xy[ind: ind + self.nb_steps] + t_b[i * nb_steps: (i + 1) * self.nb_steps] = self.xy[ + ind + 1: ind + nb_steps + 1 + ] return torch.from_numpy(t_1_b).float(), torch.from_numpy(t_b).float() def train(self): @@ -287,6 +290,7 @@ def plot(self, interactive=True): self._plot() return self.fig + def make_mlp(dim_list, activation="relu", batch_norm=True, dropout=0): layers = [] for dim_in, dim_out in zip(dim_list[:-1], dim_list[1:]): @@ -315,7 +319,7 @@ class Encoder(nn.Module): TrajectoryDiscriminator""" def __init__( - self, embedding_dim=64, h_dim=64, mlp_dim=1024, num_layers=1, dropout=0.0 + self, embedding_dim=64, h_dim=64, mlp_dim=1024, num_layers=1, dropout=0.0 ): super(Encoder, self).__init__() @@ -355,20 +359,20 @@ class Decoder(nn.Module): """Decoder is part of TrajectoryGenerator""" def __init__( - self, - seq_len, - embedding_dim=64, - h_dim=128, - mlp_dim=1024, - num_layers=1, - pool_every_timestep=True, - dropout=0.0, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - pooling_type="pool_net", - neighborhood_size=2.0, - grid_size=8, + self, + seq_len, + embedding_dim=64, + h_dim=128, + mlp_dim=1024, + num_layers=1, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + pooling_type="pool_net", + neighborhood_size=2.0, + grid_size=8, ): super(Decoder, self).__init__() @@ -452,14 +456,14 @@ class PoolHiddenNet(nn.Module): """Pooling module as proposed in our paper""" def __init__( - self, - embedding_dim=64, - h_dim=64, - mlp_dim=1024, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - dropout=0.0, + self, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + dropout=0.0, ): super(PoolHiddenNet, self).__init__() @@ -529,14 +533,14 @@ class SocialPooling(nn.Module): http://cvgl.stanford.edu/papers/CVPR16_Social_LSTM.pdf""" def __init__( - self, - h_dim=64, - activation="relu", - batch_norm=True, - dropout=0.0, - neighborhood_size=2.0, - grid_size=8, - pool_dim=None, + self, + h_dim=64, + activation="relu", + batch_norm=True, + dropout=0.0, + neighborhood_size=2.0, + grid_size=8, + pool_dim=None, ): super(SocialPooling, self).__init__() self.h_dim = h_dim @@ -620,14 +624,14 @@ def forward(self, h_states, seq_start_end, end_pos): # Make all positions to exclude as non-zero # Find which peds to exclude x_bound = (curr_end_pos[:, 0] >= bottom_right[:, 0]) + ( - curr_end_pos[:, 0] <= top_left[:, 0] + curr_end_pos[:, 0] <= top_left[:, 0] ) y_bound = (curr_end_pos[:, 1] >= top_left[:, 1]) + ( - curr_end_pos[:, 1] <= bottom_right[:, 1] + curr_end_pos[:, 1] <= bottom_right[:, 1] ) within_bound = x_bound + y_bound - within_bound[0 :: num_ped + 1] = 1 # Don't include the ped itself + within_bound[0:: num_ped + 1] = 1 # Don't include the ped itself within_bound = within_bound.view(-1) # This is a tricky way to get scatter add to work. Helps me avoid a @@ -657,25 +661,25 @@ class TrajectoryGenerator(nn.Module): """Modified from @agrimgupta92's https://github.com/agrimgupta92/sgan/blob/master/sgan/models.py.""" def __init__( - self, - obs_len, - pred_len, - embedding_dim=64, - encoder_h_dim=64, - decoder_h_dim=128, - mlp_dim=1024, - num_layers=1, - noise_dim=(0,), - noise_type="gaussian", - noise_mix_type="ped", - pooling_type=None, - pool_every_timestep=True, - dropout=0.0, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - neighborhood_size=2.0, - grid_size=8, + self, + obs_len, + pred_len, + embedding_dim=64, + encoder_h_dim=64, + decoder_h_dim=128, + mlp_dim=1024, + num_layers=1, + noise_dim=(0,), + noise_type="gaussian", + noise_mix_type="ped", + pooling_type=None, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + neighborhood_size=2.0, + grid_size=8, ): super(TrajectoryGenerator, self).__init__() @@ -805,9 +809,9 @@ def add_noise(self, _input, seq_start_end, user_noise=None): def mlp_decoder_needed(self): if ( - self.noise_dim - or self.pooling_type - or self.encoder_h_dim != self.decoder_h_dim + self.noise_dim + or self.pooling_type + or self.encoder_h_dim != self.decoder_h_dim ): return True else: @@ -858,19 +862,20 @@ def forward(self, obs_traj, obs_traj_rel, seq_start_end, user_noise=None): return pred_traj_fake_rel + class TrajectoryDiscriminator(nn.Module): def __init__( - self, - obs_len, - pred_len, - embedding_dim=64, - h_dim=64, - mlp_dim=1024, - num_layers=1, - activation="relu", - batch_norm=True, - dropout=0.0, - d_type="local", + self, + obs_len, + pred_len, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + num_layers=1, + activation="relu", + batch_norm=True, + dropout=0.0, + d_type="local", ): super(TrajectoryDiscriminator, self).__init__() diff --git a/traja/models/generator.py b/traja/models/generator.py index 6df87234..32949852 100644 --- a/traja/models/generator.py +++ b/traja/models/generator.py @@ -11,52 +11,53 @@ device = 'cuda' if torch.cuda.is_available() else 'cpu' -def timeseries(model_type:str, model_hyperparameters:dict, model_path:str, batch_size:int, num_future:int, ): + +def timeseries(model_type: str, model_hyperparameters: dict, model_path: str, batch_size: int, num_future: int, ): # Generating few samples - batch_size = model_hyperparameters.batch_size # Number of samples - num_future = model_hyperparameters.num_future # Number of time steps in each sample + batch_size = model_hyperparameters.batch_size # Number of samples + num_future = model_hyperparameters.num_future # Number of time steps in each sample if model_type == 'ae': - model = MultiModelAE(**model_hyperparameters) - + model = MultiModelAE(, + if model_type == 'vae': model = MultiModelVAE(**model_hyperparameters) - + if model_type == 'vaegan': model = MultiModelVAEGAN(**model_hyperparameters) return NotImplementedError - + if model_type == 'irl': model = MultiModelIRL(**model_hyperparameters) return NotImplementedError - + # Load the model from model path: model = load_model(model, model_hyperparameters, model_path) # z = torch.randn((batch_size, latent_size)).to(device) - z = torch.empty(10, model_hyperparameters.latent_size).normal_(mean=0,std=.1).to(device) + z = torch.empty(10, model_hyperparameters.latent_size).normal_(mean=0, std=.1).to(device) # Category from the noise cat = model.classifier(z) # Generate trajectories from the noise - out = model.decoder(z,num_future).cpu().detach().numpy() - out = out.reshape(out.shape[0]*out.shape[1],out.shape[2]) + out = model.decoder(z, num_future).cpu().detach().numpy() + out = out.reshape(out.shape[0] * out.shape[1], out.shape[2]) # for index, i in enumerate(train_df.columns): # scaler = scalers['scaler_'+i] # out[:,index] = scaler.inverse_transform(out[:,index].reshape(1, -1)) - print('IDs in this batch of synthetic data',torch.max(cat,1).indices+1) - plt.figure(figsize=(12,4)) - plt.plot(out[:,0], label='Generated x: Longitude') - plt.plot(out[:,1], label='Generated y: Latitude') + print('IDs in this batch of synthetic data', torch.max(cat, 1).indices + 1) + plt.figure(figsize=(12, 4)) + plt.plot(out[:, 0], label='Generated x: Longitude') + plt.plot(out[:, 1], label='Generated y: Latitude') plt.legend() - fig, ax = plt.subplots(nrows=2, ncols= 5, figsize=(16, 5), sharey=True) + fig, ax = plt.subplots(nrows=2, ncols=5, figsize=(16, 5), sharey=True) # plt.ticklabel_format(useOffset=False) - fig.set_size_inches(20,5) + fig.set_size_inches(20, 5) for i in range(2): for j in range(5): - ax[i,j].plot(out[:,0][(i+j)*num_future:(i+j)*num_future + num_future],out[:,1][(i+j)*num_future:(i+j)*num_future+ num_future],label = 'Animal ID {}'.format((torch.max(cat,1).indices+1).detach()[i+j]),color='g') - ax[i,j].legend() + ax[i, j].plot(out[:, 0][(i + j) * num_future:(i + j) * num_future + num_future], + out[:, 1][(i + j) * num_future:(i + j) * num_future + num_future], + label='Animal ID {}'.format((torch.max(cat, 1).indices + 1).detach()[i + j]), color='g') + ax[i, j].legend() plt.show() - - return out - + return out diff --git a/traja/models/interpretor.py b/traja/models/interpretor.py index 7ac0eb91..b4c3a1b9 100644 --- a/traja/models/interpretor.py +++ b/traja/models/interpretor.py @@ -6,6 +6,7 @@ from .vaegan import MultiModelVAEGAN from .irl import MultiModelIRL + def DisplayLatentDynamics(latent): r"""Visualize the dynamics of combination of latents Args: @@ -13,14 +14,14 @@ def DisplayLatentDynamics(latent): Latent shape (batch_size, latent_dim) Usage: DisplayLatentDynamics(latent)""" - + latents = {} - latents.fromkeys(list(range(latent.shape[1]))) + latents.fromkeys(list(range(latent.shape[1]))) for i in range(latent.shape[1]): - latents[f'{i}']=latent[:,i].cpu().detach().numpy() - fig= px.scatter_matrix(latents) + latents[f'{i}'] = latent[:, i].cpu().detach().numpy() + fig = px.scatter_matrix(latents) fig.update_layout( - autosize=False, - width=1600, - height=1000,) - return fig.show() \ No newline at end of file + autosize=False, + width=1600, + height=1000, ) + return fig.show() diff --git a/traja/models/irl.py b/traja/models/irl.py index 2fac5848..f7d5b593 100644 --- a/traja/models/irl.py +++ b/traja/models/irl.py @@ -1,20 +1,19 @@ """ Implementation of Inverse Reinforcement Learning algorithm for Time series""" import torch + class MultiModelIRL(torch.nn.Module): - def __init__(self,*model_hyperparameters, **kwargs): - super(MultiModelIRL,self).__init__() - + def __init__(self, *model_hyperparameters, **kwargs): + super(MultiModelIRL, self).__init__() + for dictionary in model_hyperparameters: for key in dictionary: - setattr(self, key, dictionary[key]) + setattr(self, key, dictionary[key]) for key in kwargs: setattr(self, key, kwargs[key]) - + def __new__(cls): pass - - def forward(self, *input:None, **kwargs: None): + + def forward(self, *input: None, **kwargs: None): return NotImplementedError - - \ No newline at end of file diff --git a/traja/models/losses.py b/traja/models/losses.py index f1019c1e..90060e70 100644 --- a/traja/models/losses.py +++ b/traja/models/losses.py @@ -1,69 +1,62 @@ import torch -class Criterion(object): - - def __init__(self, model_type): - self.model_type = model_type - - @staticmethod - def ae_criterion(recon_x, x, loss_type='huber'): - """[summary] - Args: - recon_x ([type]): [description] - x ([type]): [description] - loss_type(str): Type of Loss; huber or RMSE +class Criterion: + """Implements the loss functions of Autoencoders, Variational Autoencoders and LSTM models + Huber loss is set as default for reconstruction loss, alternative is to use rmse, + Cross entropy loss used for classification + Variational loss used huber loss and unweighted KL Divergence loss""" - Returns: - [type]: [description] - """ - if loss_type == 'huber': - - huber_loss = torch.nn.SmoothL1Loss(reduction='sum') - dist_x = huber_loss(recon_x,x) - return dist_x - else: # RMSE - return torch.sqrt(torch.mean((recon_x-x)**2)) - - @staticmethod - def vae_criterion(recon_x, x, mu, logvar, loss_type='huber'): - r"""Time series generative model loss function + def __init__(self): - Args: - recon_x ([type]): [description] - x ([type]): [description] - mu ([type]): [description] - logvar ([type]): [description] + self.huber_loss = torch.nn.SmoothL1Loss(reduction='sum') + self.crossentropy_loss = torch.nn.CrossEntropyLoss() - Returns: - [type]: [description] + def ae_criterion(self, predicted, target, loss_type='huber'): + """ Implements the Autoencoder loss for time series forecasting + :param predicted: Predicted time series by the model + :param target: Target time series + :param loss_type: Type of criterion; Defaults: 'huber' + :return: """ - if loss_type=='huber': - huber_loss = torch.nn.SmoothL1Loss(reduction='sum') - dist_x = huber_loss(recon_x,x) - else: - dist_x = torch.sqrt(torch.mean((recon_x-x)**2)) - - KLD = -0.5 * torch.sum(1 + logvar - mu**2 - logvar.exp()) - + + if loss_type == 'huber': + loss = self.huber_loss(predicted, target) + return loss + else: # Root MSE + return torch.sqrt(torch.mean((predicted - target) ** 2)) + + def vae_criterion(self, predicted, target, mu, logvar, loss_type='huber'): + """ Time series generative model loss function + :param predicted: Predicted time series by the model + :param target: Target time series + :param mu: Latent variable, Mean + :param logvar: Latent variable, Log(Variance) + :param loss_type: Type of criterion; Defaults: 'huber' + :return: Reconstruction loss + KLD loss + """ + if loss_type == 'huber': + dist_x = self.huber_loss(predicted, target) + else: + dist_x = torch.sqrt(torch.mean((predicted - target) ** 2)) + KLD = -0.5 * torch.sum(1 + logvar - mu ** 2 - logvar.exp()) return dist_x + KLD - - @staticmethod - def classifier_criterion(): - """Classifier loss function""" - classifier_criterion = torch.nn.CrossEntropyLoss() - return classifier_criterion - - def vaegan_criterion(): - return NotImplementedError - - def lstm_criterion(): - return NotImplementedError - - -# VAE loss + def classifier_criterion(self, predicted, target): + """ + Classifier loss function + :param predicted: Predicted label + :param target: Target label + :return: Cross entropy loss + """ + + loss = self.crossentropy_loss(predicted, target) + return loss + + def lstm_criterion(self, predicted, target): -# VAE-GAN loss + loss = self.huber_loss(predicted, target) + return loss -# LSTM \ No newline at end of file + def vaegan_criterion(self): + return NotImplementedError diff --git a/traja/models/lstm.py b/traja/models/lstm.py index 7ecdd287..d5f01d51 100644 --- a/traja/models/lstm.py +++ b/traja/models/lstm.py @@ -1,22 +1,62 @@ """Implementation of Multimodel LSTM""" +import torch +from traja.models.utils import TimeDistributed -import torch - -class MultiModelLSTM(torch.nn.Module): - - def __init__(self,*model_hyperparameters, **kwargs): - super(MultiModelLSTM,self).__init__() - - for dictionary in model_hyperparameters: - for key in dictionary: - setattr(self, key, dictionary[key]) - for key in kwargs: - setattr(self, key, kwargs[key]) - - def __new__(cls): - pass - - def forward(self, *input:None, **kwargs: None): - return NotImplementedError - - +device = 'cuda' if torch.cuda.is_available() else 'cpu' + + +class LSTM(torch.nn.Module): + """ Deep LSTM network. This implementation + returns output_size outputs. + Args: + input_size: The number of expected features in the input `x` + batch_size: + sequence_length: The number of in each sample + hidden_size: The number of features in the hidden state `h` + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` + would mean stacking two LSTMs together to form a `stacked LSTM`, + with the second LSTM taking in outputs of the first LSTM and + computing the final results. Default: 1 + output_size: The number of output dimensions + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + LSTM layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + """ + + def __init__(self, batch_size: int, num_future: int, hidden_size: int, num_layers: int, + output_size: int, input_size: int, batch_first: bool, dropout: float, + reset_state: bool, bidirectional: bool): + super(LSTM, self).__init__() + + self.batch_size = batch_size + self.input_size = input_size + self.num_future = num_future + self.hidden_size = hidden_size + self.num_layers = num_layers + self.output_size = output_size + self.batch_first = batch_first + self.dropout = dropout + self.reset_state = reset_state + self.bidirectional = bidirectional + + # RNN decoder + self.lstm = torch.nn.LSTM(input_size=self.input_size, hidden_size=self.hidden_size, + num_layers=self.num_layers, dropout=self.dropout, + bidirectional=self.bidirectional, batch_first=True) + self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, self.output_size)) + + def _init_hidden(self): + return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device), + torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) + + def forward(self, x): + # To feed the latent states into lstm decoder, repeat the tensor n_future times at second dim + _init_hidden = self._init_hidden() + + # Decoder input Shape(batch_size, num_futures, latent_size) + out, (dec_hidden, dec_cell) = self.lstm(x, _init_hidden) + + # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) to Time Dsitributed Linear Layer + out = self.output(out) + return out diff --git a/traja/models/nn.py b/traja/models/nn.py index 3e8a1d44..d2f8ebfc 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -24,7 +24,8 @@ nb_steps = 10 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - + + class LossMse: """ Calculate the Mean Squared Error between y_true and y_pred @@ -32,15 +33,17 @@ class LossMse: y_true is the desired output. y_pred is the model's output. """ + def __init__(self) -> None: pass - def __call__(self, y_pred, y_true): + def __call__(self, y_pred, y_true): # Calculate the Mean Squared Error and use it as loss. mse = torch.mean(torch.square(y_true - y_pred)) return mse + class Trainer: def __init__(self, model, train_loader, @@ -92,14 +95,16 @@ def __init__(self, model, if not os.path.exists(save_dir): os.makedirs(save_dir) - self.savepath = os.path.join(save_dir, f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') + self.savepath = os.path.join(save_dir, + f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') self.experiment_done = False if os.path.exists(self.savepath): trained_epochs = len(pd.read_csv(self.savepath, sep=';')) if trained_epochs >= epochs: self.experiment_done = True - print(f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') + print( + f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') if os.path.exists((self.savepath.replace('.csv', '.pt'))): self.model.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['model_state_dict']) self.model = self.model.to(self.device) @@ -111,13 +116,12 @@ def __init__(self, model, self.start_epoch = 0 self.model = self.model.to(self.device) - def _infer_initial_epoch(self, savepath): if not os.path.exists(savepath): return 0 else: df = pd.read_csv(savepath, sep=';', index_col=0) - print(len(df)+1) + print(len(df) + 1) return len(df) def train(self): @@ -131,7 +135,7 @@ def train(self): if self.opt_name == "LRS": print('LRS step') self.lr_scheduler.step() - return self.savepath+'.csv' + return self.savepath + '.csv' def train_epoch(self): self.model.train() @@ -139,8 +143,8 @@ def train_epoch(self): running_loss = 0 old_time = time() for batch, data in enumerate(self.train_loader): - - inputs, targets= data[0].to(self.device).float(), data[1].to(self.device).float() + + inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() self.optimizer.zero_grad() outputs = self.model(inputs) loss = self.criterion(outputs, targets) @@ -149,12 +153,13 @@ def train_epoch(self): running_loss += loss.item() if batch % 10 == 0 and batch != 0: - print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'loss:', running_loss/total) + print(batch, 'of', len(self.train_loader), 'processing time', time() - old_time, 'loss:', + running_loss / total) old_time = time() # Increment number of batches total += 1 - return running_loss/total + return running_loss / total def test(self, epoch, save=True): self.model.eval() @@ -163,7 +168,7 @@ def test(self, epoch, save=True): with torch.no_grad(): for batch, data in enumerate(self.test_loader): if batch % 10 == 0: - print('Processing eval batch', batch,'of', len(self.test_loader)) + print('Processing eval batch', batch, 'of', len(self.test_loader)) inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() outputs = self.model(inputs) loss = self.criterion(outputs, targets) @@ -434,22 +439,16 @@ def __init__(self, input_size: int, hidden_size: int, num_layers: int, self.head = nn.Linear(hidden_size, output_size) def forward(self, x): - x, states = self.lstm(x) - #x = x.permute([1, 0, 2]) - #x = x.reshape(x.shape[0], x.shape[1] * x.shape[2]) - x = self.head(x) - return x - - def forward(self, x): x, state = self.lstm(x) # Use the last hidden state of last layer - x = state[0][-1] + x = state[0][-1] x = self.head(x) return x + class TrajectoryLSTM: def __init__( - self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() + self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() ): fig, ax = plt.subplots(2, 1) self.fig = fig @@ -468,10 +467,10 @@ def load_batch(self, batch_size=32): inds = np.random.randint(0, len(self.xy) - self.nb_steps, (self.batch_size)) for i, ind in enumerate(inds): - t_1_b[:, i] = self.xy[ind : ind + self.nb_steps] - t_b[i * nb_steps : (i + 1) * self.nb_steps] = self.xy[ - ind + 1 : ind + nb_steps + 1 - ] + t_1_b[:, i] = self.xy[ind: ind + self.nb_steps] + t_b[i * nb_steps: (i + 1) * self.nb_steps] = self.xy[ + ind + 1: ind + nb_steps + 1 + ] return torch.from_numpy(t_1_b).float(), torch.from_numpy(t_b).float() def train(self): @@ -531,6 +530,7 @@ def plot(self, interactive=True): self._plot() return self.fig + def make_mlp(dim_list, activation="relu", batch_norm=True, dropout=0): layers = [] for dim_in, dim_out in zip(dim_list[:-1], dim_list[1:]): @@ -559,7 +559,7 @@ class Encoder(nn.Module): TrajectoryDiscriminator""" def __init__( - self, embedding_dim=64, h_dim=64, mlp_dim=1024, num_layers=1, dropout=0.0 + self, embedding_dim=64, h_dim=64, mlp_dim=1024, num_layers=1, dropout=0.0 ): super(Encoder, self).__init__() @@ -599,20 +599,20 @@ class Decoder(nn.Module): """Decoder is part of TrajectoryGenerator""" def __init__( - self, - seq_len, - embedding_dim=64, - h_dim=128, - mlp_dim=1024, - num_layers=1, - pool_every_timestep=True, - dropout=0.0, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - pooling_type="pool_net", - neighborhood_size=2.0, - grid_size=8, + self, + seq_len, + embedding_dim=64, + h_dim=128, + mlp_dim=1024, + num_layers=1, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + pooling_type="pool_net", + neighborhood_size=2.0, + grid_size=8, ): super(Decoder, self).__init__() @@ -696,14 +696,14 @@ class PoolHiddenNet(nn.Module): """Pooling module as proposed in our paper""" def __init__( - self, - embedding_dim=64, - h_dim=64, - mlp_dim=1024, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - dropout=0.0, + self, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + dropout=0.0, ): super(PoolHiddenNet, self).__init__() @@ -773,14 +773,14 @@ class SocialPooling(nn.Module): http://cvgl.stanford.edu/papers/CVPR16_Social_LSTM.pdf""" def __init__( - self, - h_dim=64, - activation="relu", - batch_norm=True, - dropout=0.0, - neighborhood_size=2.0, - grid_size=8, - pool_dim=None, + self, + h_dim=64, + activation="relu", + batch_norm=True, + dropout=0.0, + neighborhood_size=2.0, + grid_size=8, + pool_dim=None, ): super(SocialPooling, self).__init__() self.h_dim = h_dim @@ -864,14 +864,14 @@ def forward(self, h_states, seq_start_end, end_pos): # Make all positions to exclude as non-zero # Find which peds to exclude x_bound = (curr_end_pos[:, 0] >= bottom_right[:, 0]) + ( - curr_end_pos[:, 0] <= top_left[:, 0] + curr_end_pos[:, 0] <= top_left[:, 0] ) y_bound = (curr_end_pos[:, 1] >= top_left[:, 1]) + ( - curr_end_pos[:, 1] <= bottom_right[:, 1] + curr_end_pos[:, 1] <= bottom_right[:, 1] ) within_bound = x_bound + y_bound - within_bound[0 :: num_ped + 1] = 1 # Don't include the ped itself + within_bound[0:: num_ped + 1] = 1 # Don't include the ped itself within_bound = within_bound.view(-1) # This is a tricky way to get scatter add to work. Helps me avoid a @@ -901,25 +901,25 @@ class TrajectoryGenerator(nn.Module): """Modified from @agrimgupta92's https://github.com/agrimgupta92/sgan/blob/master/sgan/models.py.""" def __init__( - self, - obs_len, - pred_len, - embedding_dim=64, - encoder_h_dim=64, - decoder_h_dim=128, - mlp_dim=1024, - num_layers=1, - noise_dim=(0,), - noise_type="gaussian", - noise_mix_type="ped", - pooling_type=None, - pool_every_timestep=True, - dropout=0.0, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - neighborhood_size=2.0, - grid_size=8, + self, + obs_len, + pred_len, + embedding_dim=64, + encoder_h_dim=64, + decoder_h_dim=128, + mlp_dim=1024, + num_layers=1, + noise_dim=(0,), + noise_type="gaussian", + noise_mix_type="ped", + pooling_type=None, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + neighborhood_size=2.0, + grid_size=8, ): super(TrajectoryGenerator, self).__init__() @@ -1049,9 +1049,9 @@ def add_noise(self, _input, seq_start_end, user_noise=None): def mlp_decoder_needed(self): if ( - self.noise_dim - or self.pooling_type - or self.encoder_h_dim != self.decoder_h_dim + self.noise_dim + or self.pooling_type + or self.encoder_h_dim != self.decoder_h_dim ): return True else: @@ -1102,19 +1102,20 @@ def forward(self, obs_traj, obs_traj_rel, seq_start_end, user_noise=None): return pred_traj_fake_rel + class TrajectoryDiscriminator(nn.Module): def __init__( - self, - obs_len, - pred_len, - embedding_dim=64, - h_dim=64, - mlp_dim=1024, - num_layers=1, - activation="relu", - batch_norm=True, - dropout=0.0, - d_type="local", + self, + obs_len, + pred_len, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + num_layers=1, + activation="relu", + batch_norm=True, + dropout=0.0, + d_type="local", ): super(TrajectoryDiscriminator, self).__init__() diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py new file mode 100644 index 00000000..5bd3f902 --- /dev/null +++ b/traja/models/optimizers.py @@ -0,0 +1,89 @@ +import torch +from torch.optim.lr_scheduler import ReduceLROnPlateau +from traja.models.ae import MultiModelAE + + +class Optimizer: + + def __init__(self, model_type, model, optimizer_type): + """ + Wrapper for setting the model optimizer and learning rate schedulers using ReduceLROnPlateau; + If the model type is 'ae' or 'vae' - var optimizers is a dict with separate optimizers for encoder, decoder, + latent and classifier. In case of 'lstm', var optimizers is an optimizer for lstm and TimeDistributed(linear layer) + :param model_type: Type of model 'ae', 'vae' or 'lstm' + :param model: Model instance + :param optimizer_type: Optimizer to be used; Should be one in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', + 'LBFGS', 'ASGD', 'Adamax'] + """ + + assert isinstance(model, torch.nn.Module) + assert str(optimizer_type) in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', + 'LBFGS', 'ASGD', 'Adamax'] + + self.model_type = model_type + self.model = model + self.optimizer_type = optimizer_type + self.optimizers = {} + self.schedulers = {} + + def get_optimizers(self, lr=0.0001): + """Optimizers for each network in the model + + Args: + model_type ([type]): [description] + model ([type]): [description] + lr (float, optional): [description]. Defaults to 0.0001. + + Returns: + [type]: [description] + """ + + if self.model_type == 'lstm': + self.optimizers = getattr(torch.optim, f'{self.optimizer_type}')(self.model.parameters(), lr=lr) + + elif self.model_type == 'ae' or 'vae': + keys = ['encoder', 'decoder', 'latent', 'classifier'] + for network in keys: + self.optimizers[network] = getattr(torch.optim, f'{self.optimizer_type}')( + getattr(self.model, f'{network}').parameters(), lr=lr) + + elif self.model_type == 'vaegan': + return NotImplementedError + return self.optimizers + + def get_lrschedulers(self, factor: float, patience: int): + + """Learning rate scheduler for each network in the model + NOTE: Scheduler metric should be test set loss + + Args: + factor (float, optional): [description]. Defaults to 0.1. + patience (int, optional): [description]. Defaults to 10. + + Returns: + [dict]: [description] + """ + if self.model_type == 'lstm': + assert not isinstance(self.optimizers, dict) + self.schedulers = ReduceLROnPlateau(self.optimizers, mode='max', factor=factor, + patience=patience, verbose=True) + else: + for network in self.optimizers.keys(): + self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, + patience=patience, verbose=True) + return self.schedulers + + +if __name__ == '__main__': + # Test + model_type = 'ae' + model = MultiModelAE(input_size=2, num_past=10, batch_size=5, num_future=5, lstm_hidden_size=32, num_lstm_layers=2, + classifier_hidden_size=32, num_classifier_layers=4, output_size=2, num_classes=10, + latent_size=10, batch_first=True, dropout=0.2, reset_state=True, bidirectional=True) + + # Get the optimizers + opt = Optimizer(model_type, model, optimizer_type='RMSprop') + model_optimizers = opt.get_optimizers(lr=0.1) + model_schedulers = opt.get_lrschedulers(factor=0.1, patience=10) + + print(model_optimizers, model_schedulers) diff --git a/traja/models/test.py b/traja/models/test.py new file mode 100644 index 00000000..14d38fa2 --- /dev/null +++ b/traja/models/test.py @@ -0,0 +1,44 @@ +import pandas as pd +from models.train import Trainer +from datasets.dataset import MultiModalDataLoader + +# Downlaod dataset (9 jaguars) from github +data_url = "https://raw.githubusercontent.com/traja-team/traja-research/dataset_und_notebooks/dataset_analysis/jaguar5.csv" +df = pd.read_csv(data_url, error_bad_lines=False) + +# Hyperparameters +batch_size = 10 +num_past = 10 +num_future = 5 +model_save_path = './model.pt' + +# Prepare the dataloader +train_loader, test_loader = MultiModalDataLoader(df, + batch_size=batch_size, + n_past=num_past, + n_future=num_future, + num_workers=1) + +# Initialize the models +trainer = Trainer(model_type='vae', + device='cpu', + input_size=2, + output_size=2, + lstm_hidden_size=512, + lstm_num_layers=4, + reset_state=True, + num_classes=9, + latent_size=10, + dropout=0.1, + num_layers=4, + epochs=10, + batch_size=batch_size, + num_future=num_future, + sequence_length=num_past, + bidirectional=False, + batch_first=True, + loss_type='huber') + +# Train the model +trainer.train_latent_model(train_loader, test_loader, model_save_path) + diff --git a/traja/models/train.py b/traja/models/train.py index 6fd4bdc5..898989c3 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -1,169 +1,221 @@ from .ae import MultiModelAE from .vae import MultiModelVAE -from .vaegan import MultiModelVAEGAN from .lstm import LSTM -import torch -from functools import wraps -import inspect from . import utils from .losses import Criterion +from .optimizers import Optimizer +import torch device = 'cuda' if torch.cuda.is_available() else 'cpu' +class HybridTrainer(object): + """ + Wrapper for training and testing the LSTM model + + :param model_type: Type of model should be "LSTM" + :param optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', + 'AdamW', 'SparseAdam', 'RMSprop', ' + Rprop', 'LBFGS', 'ASGD', 'Adamax'] + :param device: Selected device; 'cuda' or 'cpu' + :param input_size: The number of expected features in the input x + :param output_size: Output feature dimension + :param lstm_hidden_size: The number of features in the hidden state h + :param num_lstm_layers: Number of layers in the LSTM model + :param reset_state: If True, will reset the hidden and cell state for each batch of data + :param num_classes: Number of categories/labels + :param latent_size: Latent space dimension + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param num_classifier_layers: Number of layers in the classifier + :param epochs: Number of epochs to train the network + :param batch_size: Number of samples in a batch + :param num_future: Number of time steps to be predicted forward + :param num_past: Number of past time steps otherwise, length of sequences in each batch of data. + :param bidirectional: If True, becomes a bidirectional LSTM + :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + :param loss_type: Type of reconstruction loss to apply, 'huber' or 'rmse'. Default:'huber' + :param lr_factor: Factor by which the learning rate will be reduced + :param scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. + For example, if patience = 2, then we will ignore the first 2 epochs with no + improvement, and will only decrease the LR after the 3rd epoch if the loss still + hasn’t improved then. + + """ + + def __init__(self, model_type: str, + optimizer_type: str, + device: str, + input_size: int, + output_size: int, + lstm_hidden_size: int, + num_lstm_layers: int, + classifier_hidden_size: int, + num_classifier_layers: int, + reset_state: bool, + num_classes: int, + latent_size: int, + dropout: float, + epochs: int, + batch_size: int, + num_future: int, + num_past: int, + bidirectional: bool = False, + batch_first: bool = True, + loss_type: str = 'huber', + lr_factor: float = 0.1, + scheduler_patience: int = 10): + + white_keys = ['ae', 'vae'] + assert model_type in white_keys, "Valid models are {}".format(white_keys) -class Trainer(object): - - def __init__(self, model_type:str, device:str, - input_size:int, output_size:int, - lstm_hidden_size:int, lstm_num_layers:int, - reset_state:bool, num_classes:int, - latent_size:int, dropout:float, - num_layers:int, epochs:int, batch_size:int, - num_future:int, sequence_length:int, - bidirectional:bool =False, - batch_first:bool =True, - loss_type:str = 'huber'): - - white_keys = ['ae','vae','lstm','vaeg an', 'irl'] - assert model_type in white_keys, "Valid models are {}".format(white_keys) self.model_type = model_type self.device = device self.input_size = input_size self.lstm_hidden_size = lstm_hidden_size - self.lstm_num_layers = lstm_num_layers - self.num_layers = lstm_num_layers - self.hidden_size = lstm_hidden_size # For classifiers too + self.num_lstm_layers = num_lstm_layers + self.classifier_hidden_size = classifier_hidden_size + self.num_classifier_layers = num_classifier_layers self.batch_first = batch_first self.reset_state = reset_state self.output_size = output_size self.num_classes = num_classes self.latent_size = latent_size - self.num_layers = num_layers + self.num_classifier_layers = num_classifier_layers self.num_future = num_future self.epochs = epochs self.batch_size = batch_size - self.sequence_length = sequence_length + self.num_past = num_past self.dropout = dropout - self.bidirectional= bidirectional + self.bidirectional = bidirectional self.loss_type = loss_type - - self.model_hyperparameters = {'input_size':self.input_size, - 'sequence_length':self.sequence_length, - 'epochs':self.epochs, - 'batch_size':self.batch_size, - 'batch_first':self.batch_first, - 'lstm_hidden_size':self.lstm_hidden_size, - 'hidden_size':self.lstm_hidden_size, - 'num_future':self.num_future, - 'lstm_num_layers':self.lstm_num_layers, - 'num_layers':self.lstm_num_layers, - 'latent_size':self.latent_size, - 'output_size':self.output_size, - 'num_classes':self.num_classes, - 'batch_first':self.batch_first, - 'dropout':self.dropout, - 'reset_state':self.reset_state, - 'bidirectional':self.bidirectional, - 'dropout':self.dropout, - 'loss_type':self.loss_type - } - - if self.model_type == 'lstm': - self.model = LSTM(self.model_hyperparameters) - + self.optimizer_type = optimizer_type + self.lr_factor = lr_factor + self.scheduler_patience = scheduler_patience + self.model_hyperparameters = {'input_size': self.input_size, + 'num_past': self.num_past, + 'batch_size': self.batch_size, + 'lstm_hidden_size': self.lstm_hidden_size, + 'num_lstm_layers':self.num_lstm_layers, + 'classifier_hidden_size':self.classifier_hidden_size, + 'num_classifier_layers':self.num_classifier_layers, + 'num_future': self.num_future, + 'latent_size': self.latent_size, + 'output_size': self.output_size, + 'num_classes': self.num_classes, + 'batch_first': self.batch_first, + 'reset_state': self.reset_state, + 'bidirectional': self.bidirectional, + 'dropout': self.dropout + } + + # Instantiate model instance based on model_type if self.model_type == 'ae': - self.model = MultiModelAE(self.model_hyperparameters) - + self.model = MultiModelAE(**self.model_hyperparameters) + if self.model_type == 'vae': - self.model = MultiModelVAE(self.model_hyperparameters) - - if self.model_type == 'vaegan': - self.model = MultiModelVAEGAN(self.model_hyperparameters) - - if self.model_type == 'irl': - return NotImplementedError - - # Get the optimizers for each network in the model - [self.encoder_optimizer, self.latent_optimizer, self.decoder_optimizer, self.classifier_optimizer] = utils.set_optimizers(self.model_type, self.model) + self.model = MultiModelVAE(**self.model_hyperparameters) - # Learning rate schedulers for the models - [self.encoder_scheduler, self.latent_scheduler, self.decoder_scheduler, self.classifier_scheduler] = utils.get_lrschedulers(self.model_type, self.model, factor=0.1, patience=10) - + # Model optimizer and the learning rate scheduler based on user defined optimizer_type + # and learning rate parameters + optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) + self.model_optimizers = optimizer.get_optimizers(lr=0.001) + self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) def __str__(self): return "Training model type {}".format(self.model_type) - - # TRAIN AUTOENCODERS/VARIATIONAL AUTOENCODERS - def train_latent_model(self, train_loader, test_loader, model_save_path): - + + def train(self, train_loader, test_loader, model_save_path): + """ + This method implements the batch- wise training and testing protocol for both time series forecasting and + classification of the timeseries + + :param train_loader: Dataloader object of train dataset with batch data [data,target,category] + :param test_loader: Dataloader object of test dataset with [data,target,category] + :param model_save_path: Directory path to save the model + :return: None + """ + assert self.model_type == 'ae' or 'vae' - # Move the model to target device self.model.to(device) + encoder_optimizer, latent_optimizer, decoder_optimizer, classifier_optimizer = self.model_optimizers.values() + encoder_scheduler, latent_scheduler, decoder_scheduler, classifier_scheduler = self.model_lrschedulers.values() + # Training mode: Switch from Generative to classifier training mode training_mode = 'forecasting' # Training - for epoch in range(self.epochs*2): # First half for generative model and next for classifier - if epoch>0: # Initial step is to test and set LR schduler + for epoch in range(self.epochs * 2): # First half for generative model and next for classifier + test_loss_forecasting = 0 + test_loss_classification = 0 + if epoch > 0: # Initial step is to test and set LR schduler # Training self.model.train() total_loss = 0 - for idx, (data, target,category) in enumerate(train_loader): + for idx, (data, target, category) in enumerate(train_loader): # Reset optimizer states - self.encoder_optimizer.zero_grad() - self.latent_optimizer.zero_grad() - self.decoder_optimizer.zero_grad() - self.classifier_optimizer.zero_grad() - - data, target,category = data.float().to(device), target.float().to(device), category.to(device) - - if training_mode =='forecasting': + encoder_optimizer.zero_grad() + latent_optimizer.zero_grad() + decoder_optimizer.zero_grad() + classifier_optimizer.zero_grad() + + data, target, category = data.float().to(device), target.float().to(device), category.to(device) + + if training_mode == 'forecasting': if self.model_type == 'ae': decoder_out, latent_out = self.model(data, training=True, is_classification=False) - loss = Criterion.ae_criterion(decoder_out, target) - - else: # vae - decoder_out, latent_out, mu, logvar= self.model(data, training=True, is_classification=False) - loss = Criterion.vae_criterion(decoder_out, target, mu, logvar) - - loss.backward() + loss = Criterion().ae_criterion(decoder_out, target) - self.encoder_optimizer.step() - self.decoder_optimizer.step() - self.latent_optimizer.step() + else: # vae + decoder_out, latent_out, mu, logvar = self.model(data, training=True, + is_classification=False) + loss = Criterion().vae_criterion(decoder_out, target, mu, logvar) - else: # training_mode == 'classification' - - classifier_out = self.model(data, training=True, is_classification=True) - loss = Criterion.classifier_criterion(classifier_out, category-1) loss.backward() - self.classifier_optimizer.step() - total_loss+=loss - - print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss/(idx+1))) - - if epoch+1 == self.epochs: # + encoder_optimizer.step() + decoder_optimizer.step() + latent_optimizer.step() + + else: # training_mode == 'classification' + if self.model_type == 'vae': + classifier_out, latent_out, mu, logvar = self.model(data, training=True, is_classification=True) + else: + classifier_out = self.model(data, training=True, + is_classification=True) + loss = Criterion().classifier_criterion(classifier_out, category - 1) + loss.backward() + classifier_optimizer.step() + total_loss += loss + + print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss / (idx + 1))) + + if epoch + 1 == self.epochs: training_mode = 'classification' # Testing - if epoch%10==0: + if epoch % 10 == 0: with torch.no_grad(): self.model.eval() - test_loss_forecasting = 0 - test_loss_classification = 0 - for idx, (data, target,category) in enumerate(list(test_loader)): + for idx, (data, target, category) in enumerate(list(test_loader)): data, target, category = data.float().to(device), target.float().to(device), category.to(device) - # Time seriesforecasting test - if self.model_type=='ae': + # Time series forecasting test + if self.model_type == 'ae': out, latent = self.model(data, training=False, is_classification=False) - test_loss_forecasting += Criterion.ae_criterion(out,target).item() + test_loss_forecasting += Criterion().ae_criterion(out, target).item() else: - decoder_out,latent_out,mu,logvar= self.model(data,training=False, is_classification=False) - test_loss_forecasting += Criterion.vae_criterion(decoder_out, target,mu,logvar) + decoder_out, latent_out, mu, logvar = self.model(data, training=False, + is_classification=False) + test_loss_forecasting += Criterion().vae_criterion(decoder_out, target, mu, logvar) # Classification test - classifier_out= self.model(data,training=False, is_classification=True) - test_loss_classification += Criterion.classifier_criterion(classifier_out, category-1).item() + if self.model_type == 'ae': + classifier_out = self.model(data, training=False, + is_classification=True) + else: + classifier_out, latent_out, mu, logvar = self.model(data, training=False, + is_classification=True) + + test_loss_classification += Criterion().classifier_criterion(classifier_out, + category - 1).item() test_loss_forecasting /= len(test_loader.dataset) print(f'====> Mean test set generator loss: {test_loss_forecasting:.4f}') @@ -171,28 +223,154 @@ def train_latent_model(self, train_loader, test_loader, model_save_path): print(f'====> Mean test set classifier loss: {test_loss_classification:.4f}') # Scheduler metric is test set loss - if training_mode =='forecasting': - self.encoder_scheduler.step(self.test_loss_forecasting) - self.decoder_scheduler.step(self.test_loss_forecasting) - self.latent_scheduler.step(self.test_loss_forecasting) + if training_mode == 'forecasting': + encoder_scheduler.step(test_loss_forecasting) + decoder_scheduler.step(test_loss_forecasting) + latent_scheduler.step(test_loss_forecasting) else: - self.classifier_scheduler.step(self.test_loss_classification) + classifier_scheduler.step(test_loss_classification) + + # Save the model at target path + utils.save_model(self.model, PATH=model_save_path) + + +class LSTMTrainer: + """ + Wrapper for training and testing the LSTM model + + :param model_type: Type of model should be "LSTM" + :param optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', + 'AdamW', 'SparseAdam', 'RMSprop', ' + Rprop', 'LBFGS', 'ASGD', 'Adamax'] + :param device: Selected device; 'cuda' or 'cpu' + :param epochs: Number of epochs to train the network + :param input_size: The number of expected features in the input x + :param batch_size: Number of samples in a batch + :param hidden_size: The number of features in the hidden state h + :param num_future: Number of time steps to be predicted forward + :param num_layers: Number of layers in the LSTM model + :param output_size: Output feature dimension + :param lr_factor: Factor by which the learning rate will be reduced + :param scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. + For example, if patience = 2, then we will ignore the first 2 epochs with no + improvement, and will only decrease the LR after the 3rd epoch if the loss still + hasn’t improved then. + :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param reset_state: If True, will reset the hidden and cell state for each batch of data + :param bidirectional: If True, becomes a bidirectional LSTM + + """ + + def __init__(self, + model_type: str, + optimizer_type: str, + device: str, + epochs: int, + input_size: int, + batch_size: int, + hidden_size: int, + num_future: int, + num_layers: int, + output_size: int, + lr_factor: float, + scheduler_patience: int, + batch_first: True, + dropout: float, + reset_state: bool, + bidirectional: bool): + self.model_type = model_type + self.optimizer_type = optimizer_type + self.device = device + self.epochs = epochs + self.input_size = input_size + self.batch_size = batch_size + self.hidden_size = hidden_size + self.num_future = num_future + self.num_layers = num_layers + self.output_size = output_size + self.batch_first = batch_first + self.lr_factor = lr_factor + self.scheduler_patience = scheduler_patience + self.dropout = dropout + self.reset_state = reset_state + self.bidirectional = bidirectional + + self.model_hyperparameters = {'input_size': self.input_size, + 'batch_size': self.batch_size, + 'hidden_size': self.hidden_size, + 'num_future': self.num_future, + 'num_layers': self.num_layers, + 'output_size': self.output_size, + 'batch_first': self.batch_first, + 'reset_state': self.reset_state, + 'bidirectional': self.bidirectional, + 'dropout': self.dropout + } + + self.model = LSTM(**self.model_hyperparameters) + optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) + self.optimizer = optimizer.get_optimizers(lr=0.001) + self.scheduler = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) + + def train(self, train_loader, test_loader, model_save_path): + + """ Implements the batch wise training and testing for time series forecasting + :param train_loader: Dataloader object of train dataset with batch data [data,target,category] + :param test_loader: Dataloader object of test dataset with [data,target,category] + :param model_save_path: Directory path to save the model + :return: None""" + + assert self.model_type == 'lstm' + self.model.to(device) + + for epoch in range(self.epochs): + if epoch > 0: + self.model.train() + total_loss = 0 + for idx, (data, target, _) in enumerate(train_loader): + self.optimizer.zero_grad() + data, target = data.float().to(device), target.float().to(device) + output = self.model(data) + loss = Criterion().lstm_criterion(output, target) + loss.backward() + self.optimizer.step() + total_loss += loss + + print('Epoch {} | loss {}'.format(epoch, total_loss / (idx + 1))) + + # Testing + if epoch % 10 == 0: + with torch.no_grad(): + self.model.eval() + test_loss_forecasting = 0 + for idx, (data, target, _) in enumerate(list(test_loader)): + data, target = data.float().to(device), target.float().to(device) + out = self.model(data) + test_loss_forecasting += Criterion().lstm_criterion(out, target).item() + + test_loss_forecasting /= len(test_loader.dataset) + print(f'====> Test set generator loss: {test_loss_forecasting:.4f}') + + # Scheduler metric is test set loss + self.scheduler.step(test_loss_forecasting) # Save the model at target path - utils.save_model(self.model,PATH = model_save_path) - - - # TRAIN VARIATIONAL AUTOENCODERS-GAN - def train_vaegan(self): - assert self.model_type == 'vaegan' + utils.save_model(self.model, PATH=model_save_path) + + +class VAEGANTrainer: + def __init__(self): + pass + + def train(self): return NotImplementedError - - # TRAIN INVERSE RL - def train_irl(self): - assert self.model_type == 'irl' + + +class IRLTrainer: + def __init__(self): + pass + + def train(self): return NotImplementedError - - # TRAIN LSTM - def train_lstm(self): - assert self.model_type == 'lstm' - return NotImplementedError \ No newline at end of file diff --git a/traja/models/utils.py b/traja/models/utils.py index 5425977b..5b670334 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -1,9 +1,13 @@ import torch import matplotlib.pyplot as plt -from torch.optim.lr_scheduler import ReduceLROnPlateau +import numpy as np +import collections +from numpy import math + class TimeDistributed(torch.nn.Module): """ Time distributed wrapper compatible with linear/dense pytorch layer modules""" + def __init__(self, module, batch_first=True): super(TimeDistributed, self).__init__() self.module = module @@ -26,76 +30,24 @@ def forward(self, x): out = out.view(-1, x.size(1), out.size(-1)) # (timesteps, samples, output_size) return out - -def get_optimizers(model_type, model, lr=0.0001): - r"""Optimizers for each network in the model - - Args: - model_type ([type]): [description] - model ([type]): [description] - lr (float, optional): [description]. Defaults to 0.0001. - Returns: - [type]: [description] - """ - - if model_type == 'ae' or 'vae': - # Optimizers for each network in the model - encoder_optimizer = torch.optim.Adam(model.encoder.parameters(), lr=lr) - latent_optimizer = torch.optim.Adam(model.latent.parameters(), lr=lr) - decoder_optimizer = torch.optim.Adam(model.decoder.parameters(), lr=lr) - classifier_optimizer = torch.optim.Adam(model.classifier.parameters(), lr=lr) - return [encoder_optimizer, latent_optimizer, decoder_optimizer, classifier_optimizer] - - elif model_type == 'vaegan': - return NotImplementedError - - else: # LSTM - return NotImplementedError - -def get_lrschedulers(model_type, model, factor=0.1, patience=10 ): - r"""Learning rate scheduler for each network in the model - NOTE: Scheduler metric should be test set loss - - Args: - model_type ([type]): [description] - model ([type]): [description] - factor (float, optional): [description]. Defaults to 0.1. - patience (int, optional): [description]. Defaults to 10. - Returns: - [type]: [description] - """ - if model_type == 'ae' or 'vae': - encoder_scheduler = ReduceLROnPlateau(model.encoder_optimizer, mode='max', factor=factor, patience=patience, verbose=True) - decoder_scheduler = ReduceLROnPlateau(model.decoder_optimizer, mode='max', factor=factor, patience=patience, verbose=True) - latent_scheduler = ReduceLROnPlateau(model.latent_optimizer, mode='max', factor=factor, patience=patience, verbose=True) - classifier_scheduler = ReduceLROnPlateau(model.classifier_optimizer, mode='max', factor=factor, patience=patience, verbose=True) - return [encoder_scheduler, decoder_scheduler, latent_scheduler, classifier_scheduler] - - elif model_type == 'vaegan': - return NotImplementedError - - else: # LSTM - return NotImplementedError - - - def save_model(model, PATH): - r"""[summary] + """[summary] Args: model ([type]): [description] PATH ([type]): [description] """ - + # PATH = "state_dict_model.pt" # Save torch.save(model.state_dict(), PATH) print('Model saved at {}'.format(PATH)) - -def load_model(model,model_hyperparameters, PATH): - r"""[summary] + + +def load_model(model, model_hyperparameters, PATH): + """[summary] Args: model ([type]): [description] @@ -108,7 +60,5 @@ def load_model(model,model_hyperparameters, PATH): # Load model = model(model_hyperparameters) model.load_state_dict(torch.load(PATH)) - - return model - \ No newline at end of file + return model diff --git a/traja/models/vae.py b/traja/models/vae.py index a0126267..2aeec226 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -16,7 +16,7 @@ epochs=epochs, batch_size=batch_size, num_future=num_future, - sequence_length=sequence_length, + num_past=num_past, bidirectional =False, batch_first =True, loss_type = 'huber') @@ -25,73 +25,72 @@ import torch from .utils import TimeDistributed -from .utils import load_model +from torch import nn device = 'cuda' if torch.cuda.is_available() else 'cpu' class LSTMEncoder(torch.nn.Module): - """ Deep LSTM network. This implementation - returns output_size hidden size. - Args: - input_size: The number of expected features in the input `x` - batch_size: - sequence_length: The number of in each sample - hidden_size: The number of features in the hidden state `h` - num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` - would mean stacking two LSTMs together to form a `stacked LSTM`, - with the second LSTM taking in outputs of the first LSTM and - computing the final results. Default: 1 - output_size: The number of output dimensions - dropout: If non-zero, introduces a `Dropout` layer on the outputs of each - LSTM layer except the last layer, with dropout probability equal to - :attr:`dropout`. Default: 0 - bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` - """ - - def __init__(self, input_size: int, sequence_length: int, batch_size: int, - hidden_size: int, num_layers: int, + """ Implementation of Encoder network using LSTM layers + :param input_size: The number of expected features in the input x + :param num_past: Number of time steps to look backwards to predict num_future steps forward + :param batch_size: Number of samples in a batch + :param hidden_size: The number of features in the hidden state h + :param num_lstm_layers: Number of layers in the LSTM model + + :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param reset_state: If True, will reset the hidden and cell state for each batch of data + :param bidirectional: If True, becomes a bidirectional LSTM + """ + def __init__(self, input_size: int, num_past: int, batch_size: int, + hidden_size: int, num_lstm_layers: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): - super(LSTMEncoder, self).__init__() self.input_size = input_size - self.sequence_length = sequence_length + self.num_past = num_past self.batch_size = batch_size self.hidden_size = hidden_size - self.num_layers = num_layers + self.num_lstm_layers = num_lstm_layers self.batch_first = batch_first self.dropout = dropout self.reset_state = reset_state self.bidirectional = bidirectional self.lstm_encoder = torch.nn.LSTM(input_size=input_size, hidden_size=self.hidden_size, - num_layers=num_layers, dropout=dropout, + num_layers=num_lstm_layers, dropout=dropout, bidirectional=self.bidirectional, batch_first=True) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size), torch.zeros(self.num_layers, self.batch_size, self.hidden_size)) + return (torch.zeros(self.num_lstm_layers, self.batch_size, + self.hidden_size).to(device), + torch.zeros(self.num_lstm_layers, self.batch_size, + self.hidden_size).to(device)) def forward(self, x): - enc_init_hidden = self._init_hidden() enc_output, _ = self.lstm_encoder(x, enc_init_hidden) - # RNNs obeys, Markovian. Consider the last state of the hidden is the markovian of the entire sequence in that batch. + # RNNs obeys, Markovian. So, the last state of the hidden is the markovian state for the entire + # sequence in that batch. enc_output = enc_output[:, -1, :] # Shape(batch_size,hidden_dim) return enc_output class DisentangledAELatent(torch.nn.Module): """Dense Dientangled Latent Layer between encoder and decoder""" - def __init__(self, hidden_size: int, latent_size: int, dropout: float): + + def __init__(self, hidden_size: int, latent_size: int, dropout: float): super(DisentangledAELatent, self).__init__() self.latent_size = latent_size self.hidden_size = hidden_size self.dropout = dropout - self.latent = torch.nn.Linear(self.hidden_size, self.latent_size) - - def reparameterize(self, mu, logvar, training= True): + self.latent = torch.nn.Linear(self.hidden_size, self.latent_size * 2) + + @staticmethod + def reparameterize(mu, logvar, training=True): if training: std = logvar.mul(0.5).exp_() eps = std.data.new(std.size()).normal_() @@ -99,6 +98,7 @@ def reparameterize(self, mu, logvar, training= True): return mu def forward(self, x, training=True): + z_variables = self.latent(x) # [batch_size, latent_size*2] mu, logvar = torch.chunk(z_variables, 2, dim=1) # [batch_size,latent_size] # Reparameterize @@ -126,8 +126,9 @@ class LSTMDecoder(torch.nn.Module): reset_state: If ``True``, the hidden and cell states of the LSTM will be reset at the beginning of each batch of input """ - def __init__(self, batch_size: int, num_future: int, hidden_size: int, - num_layers: int, output_size: int, latent_size: int, + + def __init__(self, batch_size: int, num_future: int, hidden_size: int, + num_lstm_layers: int, output_size: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMDecoder, self).__init__() @@ -135,7 +136,7 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, self.latent_size = latent_size self.num_future = num_future self.hidden_size = hidden_size - self.num_layers = num_layers + self.num_lstm_layers = num_lstm_layers self.output_size = output_size self.batch_first = batch_first self.dropout = dropout @@ -145,23 +146,23 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, # RNN decoder self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, hidden_size=self.hidden_size, - num_layers=self.num_layers, + num_layers=self.num_lstm_layers, dropout=self.dropout, bidirectional=self.bidirectional, batch_first=True) - self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, + self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, self.output_size)) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, - self.hidden_size).to(device), - torch.zeros(self.num_layers, self.batch_size, + return (torch.zeros(self.num_lstm_layers, self.batch_size, + self.hidden_size).to(device), + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device)) def forward(self, x, num_future=None): - # To feed the latent states into lstm decoder, repeat the - # tensor n_future times at second dim + # To feed the latent states into lstm decoder, + # repeat the tensor n_future times at second dim _init_hidden = self._init_hidden() decoder_inputs = x.unsqueeze(1) @@ -182,54 +183,60 @@ def forward(self, x, num_future=None): class MLPClassifier(torch.nn.Module): """ MLP classifier """ - def __init__(self, hidden_size: int, num_classes: int, latent_size: int, + + def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_size: int, num_classifier_layers: int, dropout: float): super(MLPClassifier, self).__init__() self.latent_size = latent_size + self.input_size = input_size self.hidden_size = hidden_size self.num_classes = num_classes + self.num_classifier_layers = num_classifier_layers self.dropout = dropout # Classifier layers - self.classifier1 = torch.nn.Linear(self.latent_size, self.hidden_size) - self.classifier2 = torch.nn.Linear(self.hidden_size, self.hidden_size) - self.classifier3 = torch.nn.Linear(self.hidden_size, self.hidden_size) - self.classifier4 = torch.nn.Linear(self.hidden_size, self.num_classes) + self.hidden = nn.ModuleList([nn.Linear(self.input_size, self.hidden_size)]) + self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_classifier_layers - 1)]) + self.hidden = nn.Sequential(*self.hidden) + self.out = nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) def forward(self, x): + x = self.dropout(self.hidden(x)) + out = self.out(x) + return out - classifier1 = self.dropout(self.classifier1(x)) - classifier2 = self.dropout(self.classifier2(classifier1)) - classifier3 = self.dropout(self.classifier3(classifier2)) - classifier4 = self.classifier4(classifier3) - return classifier4 class MultiModelVAE(torch.nn.Module): """Implementation of Multimodel Variational autoencoders; """ - def __init__(self, input_size: int, - sequence_length: int, - batch_size: int, - num_future: int, - hidden_size: int, - num_layers: int, - output_size: int, - num_classes: int, - latent_size: int, - batch_first: bool, - dropout: float, - reset_state: bool, - bidirectional: bool ): + + def __init__(self, input_size: int, + num_past: int, + batch_size: int, + num_future: int, + lstm_hidden_size: int, + num_lstm_layers: int, + classifier_hidden_size: int, + num_classifier_layers: int, + output_size: int, + num_classes: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool): super(MultiModelVAE, self).__init__() self.input_size = input_size - self.sequence_length = sequence_length + self.num_past = num_past self.batch_size = batch_size self.latent_size = latent_size self.num_future = num_future - self.hidden_size = hidden_size - self.num_layers = num_layers + self.lstm_hidden_size = lstm_hidden_size + self.num_lstm_layers = num_lstm_layers + self.classifier_hidden_size = classifier_hidden_size + self.num_classifier_layers = num_classifier_layers self.output_size = output_size self.num_classes = num_classes self.batch_first = batch_first @@ -237,34 +244,36 @@ def __init__(self, input_size: int, self.reset_state = reset_state self.bidirectional = bidirectional - self.encoder = LSTMEncoder(input_size=self.input_size, - sequence_length=self.sequence_length, + self.encoder = LSTMEncoder(input_size=self.input_size, + num_past=self.num_past, batch_size=self.batch_size, - hidden_size=self.hidden_size, - num_layers=self.num_layers, - batch_first=self.batch_first, + hidden_size=self.lstm_hidden_size, + num_lstm_layers=self.num_lstm_layers, + batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, + reset_state=True, bidirectional=self.bidirectional) - self.latent = DisentangledAELatent(hidden_size=self.hidden_size, - latent_size=self.latent_size, + self.latent = DisentangledAELatent(hidden_size=self.lstm_hidden_size, + latent_size=self.latent_size, dropout=self.dropout) - self.decoder = LSTMDecoder(batch_size=self.batch_size, + self.decoder = LSTMDecoder(batch_size=self.batch_size, num_future=self.num_future, - hidden_size=self.hidden_size, - num_layers=self.num_layers, + hidden_size=self.lstm_hidden_size, + num_lstm_layers=self.num_lstm_layers, output_size=self.output_size, - latent_size=self.latent_size, - batch_first=self.batch_first, + latent_size=self.latent_size, + batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, + reset_state=True, bidirectional=self.bidirectional) - self.classifier = MLPClassifier(hidden_size=self.hidden_size, - num_classes=self.num_classes, - latent_size=self.latent_size, + self.classifier = MLPClassifier(input_size=self.latent_size, + hidden_size = self.classifier_hidden_size, + num_classes=self.num_classes, + latent_size=self.latent_size, + num_classifier_layers=self.num_classifier_layers, dropout=self.dropout) def forward(self, data, training=True, is_classification=False): @@ -279,14 +288,14 @@ def forward(self, data, training=True, is_classification=False): param.requires_grad = True for param in self.latent.parameters(): param.requires_grad = True - + # Encoder enc_out = self.encoder(data) # Latent - latent_out = self.latent(enc_out) + latent_out, mu, logvar = self.latent(enc_out) # Decoder decoder_out = self.decoder(latent_out) - return decoder_out, latent_out + return decoder_out, latent_out, mu, logvar else: # training_mode = 'classification' # Unfreeze classifier parameters and freeze all other @@ -299,11 +308,11 @@ def forward(self, data, training=True, is_classification=False): param.requires_grad = False for param in self.latent.parameters(): param.requires_grad = False - + # Encoder enc_out = self.encoder(data) # Latent latent_out, mu, logvar = self.latent(enc_out, training=training) # Classifier - classifier_out = self.classifier(latent_out) # Deterministic + classifier_out = self.classifier(mu) # Deterministic return classifier_out, latent_out, mu, logvar diff --git a/traja/models/vaegan.py b/traja/models/vaegan.py index ce47a12f..5fb817bc 100644 --- a/traja/models/vaegan.py +++ b/traja/models/vaegan.py @@ -6,22 +6,22 @@ 1. MSE 2. Huber Loss""" - import torch + class MultiModelVAEGAN(torch.nn.Module): - - def __init__(self,*model_hyperparameters, **kwargs): - super(MultiModelVAEGAN,self).__init__() - + + def __init__(self, *model_hyperparameters, **kwargs): + super(MultiModelVAEGAN, self).__init__() + for dictionary in model_hyperparameters: for key in dictionary: - setattr(self, key, dictionary[key]) + setattr(self, key, dictionary[key]) for key in kwargs: setattr(self, key, kwargs[key]) - + def __new__(cls): pass - - def forward(self, *input:None, **kwargs: None): - return NotImplementedError \ No newline at end of file + + def forward(self, *input: None, **kwargs: None): + return NotImplementedError diff --git a/traja/models/visualizer.py b/traja/models/visualizer.py new file mode 100644 index 00000000..9e1462bd --- /dev/null +++ b/traja/models/visualizer.py @@ -0,0 +1,266 @@ +import networkx as nx +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +import scipy +import sklearn +from sklearn import neighbors +from scipy.sparse import csgraph +from sklearn.neighbors import radius_neighbors_graph +from sklearn.neighbors import kneighbors_graph +from mpl_toolkits.mplot3d import Axes3D +from matplotlib.axes import Axes +from matplotlib import cm +# import plotly +# import plotly.graph_objs as go +# import plotly.io as pio +import os, sys +import seaborn as sns +import matplotlib +from matplotlib import style +from scipy.sparse import csgraph +import argparse, copy, h5py, os, sys, time, socket +import tensorflow as tf +import torch, torchvision, torch.nn as nn +import torch.optim as optim +import torchvision.transforms as transforms +# from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot +from matplotlib import ticker, colors + +# print(matplotlib.get_backend()) +# matplotlib.rcParams["backend"] = "Gtk3Agg" +# print(matplotlib.get_backend()) +# matplotlib.use('Gtk3Agg') +import matplotlib.pyplot as plt + +# plt.switch_backend('Qt4Agg') +# print(matplotlib.get_backend()) +plt.switch_backend("TkAgg") + +# seed value and plotly +# init_notebook_mode(connected=True) +np.set_printoptions(suppress=True, precision=3, ) +style.use('ggplot') + + +class DirectedNetwork(object): + + def __init__(self): + super().__init__() + pass + + def show(self, states, weight, fig): + """ + + :param states: list - Hidden states + :param weight: numpy.ndarray - Array of connection weights + :param fig: Figure number + + :return: boolean: Figure close status : Open - False/ Close - True + + """ + np.random.seed(70001) + # Set up hidden states + state_dict = {i: states[i] for i in range(0, len(states))} + + # Set up links + self_connections = [weight[i][i] for i in range(len(weight))] + + # Intialize graph + G = nx.from_numpy_matrix(weight, create_using=nx.MultiDiGraph, parallel_edges=True) + + edge_colors = weight.tolist() + edge_colors_ = [float('%.8f' % j) for i in edge_colors for j in i] + + # Set up nodes + neuron_color = [state_dict.get(node, 0.25) for node in G.nodes()] + + # Set colrmap + vmin = np.min(states) + vmax = np.max(states) + cmap = plt.cm.coolwarm + edge_cmap = plt.cm.Spectral + nx.draw(G, with_labels=True, + cmap=cmap, node_color=neuron_color, + node_size=200, linewidths=5, + edge_color=edge_colors_, + edge_cmap=edge_cmap, font_size=10, + connectionstyle='arc3, rad=0.3') + + sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=vmin, vmax=vmax)) + sm.set_array([]) + cbar = plt.colorbar(sm, orientation="vertical", pad=0.1) + + # State of streaming plot + if plt.fignum_exists(fig.number): + fig.canvas.draw() + fig.canvas.flush_events() + fig.clear() + + # Plot is not closed + return False + else: + return True + + +class LocalLinearEmbedding(object): + + def __init__(self): + super(LocalLinearEmbedding, self).__init__() + pass + + def local_linear_embedding(self, X, d, k, alpha=0.1): + """ + Local Linear Embeddings + + :param X: numpy.ndarray - input data matrix mxD , m data points with D dimensions + :param d: int - target dimensions + :param k: int -number of neighbors + :param alpha: float - Tikhonov coefficient regularization + + :return Y: numpy.ndarray - matrix m row, d attributes are reduced dimensional + """ + # Find the nearest neighbor + x_neighbors = neighbors.kneighbors_graph(X, n_neighbors=k) + + m = len(X) + + # Init weights + W = np.zeros(shape=(m, m)) + + for i, nbor_row in enumerate(x_neighbors): + # Get the kneighboring indexes of i + k_indices = nbor_row.indices + + # Calculate the Z matrix + Z_i = X[k_indices] - X[i] + + # Calculate the matrix G + G_i = Z_i @ Z_i.T + + # Weights between neigbors + w_i = scipy.linalg.pinv(G_i + alpha * np.eye(k)) @ np.ones(k) + W[i, k_indices] = w_i / w_i.sum() + + # Calculate matrix M + M = (np.eye(m) - W).T @ (np.eye(m) - W) + M = M.T + + # Calculate Eigen vectors + _, vectors = scipy.linalg.eigh(M, eigvals=(0, d)) + + # Return the vectors and discard the first column of the matrix + return vectors[:, 1:] + + def show(self, pc, fig2): + """[summary] + + Args: + pc ([type]): [description] + fig2 ([type]): [description] + """ + + ax = Axes3D(fig2) + f = ax.scatter(pc[:, 0], pc[:, 1], pc[:, 2], s=40, c=pc[:, 2]) + # ax.set_xlim(-1,1) + # ax.set_ylim(-1,1) + # ax.set_zlim(-1,1) + for i in range(len(pc)): + ax.plot3D(pc[i:, 0], pc[i:, 1], pc[i:, 2], + alpha=i / len(pc), color='red', linewidth=1) + fig2.colorbar(f) + # plt.pause(0.0001) + # State of streaming plot + if plt.fignum_exists(fig2.number): + fig2.canvas.draw() + fig2.canvas.flush_events() + fig2.clear() + + # Plot is not closed + return False + else: + return True + + +class SpectralEmbedding(object): + + def __init__(self): + super(SpectralEmbedding, self).__init__() + pass + + def spectral_embedding(self, X, rad): + """ + Spectral Clustering + + :param X: numpy.ndarray - input data matrix mxn , m data points with n dimensions + :param rad: float -radius for neighbor search + + :return Y: numpy.ndarray - matrix m row, d attributes are reduced dimensional + """ + # Get the adjacency matrix/nearest neighbor graph; neighbors within the radius of 0.4 + A = radius_neighbors_graph(X.T, rad, mode='distance', + metric='minkowski', p=2, + metric_params=None, include_self=False) + A = A.toarray() + + # Find the laplacian of the neighbour graph + # L = D - A ; where D is the diagonal degree matrix + L = csgraph.laplacian(A, normed=False) + # Embedd the data points i low dimension using the Eigen values/vectos + # of the laplacian graph to get the most optimal partition of the graph + eigval, eigvec = np.linalg.eig(L) + # the second smallest eigenvalue represents sparsest cut of the graph. + np.where(eigval == np.partition(eigval, 1)[1]) + # Partition the graph using the smallest eigen value + y_spec = eigvec[:, 1].copy() + y_spec[y_spec < 0] = 0 + y_spec[y_spec > 0] = 1 + return y_spec + + def show(self, X, spec_embed, fig3): + """[summary] + + Args: + X ([type]): [description] + spec_embed ([type]): [description] + fig3 ([type]): [description] + + Returns: + [type]: [description] + """ + + ax3 = fig3.add_subplot() + X = X.T + fi = ax3.scatter(x=X[:, 0], y=X[:, 1], c=spec_embed, s=30, cmap=plt.cm.Spectral) + for i in range(len(X[:, 0])): + ax3.annotate(i, (X[:, 0][i], X[:, 1][i])) + fig3.colorbar(fi) + + # State of streaming plot + if plt.fignum_exists(fig3.number): + fig3.canvas.draw() + fig3.canvas.flush_events() + fig3.clear() + + # Plot is not closed + return False + else: + return True + + +if __name__ == '__main__': + # create the coordinates + numebr_of_points = 21; + small_range = -1.0; + large_range = 1.0 + + xcoordinates = np.linspace(small_range, large_range, num=numebr_of_points) + ycoordinates = np.linspace(small_range, large_range, num=numebr_of_points) + + xcoord_mesh, ycoord_mesh = np.meshgrid(xcoordinates, ycoordinates) + inds = np.array(range(numebr_of_points ** 2)) + s1 = xcoord_mesh.ravel()[inds] + s2 = ycoord_mesh.ravel()[inds] + coordinate = np.c_[s1, s2] + print('From ', small_range, ' to ', large_range, ' with ', numebr_of_points, ' total number of coordinate: ', + numebr_of_points ** 2) diff --git a/traja/tests/test_models.py b/traja/tests/test_models.py new file mode 100644 index 00000000..ed9fb3fe --- /dev/null +++ b/traja/tests/test_models.py @@ -0,0 +1,92 @@ +import pandas as pd +from traja.datasets import dataset +from traja.models.train import LSTMTrainer, LatentModelTrainer + +# Sample data +data_url = "https://raw.githubusercontent.com/traja-team/traja-research/dataset_und_notebooks/dataset_analysis/jaguar5.csv" +df = pd.read_csv(data_url, error_bad_lines=False) + +def test_aevae(): + """ + Test Autoencoder and variational auto encoder models for training/testing/generative network and + classification networks + + """ + # Hyperparameters + batch_size = 10 + num_past = 10 + num_future = 5 + # Prepare the dataloader + train_loader, test_loader = dataset.MultiModalDataLoader(df, + batch_size=batch_size, + n_past=num_past, + n_future=num_future, + num_workers=1) + + model_save_path = './model.pt' + # Model Trainer + # Model types; "ae" or "vae" + trainer = LatentModelTrainer(model_type='ae', + optimizer_type='Adam', + device='cpu', + input_size=2, + output_size=2, + lstm_hidden_size=32, + num_lstm_layers=2, + reset_state=True, + num_classes=9, + latent_size=10, + dropout=0.1, + num_classifier_layers=4, + classifier_hidden_size=32, + epochs=10, + batch_size=batch_size, + num_future=num_future, + num_past=num_past, + bidirectional=False, + batch_first=True, + loss_type='huber') + + # Train the model + trainer.train(train_loader, test_loader, model_save_path) + + +def test_lstm(): + """ + Testing method for lstm model used for forecasting. + """ + # Hyperparameters + batch_size = 10 + num_past = 10 + num_future = 10 + + # For timeseries prediction + assert num_past == num_future + + # Prepare the dataloader + train_loader, test_loader = dataset.MultiModalDataLoader(df, + batch_size=batch_size, + n_past=num_past, + n_future=num_future, + num_workers=1) + + model_save_path = './model.pt' + # Model Trainer + trainer = LSTMTrainer(model_type='lstm', + optimizer_type='Adam', + device='cuda', + epochs=10, + input_size=2, + batch_size=batch_size, + hidden_size=32, + num_future=num_future, + num_layers=2, + output_size=2, + lr_factor=0.1, + scheduler_patience=3, + batch_first=True, + dropout=0.1, + reset_state=True, + bidirectional=False) + # Train the model + trainer.train(train_loader, test_loader, model_save_path) \ No newline at end of file From c3cc8f1ef3fe1d228e48cce40267d37343962131 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sun, 20 Dec 2020 17:38:02 +0100 Subject: [PATCH 033/211] update classifier hybrid networks --- docs/neuralnets/train_lstm.py | 17 +++++ traja/models/ae.py | 128 ++++++++++++++++++-------------- traja/models/optimizers.py | 13 +++- traja/models/train.py | 34 ++++++--- traja/models/vae.py | 133 ++++++++++++++++------------------ 5 files changed, 186 insertions(+), 139 deletions(-) create mode 100644 docs/neuralnets/train_lstm.py diff --git a/docs/neuralnets/train_lstm.py b/docs/neuralnets/train_lstm.py new file mode 100644 index 00000000..ca84b243 --- /dev/null +++ b/docs/neuralnets/train_lstm.py @@ -0,0 +1,17 @@ + +""" +Train LSTM model for time series forecasting +""" +import traja +from traja.model import LSTM +from traja.datasets import dataset + +df = traja.TrajaDataFrame({"x": [0, 1, 2, 3, 4], "y": [1, 3, 2, 4, 5]}) + +# Dataloader + +# Model instance + +# Trainer + + diff --git a/traja/models/ae.py b/traja/models/ae.py index 648e6b82..c548344e 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -70,24 +70,18 @@ def forward(self, x): class LSTMDecoder(torch.nn.Module): - """ Deep LSTM network. This implementation - returns output_size outputs. - Args: - latent_size: The number of dimensions of the latent layer - batch_size: Number of samples in each batch of training data - hidden_size: The number of features in the hidden state `h` - num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` - would mean stacking two LSTMs together to form a `stacked LSTM`, - with the second LSTM taking in outputs of the first LSTM and - computing the final results. Default: 1 - output_size: The number of output/input dimensions - num_future: The number of time steps in future predictions - dropout: If non-zero, introduces a `Dropout` layer on the outputs of each - LSTM layer except the last layer, with dropout probability equal to - :attr:`dropout`. Default: 0 - bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` - reset_state: If ``True``, the hidden and cell states of the LSTM will - be reset at the beginning of each batch of input + """ Implementation of Decoder network using LSTM layers + :param input_size: The number of expected features in the input x + :param num_future: Number of time steps to be predicted given the num_past steps + :param batch_size: Number of samples in a batch + :param hidden_size: The number of features in the hidden state h + :param num_lstm_layers: Number of layers in the LSTM model + :param output_size: Number of expectd features in the output x_ + :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param reset_state: If True, will reset the hidden and cell state for each batch of data + :param bidirectional: If True, becomes a bidirectional LSTM """ def __init__(self, batch_size: int, num_future: int, hidden_size: int, @@ -144,13 +138,17 @@ def forward(self, x, num_future=None): class MLPClassifier(torch.nn.Module): - """ MLP classifier - """ - + """ MLP classifier: Classify the input data using the latent embeddings + :param input_size: The number of expected latent size + :param hidden_size: The number of features in the hidden state h + :param num_classes: Size of labels or the number of categories in the data + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param num_classifier_layers: Number of hidden layers in the classifier + """ def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_size: int, num_classifier_layers: int, dropout: float): super(MLPClassifier, self).__init__() - self.latent_size = latent_size self.input_size = input_size self.hidden_size = hidden_size self.num_classes = num_classes @@ -171,11 +169,25 @@ def forward(self, x): class MultiModelAE(torch.nn.Module): + """Implementation of Multimodel autoencoders; This Module wraps the Autoencoder + models [Encoder,Latent,Decoder]. If classify=True, then the wrapper also include classification layers + :param input_size: The number of expected features in the input x + :param num_future: Number of time steps to be predicted given the num_past steps + :param batch_size: Number of samples in a batch + :param hidden_size: The number of features in the hidden state h + :param num_lstm_layers: Number of layers in the LSTM model + :param output_size: Number of expectd features in the output x_ + :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param reset_state: If True, will reset the hidden and cell state for each batch of data + :param bidirectional: If True, becomes a bidirectional LSTM + """ def __init__(self, input_size: int, num_past: int, batch_size: int, num_future: int, lstm_hidden_size: int, - num_lstm_layers: int, classifier_hidden_size: int, num_classifier_layers: int, output_size: int, - num_classes: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, - bidirectional: bool): + num_lstm_layers: int , output_size: int, latent_size: int, batch_first: bool, dropout: float, + reset_state: bool, bidirectional: bool=False, num_classifier_layers: int= None, + classifier_hidden_size: int=None, num_classes: int=None): super(MultiModelAE, self).__init__() self.input_size = input_size @@ -218,38 +230,48 @@ def __init__(self, input_size: int, num_past: int, batch_size: int, num_future: dropout=self.dropout, reset_state=True, bidirectional=self.bidirectional) - - self.classifier = MLPClassifier(input_size=self.latent_size, - hidden_size=self.classifier_hidden_size, - num_classes=self.num_classes, - latent_size=self.latent_size, - num_classifier_layers=self.num_classifier_layers, - dropout=self.dropout) - - def forward(self, data, training=True, is_classification=False): - if not is_classification: + if self.num_classes is not None: + self.classifier = MLPClassifier(input_size=self.latent_size, + hidden_size=self.classifier_hidden_size, + num_classes=self.num_classes, + latent_size=self.latent_size, + num_classifier_layers=self.num_classifier_layers, + dropout=self.dropout) + + def get_ae_parameters(self): + """ + :return: Tuple of parameters of the encoder, latent and decoder networks + """ + return [self.encoder.parameters(),self.latent.parameters(),self.decoder.parameters()] + + def get_classifier_parameters(self): + """ + :return: Tuple of parameters of classifier network + """ + assert self.classifier_hidden_size is not None,"Classifier not found" + return [self.classifier.parameters()] + + def forward(self, data, classify=False, training=True): + """ + :param data: Train or test data + :param training: If Training= False, latents are deterministic; This arg is unused; + :param classify: If True, perform classification of input data using the latent embeddings + :return: decoder_out,latent_out or classifier out + """ + if not classify: # Set the classifier grad off - for param in self.classifier.parameters(): - param.requires_grad = False - for param in self.encoder.parameters(): - param.requires_grad = True - for param in self.decoder.parameters(): - param.requires_grad = True - for param in self.latent.parameters(): - param.requires_grad = True - - # Encoder + if self.num_classes is not None: + for param in self.classifier.parameters(): + param.requires_grad = False + # Encoder -->Latent --> Decoder enc_out = self.encoder(data) - # Latent latent_out = self.latent(enc_out) - # Decoder decoder_out = self.decoder(latent_out) - return decoder_out, latent_out - else: # training_mode = 'classification' - # Unfreeze classifier parameters and freeze all other - # network parameters + else: # Classify + # Unfreeze classifier and freeze the rest + assert self.num_classifier_layers is not None, "Classifier not found" for param in self.classifier.parameters(): param.requires_grad = True for param in self.encoder.parameters(): @@ -259,10 +281,8 @@ def forward(self, data, training=True, is_classification=False): for param in self.latent.parameters(): param.requires_grad = False - # Encoder + # Encoder-->Latent-->Classifier enc_out = self.encoder(data) - # Latent latent_out = self.latent(enc_out) - # Classifier classifier_out = self.classifier(latent_out) # Deterministic return classifier_out diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 5bd3f902..d2a5a44e 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -5,13 +5,14 @@ class Optimizer: - def __init__(self, model_type, model, optimizer_type): + def __init__(self, model_type, model, optimizer_type, classify=False): """ Wrapper for setting the model optimizer and learning rate schedulers using ReduceLROnPlateau; If the model type is 'ae' or 'vae' - var optimizers is a dict with separate optimizers for encoder, decoder, latent and classifier. In case of 'lstm', var optimizers is an optimizer for lstm and TimeDistributed(linear layer) :param model_type: Type of model 'ae', 'vae' or 'lstm' :param model: Model instance + :param classify: If True, will return the Optimizer and scheduler for classifier :param optimizer_type: Optimizer to be used; Should be one in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', 'LBFGS', 'ASGD', 'Adamax'] """ @@ -23,6 +24,7 @@ def __init__(self, model_type, model, optimizer_type): self.model_type = model_type self.model = model self.optimizer_type = optimizer_type + self.classify = classify self.optimizers = {} self.schedulers = {} @@ -46,9 +48,12 @@ def get_optimizers(self, lr=0.0001): for network in keys: self.optimizers[network] = getattr(torch.optim, f'{self.optimizer_type}')( getattr(self.model, f'{network}').parameters(), lr=lr) + if not self.classify: + self.optimizers['classifier'] = None elif self.model_type == 'vaegan': return NotImplementedError + return self.optimizers def get_lrschedulers(self, factor: float, patience: int): @@ -69,8 +74,10 @@ def get_lrschedulers(self, factor: float, patience: int): patience=patience, verbose=True) else: for network in self.optimizers.keys(): - self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, - patience=patience, verbose=True) + if self.optimizers[network] is not None: + self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, + if not self.classify: patience=patience, verbose=True) + self.schedulers['classifier'] = None return self.schedulers diff --git a/traja/models/train.py b/traja/models/train.py index 898989c3..79f752ee 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -8,6 +8,7 @@ device = 'cuda' if torch.cuda.is_available() else 'cpu' + class HybridTrainer(object): """ Wrapper for training and testing the LSTM model @@ -49,16 +50,16 @@ def __init__(self, model_type: str, output_size: int, lstm_hidden_size: int, num_lstm_layers: int, - classifier_hidden_size: int, - num_classifier_layers: int, reset_state: bool, - num_classes: int, latent_size: int, dropout: float, epochs: int, batch_size: int, num_future: int, num_past: int, + num_classes: int = None, + classifier_hidden_size: int =None, + num_classifier_layers: int = None, bidirectional: bool = False, batch_first: bool = True, loss_type: str = 'huber', @@ -95,9 +96,9 @@ def __init__(self, model_type: str, 'num_past': self.num_past, 'batch_size': self.batch_size, 'lstm_hidden_size': self.lstm_hidden_size, - 'num_lstm_layers':self.num_lstm_layers, - 'classifier_hidden_size':self.classifier_hidden_size, - 'num_classifier_layers':self.num_classifier_layers, + 'num_lstm_layers': self.num_lstm_layers, + 'classifier_hidden_size': self.classifier_hidden_size, + 'num_classifier_layers': self.num_classifier_layers, 'num_future': self.num_future, 'latent_size': self.latent_size, 'output_size': self.output_size, @@ -117,14 +118,17 @@ def __init__(self, model_type: str, # Model optimizer and the learning rate scheduler based on user defined optimizer_type # and learning rate parameters - optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) + + classify = True if self.classifier_hidden_size is not None else False + + optimizer = Optimizer(self.model_type, self.model, self.optimizer_type, classify = classify) self.model_optimizers = optimizer.get_optimizers(lr=0.001) self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) def __str__(self): return "Training model type {}".format(self.model_type) - def train(self, train_loader, test_loader, model_save_path): + def train(self, train_loader, test_loader, model_save_path=None): """ This method implements the batch- wise training and testing protocol for both time series forecasting and classification of the timeseries @@ -136,6 +140,7 @@ def train(self, train_loader, test_loader, model_save_path): """ assert self.model_type == 'ae' or 'vae' + assert model_save_path is not None, "Model path unknown" self.model.to(device) encoder_optimizer, latent_optimizer, decoder_optimizer, classifier_optimizer = self.model_optimizers.values() @@ -144,8 +149,11 @@ def train(self, train_loader, test_loader, model_save_path): # Training mode: Switch from Generative to classifier training mode training_mode = 'forecasting' + if self.classifier_hidden_size is not None: + self.epochs *= 2 + # Training - for epoch in range(self.epochs * 2): # First half for generative model and next for classifier + for epoch in range(self.epochs): # First half for generative model and next for classifier test_loss_forecasting = 0 test_loss_classification = 0 if epoch > 0: # Initial step is to test and set LR schduler @@ -176,9 +184,11 @@ def train(self, train_loader, test_loader, model_save_path): decoder_optimizer.step() latent_optimizer.step() - else: # training_mode == 'classification' + elif self.classifier_hidden_size \ + and training_mode is not 'forecasting': # training_mode == 'classification' if self.model_type == 'vae': - classifier_out, latent_out, mu, logvar = self.model(data, training=True, is_classification=True) + classifier_out, latent_out, mu, logvar = self.model(data, training=True, + is_classification=True) else: classifier_out = self.model(data, training=True, is_classification=True) @@ -200,7 +210,7 @@ def train(self, train_loader, test_loader, model_save_path): data, target, category = data.float().to(device), target.float().to(device), category.to(device) # Time series forecasting test if self.model_type == 'ae': - out, latent = self.model(data, training=False, is_classification=False) + out, latent = self.model(data, training=False, classify=False) test_loss_forecasting += Criterion().ae_criterion(out, target).item() else: decoder_out, latent_out, mu, logvar = self.model(data, training=False, diff --git a/traja/models/vae.py b/traja/models/vae.py index 2aeec226..623ffaab 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -107,25 +107,19 @@ def forward(self, x, training=True): class LSTMDecoder(torch.nn.Module): - """ Deep LSTM network. This implementation - returns output_size outputs. - Args: - latent_size: The number of dimensions of the latent layer - batch_size: Number of samples in each batch of training data - hidden_size: The number of features in the hidden state `h` - num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` - would mean stacking two LSTMs together to form a `stacked LSTM`, - with the second LSTM taking in outputs of the first LSTM and - computing the final results. Default: 1 - output_size: The number of output/input dimensions - num_future: The number of time steps in future predictions - dropout: If non-zero, introduces a `Dropout` layer on the outputs of each - LSTM layer except the last layer, with dropout probability equal to - :attr:`dropout`. Default: 0 - bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` - reset_state: If ``True``, the hidden and cell states of the LSTM will - be reset at the beginning of each batch of input - """ + """ Implementation of Decoder network using LSTM layers + :param input_size: The number of expected features in the input x + :param num_future: Number of time steps to be predicted given the num_past steps + :param batch_size: Number of samples in a batch + :param hidden_size: The number of features in the hidden state h + :param num_lstm_layers: Number of layers in the LSTM model + :param output_size: Number of expectd features in the output x_ + :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param reset_state: If True, will reset the hidden and cell state for each batch of data + :param bidirectional: If True, becomes a bidirectional LSTM + """ def __init__(self, batch_size: int, num_future: int, hidden_size: int, num_lstm_layers: int, output_size: int, latent_size: int, @@ -181,13 +175,17 @@ def forward(self, x, num_future=None): class MLPClassifier(torch.nn.Module): - """ MLP classifier - """ - + """ MLP classifier: Classify the input data using the latent embeddings + :param input_size: The number of expected latent size + :param hidden_size: The number of features in the hidden state h + :param num_classes: Size of labels or the number of categories in the data + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param num_classifier_layers: Number of hidden layers in the classifier + """ def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_size: int, num_classifier_layers: int, dropout: float): super(MLPClassifier, self).__init__() - self.latent_size = latent_size self.input_size = input_size self.hidden_size = hidden_size self.num_classes = num_classes @@ -208,24 +206,24 @@ def forward(self, x): class MultiModelVAE(torch.nn.Module): - """Implementation of Multimodel Variational autoencoders; + """Implementation of Multimodel Variational autoencoders; This Module wraps the Variational Autoencoder + models [Encoder,Latent[Sampler],Decoder]. If classify=True, then the wrapper also include classification layers + + :param input_size: The number of expected features in the input x + :param num_future: Number of time steps to be predicted given the num_past steps + :param batch_size: Number of samples in a batch + :param hidden_size: The number of features in the hidden state h + :param num_lstm_layers: Number of layers in the LSTM model + :param output_size: Number of expectd features in the output x_ + :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param reset_state: If True, will reset the hidden and cell state for each batch of data + :param bidirectional: If True, becomes a bidirectional LSTM """ - - def __init__(self, input_size: int, - num_past: int, - batch_size: int, - num_future: int, - lstm_hidden_size: int, - num_lstm_layers: int, - classifier_hidden_size: int, - num_classifier_layers: int, - output_size: int, - num_classes: int, - latent_size: int, - batch_first: bool, - dropout: float, - reset_state: bool, - bidirectional: bool): + def __init__(self, input_size: int, num_past: int, batch_size: int, num_future: int, lstm_hidden_size: int, + num_lstm_layers: int , output_size: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, + bidirectional: bool=False, num_classifier_layers: int= None, classifier_hidden_size: int=None, num_classes: int=None): super(MultiModelVAE, self).__init__() self.input_size = input_size @@ -268,38 +266,35 @@ def __init__(self, input_size: int, dropout=self.dropout, reset_state=True, bidirectional=self.bidirectional) - - self.classifier = MLPClassifier(input_size=self.latent_size, - hidden_size = self.classifier_hidden_size, - num_classes=self.num_classes, - latent_size=self.latent_size, - num_classifier_layers=self.num_classifier_layers, - dropout=self.dropout) - - def forward(self, data, training=True, is_classification=False): - - if not is_classification: - # Set the classifier grad off - for param in self.classifier.parameters(): - param.requires_grad = False - for param in self.encoder.parameters(): - param.requires_grad = True - for param in self.decoder.parameters(): - param.requires_grad = True - for param in self.latent.parameters(): - param.requires_grad = True - - # Encoder + if self.num_classes is not None: + self.classifier = MLPClassifier(input_size=self.latent_size, + hidden_size = self.classifier_hidden_size, + num_classes=self.num_classes, + latent_size=self.latent_size, + num_classifier_layers=self.num_classifier_layers, + dropout=self.dropout) + + def forward(self, data, training=True, classify=False): + """ + :param data: Train or test data + :param training: If Training= False, latents are deterministic + :param classify: If True, perform classification of input data using the latent embeddings + :return: decoder_out,latent_out or classifier out + """ + if not classify: + if self.num_classes is not None: + # Set the classifier grad off + for param in self.classifier.parameters(): + param.requires_grad = False + # Encoder -->Latent --> Decoder enc_out = self.encoder(data) - # Latent latent_out, mu, logvar = self.latent(enc_out) - # Decoder decoder_out = self.decoder(latent_out) return decoder_out, latent_out, mu, logvar - else: # training_mode = 'classification' - # Unfreeze classifier parameters and freeze all other - # network parameters + else: + # Unfreeze classifier and freeze the rest + assert self.num_classes is not None, "Classifier not found" for param in self.classifier.parameters(): param.requires_grad = True for param in self.encoder.parameters(): @@ -309,10 +304,8 @@ def forward(self, data, training=True, is_classification=False): for param in self.latent.parameters(): param.requires_grad = False - # Encoder + # Encoder -->Latent --> Classifier enc_out = self.encoder(data) - # Latent latent_out, mu, logvar = self.latent(enc_out, training=training) - # Classifier classifier_out = self.classifier(mu) # Deterministic return classifier_out, latent_out, mu, logvar From 4b201679078e26781969c62d917ea676017b0425 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sun, 20 Dec 2020 17:40:55 +0100 Subject: [PATCH 034/211] Update optimizers.py --- traja/models/optimizers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index d2a5a44e..104ebb5b 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -76,7 +76,8 @@ def get_lrschedulers(self, factor: float, patience: int): for network in self.optimizers.keys(): if self.optimizers[network] is not None: self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, - if not self.classify: patience=patience, verbose=True) + patience=patience, verbose=True) + if not self.classify: self.schedulers['classifier'] = None return self.schedulers From 53579d51b089ad467598b0c1fa37c225ba8ab886 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sun, 20 Dec 2020 17:44:00 +0100 Subject: [PATCH 035/211] Update train.py --- traja/models/train.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 79f752ee..4dda4c0f 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -171,12 +171,12 @@ def train(self, train_loader, test_loader, model_save_path=None): if training_mode == 'forecasting': if self.model_type == 'ae': - decoder_out, latent_out = self.model(data, training=True, is_classification=False) + decoder_out, latent_out = self.model(data, training=True, classify=False) loss = Criterion().ae_criterion(decoder_out, target) else: # vae decoder_out, latent_out, mu, logvar = self.model(data, training=True, - is_classification=False) + classify=False) loss = Criterion().vae_criterion(decoder_out, target, mu, logvar) loss.backward() @@ -188,10 +188,10 @@ def train(self, train_loader, test_loader, model_save_path=None): and training_mode is not 'forecasting': # training_mode == 'classification' if self.model_type == 'vae': classifier_out, latent_out, mu, logvar = self.model(data, training=True, - is_classification=True) + classify=True) else: classifier_out = self.model(data, training=True, - is_classification=True) + classify=True) loss = Criterion().classifier_criterion(classifier_out, category - 1) loss.backward() classifier_optimizer.step() @@ -214,15 +214,15 @@ def train(self, train_loader, test_loader, model_save_path=None): test_loss_forecasting += Criterion().ae_criterion(out, target).item() else: decoder_out, latent_out, mu, logvar = self.model(data, training=False, - is_classification=False) + classify=False) test_loss_forecasting += Criterion().vae_criterion(decoder_out, target, mu, logvar) # Classification test if self.model_type == 'ae': classifier_out = self.model(data, training=False, - is_classification=True) + classify=True) else: classifier_out, latent_out, mu, logvar = self.model(data, training=False, - is_classification=True) + classify=True) test_loss_classification += Criterion().classifier_criterion(classifier_out, category - 1).item() From 692b754cc0500929a667b34d519d12fb4718080f Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sun, 20 Dec 2020 19:26:49 +0100 Subject: [PATCH 036/211] a fix grad_fn --- traja/models/ae.py | 7 +++++++ traja/models/train.py | 2 +- traja/models/vae.py | 10 +++++++++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index c548344e..cce67e65 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -263,6 +263,13 @@ def forward(self, data, classify=False, training=True): if self.num_classes is not None: for param in self.classifier.parameters(): param.requires_grad = False + for param in self.encoder.parameters(): + param.requires_grad = True + for param in self.decoder.parameters(): + param.requires_grad = True + for param in self.latent.parameters(): + param.requires_grad = True + # Encoder -->Latent --> Decoder enc_out = self.encoder(data) latent_out = self.latent(enc_out) diff --git a/traja/models/train.py b/traja/models/train.py index 4dda4c0f..e0163c44 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -131,7 +131,7 @@ def __str__(self): def train(self, train_loader, test_loader, model_save_path=None): """ This method implements the batch- wise training and testing protocol for both time series forecasting and - classification of the timeseries + classification of the timeseriesis_classification :param train_loader: Dataloader object of train dataset with batch data [data,target,category] :param test_loader: Dataloader object of test dataset with [data,target,category] diff --git a/traja/models/vae.py b/traja/models/vae.py index 623ffaab..907eb524 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -282,10 +282,18 @@ def forward(self, data, training=True, classify=False): :return: decoder_out,latent_out or classifier out """ if not classify: + + # Set the classifier grad off if self.num_classes is not None: - # Set the classifier grad off for param in self.classifier.parameters(): param.requires_grad = False + for param in self.encoder.parameters(): + param.requires_grad = True + for param in self.decoder.parameters(): + param.requires_grad = True + for param in self.latent.parameters(): + param.requires_grad = True + # Encoder -->Latent --> Decoder enc_out = self.encoder(data) latent_out, mu, logvar = self.latent(enc_out) From 7106b23acf0de02e4e39239636a33b0a1081151d Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sun, 20 Dec 2020 19:34:29 +0100 Subject: [PATCH 037/211] cuda ae grad err, user defined forecast,classification task --- traja/models/ae.py | 5 +++-- traja/models/vae.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index cce67e65..ae24d105 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -1,6 +1,7 @@ import torch from traja.models.utils import TimeDistributed from torch import nn + device = 'cuda' if torch.cuda.is_available() else 'cpu' @@ -42,8 +43,8 @@ def __init__(self, input_size: int, num_past: int, batch_size: int, bidirectional=self.bidirectional, batch_first=True) def _init_hidden(self): - return (torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size), - torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size)) + return (torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device), + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device)) def forward(self, x): enc_init_hidden = self._init_hidden() diff --git a/traja/models/vae.py b/traja/models/vae.py index 907eb524..a0f474b1 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -282,7 +282,7 @@ def forward(self, data, training=True, classify=False): :return: decoder_out,latent_out or classifier out """ if not classify: - + # Set the classifier grad off if self.num_classes is not None: for param in self.classifier.parameters(): From deead55b5ba5d49158310fc89463acd5117dacc0 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 21 Dec 2020 02:12:15 +0100 Subject: [PATCH 038/211] Update optimizers.py --- traja/models/optimizers.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 104ebb5b..54042384 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -46,10 +46,13 @@ def get_optimizers(self, lr=0.0001): elif self.model_type == 'ae' or 'vae': keys = ['encoder', 'decoder', 'latent', 'classifier'] for network in keys: - self.optimizers[network] = getattr(torch.optim, f'{self.optimizer_type}')( - getattr(self.model, f'{network}').parameters(), lr=lr) - if not self.classify: - self.optimizers['classifier'] = None + if network == 'classifier': + + self.optimizers[network] = getattr(torch.optim, f'{self.optimizer_type}')( + getattr(self.model, f'{network}').parameters(), lr=lr) + + if self.classify: + self.optimizers['classifier'] = elif self.model_type == 'vaegan': return NotImplementedError From 285f4f0ad21cd2072214b3e8aeef487809007948 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 21 Dec 2020 02:17:47 +0100 Subject: [PATCH 039/211] Update optimizers.py --- traja/models/optimizers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 54042384..b5de9a02 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -46,13 +46,13 @@ def get_optimizers(self, lr=0.0001): elif self.model_type == 'ae' or 'vae': keys = ['encoder', 'decoder', 'latent', 'classifier'] for network in keys: - if network == 'classifier': - + if network != 'classifier': self.optimizers[network] = getattr(torch.optim, f'{self.optimizer_type}')( getattr(self.model, f'{network}').parameters(), lr=lr) if self.classify: - self.optimizers['classifier'] = + self.optimizers['classifier'] = self.optimizers['classifier'] = getattr(torch.optim, f'{self.optimizer_type}')( + getattr(self.model, 'classifier').parameters(), lr=lr) elif self.model_type == 'vaegan': return NotImplementedError From 4be367b712bff55083809278e84e96ebf154fc93 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 21 Dec 2020 02:21:37 +0100 Subject: [PATCH 040/211] Update optimizers.py --- traja/models/optimizers.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index b5de9a02..d1e66e15 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -32,12 +32,10 @@ def get_optimizers(self, lr=0.0001): """Optimizers for each network in the model Args: - model_type ([type]): [description] - model ([type]): [description] lr (float, optional): [description]. Defaults to 0.0001. Returns: - [type]: [description] + dict: Optimizers """ if self.model_type == 'lstm': @@ -51,8 +49,9 @@ def get_optimizers(self, lr=0.0001): getattr(self.model, f'{network}').parameters(), lr=lr) if self.classify: - self.optimizers['classifier'] = self.optimizers['classifier'] = getattr(torch.optim, f'{self.optimizer_type}')( - getattr(self.model, 'classifier').parameters(), lr=lr) + self.optimizers['classifier'] = getattr(torch.optim, f'{self.optimizer_type}')(getattr(self.model, 'classifier').parameters(), lr=lr) + else: + self.optimizers['classifier'] = None elif self.model_type == 'vaegan': return NotImplementedError @@ -69,7 +68,7 @@ def get_lrschedulers(self, factor: float, patience: int): patience (int, optional): [description]. Defaults to 10. Returns: - [dict]: [description] + [dict]: Learning rate schedulers """ if self.model_type == 'lstm': assert not isinstance(self.optimizers, dict) From 37dcf143341b6be4af66d517b253bff97124951d Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 21 Dec 2020 13:27:22 +0100 Subject: [PATCH 041/211] Update train.py --- traja/models/train.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index e0163c44..d9bb0f94 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -216,21 +216,24 @@ def train(self, train_loader, test_loader, model_save_path=None): decoder_out, latent_out, mu, logvar = self.model(data, training=False, classify=False) test_loss_forecasting += Criterion().vae_criterion(decoder_out, target, mu, logvar) + # Classification test - if self.model_type == 'ae': - classifier_out = self.model(data, training=False, - classify=True) - else: - classifier_out, latent_out, mu, logvar = self.model(data, training=False, - classify=True) + if self.classifier_hidden_size is not None: + if self.model_type == 'ae': + classifier_out = self.model(data, training=False, + classify=True) + else: + classifier_out, latent_out, mu, logvar = self.model(data, training=False, + classify=True) - test_loss_classification += Criterion().classifier_criterion(classifier_out, - category - 1).item() + test_loss_classification += Criterion().classifier_criterion(classifier_out, + category - 1).item() test_loss_forecasting /= len(test_loader.dataset) print(f'====> Mean test set generator loss: {test_loss_forecasting:.4f}') - test_loss_classification /= len(test_loader.dataset) - print(f'====> Mean test set classifier loss: {test_loss_classification:.4f}') + if test_loss_classification != 0: + test_loss_classification /= len(test_loader.dataset) + print(f'====> Mean test set classifier loss: {test_loss_classification:.4f}') # Scheduler metric is test set loss if training_mode == 'forecasting': @@ -238,7 +241,8 @@ def train(self, train_loader, test_loader, model_save_path=None): decoder_scheduler.step(test_loss_forecasting) latent_scheduler.step(test_loss_forecasting) else: - classifier_scheduler.step(test_loss_classification) + if self.classifier_hidden_size is not None: + classifier_scheduler.step(test_loss_classification) # Save the model at target path utils.save_model(self.model, PATH=model_save_path) From 84111ecb3b7a6154da80fed88a27e5217f5e4d5d Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 21 Dec 2020 13:30:05 +0100 Subject: [PATCH 042/211] Update train.py --- traja/models/train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index d9bb0f94..17713cf7 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -165,7 +165,8 @@ def train(self, train_loader, test_loader, model_save_path=None): encoder_optimizer.zero_grad() latent_optimizer.zero_grad() decoder_optimizer.zero_grad() - classifier_optimizer.zero_grad() + if classifier_optimizer is not None: + classifier_optimizer.zero_grad() data, target, category = data.float().to(device), target.float().to(device), category.to(device) From 969e32b7b1d309c8caa343ac5ce9651e23086877 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 21 Dec 2020 13:46:45 +0100 Subject: [PATCH 043/211] Update train.py --- traja/models/train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/traja/models/train.py b/traja/models/train.py index 17713cf7..225ddb37 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -202,6 +202,7 @@ def train(self, train_loader, test_loader, model_save_path=None): if epoch + 1 == self.epochs: training_mode = 'classification' + print('Training classifier') # Testing if epoch % 10 == 0: From 5e1ff12dfd7d3c6296442de31d1a55faaf3541dd Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 21 Dec 2020 13:49:44 +0100 Subject: [PATCH 044/211] Update train.py --- traja/models/train.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 225ddb37..058c896d 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -202,8 +202,7 @@ def train(self, train_loader, test_loader, model_save_path=None): if epoch + 1 == self.epochs: training_mode = 'classification' - print('Training classifier') - + # Testing if epoch % 10 == 0: with torch.no_grad(): From 0a79f95c8d444b5735924521951d3794b79d825b Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 21 Dec 2020 13:51:50 +0100 Subject: [PATCH 045/211] Update train.py --- traja/models/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 058c896d..bb42fcf9 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -200,9 +200,9 @@ def train(self, train_loader, test_loader, model_save_path=None): print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss / (idx + 1))) - if epoch + 1 == self.epochs: + if epoch + 1 == self.epochs//2: training_mode = 'classification' - + # Testing if epoch % 10 == 0: with torch.no_grad(): From e80f2bb1f0d9645f87de7c513515342fa8c3b611 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 21 Dec 2020 13:54:11 +0100 Subject: [PATCH 046/211] Update train.py --- traja/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index bb42fcf9..86b77623 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -200,7 +200,7 @@ def train(self, train_loader, test_loader, model_save_path=None): print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss / (idx + 1))) - if epoch + 1 == self.epochs//2: + if epoch == self.epochs//2: training_mode = 'classification' # Testing From 516750b4b28d2af3c57ba90aba5513974a1ef9fd Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 21 Dec 2020 15:03:34 +0100 Subject: [PATCH 047/211] rmv cache, gitignore vscode --- .gitignore | 3 +++ .vscode/settings.json | 6 ------ 2 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index be47ab2b..39b5ff97 100644 --- a/.gitignore +++ b/.gitignore @@ -77,6 +77,9 @@ target/ .DS_Store +.vscode +.vscode/ + # celery beat schedule file celerybeat-schedule diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 18837826..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "python.pythonPath":"C:\\Users\\saran\\Anaconda3\\envs\\tfgpu\\python.exe", - "python.linting.pycodestyleEnabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.provider": "autopep8" -} \ No newline at end of file From 11e16a67c5fd9daa4a657d635211b5739a534caf Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 25 Dec 2020 17:34:56 +0100 Subject: [PATCH 048/211] inference, models tests, uml update --- traja/models/train.py | 447 ++++++++++++++++++++++++------------------ 1 file changed, 255 insertions(+), 192 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 86b77623..2ee2824d 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -6,67 +6,70 @@ from .optimizers import Optimizer import torch -device = 'cuda' if torch.cuda.is_available() else 'cpu' +device = "cuda" if torch.cuda.is_available() else "cpu" class HybridTrainer(object): """ Wrapper for training and testing the LSTM model - - :param model_type: Type of model should be "LSTM" - :param optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', - 'AdamW', 'SparseAdam', 'RMSprop', ' - Rprop', 'LBFGS', 'ASGD', 'Adamax'] - :param device: Selected device; 'cuda' or 'cpu' - :param input_size: The number of expected features in the input x - :param output_size: Output feature dimension - :param lstm_hidden_size: The number of features in the hidden state h - :param num_lstm_layers: Number of layers in the LSTM model - :param reset_state: If True, will reset the hidden and cell state for each batch of data - :param num_classes: Number of categories/labels - :param latent_size: Latent space dimension - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - :param num_classifier_layers: Number of layers in the classifier - :param epochs: Number of epochs to train the network - :param batch_size: Number of samples in a batch - :param num_future: Number of time steps to be predicted forward - :param num_past: Number of past time steps otherwise, length of sequences in each batch of data. - :param bidirectional: If True, becomes a bidirectional LSTM - :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - :param loss_type: Type of reconstruction loss to apply, 'huber' or 'rmse'. Default:'huber' - :param lr_factor: Factor by which the learning rate will be reduced - :param scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. - For example, if patience = 2, then we will ignore the first 2 epochs with no - improvement, and will only decrease the LR after the 3rd epoch if the loss still - hasn’t improved then. + Args: + model_type: Type of model should be "LSTM" + optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', + 'AdamW', 'SparseAdam', 'RMSprop', ' + Rprop', 'LBFGS', 'ASGD', 'Adamax'] + device: Selected device; 'cuda' or 'cpu' + input_size: The number of expected features in the input x + output_size: Output feature dimension + lstm_hidden_size: The number of features in the hidden state h + num_lstm_layers: Number of layers in the LSTM model + reset_state: If True, will reset the hidden and cell state for each batch of data + num_classes: Number of categories/labels + latent_size: Latent space dimension + dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + num_classifier_layers: Number of layers in the classifier + epochs: Number of epochs to train the network + batch_size: Number of samples in a batch + num_future: Number of time steps to be predicted forward + num_past: Number of past time steps otherwise, length of sequences in each batch of data. + bidirectional: If True, becomes a bidirectional LSTM + batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + loss_type: Type of reconstruction loss to apply, 'huber' or 'rmse'. Default:'huber' + lr_factor: Factor by which the learning rate will be reduced + scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. + For example, if patience = 2, then we will ignore the first 2 epochs with no + improvement, and will only decrease the LR after the 3rd epoch if the loss still + hasn’t improved then. - """ - - def __init__(self, model_type: str, - optimizer_type: str, - device: str, - input_size: int, - output_size: int, - lstm_hidden_size: int, - num_lstm_layers: int, - reset_state: bool, - latent_size: int, - dropout: float, - epochs: int, - batch_size: int, - num_future: int, - num_past: int, - num_classes: int = None, - classifier_hidden_size: int =None, - num_classifier_layers: int = None, - bidirectional: bool = False, - batch_first: bool = True, - loss_type: str = 'huber', - lr_factor: float = 0.1, - scheduler_patience: int = 10): - - white_keys = ['ae', 'vae'] + """ + + def __init__( + self, + model_type: str, + optimizer_type: str, + device: str, + input_size: int, + output_size: int, + lstm_hidden_size: int, + num_lstm_layers: int, + reset_state: bool, + latent_size: int, + dropout: float, + epochs: int, + batch_size: int, + num_future: int, + num_past: int, + num_classes: int = None, + classifier_hidden_size: int = None, + num_classifier_layers: int = None, + bidirectional: bool = False, + batch_first: bool = True, + loss_type: str = "huber", + lr_factor: float = 0.1, + scheduler_patience: int = 10, + ): + + white_keys = ["ae", "vae"] assert model_type in white_keys, "Valid models are {}".format(white_keys) self.model_type = model_type @@ -92,38 +95,42 @@ def __init__(self, model_type: str, self.optimizer_type = optimizer_type self.lr_factor = lr_factor self.scheduler_patience = scheduler_patience - self.model_hyperparameters = {'input_size': self.input_size, - 'num_past': self.num_past, - 'batch_size': self.batch_size, - 'lstm_hidden_size': self.lstm_hidden_size, - 'num_lstm_layers': self.num_lstm_layers, - 'classifier_hidden_size': self.classifier_hidden_size, - 'num_classifier_layers': self.num_classifier_layers, - 'num_future': self.num_future, - 'latent_size': self.latent_size, - 'output_size': self.output_size, - 'num_classes': self.num_classes, - 'batch_first': self.batch_first, - 'reset_state': self.reset_state, - 'bidirectional': self.bidirectional, - 'dropout': self.dropout - } + self.model_hyperparameters = { + "input_size": self.input_size, + "num_past": self.num_past, + "batch_size": self.batch_size, + "lstm_hidden_size": self.lstm_hidden_size, + "num_lstm_layers": self.num_lstm_layers, + "classifier_hidden_size": self.classifier_hidden_size, + "num_classifier_layers": self.num_classifier_layers, + "num_future": self.num_future, + "latent_size": self.latent_size, + "output_size": self.output_size, + "num_classes": self.num_classes, + "batch_first": self.batch_first, + "reset_state": self.reset_state, + "bidirectional": self.bidirectional, + "dropout": self.dropout, + } # Instantiate model instance based on model_type - if self.model_type == 'ae': + if self.model_type == "ae": self.model = MultiModelAE(**self.model_hyperparameters) - if self.model_type == 'vae': + if self.model_type == "vae": self.model = MultiModelVAE(**self.model_hyperparameters) - # Model optimizer and the learning rate scheduler based on user defined optimizer_type - # and learning rate parameters + # Classification task check + self.classify = True if self.classifier_hidden_size is not None else False - classify = True if self.classifier_hidden_size is not None else False - - optimizer = Optimizer(self.model_type, self.model, self.optimizer_type, classify = classify) + # Model optimizer and the learning rate scheduler + optimizer = Optimizer( + self.model_type, self.model, self.optimizer_type, classify=self.classify + ) self.model_optimizers = optimizer.get_optimizers(lr=0.001) - self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) + self.model_lrschedulers = optimizer.get_lrschedulers( + factor=self.lr_factor, patience=self.scheduler_patience + ) def __str__(self): return "Training model type {}".format(self.model_type) @@ -133,27 +140,37 @@ def train(self, train_loader, test_loader, model_save_path=None): This method implements the batch- wise training and testing protocol for both time series forecasting and classification of the timeseriesis_classification - :param train_loader: Dataloader object of train dataset with batch data [data,target,category] - :param test_loader: Dataloader object of test dataset with [data,target,category] - :param model_save_path: Directory path to save the model + train_loader: Dataloader object of train dataset with batch data [data,target,category] + test_loader: Dataloader object of test dataset with [data,target,category] + model_save_path: Directory path to save the model :return: None """ - assert self.model_type == 'ae' or 'vae' + assert self.model_type == "ae" or "vae" assert model_save_path is not None, "Model path unknown" self.model.to(device) - encoder_optimizer, latent_optimizer, decoder_optimizer, classifier_optimizer = self.model_optimizers.values() - encoder_scheduler, latent_scheduler, decoder_scheduler, classifier_scheduler = self.model_lrschedulers.values() + ( + encoder_optimizer, + latent_optimizer, + decoder_optimizer, + classifier_optimizer, + ) = self.model_optimizers.values() + ( + encoder_scheduler, + latent_scheduler, + decoder_scheduler, + classifier_scheduler, + ) = self.model_lrschedulers.values() # Training mode: Switch from Generative to classifier training mode - training_mode = 'forecasting' + training_mode = "forecasting" - if self.classifier_hidden_size is not None: - self.epochs *= 2 + if self.classify: + self.epochs *= 2 # Forecasting + Classification # Training - for epoch in range(self.epochs): # First half for generative model and next for classifier + for epoch in range(self.epochs): test_loss_forecasting = 0 test_loss_classification = 0 if epoch > 0: # Initial step is to test and set LR schduler @@ -165,86 +182,122 @@ def train(self, train_loader, test_loader, model_save_path=None): encoder_optimizer.zero_grad() latent_optimizer.zero_grad() decoder_optimizer.zero_grad() - if classifier_optimizer is not None: + if self.classify and classifier_optimizer is not None: classifier_optimizer.zero_grad() - data, target, category = data.float().to(device), target.float().to(device), category.to(device) - - if training_mode == 'forecasting': - if self.model_type == 'ae': - decoder_out, latent_out = self.model(data, training=True, classify=False) + data, target, category = ( + data.float().to(device), + target.float().to(device), + category.to(device), + ) + + if training_mode == "forecasting": + if self.model_type == "ae": + decoder_out, latent_out = self.model( + data, training=True, classify=False + ) loss = Criterion().ae_criterion(decoder_out, target) else: # vae - decoder_out, latent_out, mu, logvar = self.model(data, training=True, - classify=False) - loss = Criterion().vae_criterion(decoder_out, target, mu, logvar) + decoder_out, latent_out, mu, logvar = self.model( + data, training=True, classify=False + ) + loss = Criterion().vae_criterion( + decoder_out, target, mu, logvar + ) loss.backward() encoder_optimizer.step() decoder_optimizer.step() latent_optimizer.step() - elif self.classifier_hidden_size \ - and training_mode is not 'forecasting': # training_mode == 'classification' - if self.model_type == 'vae': - classifier_out, latent_out, mu, logvar = self.model(data, training=True, - classify=True) - else: - classifier_out = self.model(data, training=True, - classify=True) - loss = Criterion().classifier_criterion(classifier_out, category - 1) + elif self.classify and training_mode is not "forecasting": + if self.model_type == "vae": + classifier_out, latent_out, mu, logvar = self.model( + data, training=True, classify=True + ) + else: # "ae" + classifier_out = self.model( + data, training=True, classify=True + ) + loss = Criterion().classifier_criterion( + classifier_out, category - 1 + ) loss.backward() classifier_optimizer.step() total_loss += loss - print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss / (idx + 1))) + print( + "Epoch {} | {} loss {}".format( + epoch, training_mode, total_loss / (idx + 1) + ) + ) - if epoch == self.epochs//2: - training_mode = 'classification' + if epoch == self.epochs // 2 and self.classify: + training_mode = "classification" # Testing if epoch % 10 == 0: with torch.no_grad(): self.model.eval() for idx, (data, target, category) in enumerate(list(test_loader)): - data, target, category = data.float().to(device), target.float().to(device), category.to(device) + data, target, category = ( + data.float().to(device), + target.float().to(device), + category.to(device), + ) # Time series forecasting test - if self.model_type == 'ae': - out, latent = self.model(data, training=False, classify=False) - test_loss_forecasting += Criterion().ae_criterion(out, target).item() + if self.model_type == "ae": + out, latent = self.model( + data, training=False, classify=False + ) + test_loss_forecasting += ( + Criterion().ae_criterion(out, target).item() + ) else: - decoder_out, latent_out, mu, logvar = self.model(data, training=False, - classify=False) - test_loss_forecasting += Criterion().vae_criterion(decoder_out, target, mu, logvar) + decoder_out, latent_out, mu, logvar = self.model( + data, training=False, classify=False + ) + test_loss_forecasting += Criterion().vae_criterion( + decoder_out, target, mu, logvar + ) # Classification test - if self.classifier_hidden_size is not None: - if self.model_type == 'ae': - classifier_out = self.model(data, training=False, - classify=True) + if self.classify: + if self.model_type == "ae": + classifier_out = self.model( + data, training=False, classify=True + ) else: - classifier_out, latent_out, mu, logvar = self.model(data, training=False, - classify=True) + classifier_out, latent_out, mu, logvar = self.model( + data, training=False, classify=True + ) - test_loss_classification += Criterion().classifier_criterion(classifier_out, - category - 1).item() + test_loss_classification += ( + Criterion() + .classifier_criterion(classifier_out, category - 1) + .item() + ) test_loss_forecasting /= len(test_loader.dataset) - print(f'====> Mean test set generator loss: {test_loss_forecasting:.4f}') - if test_loss_classification != 0: - test_loss_classification /= len(test_loader.dataset) - print(f'====> Mean test set classifier loss: {test_loss_classification:.4f}') + print( + f"====> Mean test set generator loss: {test_loss_forecasting:.4f}" + ) + if self.classify: + if test_loss_classification != 0: + test_loss_classification /= len(test_loader.dataset) + print( + f"====> Mean test set classifier loss: {test_loss_classification:.4f}" + ) # Scheduler metric is test set loss - if training_mode == 'forecasting': + if training_mode == "forecasting": encoder_scheduler.step(test_loss_forecasting) decoder_scheduler.step(test_loss_forecasting) latent_scheduler.step(test_loss_forecasting) else: - if self.classifier_hidden_size is not None: + if self.classify: classifier_scheduler.step(test_loss_classification) - # Save the model at target path utils.save_model(self.model, PATH=model_save_path) @@ -252,49 +305,51 @@ def train(self, train_loader, test_loader, model_save_path=None): class LSTMTrainer: """ Wrapper for training and testing the LSTM model - - :param model_type: Type of model should be "LSTM" - :param optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', - 'AdamW', 'SparseAdam', 'RMSprop', ' - Rprop', 'LBFGS', 'ASGD', 'Adamax'] - :param device: Selected device; 'cuda' or 'cpu' - :param epochs: Number of epochs to train the network - :param input_size: The number of expected features in the input x - :param batch_size: Number of samples in a batch - :param hidden_size: The number of features in the hidden state h - :param num_future: Number of time steps to be predicted forward - :param num_layers: Number of layers in the LSTM model - :param output_size: Output feature dimension - :param lr_factor: Factor by which the learning rate will be reduced - :param scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. - For example, if patience = 2, then we will ignore the first 2 epochs with no - improvement, and will only decrease the LR after the 3rd epoch if the loss still - hasn’t improved then. - :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - :param reset_state: If True, will reset the hidden and cell state for each batch of data - :param bidirectional: If True, becomes a bidirectional LSTM + Args: + model_type: Type of model should be "LSTM" + optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', + 'AdamW', 'SparseAdam', 'RMSprop', ' + Rprop', 'LBFGS', 'ASGD', 'Adamax'] + device: Selected device; 'cuda' or 'cpu' + epochs: Number of epochs to train the network + input_size: The number of expected features in the input x + batch_size: Number of samples in a batch + hidden_size: The number of features in the hidden state h + num_future: Number of time steps to be predicted forward + num_layers: Number of layers in the LSTM model + output_size: Output feature dimension + lr_factor: Factor by which the learning rate will be reduced + scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. + For example, if patience = 2, then we will ignore the first 2 epochs with no + improvement, and will only decrease the LR after the 3rd epoch if the loss still + hasn’t improved then. + batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + reset_state: If True, will reset the hidden and cell state for each batch of data + bidirectional: If True, becomes a bidirectional LSTM """ - def __init__(self, - model_type: str, - optimizer_type: str, - device: str, - epochs: int, - input_size: int, - batch_size: int, - hidden_size: int, - num_future: int, - num_layers: int, - output_size: int, - lr_factor: float, - scheduler_patience: int, - batch_first: True, - dropout: float, - reset_state: bool, - bidirectional: bool): + def __init__( + self, + model_type: str, + optimizer_type: str, + device: str, + epochs: int, + input_size: int, + batch_size: int, + hidden_size: int, + num_future: int, + num_layers: int, + output_size: int, + lr_factor: float, + scheduler_patience: int, + batch_first: True, + dropout: float, + reset_state: bool, + bidirectional: bool, + ): self.model_type = model_type self.optimizer_type = optimizer_type self.device = device @@ -312,32 +367,35 @@ def __init__(self, self.reset_state = reset_state self.bidirectional = bidirectional - self.model_hyperparameters = {'input_size': self.input_size, - 'batch_size': self.batch_size, - 'hidden_size': self.hidden_size, - 'num_future': self.num_future, - 'num_layers': self.num_layers, - 'output_size': self.output_size, - 'batch_first': self.batch_first, - 'reset_state': self.reset_state, - 'bidirectional': self.bidirectional, - 'dropout': self.dropout - } + self.model_hyperparameters = { + "input_size": self.input_size, + "batch_size": self.batch_size, + "hidden_size": self.hidden_size, + "num_future": self.num_future, + "num_layers": self.num_layers, + "output_size": self.output_size, + "batch_first": self.batch_first, + "reset_state": self.reset_state, + "bidirectional": self.bidirectional, + "dropout": self.dropout, + } self.model = LSTM(**self.model_hyperparameters) optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) self.optimizer = optimizer.get_optimizers(lr=0.001) - self.scheduler = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) + self.scheduler = optimizer.get_lrschedulers( + factor=self.lr_factor, patience=self.scheduler_patience + ) def train(self, train_loader, test_loader, model_save_path): """ Implements the batch wise training and testing for time series forecasting - :param train_loader: Dataloader object of train dataset with batch data [data,target,category] - :param test_loader: Dataloader object of test dataset with [data,target,category] - :param model_save_path: Directory path to save the model + train_loader: Dataloader object of train dataset with batch data [data,target,category] + test_loader: Dataloader object of test dataset with [data,target,category] + model_save_path: Directory path to save the model :return: None""" - assert self.model_type == 'lstm' + assert self.model_type == "lstm" self.model.to(device) for epoch in range(self.epochs): @@ -353,7 +411,7 @@ def train(self, train_loader, test_loader, model_save_path): self.optimizer.step() total_loss += loss - print('Epoch {} | loss {}'.format(epoch, total_loss / (idx + 1))) + print("Epoch {} | loss {}".format(epoch, total_loss / (idx + 1))) # Testing if epoch % 10 == 0: @@ -361,12 +419,17 @@ def train(self, train_loader, test_loader, model_save_path): self.model.eval() test_loss_forecasting = 0 for idx, (data, target, _) in enumerate(list(test_loader)): - data, target = data.float().to(device), target.float().to(device) + data, target = ( + data.float().to(device), + target.float().to(device), + ) out = self.model(data) - test_loss_forecasting += Criterion().lstm_criterion(out, target).item() + test_loss_forecasting += ( + Criterion().lstm_criterion(out, target).item() + ) test_loss_forecasting /= len(test_loader.dataset) - print(f'====> Test set generator loss: {test_loss_forecasting:.4f}') + print(f"====> Test set generator loss: {test_loss_forecasting:.4f}") # Scheduler metric is test set loss self.scheduler.step(test_loss_forecasting) From e62de86a15d67ac586371e38b7005ced7e1743da Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 25 Dec 2020 17:39:47 +0100 Subject: [PATCH 049/211] train update --- traja/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index 2ee2824d..9f8cb85e 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -233,7 +233,7 @@ def train(self, train_loader, test_loader, model_save_path=None): ) ) - if epoch == self.epochs // 2 and self.classify: + if epoch + 1 == self.epochs // 2 and self.classify: training_mode = "classification" # Testing From 7c738a9a21bdac2d6497a95a8ad2481033291c5b Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 25 Dec 2020 17:42:38 +0100 Subject: [PATCH 050/211] Update train.py --- traja/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index 9f8cb85e..ddbf453f 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -233,7 +233,7 @@ def train(self, train_loader, test_loader, model_save_path=None): ) ) - if epoch + 1 == self.epochs // 2 and self.classify: + if epoch == (self.epochs + 1) // 2 and self.classify: training_mode = "classification" # Testing From 48fc6636bacb80cb8c5d6a504ed9f6c09208aef5 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 25 Dec 2020 17:47:34 +0100 Subject: [PATCH 051/211] Update train.py --- traja/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index ddbf453f..d82879de 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -233,7 +233,7 @@ def train(self, train_loader, test_loader, model_save_path=None): ) ) - if epoch == (self.epochs + 1) // 2 and self.classify: + if epoch == (self.epochs) // 2 and self.classify: training_mode = "classification" # Testing From d9af683acf835d32ccd9a77b004f88b9e5810b9e Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 25 Dec 2020 17:49:41 +0100 Subject: [PATCH 052/211] Update train.py --- traja/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index d82879de..ae43bb13 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -237,7 +237,7 @@ def train(self, train_loader, test_loader, model_save_path=None): training_mode = "classification" # Testing - if epoch % 10 == 0: + if epoch + 1 % 10 == 0: with torch.no_grad(): self.model.eval() for idx, (data, target, category) in enumerate(list(test_loader)): From f31d42ab14daa2cdae05be9d89e391c5ba77945c Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 25 Dec 2020 17:52:50 +0100 Subject: [PATCH 053/211] Update train.py --- traja/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index ae43bb13..d82879de 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -237,7 +237,7 @@ def train(self, train_loader, test_loader, model_save_path=None): training_mode = "classification" # Testing - if epoch + 1 % 10 == 0: + if epoch % 10 == 0: with torch.no_grad(): self.model.eval() for idx, (data, target, category) in enumerate(list(test_loader)): From ccd0143cdf3edc250ef3012a4fa19c257543dad5 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 9 Dec 2020 20:46:01 +0100 Subject: [PATCH 054/211] added latent viz, embeddings, lstm, rescalers --- traja/__init__.py | 5 +- traja/datasets/__init__.py | 3 +- traja/datasets/dataset.py | 19 +- traja/models/__init__.py | 2 +- traja/models/ae.py | 292 +++++++++--------- traja/models/experiment.py | 175 ++++++----- traja/models/generator.py | 45 ++- traja/models/interpretor.py | 17 +- traja/models/irl.py | 17 +- traja/models/losses.py | 109 +++---- traja/models/lstm.py | 80 ++--- traja/models/nn.py | 409 ++++++-------------------- traja/models/train.py | 572 +++++++++++------------------------- traja/models/utils.py | 80 ++++- traja/models/vae.py | 271 +++++++++-------- traja/models/vaegan.py | 20 +- traja/models/visualizer.py | 136 ++++----- 17 files changed, 906 insertions(+), 1346 deletions(-) diff --git a/traja/__init__.py b/traja/__init__.py index 5a3f5355..2e884412 100644 --- a/traja/__init__.py +++ b/traja/__init__.py @@ -1,10 +1,11 @@ +from . import models +from . import datasets + from .accessor import TrajaAccessor from .frame import TrajaDataFrame, TrajaCollection from .parsers import read_file, from_df from .plotting import * from .trajectory import * -from traja import models -from traja import datasets import logging diff --git a/traja/datasets/__init__.py b/traja/datasets/__init__.py index 665235a1..d195115f 100644 --- a/traja/datasets/__init__.py +++ b/traja/datasets/__init__.py @@ -2,8 +2,9 @@ import glob import os from typing import List + import pandas as pd -from traja.datasets import dataset + import traja diff --git a/traja/datasets/dataset.py b/traja/datasets/dataset.py index 8742da2f..7e75253c 100644 --- a/traja/datasets/dataset.py +++ b/traja/datasets/dataset.py @@ -1,3 +1,16 @@ +""" +Modified from https://github.com/agrimgupta92/sgan/blob/master/sgan/data/trajectories.py. + +This module contains: + +Classes: +1. Pytorch Time series dataset class instance +2. Weighted train and test dataset loader with respect to class distribution + +Helpers: +1. Class distribution in the dataset + +""" import logging import os import math @@ -8,7 +21,7 @@ from torch.utils.data.sampler import WeightedRandomSampler import pandas as pd from sklearn.utils import shuffle -from traja.datasets import utils +from datasets import utils logger = logging.getLogger(__name__) @@ -236,7 +249,7 @@ def __getitem__(self, index): def __len__(self): return len(self.data) -class MultiModalDataLoader: +class MultiModalDataLoader(object): """ MultiModalDataLoader wraps the following data preparation steps, @@ -256,7 +269,7 @@ class MultiModalDataLoader: batch_size (int): Number of samples per batch of data n_past (int): Input sequence length. Number of time steps from the past. n_future (int): Target sequence length. Number of time steps to the future. - num_workers (int): Number of cpu subprocess occupied during data loading process + num_workers (int): Number of cpu subprocess to be occupied during data loading process Usage: train_dataloader, test_dataloader = MultiModalDataLoader(df = data_frame, batch_size=32, diff --git a/traja/models/__init__.py b/traja/models/__init__.py index 63328ec4..82d110dd 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1,2 +1,2 @@ from .nn import LSTM -from .vae import MultiModelVAE + diff --git a/traja/models/ae.py b/traja/models/ae.py index ae24d105..3d4bb615 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -1,92 +1,122 @@ +""" This module implement the Auto encoder model for both forecasting +and classification of time series data. + +```USAGE``` to train AE model: +trainer = Trainer(model_type='ae', + device=device, + input_size=input_size, + output_size=output_size, + lstm_hidden_size=lstm_hidden_size, + lstm_num_layers=lstm_num_layers, + reset_state=True, + num_classes=num_classes, + latent_size=latent_size, + dropout=0.1, + num_layers=num_layers, + epochs=epochs, + batch_size=batch_size, + num_future=num_future, + sequence_length=sequence_length, + bidirectional =False, + batch_first =True, + loss_type = 'huber') + +trainer.train_latent_model(train_dataloader, test_dataloader, model_save_path=PATH)""" + import torch -from traja.models.utils import TimeDistributed -from torch import nn +from .utils import TimeDistributed +from .utils import load_model device = 'cuda' if torch.cuda.is_available() else 'cpu' class LSTMEncoder(torch.nn.Module): - - """ Implementation of Encoder network using LSTM layers - :param input_size: The number of expected features in the input x - :param num_past: Number of time steps to look backwards to predict num_future steps forward - :param batch_size: Number of samples in a batch - :param hidden_size: The number of features in the hidden state h - :param num_lstm_layers: Number of layers in the LSTM model - - :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - :param reset_state: If True, will reset the hidden and cell state for each batch of data - :param bidirectional: If True, becomes a bidirectional LSTM + """ Deep LSTM network. This implementation + returns output_size hidden size. + Args: + input_size: The number of expected features in the input `x` + batch_size: + sequence_length: The number of in each sample + hidden_size: The number of features in the hidden state `h` + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` + would mean stacking two LSTMs together to form a `stacked LSTM`, + with the second LSTM taking in outputs of the first LSTM and + computing the final results. Default: 1 + output_size: The number of output dimensions + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + LSTM layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` """ - def __init__(self, input_size: int, num_past: int, batch_size: int, - hidden_size: int, num_lstm_layers: int, + def __init__(self, input_size: int, sequence_length: int, batch_size: int, + hidden_size: int, num_layers: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMEncoder, self).__init__() self.input_size = input_size - self.num_past = num_past + self.sequence_length = sequence_length self.batch_size = batch_size self.hidden_size = hidden_size - self.num_lstm_layers = num_lstm_layers + self.num_layers = num_layers self.batch_first = batch_first self.dropout = dropout self.reset_state = reset_state self.bidirectional = bidirectional self.lstm_encoder = torch.nn.LSTM(input_size=input_size, hidden_size=self.hidden_size, - num_layers=num_lstm_layers, dropout=dropout, + num_layers=num_layers, dropout=dropout, bidirectional=self.bidirectional, batch_first=True) def _init_hidden(self): - return (torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device), - torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device)) + return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size), torch.zeros(self.num_layers, self.batch_size, self.hidden_size)) def forward(self, x): + enc_init_hidden = self._init_hidden() enc_output, _ = self.lstm_encoder(x, enc_init_hidden) - # RNNs obeys, Markovian. So, the last state of the hidden is the markovian state for the entire - # sequence in that batch. + # RNNs obeys, Markovian. Consider the last state of the hidden is the markovian of the entire sequence in that batch. enc_output = enc_output[:, -1, :] # Shape(batch_size,hidden_dim) return enc_output class DisentangledAELatent(torch.nn.Module): """Dense Dientangled Latent Layer between encoder and decoder""" - - def __init__(self, hidden_size: int, latent_size: int, dropout: float): + def __init__(self, hidden_size: int, latent_size: int, dropout: float): super(DisentangledAELatent, self).__init__() self.latent_size = latent_size self.hidden_size = hidden_size self.dropout = dropout self.latent = torch.nn.Linear(self.hidden_size, self.latent_size) - def forward(self, x): z = self.latent(x) # Shape(batch_size, latent_size*2) return z class LSTMDecoder(torch.nn.Module): - """ Implementation of Decoder network using LSTM layers - :param input_size: The number of expected features in the input x - :param num_future: Number of time steps to be predicted given the num_past steps - :param batch_size: Number of samples in a batch - :param hidden_size: The number of features in the hidden state h - :param num_lstm_layers: Number of layers in the LSTM model - :param output_size: Number of expectd features in the output x_ - :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - :param reset_state: If True, will reset the hidden and cell state for each batch of data - :param bidirectional: If True, becomes a bidirectional LSTM + """ Deep LSTM network. This implementation + returns output_size outputs. + Args: + latent_size: The number of dimensions of the latent layer + batch_size: Number of samples in each batch of training data + hidden_size: The number of features in the hidden state `h` + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` + would mean stacking two LSTMs together to form a `stacked LSTM`, + with the second LSTM taking in outputs of the first LSTM and + computing the final results. Default: 1 + output_size: The number of output/input dimensions + num_future: The number of time steps in future predictions + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + LSTM layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + reset_state: If ``True``, the hidden and cell states of the LSTM will + be reset at the beginning of each batch of input """ - - def __init__(self, batch_size: int, num_future: int, hidden_size: int, - num_lstm_layers: int, output_size: int, latent_size: int, + def __init__(self, batch_size: int, num_future: int, hidden_size: int, + num_layers: int, output_size: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMDecoder, self).__init__() @@ -94,7 +124,7 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, self.latent_size = latent_size self.num_future = num_future self.hidden_size = hidden_size - self.num_lstm_layers = num_lstm_layers + self.num_layers = num_layers self.output_size = output_size self.batch_first = batch_first self.dropout = dropout @@ -104,17 +134,17 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, # RNN decoder self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, hidden_size=self.hidden_size, - num_layers=self.num_lstm_layers, + num_layers=self.num_layers, dropout=self.dropout, bidirectional=self.bidirectional, batch_first=True) - self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, + self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, self.output_size)) def _init_hidden(self): - return (torch.zeros(self.num_lstm_layers, self.batch_size, - self.hidden_size).to(device), - torch.zeros(self.num_lstm_layers, self.batch_size, + return (torch.zeros(self.num_layers, self.batch_size, + self.hidden_size).to(device), + torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) def forward(self, x, num_future=None): @@ -139,147 +169,117 @@ def forward(self, x, num_future=None): class MLPClassifier(torch.nn.Module): - """ MLP classifier: Classify the input data using the latent embeddings - :param input_size: The number of expected latent size - :param hidden_size: The number of features in the hidden state h - :param num_classes: Size of labels or the number of categories in the data - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - :param num_classifier_layers: Number of hidden layers in the classifier - """ - def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_size: int, num_classifier_layers: int, + """ MLP classifier + """ + def __init__(self, hidden_size: int, num_classes: int, latent_size: int, dropout: float): super(MLPClassifier, self).__init__() - self.input_size = input_size + self.latent_size = latent_size self.hidden_size = hidden_size self.num_classes = num_classes - self.num_classifier_layers = num_classifier_layers self.dropout = dropout # Classifier layers - self.hidden = nn.ModuleList([nn.Linear(self.input_size, self.hidden_size)]) - self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_classifier_layers - 1)]) - self.hidden = nn.Sequential(*self.hidden) - self.out = nn.Linear(self.hidden_size, self.num_classes) + self.classifier1 = torch.nn.Linear(self.latent_size, self.hidden_size) + self.classifier2 = torch.nn.Linear(self.hidden_size, self.hidden_size) + self.classifier3 = torch.nn.Linear(self.hidden_size, self.hidden_size) + self.classifier4 = torch.nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) def forward(self, x): - x = self.dropout(self.hidden(x)) - out = self.out(x) - return out + + classifier1 = self.dropout(self.classifier1(x)) + classifier2 = self.dropout(self.classifier2(classifier1)) + classifier3 = self.dropout(self.classifier3(classifier2)) + classifier4 = self.classifier4(classifier3) + return classifier4 class MultiModelAE(torch.nn.Module): - """Implementation of Multimodel autoencoders; This Module wraps the Autoencoder - models [Encoder,Latent,Decoder]. If classify=True, then the wrapper also include classification layers - - :param input_size: The number of expected features in the input x - :param num_future: Number of time steps to be predicted given the num_past steps - :param batch_size: Number of samples in a batch - :param hidden_size: The number of features in the hidden state h - :param num_lstm_layers: Number of layers in the LSTM model - :param output_size: Number of expectd features in the output x_ - :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - :param reset_state: If True, will reset the hidden and cell state for each batch of data - :param bidirectional: If True, becomes a bidirectional LSTM - """ - def __init__(self, input_size: int, num_past: int, batch_size: int, num_future: int, lstm_hidden_size: int, - num_lstm_layers: int , output_size: int, latent_size: int, batch_first: bool, dropout: float, - reset_state: bool, bidirectional: bool=False, num_classifier_layers: int= None, - classifier_hidden_size: int=None, num_classes: int=None): + + def __init__(self, input_size: int, + sequence_length: int, + batch_size: int, + num_future: int, + hidden_size: int, + num_layers: int, + output_size: int, + num_classes: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool ): super(MultiModelAE, self).__init__() self.input_size = input_size - self.num_past = num_past + self.sequence_length = sequence_length self.batch_size = batch_size self.latent_size = latent_size self.num_future = num_future - self.lstm_hidden_size = lstm_hidden_size - self.num_lstm_layers = num_lstm_layers - self.num_classifier_layers = num_classifier_layers - self.classifier_hidden_size = classifier_hidden_size + self.hidden_size = hidden_size + self.num_layers = num_layers self.output_size = output_size self.num_classes = num_classes self.batch_first = batch_first self.dropout = dropout self.reset_state = reset_state self.bidirectional = bidirectional - - self.encoder = LSTMEncoder(input_size=self.input_size, - num_past=self.num_past, + + self.encoder = LSTMEncoder(input_size=self.input_size, + sequence_length=self.sequence_length, batch_size=self.batch_size, - hidden_size=self.lstm_hidden_size, - num_lstm_layers=self.num_lstm_layers, - batch_first=self.batch_first, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, + reset_state=True, bidirectional=self.bidirectional) - self.latent = DisentangledAELatent(hidden_size=self.lstm_hidden_size, - latent_size=self.latent_size, + self.latent = DisentangledAELatent(hidden_size=self.hidden_size, + latent_size=self.latent_size, dropout=self.dropout) - self.decoder = LSTMDecoder(batch_size=self.batch_size, + self.decoder = LSTMDecoder(batch_size=self.batch_size, num_future=self.num_future, - hidden_size=self.lstm_hidden_size, - num_lstm_layers=self.num_lstm_layers, + hidden_size=self.hidden_size, + num_layers=self.num_layers, output_size=self.output_size, - latent_size=self.latent_size, - batch_first=self.batch_first, + latent_size=self.latent_size, + batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, + reset_state=True, bidirectional=self.bidirectional) - if self.num_classes is not None: - self.classifier = MLPClassifier(input_size=self.latent_size, - hidden_size=self.classifier_hidden_size, - num_classes=self.num_classes, - latent_size=self.latent_size, - num_classifier_layers=self.num_classifier_layers, - dropout=self.dropout) - - def get_ae_parameters(self): - """ - :return: Tuple of parameters of the encoder, latent and decoder networks - """ - return [self.encoder.parameters(),self.latent.parameters(),self.decoder.parameters()] - - def get_classifier_parameters(self): - """ - :return: Tuple of parameters of classifier network - """ - assert self.classifier_hidden_size is not None,"Classifier not found" - return [self.classifier.parameters()] - - def forward(self, data, classify=False, training=True): - """ - :param data: Train or test data - :param training: If Training= False, latents are deterministic; This arg is unused; - :param classify: If True, perform classification of input data using the latent embeddings - :return: decoder_out,latent_out or classifier out - """ - if not classify: + + self.classifier = MLPClassifier(hidden_size=self.hidden_size, + num_classes=self.num_classes, + latent_size=self.latent_size, + dropout=self.dropout) + + def forward(self, data, training=True, is_classification=False): + if not is_classification: # Set the classifier grad off - if self.num_classes is not None: - for param in self.classifier.parameters(): - param.requires_grad = False + for param in self.classifier.parameters(): + param.requires_grad = False for param in self.encoder.parameters(): param.requires_grad = True for param in self.decoder.parameters(): param.requires_grad = True for param in self.latent.parameters(): param.requires_grad = True - - # Encoder -->Latent --> Decoder + + # Encoder enc_out = self.encoder(data) + # Latent latent_out = self.latent(enc_out) + # Decoder decoder_out = self.decoder(latent_out) + return decoder_out, latent_out - else: # Classify - # Unfreeze classifier and freeze the rest - assert self.num_classifier_layers is not None, "Classifier not found" + else: # training_mode = 'classification' + # Unfreeze classifier parameters and freeze all other + # network parameters for param in self.classifier.parameters(): param.requires_grad = True for param in self.encoder.parameters(): @@ -288,9 +288,13 @@ def forward(self, data, classify=False, training=True): param.requires_grad = False for param in self.latent.parameters(): param.requires_grad = False - - # Encoder-->Latent-->Classifier + + # Encoder enc_out = self.encoder(data) + # Latent latent_out = self.latent(enc_out) + # Classifier classifier_out = self.classifier(latent_out) # Deterministic return classifier_out + + diff --git a/traja/models/experiment.py b/traja/models/experiment.py index 5e805571..8e800790 100644 --- a/traja/models/experiment.py +++ b/traja/models/experiment.py @@ -24,8 +24,8 @@ from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler import torchvision.transforms as transforms -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") class Trainer: def __init__(self, model, @@ -47,7 +47,7 @@ def __init__(self, model, self.train_loader = train_loader self.test_loader = test_loader - self.criterion = torch.nn.MSELoss() + self.criterion =torch.nn.MSELoss() print('Checking for optimizer for {}'.format(optimizer)) if optimizer == "adam": print('Using adam') @@ -78,16 +78,14 @@ def __init__(self, model, if not os.path.exists(save_dir): os.makedirs(save_dir) - self.savepath = os.path.join(save_dir, - f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') + self.savepath = os.path.join(save_dir, f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') self.experiment_done = False if os.path.exists(self.savepath): trained_epochs = len(pd.read_csv(self.savepath, sep=';')) if trained_epochs >= epochs: self.experiment_done = True - print( - f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') + print(f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') if os.path.exists((self.savepath.replace('.csv', '.pt'))): self.model.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['model_state_dict']) self.model = self.model.to(self.device) @@ -99,12 +97,13 @@ def __init__(self, model, self.start_epoch = 0 self.model = self.model.to(self.device) + def _infer_initial_epoch(self, savepath): if not os.path.exists(savepath): return 0 else: df = pd.read_csv(savepath, sep=';', index_col=0) - print(len(df) + 1) + print(len(df)+1) return len(df) def train(self): @@ -118,7 +117,7 @@ def train(self): if self.opt_name == "LRS": print('LRS step') self.lr_scheduler.step() - return self.savepath + '.csv' + return self.savepath+'.csv' def train_epoch(self): self.model.train() @@ -126,7 +125,7 @@ def train_epoch(self): running_loss = 0 old_time = time() for batch, data in enumerate(self.train_loader): - inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() + inputs, targets= data[0].to(self.device).float(), data[1].to(self.device).float() self.optimizer.zero_grad() outputs = self.model(inputs) loss = self.criterion(outputs, targets) @@ -135,13 +134,12 @@ def train_epoch(self): running_loss += loss.item() if batch % 10 == 0 and batch != 0: - print(batch, 'of', len(self.train_loader), 'processing time', time() - old_time, 'loss:', - running_loss / total) + print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'loss:', running_loss/total) old_time = time() # Increment number of batches total += 1 - return running_loss / total + return running_loss/total def test(self, epoch, save=True): self.model.eval() @@ -150,7 +148,7 @@ def test(self, epoch, save=True): with torch.no_grad(): for batch, data in enumerate(self.test_loader): if batch % 10 == 0: - print('Processing eval batch', batch, 'of', len(self.test_loader)) + print('Processing eval batch', batch,'of', len(self.test_loader)) inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() outputs = self.model(inputs) loss = self.criterion(outputs, targets) @@ -198,17 +196,16 @@ def __init__(self, input_size: int, hidden_size: int, num_layers: int, self.head = nn.Linear(hidden_size, output_size) - def forward(self, x): + def forward(self, x): x, state = self.lstm(x) # Use the last hidden state of last layer - x = state[0][-1] + x = state[0][-1] x = self.head(x) return x - class TrajectoryLSTM: def __init__( - self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() + self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() ): fig, ax = plt.subplots(2, 1) self.fig = fig @@ -227,10 +224,10 @@ def load_batch(self, batch_size=32): inds = np.random.randint(0, len(self.xy) - self.nb_steps, (self.batch_size)) for i, ind in enumerate(inds): - t_1_b[:, i] = self.xy[ind: ind + self.nb_steps] - t_b[i * nb_steps: (i + 1) * self.nb_steps] = self.xy[ - ind + 1: ind + nb_steps + 1 - ] + t_1_b[:, i] = self.xy[ind : ind + self.nb_steps] + t_b[i * nb_steps : (i + 1) * self.nb_steps] = self.xy[ + ind + 1 : ind + nb_steps + 1 + ] return torch.from_numpy(t_1_b).float(), torch.from_numpy(t_b).float() def train(self): @@ -290,7 +287,6 @@ def plot(self, interactive=True): self._plot() return self.fig - def make_mlp(dim_list, activation="relu", batch_norm=True, dropout=0): layers = [] for dim_in, dim_out in zip(dim_list[:-1], dim_list[1:]): @@ -319,7 +315,7 @@ class Encoder(nn.Module): TrajectoryDiscriminator""" def __init__( - self, embedding_dim=64, h_dim=64, mlp_dim=1024, num_layers=1, dropout=0.0 + self, embedding_dim=64, h_dim=64, mlp_dim=1024, num_layers=1, dropout=0.0 ): super(Encoder, self).__init__() @@ -359,20 +355,20 @@ class Decoder(nn.Module): """Decoder is part of TrajectoryGenerator""" def __init__( - self, - seq_len, - embedding_dim=64, - h_dim=128, - mlp_dim=1024, - num_layers=1, - pool_every_timestep=True, - dropout=0.0, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - pooling_type="pool_net", - neighborhood_size=2.0, - grid_size=8, + self, + seq_len, + embedding_dim=64, + h_dim=128, + mlp_dim=1024, + num_layers=1, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + pooling_type="pool_net", + neighborhood_size=2.0, + grid_size=8, ): super(Decoder, self).__init__() @@ -456,14 +452,14 @@ class PoolHiddenNet(nn.Module): """Pooling module as proposed in our paper""" def __init__( - self, - embedding_dim=64, - h_dim=64, - mlp_dim=1024, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - dropout=0.0, + self, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + dropout=0.0, ): super(PoolHiddenNet, self).__init__() @@ -533,14 +529,14 @@ class SocialPooling(nn.Module): http://cvgl.stanford.edu/papers/CVPR16_Social_LSTM.pdf""" def __init__( - self, - h_dim=64, - activation="relu", - batch_norm=True, - dropout=0.0, - neighborhood_size=2.0, - grid_size=8, - pool_dim=None, + self, + h_dim=64, + activation="relu", + batch_norm=True, + dropout=0.0, + neighborhood_size=2.0, + grid_size=8, + pool_dim=None, ): super(SocialPooling, self).__init__() self.h_dim = h_dim @@ -624,14 +620,14 @@ def forward(self, h_states, seq_start_end, end_pos): # Make all positions to exclude as non-zero # Find which peds to exclude x_bound = (curr_end_pos[:, 0] >= bottom_right[:, 0]) + ( - curr_end_pos[:, 0] <= top_left[:, 0] + curr_end_pos[:, 0] <= top_left[:, 0] ) y_bound = (curr_end_pos[:, 1] >= top_left[:, 1]) + ( - curr_end_pos[:, 1] <= bottom_right[:, 1] + curr_end_pos[:, 1] <= bottom_right[:, 1] ) within_bound = x_bound + y_bound - within_bound[0:: num_ped + 1] = 1 # Don't include the ped itself + within_bound[0 :: num_ped + 1] = 1 # Don't include the ped itself within_bound = within_bound.view(-1) # This is a tricky way to get scatter add to work. Helps me avoid a @@ -661,25 +657,25 @@ class TrajectoryGenerator(nn.Module): """Modified from @agrimgupta92's https://github.com/agrimgupta92/sgan/blob/master/sgan/models.py.""" def __init__( - self, - obs_len, - pred_len, - embedding_dim=64, - encoder_h_dim=64, - decoder_h_dim=128, - mlp_dim=1024, - num_layers=1, - noise_dim=(0,), - noise_type="gaussian", - noise_mix_type="ped", - pooling_type=None, - pool_every_timestep=True, - dropout=0.0, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - neighborhood_size=2.0, - grid_size=8, + self, + obs_len, + pred_len, + embedding_dim=64, + encoder_h_dim=64, + decoder_h_dim=128, + mlp_dim=1024, + num_layers=1, + noise_dim=(0,), + noise_type="gaussian", + noise_mix_type="ped", + pooling_type=None, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + neighborhood_size=2.0, + grid_size=8, ): super(TrajectoryGenerator, self).__init__() @@ -809,9 +805,9 @@ def add_noise(self, _input, seq_start_end, user_noise=None): def mlp_decoder_needed(self): if ( - self.noise_dim - or self.pooling_type - or self.encoder_h_dim != self.decoder_h_dim + self.noise_dim + or self.pooling_type + or self.encoder_h_dim != self.decoder_h_dim ): return True else: @@ -862,20 +858,19 @@ def forward(self, obs_traj, obs_traj_rel, seq_start_end, user_noise=None): return pred_traj_fake_rel - class TrajectoryDiscriminator(nn.Module): def __init__( - self, - obs_len, - pred_len, - embedding_dim=64, - h_dim=64, - mlp_dim=1024, - num_layers=1, - activation="relu", - batch_norm=True, - dropout=0.0, - d_type="local", + self, + obs_len, + pred_len, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + num_layers=1, + activation="relu", + batch_norm=True, + dropout=0.0, + d_type="local", ): super(TrajectoryDiscriminator, self).__init__() diff --git a/traja/models/generator.py b/traja/models/generator.py index 32949852..6df87234 100644 --- a/traja/models/generator.py +++ b/traja/models/generator.py @@ -11,53 +11,52 @@ device = 'cuda' if torch.cuda.is_available() else 'cpu' - -def timeseries(model_type: str, model_hyperparameters: dict, model_path: str, batch_size: int, num_future: int, ): +def timeseries(model_type:str, model_hyperparameters:dict, model_path:str, batch_size:int, num_future:int, ): # Generating few samples - batch_size = model_hyperparameters.batch_size # Number of samples - num_future = model_hyperparameters.num_future # Number of time steps in each sample + batch_size = model_hyperparameters.batch_size # Number of samples + num_future = model_hyperparameters.num_future # Number of time steps in each sample if model_type == 'ae': - model = MultiModelAE(, - + model = MultiModelAE(**model_hyperparameters) + if model_type == 'vae': model = MultiModelVAE(**model_hyperparameters) - + if model_type == 'vaegan': model = MultiModelVAEGAN(**model_hyperparameters) return NotImplementedError - + if model_type == 'irl': model = MultiModelIRL(**model_hyperparameters) return NotImplementedError - + # Load the model from model path: model = load_model(model, model_hyperparameters, model_path) # z = torch.randn((batch_size, latent_size)).to(device) - z = torch.empty(10, model_hyperparameters.latent_size).normal_(mean=0, std=.1).to(device) + z = torch.empty(10, model_hyperparameters.latent_size).normal_(mean=0,std=.1).to(device) # Category from the noise cat = model.classifier(z) # Generate trajectories from the noise - out = model.decoder(z, num_future).cpu().detach().numpy() - out = out.reshape(out.shape[0] * out.shape[1], out.shape[2]) + out = model.decoder(z,num_future).cpu().detach().numpy() + out = out.reshape(out.shape[0]*out.shape[1],out.shape[2]) # for index, i in enumerate(train_df.columns): # scaler = scalers['scaler_'+i] # out[:,index] = scaler.inverse_transform(out[:,index].reshape(1, -1)) - print('IDs in this batch of synthetic data', torch.max(cat, 1).indices + 1) - plt.figure(figsize=(12, 4)) - plt.plot(out[:, 0], label='Generated x: Longitude') - plt.plot(out[:, 1], label='Generated y: Latitude') + print('IDs in this batch of synthetic data',torch.max(cat,1).indices+1) + plt.figure(figsize=(12,4)) + plt.plot(out[:,0], label='Generated x: Longitude') + plt.plot(out[:,1], label='Generated y: Latitude') plt.legend() - fig, ax = plt.subplots(nrows=2, ncols=5, figsize=(16, 5), sharey=True) + fig, ax = plt.subplots(nrows=2, ncols= 5, figsize=(16, 5), sharey=True) # plt.ticklabel_format(useOffset=False) - fig.set_size_inches(20, 5) + fig.set_size_inches(20,5) for i in range(2): for j in range(5): - ax[i, j].plot(out[:, 0][(i + j) * num_future:(i + j) * num_future + num_future], - out[:, 1][(i + j) * num_future:(i + j) * num_future + num_future], - label='Animal ID {}'.format((torch.max(cat, 1).indices + 1).detach()[i + j]), color='g') - ax[i, j].legend() + ax[i,j].plot(out[:,0][(i+j)*num_future:(i+j)*num_future + num_future],out[:,1][(i+j)*num_future:(i+j)*num_future+ num_future],label = 'Animal ID {}'.format((torch.max(cat,1).indices+1).detach()[i+j]),color='g') + ax[i,j].legend() plt.show() - + return out + + diff --git a/traja/models/interpretor.py b/traja/models/interpretor.py index b4c3a1b9..7ac0eb91 100644 --- a/traja/models/interpretor.py +++ b/traja/models/interpretor.py @@ -6,7 +6,6 @@ from .vaegan import MultiModelVAEGAN from .irl import MultiModelIRL - def DisplayLatentDynamics(latent): r"""Visualize the dynamics of combination of latents Args: @@ -14,14 +13,14 @@ def DisplayLatentDynamics(latent): Latent shape (batch_size, latent_dim) Usage: DisplayLatentDynamics(latent)""" - + latents = {} - latents.fromkeys(list(range(latent.shape[1]))) + latents.fromkeys(list(range(latent.shape[1]))) for i in range(latent.shape[1]): - latents[f'{i}'] = latent[:, i].cpu().detach().numpy() - fig = px.scatter_matrix(latents) + latents[f'{i}']=latent[:,i].cpu().detach().numpy() + fig= px.scatter_matrix(latents) fig.update_layout( - autosize=False, - width=1600, - height=1000, ) - return fig.show() + autosize=False, + width=1600, + height=1000,) + return fig.show() \ No newline at end of file diff --git a/traja/models/irl.py b/traja/models/irl.py index f7d5b593..2fac5848 100644 --- a/traja/models/irl.py +++ b/traja/models/irl.py @@ -1,19 +1,20 @@ """ Implementation of Inverse Reinforcement Learning algorithm for Time series""" import torch - class MultiModelIRL(torch.nn.Module): - def __init__(self, *model_hyperparameters, **kwargs): - super(MultiModelIRL, self).__init__() - + def __init__(self,*model_hyperparameters, **kwargs): + super(MultiModelIRL,self).__init__() + for dictionary in model_hyperparameters: for key in dictionary: - setattr(self, key, dictionary[key]) + setattr(self, key, dictionary[key]) for key in kwargs: setattr(self, key, kwargs[key]) - + def __new__(cls): pass - - def forward(self, *input: None, **kwargs: None): + + def forward(self, *input:None, **kwargs: None): return NotImplementedError + + \ No newline at end of file diff --git a/traja/models/losses.py b/traja/models/losses.py index 90060e70..f1019c1e 100644 --- a/traja/models/losses.py +++ b/traja/models/losses.py @@ -1,62 +1,69 @@ import torch +class Criterion(object): + + def __init__(self, model_type): + self.model_type = model_type + + @staticmethod + def ae_criterion(recon_x, x, loss_type='huber'): + """[summary] -class Criterion: - """Implements the loss functions of Autoencoders, Variational Autoencoders and LSTM models - Huber loss is set as default for reconstruction loss, alternative is to use rmse, - Cross entropy loss used for classification - Variational loss used huber loss and unweighted KL Divergence loss""" + Args: + recon_x ([type]): [description] + x ([type]): [description] + loss_type(str): Type of Loss; huber or RMSE - def __init__(self): - - self.huber_loss = torch.nn.SmoothL1Loss(reduction='sum') - self.crossentropy_loss = torch.nn.CrossEntropyLoss() - - def ae_criterion(self, predicted, target, loss_type='huber'): - """ Implements the Autoencoder loss for time series forecasting - :param predicted: Predicted time series by the model - :param target: Target time series - :param loss_type: Type of criterion; Defaults: 'huber' - :return: + Returns: + [type]: [description] """ - if loss_type == 'huber': - loss = self.huber_loss(predicted, target) - return loss - else: # Root MSE - return torch.sqrt(torch.mean((predicted - target) ** 2)) - - def vae_criterion(self, predicted, target, mu, logvar, loss_type='huber'): - """ Time series generative model loss function - :param predicted: Predicted time series by the model - :param target: Target time series - :param mu: Latent variable, Mean - :param logvar: Latent variable, Log(Variance) - :param loss_type: Type of criterion; Defaults: 'huber' - :return: Reconstruction loss + KLD loss - """ - if loss_type == 'huber': - dist_x = self.huber_loss(predicted, target) - else: - dist_x = torch.sqrt(torch.mean((predicted - target) ** 2)) - KLD = -0.5 * torch.sum(1 + logvar - mu ** 2 - logvar.exp()) - return dist_x + KLD + + huber_loss = torch.nn.SmoothL1Loss(reduction='sum') + dist_x = huber_loss(recon_x,x) + return dist_x + else: # RMSE + return torch.sqrt(torch.mean((recon_x-x)**2)) + + @staticmethod + def vae_criterion(recon_x, x, mu, logvar, loss_type='huber'): + r"""Time series generative model loss function - def classifier_criterion(self, predicted, target): - """ - Classifier loss function - :param predicted: Predicted label - :param target: Target label - :return: Cross entropy loss - """ + Args: + recon_x ([type]): [description] + x ([type]): [description] + mu ([type]): [description] + logvar ([type]): [description] - loss = self.crossentropy_loss(predicted, target) - return loss + Returns: + [type]: [description] + """ + if loss_type=='huber': + huber_loss = torch.nn.SmoothL1Loss(reduction='sum') + dist_x = huber_loss(recon_x,x) + else: + dist_x = torch.sqrt(torch.mean((recon_x-x)**2)) + + KLD = -0.5 * torch.sum(1 + logvar - mu**2 - logvar.exp()) + + return dist_x + KLD + + @staticmethod + def classifier_criterion(): + """Classifier loss function""" + classifier_criterion = torch.nn.CrossEntropyLoss() + return classifier_criterion + + def vaegan_criterion(): + return NotImplementedError + + def lstm_criterion(): + return NotImplementedError + + - def lstm_criterion(self, predicted, target): +# VAE loss - loss = self.huber_loss(predicted, target) - return loss +# VAE-GAN loss - def vaegan_criterion(self): - return NotImplementedError +# LSTM \ No newline at end of file diff --git a/traja/models/lstm.py b/traja/models/lstm.py index d5f01d51..7ecdd287 100644 --- a/traja/models/lstm.py +++ b/traja/models/lstm.py @@ -1,62 +1,22 @@ """Implementation of Multimodel LSTM""" -import torch -from traja.models.utils import TimeDistributed -device = 'cuda' if torch.cuda.is_available() else 'cpu' - - -class LSTM(torch.nn.Module): - """ Deep LSTM network. This implementation - returns output_size outputs. - Args: - input_size: The number of expected features in the input `x` - batch_size: - sequence_length: The number of in each sample - hidden_size: The number of features in the hidden state `h` - num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` - would mean stacking two LSTMs together to form a `stacked LSTM`, - with the second LSTM taking in outputs of the first LSTM and - computing the final results. Default: 1 - output_size: The number of output dimensions - dropout: If non-zero, introduces a `Dropout` layer on the outputs of each - LSTM layer except the last layer, with dropout probability equal to - :attr:`dropout`. Default: 0 - bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` - """ - - def __init__(self, batch_size: int, num_future: int, hidden_size: int, num_layers: int, - output_size: int, input_size: int, batch_first: bool, dropout: float, - reset_state: bool, bidirectional: bool): - super(LSTM, self).__init__() - - self.batch_size = batch_size - self.input_size = input_size - self.num_future = num_future - self.hidden_size = hidden_size - self.num_layers = num_layers - self.output_size = output_size - self.batch_first = batch_first - self.dropout = dropout - self.reset_state = reset_state - self.bidirectional = bidirectional - - # RNN decoder - self.lstm = torch.nn.LSTM(input_size=self.input_size, hidden_size=self.hidden_size, - num_layers=self.num_layers, dropout=self.dropout, - bidirectional=self.bidirectional, batch_first=True) - self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, self.output_size)) - - def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device), - torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) - - def forward(self, x): - # To feed the latent states into lstm decoder, repeat the tensor n_future times at second dim - _init_hidden = self._init_hidden() - - # Decoder input Shape(batch_size, num_futures, latent_size) - out, (dec_hidden, dec_cell) = self.lstm(x, _init_hidden) - - # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) to Time Dsitributed Linear Layer - out = self.output(out) - return out +import torch + +class MultiModelLSTM(torch.nn.Module): + + def __init__(self,*model_hyperparameters, **kwargs): + super(MultiModelLSTM,self).__init__() + + for dictionary in model_hyperparameters: + for key in dictionary: + setattr(self, key, dictionary[key]) + for key in kwargs: + setattr(self, key, kwargs[key]) + + def __new__(cls): + pass + + def forward(self, *input:None, **kwargs: None): + return NotImplementedError + + diff --git a/traja/models/nn.py b/traja/models/nn.py index d2f8ebfc..817e437a 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -19,13 +19,13 @@ from time import time from sklearn.preprocessing import MinMaxScaler from datetime import datetime - +from sklearn.model_selection import train_test_split from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler +import torchvision.transforms as transforms nb_steps = 10 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - - + class LossMse: """ Calculate the Mean Squared Error between y_true and y_pred @@ -33,17 +33,15 @@ class LossMse: y_true is the desired output. y_pred is the model's output. """ - def __init__(self) -> None: pass - def __call__(self, y_pred, y_true): + # Calculate the Mean Squared Error and use it as loss. mse = torch.mean(torch.square(y_true - y_pred)) return mse - class Trainer: def __init__(self, model, train_loader, @@ -95,16 +93,14 @@ def __init__(self, model, if not os.path.exists(save_dir): os.makedirs(save_dir) - self.savepath = os.path.join(save_dir, - f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') + self.savepath = os.path.join(save_dir, f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') self.experiment_done = False if os.path.exists(self.savepath): trained_epochs = len(pd.read_csv(self.savepath, sep=';')) if trained_epochs >= epochs: self.experiment_done = True - print( - f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') + print(f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') if os.path.exists((self.savepath.replace('.csv', '.pt'))): self.model.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['model_state_dict']) self.model = self.model.to(self.device) @@ -116,12 +112,13 @@ def __init__(self, model, self.start_epoch = 0 self.model = self.model.to(self.device) + def _infer_initial_epoch(self, savepath): if not os.path.exists(savepath): return 0 else: df = pd.read_csv(savepath, sep=';', index_col=0) - print(len(df) + 1) + print(len(df)+1) return len(df) def train(self): @@ -135,7 +132,7 @@ def train(self): if self.opt_name == "LRS": print('LRS step') self.lr_scheduler.step() - return self.savepath + '.csv' + return self.savepath+'.csv' def train_epoch(self): self.model.train() @@ -143,8 +140,8 @@ def train_epoch(self): running_loss = 0 old_time = time() for batch, data in enumerate(self.train_loader): - - inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() + + inputs, targets= data[0].to(self.device).float(), data[1].to(self.device).float() self.optimizer.zero_grad() outputs = self.model(inputs) loss = self.criterion(outputs, targets) @@ -153,13 +150,12 @@ def train_epoch(self): running_loss += loss.item() if batch % 10 == 0 and batch != 0: - print(batch, 'of', len(self.train_loader), 'processing time', time() - old_time, 'loss:', - running_loss / total) + print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'loss:', running_loss/total) old_time = time() # Increment number of batches total += 1 - return running_loss / total + return running_loss/total def test(self, epoch, save=True): self.model.eval() @@ -168,7 +164,7 @@ def test(self, epoch, save=True): with torch.no_grad(): for batch, data in enumerate(self.test_loader): if batch % 10 == 0: - print('Processing eval batch', batch, 'of', len(self.test_loader)) + print('Processing eval batch', batch,'of', len(self.test_loader)) inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() outputs = self.model(inputs) loss = self.criterion(outputs, targets) @@ -185,228 +181,6 @@ def test(self, epoch, save=True): return test_loss / total -class TimeseriesDataset(Dataset): - # Loads the dataset and splits it into equally sized chunks. - # Whereas this can lead to uneven training data, - # with sufficiently long sequence lengths the - # bias should even out. - - def __init__(self, data_frame, sequence_length): - self.data = data_frame - self.sequence_length = sequence_length - - def __len__(self): - return int((self.data.shape[0]) / self.sequence_length) - - def __getitem__(self, index): - data = (self.data[index * self.sequence_length: (index + 1) * self.sequence_length], - self.data[index * self.sequence_length : (index + 1) * self.sequence_length]) - return data - -def get_transformed_timeseries_dataloaders(data_frame: pd.DataFrame, sequence_length: int, train_fraction: float, batch_size:int): - """ Scale the timeseries dataset and generate train and test dataloaders - - Args: - data_frame (pd.DataFrame): Dataset - sequence_length (int): Sequence length of time series for a single gradient step - train_fraction (float): train data vs test data ratio - batch_size (int): Batch size of single gradient measure - - Returns: - train_loader (Dataloader) - validation_loader(Dataloader) - scaler (instance): Data scaler instance - """ - # Dataset transformation - scaler = MinMaxScaler(copy=False) - scaled_dataset = scaler.fit_transform(data_frame.values) - dataset_length = int(scaled_dataset.shape[0] / sequence_length) - indices = list(range(dataset_length)) - split = int(np.floor(train_fraction * dataset_length)) - train_indices, val_indices = indices[split:], indices[:split] - - # Creating PT data samplers and loaders: - train_sampler = SubsetRandomSampler(train_indices) - valid_sampler = SubsetRandomSampler(val_indices) - - dataset = TimeseriesDataset(scaled_dataset, sequence_length) - - train_loader = DataLoader(dataset, batch_size=batch_size, - sampler=train_sampler) - validation_loader = DataLoader(dataset, batch_size=batch_size, - sampler=valid_sampler) - train_loader.name = "time_series" - return train_loader, validation_loader, scaler - -class LossMseWarmup: - """ - Calculate the Mean Squared Error between y_true and y_pred, - but ignore the beginning "warmup" part of the sequences. - - y_true is the desired output. - y_pred is the model's output. - """ - def __init__(self, warmup_steps=50): - self.warmup_steps = warmup_steps - - def __call__(self, y_pred, y_true): - - y_true_slice = y_true[:, self.warmup_steps:, :] - y_pred_slice = y_pred[:, self.warmup_steps:, :] - - # Calculate the Mean Squared Error and use it as loss. - mse = torch.mean(torch.square(y_true_slice - y_pred_slice)) - - return mse - - -class Trainer: - def __init__(self, model, - train_loader, - test_loader, - epochs=200, - batch_size=60, - run_id=0, - logs_dir='logs', - device='cpu', - optimizer='None', - plot=True, - downsampling=None, - warmup_steps=50): - self.device = device - self.model = model - self.epochs = epochs - self.plot = plot - - self.train_loader = train_loader - self.test_loader = test_loader - - self.warmup_steps = warmup_steps - - self.criterion = LossMseWarmup(self.warmup_steps) - print('Checking for optimizer for {}'.format(optimizer)) - if optimizer == "adam": - print('Using adam') - self.optimizer = optim.Adam(model.parameters()) - elif optimizer == "adam_lr": - print("Using adam with higher learning rate") - self.optimizer = optim.Adam(model.parameters(), lr=0.01) - elif optimizer == 'adam_lr2': - print('Using adam with to large learning rate') - self.optimizer = optim.Adam(model.parameters(), lr=0.0001) - elif optimizer == "SGD": - print('Using SGD') - self.optimizer = optim.SGD(model.parameters(), momentum=0.9, weight_decay=5e-4) - elif optimizer == "LRS": - print('Using LRS') - self.optimizer = optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4) - self.lr_scheduler = optim.lr_scheduler.StepLR(self.optimizer, self.epochs // 3) - elif optimizer == "radam": - print('Using radam') - self.optimizer = RAdam(model.parameters()) - elif optimizer == "RMSprop": - print('Using RMSprop') - self.optimizer = optim.RMSprop(model.parameters()) - else: - raise ValueError('Unknown optimizer {}'.format(optimizer)) - self.opt_name = optimizer - save_dir = os.path.join(logs_dir, model.name, train_loader.name) - if not os.path.exists(save_dir): - os.makedirs(save_dir) - - self.savepath = os.path.join(save_dir, f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') - self.experiment_done = False - if os.path.exists(self.savepath): - trained_epochs = len(pd.read_csv(self.savepath, sep=';')) - - if trained_epochs >= epochs: - self.experiment_done = True - print(f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') - if os.path.exists((self.savepath.replace('.csv', '.pt'))): - self.model.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['model_state_dict']) - self.model = self.model.to(self.device) - - self.optimizer.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['optimizer']) - self.start_epoch = torch.load(self.savepath.replace('.csv', '.pt'))['epoch'] + 1 - else: - - self.start_epoch = 0 - self.model = self.model.to(self.device) - - - def _infer_initial_epoch(self, savepath): - if not os.path.exists(savepath): - return 0 - else: - df = pd.read_csv(savepath, sep=';', index_col=0) - print(len(df)+1) - return len(df) - - def train(self): - if self.experiment_done: - return - for epoch in range(self.start_epoch, self.epochs): - - print('Start training epoch', epoch) - print("{} Epoch {}, training loss: {}".format(datetime.now(), epoch, self.train_epoch())) - self.test(epoch=epoch) - if self.opt_name == "LRS": - print('LRS step') - self.lr_scheduler.step() - return self.savepath+'.csv' - - def train_epoch(self): - self.model.train() - total = 0 - running_loss = 0 - old_time = time() - for batch, data in enumerate(self.train_loader): - if batch % 10 == 0 and batch != 0: - print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'loss:', running_loss/total) - old_time = time() - inputs, labels = data - inputs, labels = inputs.to(self.device).float(), labels.to(self.device).float() - - self.optimizer.zero_grad() - outputs = self.model(inputs) - _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - - loss = self.criterion(outputs, labels) - loss.backward() - self.optimizer.step() - - running_loss += loss.item() - - return running_loss/total - - def test(self, epoch, save=True): - self.model.eval() - total = 0 - test_loss = 0 - with torch.no_grad(): - for batch, data in enumerate(self.test_loader): - if batch % 10 == 0: - print('Processing eval batch', batch,'of', len(self.test_loader)) - inputs, labels = data - inputs, labels = inputs.to(self.device).float(), labels.to(self.device).float() - - outputs = self.model(inputs) - loss = self.criterion(outputs, labels) - _, predicted = torch.max(outputs.data, 1) - total += labels.size(0) - test_loss += loss.item() - - if save: - torch.save({ - 'model_state_dict': self.model.state_dict(), - 'optimizer': self.optimizer.state_dict(), - 'epoch': epoch, - 'test_loss': test_loss / total - }, self.savepath.replace('.csv', '.pt')) - return test_loss / total - - class LSTM(nn.Module): """ Deep LSTM network. This implementation returns output_size outputs. @@ -434,26 +208,25 @@ def __init__(self, input_size: int, hidden_size: int, num_layers: int, self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers, dropout=dropout, - bidirectional=bidirectional) + bidirectional=bidirectional, ) self.head = nn.Linear(hidden_size, output_size) - def forward(self, x): + def forward(self, x): x, state = self.lstm(x) # Use the last hidden state of last layer - x = state[0][-1] + x = state[0][-1] x = self.head(x) return x - class TrajectoryLSTM: def __init__( - self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() + self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() ): fig, ax = plt.subplots(2, 1) self.fig = fig self.ax = ax - assert xy.shape[1] == 2, f"xy should be an N x 2 array, but is {xy.shape}" + assert xy.shape[1] is 2, f"xy should be an N x 2 array, but is {xy.shape}" self.xy = xy self.nb_steps = nb_steps self.epochs = epochs @@ -467,10 +240,10 @@ def load_batch(self, batch_size=32): inds = np.random.randint(0, len(self.xy) - self.nb_steps, (self.batch_size)) for i, ind in enumerate(inds): - t_1_b[:, i] = self.xy[ind: ind + self.nb_steps] - t_b[i * nb_steps: (i + 1) * self.nb_steps] = self.xy[ - ind + 1: ind + nb_steps + 1 - ] + t_1_b[:, i] = self.xy[ind : ind + self.nb_steps] + t_b[i * nb_steps : (i + 1) * self.nb_steps] = self.xy[ + ind + 1 : ind + nb_steps + 1 + ] return torch.from_numpy(t_1_b).float(), torch.from_numpy(t_b).float() def train(self): @@ -530,7 +303,6 @@ def plot(self, interactive=True): self._plot() return self.fig - def make_mlp(dim_list, activation="relu", batch_norm=True, dropout=0): layers = [] for dim_in, dim_out in zip(dim_list[:-1], dim_list[1:]): @@ -559,7 +331,7 @@ class Encoder(nn.Module): TrajectoryDiscriminator""" def __init__( - self, embedding_dim=64, h_dim=64, mlp_dim=1024, num_layers=1, dropout=0.0 + self, embedding_dim=64, h_dim=64, mlp_dim=1024, num_layers=1, dropout=0.0 ): super(Encoder, self).__init__() @@ -599,20 +371,20 @@ class Decoder(nn.Module): """Decoder is part of TrajectoryGenerator""" def __init__( - self, - seq_len, - embedding_dim=64, - h_dim=128, - mlp_dim=1024, - num_layers=1, - pool_every_timestep=True, - dropout=0.0, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - pooling_type="pool_net", - neighborhood_size=2.0, - grid_size=8, + self, + seq_len, + embedding_dim=64, + h_dim=128, + mlp_dim=1024, + num_layers=1, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + pooling_type="pool_net", + neighborhood_size=2.0, + grid_size=8, ): super(Decoder, self).__init__() @@ -696,14 +468,14 @@ class PoolHiddenNet(nn.Module): """Pooling module as proposed in our paper""" def __init__( - self, - embedding_dim=64, - h_dim=64, - mlp_dim=1024, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - dropout=0.0, + self, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + dropout=0.0, ): super(PoolHiddenNet, self).__init__() @@ -773,14 +545,14 @@ class SocialPooling(nn.Module): http://cvgl.stanford.edu/papers/CVPR16_Social_LSTM.pdf""" def __init__( - self, - h_dim=64, - activation="relu", - batch_norm=True, - dropout=0.0, - neighborhood_size=2.0, - grid_size=8, - pool_dim=None, + self, + h_dim=64, + activation="relu", + batch_norm=True, + dropout=0.0, + neighborhood_size=2.0, + grid_size=8, + pool_dim=None, ): super(SocialPooling, self).__init__() self.h_dim = h_dim @@ -864,14 +636,14 @@ def forward(self, h_states, seq_start_end, end_pos): # Make all positions to exclude as non-zero # Find which peds to exclude x_bound = (curr_end_pos[:, 0] >= bottom_right[:, 0]) + ( - curr_end_pos[:, 0] <= top_left[:, 0] + curr_end_pos[:, 0] <= top_left[:, 0] ) y_bound = (curr_end_pos[:, 1] >= top_left[:, 1]) + ( - curr_end_pos[:, 1] <= bottom_right[:, 1] + curr_end_pos[:, 1] <= bottom_right[:, 1] ) within_bound = x_bound + y_bound - within_bound[0:: num_ped + 1] = 1 # Don't include the ped itself + within_bound[0 :: num_ped + 1] = 1 # Don't include the ped itself within_bound = within_bound.view(-1) # This is a tricky way to get scatter add to work. Helps me avoid a @@ -901,25 +673,25 @@ class TrajectoryGenerator(nn.Module): """Modified from @agrimgupta92's https://github.com/agrimgupta92/sgan/blob/master/sgan/models.py.""" def __init__( - self, - obs_len, - pred_len, - embedding_dim=64, - encoder_h_dim=64, - decoder_h_dim=128, - mlp_dim=1024, - num_layers=1, - noise_dim=(0,), - noise_type="gaussian", - noise_mix_type="ped", - pooling_type=None, - pool_every_timestep=True, - dropout=0.0, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - neighborhood_size=2.0, - grid_size=8, + self, + obs_len, + pred_len, + embedding_dim=64, + encoder_h_dim=64, + decoder_h_dim=128, + mlp_dim=1024, + num_layers=1, + noise_dim=(0,), + noise_type="gaussian", + noise_mix_type="ped", + pooling_type=None, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + neighborhood_size=2.0, + grid_size=8, ): super(TrajectoryGenerator, self).__init__() @@ -1049,9 +821,9 @@ def add_noise(self, _input, seq_start_end, user_noise=None): def mlp_decoder_needed(self): if ( - self.noise_dim - or self.pooling_type - or self.encoder_h_dim != self.decoder_h_dim + self.noise_dim + or self.pooling_type + or self.encoder_h_dim != self.decoder_h_dim ): return True else: @@ -1102,20 +874,19 @@ def forward(self, obs_traj, obs_traj_rel, seq_start_end, user_noise=None): return pred_traj_fake_rel - class TrajectoryDiscriminator(nn.Module): def __init__( - self, - obs_len, - pred_len, - embedding_dim=64, - h_dim=64, - mlp_dim=1024, - num_layers=1, - activation="relu", - batch_norm=True, - dropout=0.0, - d_type="local", + self, + obs_len, + pred_len, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + num_layers=1, + activation="relu", + batch_norm=True, + dropout=0.0, + d_type="local", ): super(TrajectoryDiscriminator, self).__init__() diff --git a/traja/models/train.py b/traja/models/train.py index d82879de..6f616ac0 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -1,454 +1,210 @@ -from .ae import MultiModelAE -from .vae import MultiModelVAE -from .lstm import LSTM -from . import utils -from .losses import Criterion -from .optimizers import Optimizer +from models.ae import MultiModelAE +from models.vae import MultiModelVAE +from models.vaegan import MultiModelVAEGAN import torch - -device = "cuda" if torch.cuda.is_available() else "cpu" - - -class HybridTrainer(object): - """ - Wrapper for training and testing the LSTM model - Args: - model_type: Type of model should be "LSTM" - optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', - 'AdamW', 'SparseAdam', 'RMSprop', ' - Rprop', 'LBFGS', 'ASGD', 'Adamax'] - device: Selected device; 'cuda' or 'cpu' - input_size: The number of expected features in the input x - output_size: Output feature dimension - lstm_hidden_size: The number of features in the hidden state h - num_lstm_layers: Number of layers in the LSTM model - reset_state: If True, will reset the hidden and cell state for each batch of data - num_classes: Number of categories/labels - latent_size: Latent space dimension - dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - num_classifier_layers: Number of layers in the classifier - epochs: Number of epochs to train the network - batch_size: Number of samples in a batch - num_future: Number of time steps to be predicted forward - num_past: Number of past time steps otherwise, length of sequences in each batch of data. - bidirectional: If True, becomes a bidirectional LSTM - batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - loss_type: Type of reconstruction loss to apply, 'huber' or 'rmse'. Default:'huber' - lr_factor: Factor by which the learning rate will be reduced - scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. - For example, if patience = 2, then we will ignore the first 2 epochs with no - improvement, and will only decrease the LR after the 3rd epoch if the loss still - hasn’t improved then. - - """ - - def __init__( - self, - model_type: str, - optimizer_type: str, - device: str, - input_size: int, - output_size: int, - lstm_hidden_size: int, - num_lstm_layers: int, - reset_state: bool, - latent_size: int, - dropout: float, - epochs: int, - batch_size: int, - num_future: int, - num_past: int, - num_classes: int = None, - classifier_hidden_size: int = None, - num_classifier_layers: int = None, - bidirectional: bool = False, - batch_first: bool = True, - loss_type: str = "huber", - lr_factor: float = 0.1, - scheduler_patience: int = 10, - ): - - white_keys = ["ae", "vae"] - assert model_type in white_keys, "Valid models are {}".format(white_keys) - +from functools import wraps +import inspect +from models import utils +from models.losses import Criterion + +device = 'cuda' if torch.cuda.is_available() else 'cpu' + + +class Trainer(object): + + def __init__(self, model_type:str, + device:str, + input_size:int, + output_size:int, + lstm_hidden_size:int, + lstm_num_layers:int, + reset_state:bool, + num_classes:int, + latent_size:int, + dropout:float, + num_layers:int, + epochs:int, + batch_size:int, + num_future:int, + sequence_length:int, + bidirectional:bool =False, + batch_first:bool =True, + loss_type:str = 'huber'): + + white_keys = ['ae','vae','lstm','vaeg an', 'irl'] + assert model_type in white_keys, "Valid models are {}".format(white_keys) self.model_type = model_type self.device = device self.input_size = input_size self.lstm_hidden_size = lstm_hidden_size - self.num_lstm_layers = num_lstm_layers - self.classifier_hidden_size = classifier_hidden_size - self.num_classifier_layers = num_classifier_layers + self.lstm_num_layers = lstm_num_layers + self.num_layers = lstm_num_layers + self.hidden_size = lstm_hidden_size # For classifiers too self.batch_first = batch_first self.reset_state = reset_state self.output_size = output_size self.num_classes = num_classes self.latent_size = latent_size - self.num_classifier_layers = num_classifier_layers + self.num_layers = num_layers self.num_future = num_future self.epochs = epochs self.batch_size = batch_size - self.num_past = num_past + self.sequence_length = sequence_length self.dropout = dropout - self.bidirectional = bidirectional + self.bidirectional= bidirectional self.loss_type = loss_type - self.optimizer_type = optimizer_type - self.lr_factor = lr_factor - self.scheduler_patience = scheduler_patience - self.model_hyperparameters = { - "input_size": self.input_size, - "num_past": self.num_past, - "batch_size": self.batch_size, - "lstm_hidden_size": self.lstm_hidden_size, - "num_lstm_layers": self.num_lstm_layers, - "classifier_hidden_size": self.classifier_hidden_size, - "num_classifier_layers": self.num_classifier_layers, - "num_future": self.num_future, - "latent_size": self.latent_size, - "output_size": self.output_size, - "num_classes": self.num_classes, - "batch_first": self.batch_first, - "reset_state": self.reset_state, - "bidirectional": self.bidirectional, - "dropout": self.dropout, - } - - # Instantiate model instance based on model_type - if self.model_type == "ae": + + self.model_hyperparameters = {'input_size':self.input_size, + 'sequence_length':self.sequence_length, + 'batch_size':self.batch_size, + 'batch_first':self.batch_first, + + 'hidden_size':self.lstm_hidden_size, + 'num_future':self.num_future, + + 'num_layers':self.lstm_num_layers, + 'latent_size':self.latent_size, + 'output_size':self.output_size, + 'num_classes':self.num_classes, + 'batch_first':self.batch_first, + 'dropout':self.dropout, + 'reset_state':self.reset_state, + 'bidirectional':self.bidirectional, + 'dropout':self.dropout, + + } + + if self.model_type == 'lstm': + self.model = LSTM(self.model_hyperparameters) + + if self.model_type == 'ae': self.model = MultiModelAE(**self.model_hyperparameters) - - if self.model_type == "vae": + + if self.model_type == 'vae': self.model = MultiModelVAE(**self.model_hyperparameters) - - # Classification task check - self.classify = True if self.classifier_hidden_size is not None else False - - # Model optimizer and the learning rate scheduler - optimizer = Optimizer( - self.model_type, self.model, self.optimizer_type, classify=self.classify - ) - self.model_optimizers = optimizer.get_optimizers(lr=0.001) - self.model_lrschedulers = optimizer.get_lrschedulers( - factor=self.lr_factor, patience=self.scheduler_patience - ) - + + if self.model_type == 'vaegan': + self.model = MultiModelVAEGAN(self.model_hyperparameters) + + if self.model_type == 'irl': + return NotImplementedError + + # Get the optimizers for each network in the model + + [self.encoder_optimizer, self.latent_optimizer, self.decoder_optimizer, self.classifier_optimizer] = utils.get_optimizers(self.model_type, self.model) + + # Learning rate schedulers for the models + [self.encoder_scheduler, self.latent_scheduler, self.decoder_scheduler, self.classifier_scheduler] = utils.get_lrschedulers(self.model_type, + self.encoder_optimizer, + self.latent_optimizer, + self.decoder_optimizer, + self.classifier_optimizer, + factor=0.1, + patience=10) + def __str__(self): return "Training model type {}".format(self.model_type) - - def train(self, train_loader, test_loader, model_save_path=None): - """ - This method implements the batch- wise training and testing protocol for both time series forecasting and - classification of the timeseriesis_classification - - train_loader: Dataloader object of train dataset with batch data [data,target,category] - test_loader: Dataloader object of test dataset with [data,target,category] - model_save_path: Directory path to save the model - :return: None - """ - - assert self.model_type == "ae" or "vae" - assert model_save_path is not None, "Model path unknown" + + # TRAIN AUTOENCODERS/VARIATIONAL AUTOENCODERS + def train_latent_model(self, train_loader, test_loader, model_save_path): + + assert self.model_type == 'ae' or 'vae' + # Move the model to target device self.model.to(device) - ( - encoder_optimizer, - latent_optimizer, - decoder_optimizer, - classifier_optimizer, - ) = self.model_optimizers.values() - ( - encoder_scheduler, - latent_scheduler, - decoder_scheduler, - classifier_scheduler, - ) = self.model_lrschedulers.values() - # Training mode: Switch from Generative to classifier training mode - training_mode = "forecasting" - - if self.classify: - self.epochs *= 2 # Forecasting + Classification + training_mode = 'forecasting' # Training - for epoch in range(self.epochs): - test_loss_forecasting = 0 - test_loss_classification = 0 - if epoch > 0: # Initial step is to test and set LR schduler + for epoch in range(self.epochs*2): # First half for generative model and next for classifier + if epoch>0: # Initial step is to test and set LR schduler # Training self.model.train() total_loss = 0 - for idx, (data, target, category) in enumerate(train_loader): + for idx, (data, target,category) in enumerate(train_loader): # Reset optimizer states - encoder_optimizer.zero_grad() - latent_optimizer.zero_grad() - decoder_optimizer.zero_grad() - if self.classify and classifier_optimizer is not None: - classifier_optimizer.zero_grad() - - data, target, category = ( - data.float().to(device), - target.float().to(device), - category.to(device), - ) - - if training_mode == "forecasting": - if self.model_type == "ae": - decoder_out, latent_out = self.model( - data, training=True, classify=False - ) - loss = Criterion().ae_criterion(decoder_out, target) - - else: # vae - decoder_out, latent_out, mu, logvar = self.model( - data, training=True, classify=False - ) - loss = Criterion().vae_criterion( - decoder_out, target, mu, logvar - ) - - loss.backward() - encoder_optimizer.step() - decoder_optimizer.step() - latent_optimizer.step() - - elif self.classify and training_mode is not "forecasting": - if self.model_type == "vae": - classifier_out, latent_out, mu, logvar = self.model( - data, training=True, classify=True - ) - else: # "ae" - classifier_out = self.model( - data, training=True, classify=True - ) - loss = Criterion().classifier_criterion( - classifier_out, category - 1 - ) + self.encoder_optimizer.zero_grad() + self.latent_optimizer.zero_grad() + self.decoder_optimizer.zero_grad() + self.classifier_optimizer.zero_grad() + + data, target,category = data.float().to(device), target.float().to(device), category.to(device) + + if training_mode =='forecasting': + if self.model_type == 'ae': + decoder_out, latent_out = self.model(data, training=True, is_classification=False) + loss = Criterion.ae_criterion(decoder_out, target) + + else: # vae + decoder_out, latent_out, mu, logvar= self.model(data, training=True, is_classification=False) + loss = Criterion.vae_criterion(decoder_out, target, mu, logvar) + loss.backward() - classifier_optimizer.step() - total_loss += loss - print( - "Epoch {} | {} loss {}".format( - epoch, training_mode, total_loss / (idx + 1) - ) - ) + self.encoder_optimizer.step() + self.decoder_optimizer.step() + self.latent_optimizer.step() - if epoch == (self.epochs) // 2 and self.classify: - training_mode = "classification" + else: # training_mode == 'classification' + + classifier_out = self.model(data, training=True, is_classification=True) + loss = Criterion.classifier_criterion(classifier_out, category-1) + loss.backward() + self.classifier_optimizer.step() + total_loss+=loss + + print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss/(idx+1))) + + if epoch+1 == self.epochs: # + training_mode = 'classification' # Testing - if epoch % 10 == 0: + if epoch%10==0: with torch.no_grad(): self.model.eval() - for idx, (data, target, category) in enumerate(list(test_loader)): - data, target, category = ( - data.float().to(device), - target.float().to(device), - category.to(device), - ) - # Time series forecasting test - if self.model_type == "ae": - out, latent = self.model( - data, training=False, classify=False - ) - test_loss_forecasting += ( - Criterion().ae_criterion(out, target).item() - ) + test_loss_forecasting = 0 + test_loss_classification = 0 + for idx, (data, target,category) in enumerate(list(test_loader)): + data, target, category = data.float().to(device), target.float().to(device), category.to(device) + # Time seriesforecasting test + if self.model_type=='ae': + out, latent = self.model(data, training=False, is_classification=False) + test_loss_forecasting += Criterion.ae_criterion(out,target).item() else: - decoder_out, latent_out, mu, logvar = self.model( - data, training=False, classify=False - ) - test_loss_forecasting += Criterion().vae_criterion( - decoder_out, target, mu, logvar - ) - + decoder_out,latent_out,mu,logvar= self.model(data,training=False, is_classification=False) + test_loss_forecasting += Criterion.vae_criterion(decoder_out, target,mu,logvar) # Classification test - if self.classify: - if self.model_type == "ae": - classifier_out = self.model( - data, training=False, classify=True - ) - else: - classifier_out, latent_out, mu, logvar = self.model( - data, training=False, classify=True - ) - - test_loss_classification += ( - Criterion() - .classifier_criterion(classifier_out, category - 1) - .item() - ) + classifier_out= self.model(data,training=False, is_classification=True) + test_loss_classification += Criterion.classifier_criterion(classifier_out, category-1).item() test_loss_forecasting /= len(test_loader.dataset) - print( - f"====> Mean test set generator loss: {test_loss_forecasting:.4f}" - ) - if self.classify: - if test_loss_classification != 0: - test_loss_classification /= len(test_loader.dataset) - print( - f"====> Mean test set classifier loss: {test_loss_classification:.4f}" - ) + print(f'====> Mean test set generator loss: {test_loss_forecasting:.4f}') + test_loss_classification /= len(test_loader.dataset) + print(f'====> Mean test set classifier loss: {test_loss_classification:.4f}') # Scheduler metric is test set loss - if training_mode == "forecasting": - encoder_scheduler.step(test_loss_forecasting) - decoder_scheduler.step(test_loss_forecasting) - latent_scheduler.step(test_loss_forecasting) + if training_mode =='forecasting': + self.encoder_scheduler.step(self.test_loss_forecasting) + self.decoder_scheduler.step(self.test_loss_forecasting) + self.latent_scheduler.step(self.test_loss_forecasting) else: - if self.classify: - classifier_scheduler.step(test_loss_classification) - # Save the model at target path - utils.save_model(self.model, PATH=model_save_path) - - -class LSTMTrainer: - """ - Wrapper for training and testing the LSTM model - Args: - model_type: Type of model should be "LSTM" - optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', - 'AdamW', 'SparseAdam', 'RMSprop', ' - Rprop', 'LBFGS', 'ASGD', 'Adamax'] - device: Selected device; 'cuda' or 'cpu' - epochs: Number of epochs to train the network - input_size: The number of expected features in the input x - batch_size: Number of samples in a batch - hidden_size: The number of features in the hidden state h - num_future: Number of time steps to be predicted forward - num_layers: Number of layers in the LSTM model - output_size: Output feature dimension - lr_factor: Factor by which the learning rate will be reduced - scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. - For example, if patience = 2, then we will ignore the first 2 epochs with no - improvement, and will only decrease the LR after the 3rd epoch if the loss still - hasn’t improved then. - batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - reset_state: If True, will reset the hidden and cell state for each batch of data - bidirectional: If True, becomes a bidirectional LSTM - - """ - - def __init__( - self, - model_type: str, - optimizer_type: str, - device: str, - epochs: int, - input_size: int, - batch_size: int, - hidden_size: int, - num_future: int, - num_layers: int, - output_size: int, - lr_factor: float, - scheduler_patience: int, - batch_first: True, - dropout: float, - reset_state: bool, - bidirectional: bool, - ): - self.model_type = model_type - self.optimizer_type = optimizer_type - self.device = device - self.epochs = epochs - self.input_size = input_size - self.batch_size = batch_size - self.hidden_size = hidden_size - self.num_future = num_future - self.num_layers = num_layers - self.output_size = output_size - self.batch_first = batch_first - self.lr_factor = lr_factor - self.scheduler_patience = scheduler_patience - self.dropout = dropout - self.reset_state = reset_state - self.bidirectional = bidirectional - - self.model_hyperparameters = { - "input_size": self.input_size, - "batch_size": self.batch_size, - "hidden_size": self.hidden_size, - "num_future": self.num_future, - "num_layers": self.num_layers, - "output_size": self.output_size, - "batch_first": self.batch_first, - "reset_state": self.reset_state, - "bidirectional": self.bidirectional, - "dropout": self.dropout, - } - - self.model = LSTM(**self.model_hyperparameters) - optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) - self.optimizer = optimizer.get_optimizers(lr=0.001) - self.scheduler = optimizer.get_lrschedulers( - factor=self.lr_factor, patience=self.scheduler_patience - ) - - def train(self, train_loader, test_loader, model_save_path): - - """ Implements the batch wise training and testing for time series forecasting - train_loader: Dataloader object of train dataset with batch data [data,target,category] - test_loader: Dataloader object of test dataset with [data,target,category] - model_save_path: Directory path to save the model - :return: None""" - - assert self.model_type == "lstm" - self.model.to(device) - - for epoch in range(self.epochs): - if epoch > 0: - self.model.train() - total_loss = 0 - for idx, (data, target, _) in enumerate(train_loader): - self.optimizer.zero_grad() - data, target = data.float().to(device), target.float().to(device) - output = self.model(data) - loss = Criterion().lstm_criterion(output, target) - loss.backward() - self.optimizer.step() - total_loss += loss - - print("Epoch {} | loss {}".format(epoch, total_loss / (idx + 1))) - - # Testing - if epoch % 10 == 0: - with torch.no_grad(): - self.model.eval() - test_loss_forecasting = 0 - for idx, (data, target, _) in enumerate(list(test_loader)): - data, target = ( - data.float().to(device), - target.float().to(device), - ) - out = self.model(data) - test_loss_forecasting += ( - Criterion().lstm_criterion(out, target).item() - ) - - test_loss_forecasting /= len(test_loader.dataset) - print(f"====> Test set generator loss: {test_loss_forecasting:.4f}") - - # Scheduler metric is test set loss - self.scheduler.step(test_loss_forecasting) + self.classifier_scheduler.step(self.test_loss_classification) # Save the model at target path - utils.save_model(self.model, PATH=model_save_path) - - -class VAEGANTrainer: - def __init__(self): - pass - - def train(self): + utils.save_model(self.model,PATH = model_save_path) + + + # TRAIN VARIATIONAL AUTOENCODERS-GAN + def train_vaegan(self): + assert self.model_type == 'vaegan' return NotImplementedError - - -class IRLTrainer: - def __init__(self): - pass - - def train(self): + + # TRAIN INVERSE RL + def train_irl(self): + assert self.model_type == 'irl' return NotImplementedError + + # TRAIN LSTM + def train_lstm(self): + assert self.model_type == 'lstm' + return NotImplementedError \ No newline at end of file diff --git a/traja/models/utils.py b/traja/models/utils.py index 5b670334..b9505abc 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -1,13 +1,12 @@ import torch import matplotlib.pyplot as plt +from torch.optim.lr_scheduler import ReduceLROnPlateau import numpy as np import collections from numpy import math - class TimeDistributed(torch.nn.Module): """ Time distributed wrapper compatible with linear/dense pytorch layer modules""" - def __init__(self, module, batch_first=True): super(TimeDistributed, self).__init__() self.module = module @@ -30,24 +29,81 @@ def forward(self, x): out = out.view(-1, x.size(1), out.size(-1)) # (timesteps, samples, output_size) return out + +def get_optimizers(model_type, model, lr=0.0001): + r"""Optimizers for each network in the model + + Args: + model_type ([type]): [description] + model ([type]): [description] + lr (float, optional): [description]. Defaults to 0.0001. + + Returns: + [type]: [description] + """ + + if model_type == 'ae' or 'vae': + # Optimizers for each network in the model + encoder_optimizer = torch.optim.Adam(model.encoder.parameters(), lr=lr) + latent_optimizer = torch.optim.Adam(model.latent.parameters(), lr=lr) + decoder_optimizer = torch.optim.Adam(model.decoder.parameters(), lr=lr) + classifier_optimizer = torch.optim.Adam(model.classifier.parameters(), lr=lr) + return [encoder_optimizer, latent_optimizer, decoder_optimizer, classifier_optimizer] + + elif model_type == 'vaegan': + return NotImplementedError + + else: # LSTM + return NotImplementedError + +def get_lrschedulers(model_type, encoder_optimizer, decoder_optimizer, latent_optimizer, classifier_optimizer, factor=0.1, patience=10): + + r"""Learning rate scheduler for each network in the model + NOTE: Scheduler metric should be test set loss + Args: + model_type ([type]): [description] + encoder_optimizer ([type]): [description] + decoder_optimizer ([type]): [description] + latent_optimizer ([type]): [description] + classifier_optimizer ([type]): [description] + factor (float, optional): [description]. Defaults to 0.1. + patience (int, optional): [description]. Defaults to 10. + Returns: + [type]: [description] + """ + + if model_type == 'ae' or 'vae': + encoder_scheduler = ReduceLROnPlateau(encoder_optimizer, mode='max', factor=factor, patience=patience, verbose=True) + decoder_scheduler = ReduceLROnPlateau(decoder_optimizer, mode='max', factor=factor, patience=patience, verbose=True) + latent_scheduler = ReduceLROnPlateau(latent_optimizer, mode='max', factor=factor, patience=patience, verbose=True) + classifier_scheduler = ReduceLROnPlateau(classifier_optimizer, mode='max', factor=factor, patience=patience, verbose=True) + return [encoder_scheduler, decoder_scheduler, latent_scheduler, classifier_scheduler] + + elif model_type == 'vaegan': + return NotImplementedError + + else: # LSTM + return NotImplementedError + + + def save_model(model, PATH): - """[summary] + r"""[summary] Args: model ([type]): [description] PATH ([type]): [description] """ - + # PATH = "state_dict_model.pt" # Save torch.save(model.state_dict(), PATH) print('Model saved at {}'.format(PATH)) - - -def load_model(model, model_hyperparameters, PATH): - """[summary] + +def load_model(model,model_hyperparameters, PATH): + r"""[summary] Args: model ([type]): [description] @@ -60,5 +116,11 @@ def load_model(model, model_hyperparameters, PATH): # Load model = model(model_hyperparameters) model.load_state_dict(torch.load(PATH)) - + return model + + + + + + diff --git a/traja/models/vae.py b/traja/models/vae.py index a0f474b1..02d9bc97 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -16,7 +16,7 @@ epochs=epochs, batch_size=batch_size, num_future=num_future, - num_past=num_past, + sequence_length=sequence_length, bidirectional =False, batch_first =True, loss_type = 'huber') @@ -25,72 +25,73 @@ import torch from .utils import TimeDistributed -from torch import nn +from .utils import load_model device = 'cuda' if torch.cuda.is_available() else 'cpu' class LSTMEncoder(torch.nn.Module): - """ Implementation of Encoder network using LSTM layers - :param input_size: The number of expected features in the input x - :param num_past: Number of time steps to look backwards to predict num_future steps forward - :param batch_size: Number of samples in a batch - :param hidden_size: The number of features in the hidden state h - :param num_lstm_layers: Number of layers in the LSTM model - - :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - :param reset_state: If True, will reset the hidden and cell state for each batch of data - :param bidirectional: If True, becomes a bidirectional LSTM - """ - def __init__(self, input_size: int, num_past: int, batch_size: int, - hidden_size: int, num_lstm_layers: int, + """ Deep LSTM network. This implementation + returns output_size hidden size. + Args: + input_size: The number of expected features in the input `x` + batch_size: + sequence_length: The number of in each sample + hidden_size: The number of features in the hidden state `h` + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` + would mean stacking two LSTMs together to form a `stacked LSTM`, + with the second LSTM taking in outputs of the first LSTM and + computing the final results. Default: 1 + output_size: The number of output dimensions + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + LSTM layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + """ + + def __init__(self, input_size: int, sequence_length: int, batch_size: int, + hidden_size: int, num_layers: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): + super(LSTMEncoder, self).__init__() self.input_size = input_size - self.num_past = num_past + self.sequence_length = sequence_length self.batch_size = batch_size self.hidden_size = hidden_size - self.num_lstm_layers = num_lstm_layers + self.num_layers = num_layers self.batch_first = batch_first self.dropout = dropout self.reset_state = reset_state self.bidirectional = bidirectional self.lstm_encoder = torch.nn.LSTM(input_size=input_size, hidden_size=self.hidden_size, - num_layers=num_lstm_layers, dropout=dropout, + num_layers=num_layers, dropout=dropout, bidirectional=self.bidirectional, batch_first=True) def _init_hidden(self): - return (torch.zeros(self.num_lstm_layers, self.batch_size, - self.hidden_size).to(device), - torch.zeros(self.num_lstm_layers, self.batch_size, - self.hidden_size).to(device)) + return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size), torch.zeros(self.num_layers, self.batch_size, self.hidden_size)) def forward(self, x): + enc_init_hidden = self._init_hidden() enc_output, _ = self.lstm_encoder(x, enc_init_hidden) - # RNNs obeys, Markovian. So, the last state of the hidden is the markovian state for the entire - # sequence in that batch. + # RNNs obeys, Markovian. Consider the last state of the hidden is the markovian of the entire sequence in that batch. enc_output = enc_output[:, -1, :] # Shape(batch_size,hidden_dim) return enc_output class DisentangledAELatent(torch.nn.Module): """Dense Dientangled Latent Layer between encoder and decoder""" - - def __init__(self, hidden_size: int, latent_size: int, dropout: float): + def __init__(self, hidden_size: int, latent_size: int, dropout: float): super(DisentangledAELatent, self).__init__() self.latent_size = latent_size self.hidden_size = hidden_size self.dropout = dropout - self.latent = torch.nn.Linear(self.hidden_size, self.latent_size * 2) - - @staticmethod - def reparameterize(mu, logvar, training=True): + self.latent = torch.nn.Linear(self.hidden_size, self.latent_size) + + def reparameterize(self, mu, logvar, training= True): if training: std = logvar.mul(0.5).exp_() eps = std.data.new(std.size()).normal_() @@ -98,7 +99,6 @@ def reparameterize(mu, logvar, training=True): return mu def forward(self, x, training=True): - z_variables = self.latent(x) # [batch_size, latent_size*2] mu, logvar = torch.chunk(z_variables, 2, dim=1) # [batch_size,latent_size] # Reparameterize @@ -107,22 +107,27 @@ def forward(self, x, training=True): class LSTMDecoder(torch.nn.Module): - """ Implementation of Decoder network using LSTM layers - :param input_size: The number of expected features in the input x - :param num_future: Number of time steps to be predicted given the num_past steps - :param batch_size: Number of samples in a batch - :param hidden_size: The number of features in the hidden state h - :param num_lstm_layers: Number of layers in the LSTM model - :param output_size: Number of expectd features in the output x_ - :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - :param reset_state: If True, will reset the hidden and cell state for each batch of data - :param bidirectional: If True, becomes a bidirectional LSTM - """ - - def __init__(self, batch_size: int, num_future: int, hidden_size: int, - num_lstm_layers: int, output_size: int, latent_size: int, + """ Deep LSTM network. This implementation + returns output_size outputs. + Args: + latent_size: The number of dimensions of the latent layer + batch_size: Number of samples in each batch of training data + hidden_size: The number of features in the hidden state `h` + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` + would mean stacking two LSTMs together to form a `stacked LSTM`, + with the second LSTM taking in outputs of the first LSTM and + computing the final results. Default: 1 + output_size: The number of output/input dimensions + num_future: The number of time steps in future predictions + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + LSTM layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + reset_state: If ``True``, the hidden and cell states of the LSTM will + be reset at the beginning of each batch of input + """ + def __init__(self, batch_size: int, num_future: int, hidden_size: int, + num_layers: int, output_size: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMDecoder, self).__init__() @@ -130,7 +135,7 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, self.latent_size = latent_size self.num_future = num_future self.hidden_size = hidden_size - self.num_lstm_layers = num_lstm_layers + self.num_layers = num_layers self.output_size = output_size self.batch_first = batch_first self.dropout = dropout @@ -140,23 +145,23 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, # RNN decoder self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, hidden_size=self.hidden_size, - num_layers=self.num_lstm_layers, + num_layers=self.num_layers, dropout=self.dropout, bidirectional=self.bidirectional, batch_first=True) - self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, + self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, self.output_size)) def _init_hidden(self): - return (torch.zeros(self.num_lstm_layers, self.batch_size, - self.hidden_size).to(device), - torch.zeros(self.num_lstm_layers, self.batch_size, + return (torch.zeros(self.num_layers, self.batch_size, + self.hidden_size).to(device), + torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) def forward(self, x, num_future=None): - # To feed the latent states into lstm decoder, - # repeat the tensor n_future times at second dim + # To feed the latent states into lstm decoder, repeat the + # tensor n_future times at second dim _init_hidden = self._init_hidden() decoder_inputs = x.unsqueeze(1) @@ -175,66 +180,56 @@ def forward(self, x, num_future=None): class MLPClassifier(torch.nn.Module): - """ MLP classifier: Classify the input data using the latent embeddings - :param input_size: The number of expected latent size - :param hidden_size: The number of features in the hidden state h - :param num_classes: Size of labels or the number of categories in the data - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - :param num_classifier_layers: Number of hidden layers in the classifier - """ - def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_size: int, num_classifier_layers: int, + """ MLP classifier + """ + def __init__(self, hidden_size: int, num_classes: int, latent_size: int, dropout: float): super(MLPClassifier, self).__init__() - self.input_size = input_size + self.latent_size = latent_size self.hidden_size = hidden_size self.num_classes = num_classes - self.num_classifier_layers = num_classifier_layers self.dropout = dropout # Classifier layers - self.hidden = nn.ModuleList([nn.Linear(self.input_size, self.hidden_size)]) - self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_classifier_layers - 1)]) - self.hidden = nn.Sequential(*self.hidden) - self.out = nn.Linear(self.hidden_size, self.num_classes) + self.classifier1 = torch.nn.Linear(self.latent_size, self.hidden_size) + self.classifier2 = torch.nn.Linear(self.hidden_size, self.hidden_size) + self.classifier3 = torch.nn.Linear(self.hidden_size, self.hidden_size) + self.classifier4 = torch.nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) def forward(self, x): - x = self.dropout(self.hidden(x)) - out = self.out(x) - return out + classifier1 = self.dropout(self.classifier1(x)) + classifier2 = self.dropout(self.classifier2(classifier1)) + classifier3 = self.dropout(self.classifier3(classifier2)) + classifier4 = self.classifier4(classifier3) + return classifier4 class MultiModelVAE(torch.nn.Module): - """Implementation of Multimodel Variational autoencoders; This Module wraps the Variational Autoencoder - models [Encoder,Latent[Sampler],Decoder]. If classify=True, then the wrapper also include classification layers - - :param input_size: The number of expected features in the input x - :param num_future: Number of time steps to be predicted given the num_past steps - :param batch_size: Number of samples in a batch - :param hidden_size: The number of features in the hidden state h - :param num_lstm_layers: Number of layers in the LSTM model - :param output_size: Number of expectd features in the output x_ - :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - :param reset_state: If True, will reset the hidden and cell state for each batch of data - :param bidirectional: If True, becomes a bidirectional LSTM + """Implementation of Multimodel Variational autoencoders; """ - def __init__(self, input_size: int, num_past: int, batch_size: int, num_future: int, lstm_hidden_size: int, - num_lstm_layers: int , output_size: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, - bidirectional: bool=False, num_classifier_layers: int= None, classifier_hidden_size: int=None, num_classes: int=None): + def __init__(self, input_size: int, + sequence_length: int, + batch_size: int, + num_future: int, + hidden_size: int, + num_layers: int, + output_size: int, + num_classes: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool ): super(MultiModelVAE, self).__init__() self.input_size = input_size - self.num_past = num_past + self.sequence_length = sequence_length self.batch_size = batch_size self.latent_size = latent_size self.num_future = num_future - self.lstm_hidden_size = lstm_hidden_size - self.num_lstm_layers = num_lstm_layers - self.classifier_hidden_size = classifier_hidden_size - self.num_classifier_layers = num_classifier_layers + self.hidden_size = hidden_size + self.num_layers = num_layers self.output_size = output_size self.num_classes = num_classes self.batch_first = batch_first @@ -242,67 +237,61 @@ def __init__(self, input_size: int, num_past: int, batch_size: int, num_future: self.reset_state = reset_state self.bidirectional = bidirectional - self.encoder = LSTMEncoder(input_size=self.input_size, - num_past=self.num_past, + self.encoder = LSTMEncoder(input_size=self.input_size, + sequence_length=self.sequence_length, batch_size=self.batch_size, - hidden_size=self.lstm_hidden_size, - num_lstm_layers=self.num_lstm_layers, - batch_first=self.batch_first, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, + reset_state=True, bidirectional=self.bidirectional) - self.latent = DisentangledAELatent(hidden_size=self.lstm_hidden_size, - latent_size=self.latent_size, + self.latent = DisentangledAELatent(hidden_size=self.hidden_size, + latent_size=self.latent_size, dropout=self.dropout) - self.decoder = LSTMDecoder(batch_size=self.batch_size, + self.decoder = LSTMDecoder(batch_size=self.batch_size, num_future=self.num_future, - hidden_size=self.lstm_hidden_size, - num_lstm_layers=self.num_lstm_layers, + hidden_size=self.hidden_size, + num_layers=self.num_layers, output_size=self.output_size, - latent_size=self.latent_size, - batch_first=self.batch_first, + latent_size=self.latent_size, + batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, + reset_state=True, bidirectional=self.bidirectional) - if self.num_classes is not None: - self.classifier = MLPClassifier(input_size=self.latent_size, - hidden_size = self.classifier_hidden_size, - num_classes=self.num_classes, - latent_size=self.latent_size, - num_classifier_layers=self.num_classifier_layers, - dropout=self.dropout) - - def forward(self, data, training=True, classify=False): - """ - :param data: Train or test data - :param training: If Training= False, latents are deterministic - :param classify: If True, perform classification of input data using the latent embeddings - :return: decoder_out,latent_out or classifier out - """ - if not classify: + self.classifier = MLPClassifier(hidden_size=self.hidden_size, + num_classes=self.num_classes, + latent_size=self.latent_size, + dropout=self.dropout) + + + def forward(self, data, training=True, is_classification=False): + + if not is_classification: # Set the classifier grad off - if self.num_classes is not None: - for param in self.classifier.parameters(): - param.requires_grad = False + for param in self.classifier.parameters(): + param.requires_grad = False for param in self.encoder.parameters(): param.requires_grad = True for param in self.decoder.parameters(): param.requires_grad = True for param in self.latent.parameters(): param.requires_grad = True - - # Encoder -->Latent --> Decoder + + # Encoder enc_out = self.encoder(data) - latent_out, mu, logvar = self.latent(enc_out) + # Latent + latent_out = self.latent(enc_out) + # Decoder decoder_out = self.decoder(latent_out) - return decoder_out, latent_out, mu, logvar + return decoder_out, latent_out - else: - # Unfreeze classifier and freeze the rest - assert self.num_classes is not None, "Classifier not found" + else: # training_mode = 'classification' + # Unfreeze classifier parameters and freeze all other + # network parameters for param in self.classifier.parameters(): param.requires_grad = True for param in self.encoder.parameters(): @@ -311,9 +300,11 @@ def forward(self, data, training=True, classify=False): param.requires_grad = False for param in self.latent.parameters(): param.requires_grad = False - - # Encoder -->Latent --> Classifier + + # Encoder enc_out = self.encoder(data) + # Latent latent_out, mu, logvar = self.latent(enc_out, training=training) - classifier_out = self.classifier(mu) # Deterministic + # Classifier + classifier_out = self.classifier(latent_out) # Deterministic return classifier_out, latent_out, mu, logvar diff --git a/traja/models/vaegan.py b/traja/models/vaegan.py index 5fb817bc..ce47a12f 100644 --- a/traja/models/vaegan.py +++ b/traja/models/vaegan.py @@ -6,22 +6,22 @@ 1. MSE 2. Huber Loss""" -import torch +import torch class MultiModelVAEGAN(torch.nn.Module): - - def __init__(self, *model_hyperparameters, **kwargs): - super(MultiModelVAEGAN, self).__init__() - + + def __init__(self,*model_hyperparameters, **kwargs): + super(MultiModelVAEGAN,self).__init__() + for dictionary in model_hyperparameters: for key in dictionary: - setattr(self, key, dictionary[key]) + setattr(self, key, dictionary[key]) for key in kwargs: setattr(self, key, kwargs[key]) - + def __new__(cls): pass - - def forward(self, *input: None, **kwargs: None): - return NotImplementedError + + def forward(self, *input:None, **kwargs: None): + return NotImplementedError \ No newline at end of file diff --git a/traja/models/visualizer.py b/traja/models/visualizer.py index 9e1462bd..50496408 100644 --- a/traja/models/visualizer.py +++ b/traja/models/visualizer.py @@ -19,30 +19,28 @@ import matplotlib from matplotlib import style from scipy.sparse import csgraph -import argparse, copy, h5py, os, sys, time, socket +import argparse,copy,h5py, os,sys,time,socket import tensorflow as tf -import torch, torchvision, torch.nn as nn +import torch,torchvision,torch.nn as nn import torch.optim as optim import torchvision.transforms as transforms # from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot -from matplotlib import ticker, colors +from matplotlib import ticker,colors # print(matplotlib.get_backend()) # matplotlib.rcParams["backend"] = "Gtk3Agg" # print(matplotlib.get_backend()) # matplotlib.use('Gtk3Agg') import matplotlib.pyplot as plt - # plt.switch_backend('Qt4Agg') # print(matplotlib.get_backend()) plt.switch_backend("TkAgg") # seed value and plotly # init_notebook_mode(connected=True) -np.set_printoptions(suppress=True, precision=3, ) +np.set_printoptions(suppress=True,precision=3,) style.use('ggplot') - class DirectedNetwork(object): def __init__(self): @@ -80,36 +78,40 @@ def show(self, states, weight, fig): vmax = np.max(states) cmap = plt.cm.coolwarm edge_cmap = plt.cm.Spectral - nx.draw(G, with_labels=True, - cmap=cmap, node_color=neuron_color, - node_size=200, linewidths=5, - edge_color=edge_colors_, - edge_cmap=edge_cmap, font_size=10, + nx.draw(G, with_labels=True, + cmap=cmap, node_color = neuron_color, + node_size=200, linewidths=5, + edge_color = edge_colors_, + edge_cmap = edge_cmap, font_size=10, connectionstyle='arc3, rad=0.3') + # nx.draw(G, with_labels=True, cmap=cmap, node_color=neuron_color, edge_color=edge_colors_, + # node_size=200, linewidths=5, edge_cmap=cmap, font_size=10, + # connectionstyle='arc3, rad=0.3') + sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=vmin, vmax=vmax)) sm.set_array([]) cbar = plt.colorbar(sm, orientation="vertical", pad=0.1) - + # State of streaming plot if plt.fignum_exists(fig.number): fig.canvas.draw() fig.canvas.flush_events() fig.clear() - + # Plot is not closed return False else: return True - + class LocalLinearEmbedding(object): def __init__(self): super(LocalLinearEmbedding, self).__init__() pass - def local_linear_embedding(self, X, d, k, alpha=0.1): + def local_linear_embedding(self,X,d,k,alpha = 0.1): """ Local Linear Embeddings @@ -137,7 +139,7 @@ def local_linear_embedding(self, X, d, k, alpha=0.1): # Calculate the matrix G G_i = Z_i @ Z_i.T - + # Weights between neigbors w_i = scipy.linalg.pinv(G_i + alpha * np.eye(k)) @ np.ones(k) W[i, k_indices] = w_i / w_i.sum() @@ -152,43 +154,52 @@ def local_linear_embedding(self, X, d, k, alpha=0.1): # Return the vectors and discard the first column of the matrix return vectors[:, 1:] - def show(self, pc, fig2): - """[summary] - - Args: - pc ([type]): [description] - fig2 ([type]): [description] - """ - + def show(self,pc,fig2): + + # plotly.offline.init_notebook_mode() + + # # Configure the layout. + # layout = go.Layout( + # margin=dict(l=80, r=80, t=100, b=80), title='Trajectory of first 3 principle components') + # + # fig1 = go.FigureWidget(data=[go.Scatter3d(x=pc[:, 0], y=pc[:, 1], z=pc[:, 2], mode='markers', + # marker=dict(size=12, + # color=pc[:, 2], # set color to an array/list of desired values + # colorscale='Viridis', # choose a colorscale + # opacity=0.8))], layout=layout) + # + # plotly.offline.plot(fig1,auto_open = False, filename = 'pc') + ax = Axes3D(fig2) - f = ax.scatter(pc[:, 0], pc[:, 1], pc[:, 2], s=40, c=pc[:, 2]) - # ax.set_xlim(-1,1) - # ax.set_ylim(-1,1) - # ax.set_zlim(-1,1) + f = ax.scatter(pc[:, 0], pc[:, 1], pc[:, 2], s = 40,c = pc[:, 2]) +# ax.set_xlim(-1,1) +# ax.set_ylim(-1,1) +# ax.set_zlim(-1,1) for i in range(len(pc)): - ax.plot3D(pc[i:, 0], pc[i:, 1], pc[i:, 2], - alpha=i / len(pc), color='red', linewidth=1) + + ax.plot3D(pc[i:, 0], pc[i:, 1], pc[i:, 2], + alpha = i/len(pc), color = 'red', linewidth=1 ) fig2.colorbar(f) - # plt.pause(0.0001) +# plt.pause(0.0001) # State of streaming plot if plt.fignum_exists(fig2.number): fig2.canvas.draw() fig2.canvas.flush_events() fig2.clear() - + # Plot is not closed return False else: return True - - + + class SpectralEmbedding(object): def __init__(self): super(SpectralEmbedding, self).__init__() pass - def spectral_embedding(self, X, rad): + def spectral_embedding(self,X,rad): """ Spectral Clustering @@ -198,69 +209,58 @@ def spectral_embedding(self, X, rad): :return Y: numpy.ndarray - matrix m row, d attributes are reduced dimensional """ # Get the adjacency matrix/nearest neighbor graph; neighbors within the radius of 0.4 - A = radius_neighbors_graph(X.T, rad, mode='distance', - metric='minkowski', p=2, - metric_params=None, include_self=False) + A = radius_neighbors_graph(X.T,rad,mode='distance', + metric='minkowski', p=2, + metric_params=None, include_self=False) A = A.toarray() - + # Find the laplacian of the neighbour graph # L = D - A ; where D is the diagonal degree matrix L = csgraph.laplacian(A, normed=False) + # Embedd the data points i low dimension using the Eigen values/vectos # of the laplacian graph to get the most optimal partition of the graph + eigval, eigvec = np.linalg.eig(L) # the second smallest eigenvalue represents sparsest cut of the graph. np.where(eigval == np.partition(eigval, 1)[1]) # Partition the graph using the smallest eigen value - y_spec = eigvec[:, 1].copy() + y_spec =eigvec[:,1].copy() y_spec[y_spec < 0] = 0 y_spec[y_spec > 0] = 1 return y_spec - def show(self, X, spec_embed, fig3): - """[summary] - - Args: - X ([type]): [description] - spec_embed ([type]): [description] - fig3 ([type]): [description] - - Returns: - [type]: [description] - """ + def show(self,X, spec_embed,fig3): ax3 = fig3.add_subplot() X = X.T - fi = ax3.scatter(x=X[:, 0], y=X[:, 1], c=spec_embed, s=30, cmap=plt.cm.Spectral) - for i in range(len(X[:, 0])): + fi = ax3.scatter(x = X[:, 0], y = X[:, 1],c=spec_embed ,s=30, cmap=plt.cm.Spectral) + for i in range(len(X[:,0])): ax3.annotate(i, (X[:, 0][i], X[:, 1][i])) fig3.colorbar(fi) - + # State of streaming plot if plt.fignum_exists(fig3.number): fig3.canvas.draw() fig3.canvas.flush_events() fig3.clear() - + # Plot is not closed return False else: return True - - + if __name__ == '__main__': + # create the coordinates - numebr_of_points = 21; - small_range = -1.0; - large_range = 1.0 + numebr_of_points = 21 ; small_range = -1.0 ; large_range = 1.0 - xcoordinates = np.linspace(small_range, large_range, num=numebr_of_points) - ycoordinates = np.linspace(small_range, large_range, num=numebr_of_points) + xcoordinates = np.linspace(small_range, large_range, num=numebr_of_points) + ycoordinates = np.linspace(small_range, large_range, num=numebr_of_points) xcoord_mesh, ycoord_mesh = np.meshgrid(xcoordinates, ycoordinates) - inds = np.array(range(numebr_of_points ** 2)) - s1 = xcoord_mesh.ravel()[inds] - s2 = ycoord_mesh.ravel()[inds] - coordinate = np.c_[s1, s2] - print('From ', small_range, ' to ', large_range, ' with ', numebr_of_points, ' total number of coordinate: ', - numebr_of_points ** 2) + inds = np.array(range(numebr_of_points**2)) + s1 = xcoord_mesh.ravel()[inds] + s2 = ycoord_mesh.ravel()[inds] + coordinate = np.c_[s1,s2] + print('From ',small_range,' to ',large_range,' with ',numebr_of_points,' total number of coordinate: ', numebr_of_points**2) \ No newline at end of file From b8cb8160fcc08110a0eb9e14170aae0091e3beda Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 14 Dec 2020 14:38:54 +0100 Subject: [PATCH 055/211] aevae test setup --- .gitignore | 3 +- traja/models/losses.py | 2 +- traja/models/lstm.py | 67 ++++++++++--- traja/models/train.py | 138 ++++++++++++++++++++------ traja/models/utils.py | 125 ++++++++++++----------- traja/models/visualizer.py | 37 +++---- traja/plotting.py | 2 - traja/test/test_data/test_plotting.py | 0 8 files changed, 244 insertions(+), 130 deletions(-) create mode 100644 traja/test/test_data/test_plotting.py diff --git a/.gitignore b/.gitignore index 39b5ff97..e2a46752 100644 --- a/.gitignore +++ b/.gitignore @@ -77,9 +77,8 @@ target/ .DS_Store +# Visualstudio code file .vscode -.vscode/ - # celery beat schedule file celerybeat-schedule diff --git a/traja/models/losses.py b/traja/models/losses.py index f1019c1e..b8378e13 100644 --- a/traja/models/losses.py +++ b/traja/models/losses.py @@ -27,7 +27,7 @@ def ae_criterion(recon_x, x, loss_type='huber'): @staticmethod def vae_criterion(recon_x, x, mu, logvar, loss_type='huber'): - r"""Time series generative model loss function + """Time series generative model loss function Args: recon_x ([type]): [description] diff --git a/traja/models/lstm.py b/traja/models/lstm.py index 7ecdd287..584b512c 100644 --- a/traja/models/lstm.py +++ b/traja/models/lstm.py @@ -2,21 +2,58 @@ import torch -class MultiModelLSTM(torch.nn.Module): - - def __init__(self,*model_hyperparameters, **kwargs): - super(MultiModelLSTM,self).__init__() +class LSTM(torch.nn.Module): + """ Deep LSTM network. This implementation + returns output_size outputs. + Args: + input_size: The number of expected features in the input `x` + batch_size: + sequence_length: The number of in each sample + hidden_size: The number of features in the hidden state `h` + num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` + would mean stacking two LSTMs together to form a `stacked LSTM`, + with the second LSTM taking in outputs of the first LSTM and + computing the final results. Default: 1 + output_size: The number of output dimensions + dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + LSTM layer except the last layer, with dropout probability equal to + :attr:`dropout`. Default: 0 + bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + """ + + def __init__(self, batch_size:int, num_future:int, hidden_size: int, num_layers: int, + output_size: int, input_size:int, batch_first: bool, dropout: float, + reset_state: bool, bidirectional: bool): + super(LSTM, self).__init__() - for dictionary in model_hyperparameters: - for key in dictionary: - setattr(self, key, dictionary[key]) - for key in kwargs: - setattr(self, key, kwargs[key]) - - def __new__(cls): - pass - - def forward(self, *input:None, **kwargs: None): - return NotImplementedError + self.batch_size = batch_size + self.input_size = input_size + self.num_future = num_future + self.hidden_size = hidden_size + self.num_layers = num_layers + self.output_size = output_size + self.batch_first = batch_first + self.dropout = dropout + self.reset_state = reset_state + self.bidirectional = bidirectional + + # RNN decoder + self.lstm_decoder = torch.nn.LSTM(input_size=self.input_size, hidden_size=self.hidden_size, + num_layers=self.num_layers, dropout=self.dropout, + bidirectional=self.bidirectional, batch_first=True) + self.output = TimeDistributed(torch.nn.Linear(self.hidden_size , self.output_size)) + + def _init_hidden(self): + return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device), torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) + + def forward(self, x): + # To feed the latent states into lstm decoder, repeat the tensor n_future times at second dim + _init_hidden = self._init_hidden() + + # Decoder input Shape(batch_size, num_futures, latent_size) + dec,(dec_hidden,dec_cell) = self.lstm_decoder(x,_init_hidden) + # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) to Time Dsitributed Linear Layer + output = self.output(dec) + return output \ No newline at end of file diff --git a/traja/models/train.py b/traja/models/train.py index 6f616ac0..4e56b96e 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -1,6 +1,7 @@ from models.ae import MultiModelAE from models.vae import MultiModelVAE from models.vaegan import MultiModelVAEGAN +from models.lstm import LSTM import torch from functools import wraps import inspect @@ -29,7 +30,9 @@ def __init__(self, model_type:str, sequence_length:int, bidirectional:bool =False, batch_first:bool =True, - loss_type:str = 'huber'): + loss_type:str = 'huber', + lr_factor:float = 0.1, + scheduler_patience: int=10): white_keys = ['ae','vae','lstm','vaeg an', 'irl'] assert model_type in white_keys, "Valid models are {}".format(white_keys) @@ -53,7 +56,8 @@ def __init__(self, model_type:str, self.dropout = dropout self.bidirectional= bidirectional self.loss_type = loss_type - + self.lr_factor = lr_factor + self.scheduler_patience = scheduler_patience self.model_hyperparameters = {'input_size':self.input_size, 'sequence_length':self.sequence_length, 'batch_size':self.batch_size, @@ -89,19 +93,9 @@ def __init__(self, model_type:str, if self.model_type == 'irl': return NotImplementedError - # Get the optimizers for each network in the model - - [self.encoder_optimizer, self.latent_optimizer, self.decoder_optimizer, self.classifier_optimizer] = utils.get_optimizers(self.model_type, self.model) - - # Learning rate schedulers for the models - [self.encoder_scheduler, self.latent_scheduler, self.decoder_scheduler, self.classifier_scheduler] = utils.get_lrschedulers(self.model_type, - self.encoder_optimizer, - self.latent_optimizer, - self.decoder_optimizer, - self.classifier_optimizer, - factor=0.1, - patience=10) - + self.model_optimizers = utils.get_optimizers(self.model_type, self.model) + self.model_lrschedulers = utils.get_lrschedulers(self.model_type,self.model_optimizers, factor= self.lr_factor, patience = self.scheduler_patience) + def __str__(self): return "Training model type {}".format(self.model_type) @@ -111,22 +105,27 @@ def train_latent_model(self, train_loader, test_loader, model_save_path): assert self.model_type == 'ae' or 'vae' # Move the model to target device self.model.to(device) - + + encoder_optimizer, latent_optimizer, decoder_optimizer, classifier_optimizer = self.model_optimizers.values() + encoder_scheduler, latent_scheduler, decoder_scheduler, classifier_scheduler = self.model_lrschedulers.values() + # Training mode: Switch from Generative to classifier training mode training_mode = 'forecasting' # Training for epoch in range(self.epochs*2): # First half for generative model and next for classifier + test_loss_forecasting = 0 + test_loss_classification = 0 if epoch>0: # Initial step is to test and set LR schduler # Training self.model.train() - total_loss = 0 + total_loss = 0 for idx, (data, target,category) in enumerate(train_loader): # Reset optimizer states - self.encoder_optimizer.zero_grad() - self.latent_optimizer.zero_grad() - self.decoder_optimizer.zero_grad() - self.classifier_optimizer.zero_grad() + encoder_optimizer.zero_grad() + latent_optimizer.zero_grad() + decoder_optimizer.zero_grad() + classifier_optimizer.zero_grad() data, target,category = data.float().to(device), target.float().to(device), category.to(device) @@ -141,16 +140,16 @@ def train_latent_model(self, train_loader, test_loader, model_save_path): loss.backward() - self.encoder_optimizer.step() - self.decoder_optimizer.step() - self.latent_optimizer.step() + encoder_optimizer.step() + decoder_optimizer.step() + latent_optimizer.step() else: # training_mode == 'classification' classifier_out = self.model(data, training=True, is_classification=True) loss = Criterion.classifier_criterion(classifier_out, category-1) loss.backward() - self.classifier_optimizer.step() + classifier_optimizer.step() total_loss+=loss print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss/(idx+1))) @@ -162,8 +161,6 @@ def train_latent_model(self, train_loader, test_loader, model_save_path): if epoch%10==0: with torch.no_grad(): self.model.eval() - test_loss_forecasting = 0 - test_loss_classification = 0 for idx, (data, target,category) in enumerate(list(test_loader)): data, target, category = data.float().to(device), target.float().to(device), category.to(device) # Time seriesforecasting test @@ -184,11 +181,11 @@ def train_latent_model(self, train_loader, test_loader, model_save_path): # Scheduler metric is test set loss if training_mode =='forecasting': - self.encoder_scheduler.step(self.test_loss_forecasting) - self.decoder_scheduler.step(self.test_loss_forecasting) - self.latent_scheduler.step(self.test_loss_forecasting) + encoder_scheduler.step(test_loss_forecasting) + decoder_scheduler.step(test_loss_forecasting) + latent_scheduler.step(test_loss_forecasting) else: - self.classifier_scheduler.step(self.test_loss_classification) + classifier_scheduler.step(test_loss_classification) # Save the model at target path utils.save_model(self.model,PATH = model_save_path) @@ -207,4 +204,81 @@ def train_irl(self): # TRAIN LSTM def train_lstm(self): assert self.model_type == 'lstm' - return NotImplementedError \ No newline at end of file + + # Move the model to target device + self.model.to(device) + + # Training + for epoch in range(self.epochs): # First half for generative model and next for classifier + if epoch>0: # Initial step is to test and set LR schduler + # Training + self.model.train() + total_loss = 0 + for idx, (data, target,category) in enumerate(self.train_loader): + # Reset optimizer states + self.encoder_optimizer.zero_grad() + + data, target,category = data.float().to(device), target.float().to(device), category.to(device) + + if self.training_mode =='forecasting': + if self.model_type == 'ae': + decoder_out, latent_out = self.model(data, training=True, is_classification=False) + loss = Criterion.ae_criterion(decoder_out, target) + + else: # vae + decoder_out, latent_out, mu, logvar= self.model(data, training=True, is_classification=False) + loss = Criterion.vae_criterion(decoder_out, target, mu, logvar) + + loss.backward() + + self.encoder_optimizer.step() + self.decoder_optimizer.step() + self.latent_optimizer.step() + + else: # training_mode == 'classification' + + classifier_out = self.model(data, training=True, is_classification=True) + loss = Criterion.classifier_criterion(classifier_out, category-1) + loss.backward() + self.classifier_optimizer.step() + total_loss+=loss + + print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss/(idx+1))) + + if epoch+1 == self.epochs: # + training_mode = 'classification' + + # Testing + if epoch%10==0: + with torch.no_grad(): + self.model.eval() + test_loss_forecasting = 0 + test_loss_classification = 0 + for idx, (data, target,category) in enumerate(list(test_loader)): + data, target, category = data.float().to(device), target.float().to(device), category.to(device) + # Time seriesforecasting test + if self.model_type=='ae': + out, latent = self.model(data, training=False, is_classification=False) + test_loss_forecasting += Criterion.ae_criterion(out,target).item() + else: + decoder_out,latent_out,mu,logvar= self.model(data,training=False, is_classification=False) + test_loss_forecasting += Criterion.vae_criterion(decoder_out, target,mu,logvar) + # Classification test + classifier_out= self.model(data,training=False, is_classification=True) + test_loss_classification += Criterion.classifier_criterion(classifier_out, category-1).item() + + test_loss_forecasting /= len(test_loader.dataset) + print(f'====> Mean test set generator loss: {test_loss_forecasting:.4f}') + test_loss_classification /= len(test_loader.dataset) + print(f'====> Mean test set classifier loss: {test_loss_classification:.4f}') + + # Scheduler metric is test set loss + if training_mode =='forecasting': + self.encoder_scheduler.step(self.test_loss_forecasting) + self.decoder_scheduler.step(self.test_loss_forecasting) + self.latent_scheduler.step(self.test_loss_forecasting) + else: + self.classifier_scheduler.step(self.test_loss_classification) + + # Save the model at target path + utils.save_model(self.model,PATH = model_save_path) \ No newline at end of file diff --git a/traja/models/utils.py b/traja/models/utils.py index b9505abc..800edefa 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -29,68 +29,79 @@ def forward(self, x): out = out.view(-1, x.size(1), out.size(-1)) # (timesteps, samples, output_size) return out + +class Optimizers: -def get_optimizers(model_type, model, lr=0.0001): - r"""Optimizers for each network in the model - - Args: - model_type ([type]): [description] - model ([type]): [description] - lr (float, optional): [description]. Defaults to 0.0001. - - Returns: - [type]: [description] - """ - - if model_type == 'ae' or 'vae': - # Optimizers for each network in the model - encoder_optimizer = torch.optim.Adam(model.encoder.parameters(), lr=lr) - latent_optimizer = torch.optim.Adam(model.latent.parameters(), lr=lr) - decoder_optimizer = torch.optim.Adam(model.decoder.parameters(), lr=lr) - classifier_optimizer = torch.optim.Adam(model.classifier.parameters(), lr=lr) - return [encoder_optimizer, latent_optimizer, decoder_optimizer, classifier_optimizer] - - elif model_type == 'vaegan': - return NotImplementedError - - else: # LSTM - return NotImplementedError - -def get_lrschedulers(model_type, encoder_optimizer, decoder_optimizer, latent_optimizer, classifier_optimizer, factor=0.1, patience=10): - - r"""Learning rate scheduler for each network in the model - NOTE: Scheduler metric should be test set loss - - Args: - model_type ([type]): [description] - encoder_optimizer ([type]): [description] - decoder_optimizer ([type]): [description] - latent_optimizer ([type]): [description] - classifier_optimizer ([type]): [description] - factor (float, optional): [description]. Defaults to 0.1. - patience (int, optional): [description]. Defaults to 10. - - Returns: - [type]: [description] - """ - - if model_type == 'ae' or 'vae': - encoder_scheduler = ReduceLROnPlateau(encoder_optimizer, mode='max', factor=factor, patience=patience, verbose=True) - decoder_scheduler = ReduceLROnPlateau(decoder_optimizer, mode='max', factor=factor, patience=patience, verbose=True) - latent_scheduler = ReduceLROnPlateau(latent_optimizer, mode='max', factor=factor, patience=patience, verbose=True) - classifier_scheduler = ReduceLROnPlateau(classifier_optimizer, mode='max', factor=factor, patience=patience, verbose=True) - return [encoder_scheduler, decoder_scheduler, latent_scheduler, classifier_scheduler] - - elif model_type == 'vaegan': - return NotImplementedError + def __init__(self): + pass - else: # LSTM - return NotImplementedError + def get_optimizers(model_type, model, lr=0.0001): + """Optimizers for each network in the model + + Args: + model_type ([type]): [description] + model ([type]): [description] + lr (float, optional): [description]. Defaults to 0.0001. + + Returns: + [type]: [description] + """ + + if model_type == 'ae' or 'vae': + keys = ['encoder','decoder', 'latent', 'classifier'] + optimizers = {} + for network in keys: + optimizers[network]=torch.optim.Adam(model.keys[network].parameters(), lr=lr) + + if model_type == 'lstm': + optimizers = {} + optimizers["lstm_optimizer"] = torch.optim.Adam(model.parameters(), lr=lr) + return optimizers + + elif model_type == 'vaegan': + return NotImplementedError + + else: # LSTM + return NotImplementedError + + def get_lrschedulers(model_type, lstm_optimizer=None, encoder_optimizer=None, decoder_optimizer=None, latent_optimizer=None, classifier_optimizer=None, factor=0.1, patience=10): + + """Learning rate scheduler for each network in the model + NOTE: Scheduler metric should be test set loss + + Args: + model_type ([type]): [description] + encoder_optimizer ([type]): [description] + decoder_optimizer ([type]): [description] + latent_optimizer ([type]): [description] + classifier_optimizer ([type]): [description] + factor (float, optional): [description]. Defaults to 0.1. + patience (int, optional): [description]. Defaults to 10. + + Returns: + [type]: [description] + """ + + if model_type == 'ae' or 'vae': + keys = ['encoder_scheduler','decoder_scheduler', 'latent_scheduler', 'classifier_scheduler'] + schedulers = {} + for key in keys: + schedulers[key] = ReduceLROnPlateau(encoder_optimizer, mode='max', factor=factor, patience=patience, verbose=True) + return schedulers + if model_type == 'lstm': + scheduler = ReduceLROnPlateau(lstm_optimizer, mode='max', factor=factor, patience=patience, verbose=True) + return scheduler + + elif model_type == 'vaegan': + return NotImplementedError + + else: # irl + return NotImplementedError def save_model(model, PATH): - r"""[summary] + """[summary] Args: model ([type]): [description] @@ -103,7 +114,7 @@ def save_model(model, PATH): print('Model saved at {}'.format(PATH)) def load_model(model,model_hyperparameters, PATH): - r"""[summary] + """[summary] Args: model ([type]): [description] diff --git a/traja/models/visualizer.py b/traja/models/visualizer.py index 50496408..8b6b2bd1 100644 --- a/traja/models/visualizer.py +++ b/traja/models/visualizer.py @@ -85,10 +85,6 @@ def show(self, states, weight, fig): edge_cmap = edge_cmap, font_size=10, connectionstyle='arc3, rad=0.3') - # nx.draw(G, with_labels=True, cmap=cmap, node_color=neuron_color, edge_color=edge_colors_, - # node_size=200, linewidths=5, edge_cmap=cmap, font_size=10, - # connectionstyle='arc3, rad=0.3') - sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=vmin, vmax=vmax)) sm.set_array([]) cbar = plt.colorbar(sm, orientation="vertical", pad=0.1) @@ -104,7 +100,6 @@ def show(self, states, weight, fig): else: return True - class LocalLinearEmbedding(object): def __init__(self): @@ -155,21 +150,13 @@ def local_linear_embedding(self,X,d,k,alpha = 0.1): return vectors[:, 1:] def show(self,pc,fig2): + """[summary] - # plotly.offline.init_notebook_mode() - - # # Configure the layout. - # layout = go.Layout( - # margin=dict(l=80, r=80, t=100, b=80), title='Trajectory of first 3 principle components') - # - # fig1 = go.FigureWidget(data=[go.Scatter3d(x=pc[:, 0], y=pc[:, 1], z=pc[:, 2], mode='markers', - # marker=dict(size=12, - # color=pc[:, 2], # set color to an array/list of desired values - # colorscale='Viridis', # choose a colorscale - # opacity=0.8))], layout=layout) - # - # plotly.offline.plot(fig1,auto_open = False, filename = 'pc') - + Args: + pc ([type]): [description] + fig2 ([type]): [description] + """ + ax = Axes3D(fig2) f = ax.scatter(pc[:, 0], pc[:, 1], pc[:, 2], s = 40,c = pc[:, 2]) # ax.set_xlim(-1,1) @@ -217,10 +204,8 @@ def spectral_embedding(self,X,rad): # Find the laplacian of the neighbour graph # L = D - A ; where D is the diagonal degree matrix L = csgraph.laplacian(A, normed=False) - # Embedd the data points i low dimension using the Eigen values/vectos # of the laplacian graph to get the most optimal partition of the graph - eigval, eigvec = np.linalg.eig(L) # the second smallest eigenvalue represents sparsest cut of the graph. np.where(eigval == np.partition(eigval, 1)[1]) @@ -231,6 +216,16 @@ def spectral_embedding(self,X,rad): return y_spec def show(self,X, spec_embed,fig3): + """[summary] + + Args: + X ([type]): [description] + spec_embed ([type]): [description] + fig3 ([type]): [description] + + Returns: + [type]: [description] + """ ax3 = fig3.add_subplot() X = X.T diff --git a/traja/plotting.py b/traja/plotting.py index da508e77..3c5ee107 100644 --- a/traja/plotting.py +++ b/traja/plotting.py @@ -3,7 +3,6 @@ import logging from typing import Union, Optional, Tuple, List -import matplotlib import matplotlib.pyplot as plt from matplotlib.axes import Axes from matplotlib.collections import PathCollection @@ -288,7 +287,6 @@ def plot_3d(trj: TrajaDataFrame, **kwargs) -> matplotlib.collections.PathCollect return ax - def plot( trj: TrajaDataFrame, n_coords: Optional[int] = None, diff --git a/traja/test/test_data/test_plotting.py b/traja/test/test_data/test_plotting.py new file mode 100644 index 00000000..e69de29b From 143fc94b45af12f06a34a8b3557a43647abc8475 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 14 Dec 2020 17:33:57 +0100 Subject: [PATCH 056/211] bug in plotting --- traja/datasets/dataset.py | 4 ++-- traja/plotting.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/traja/datasets/dataset.py b/traja/datasets/dataset.py index 7e75253c..25cec13c 100644 --- a/traja/datasets/dataset.py +++ b/traja/datasets/dataset.py @@ -249,7 +249,7 @@ def __getitem__(self, index): def __len__(self): return len(self.data) -class MultiModalDataLoader(object): +class MultiModalDataLoader: """ MultiModalDataLoader wraps the following data preparation steps, @@ -269,7 +269,7 @@ class MultiModalDataLoader(object): batch_size (int): Number of samples per batch of data n_past (int): Input sequence length. Number of time steps from the past. n_future (int): Target sequence length. Number of time steps to the future. - num_workers (int): Number of cpu subprocess to be occupied during data loading process + num_workers (int): Number of cpu subprocess occupied during data loading process Usage: train_dataloader, test_dataloader = MultiModalDataLoader(df = data_frame, batch_size=32, diff --git a/traja/plotting.py b/traja/plotting.py index 3c5ee107..44e9c279 100644 --- a/traja/plotting.py +++ b/traja/plotting.py @@ -3,6 +3,7 @@ import logging from typing import Union, Optional, Tuple, List +import matplotlib import matplotlib.pyplot as plt from matplotlib.axes import Axes from matplotlib.collections import PathCollection From 9c15fb4b54a555e7e6209c33771cc178e9c90abd Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 14 Dec 2020 17:44:05 +0100 Subject: [PATCH 057/211] Update __init__.py --- traja/datasets/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/datasets/__init__.py b/traja/datasets/__init__.py index d195115f..acf2eb6f 100644 --- a/traja/datasets/__init__.py +++ b/traja/datasets/__init__.py @@ -2,7 +2,7 @@ import glob import os from typing import List - +from datasets import dataset import pandas as pd import traja From 838c40c2affe5f19dc9001833a5510bd377020f4 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 14 Dec 2020 17:52:06 +0100 Subject: [PATCH 058/211] Update __init__.py --- traja/datasets/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/datasets/__init__.py b/traja/datasets/__init__.py index acf2eb6f..9ea26f5b 100644 --- a/traja/datasets/__init__.py +++ b/traja/datasets/__init__.py @@ -2,7 +2,7 @@ import glob import os from typing import List -from datasets import dataset +from . import dataset import pandas as pd import traja From f6f71ac6c251f30cf2aac18a7fd336e85c1de135 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 14 Dec 2020 17:53:38 +0100 Subject: [PATCH 059/211] Update __init__.py --- traja/datasets/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/datasets/__init__.py b/traja/datasets/__init__.py index 9ea26f5b..a668be5b 100644 --- a/traja/datasets/__init__.py +++ b/traja/datasets/__init__.py @@ -2,7 +2,7 @@ import glob import os from typing import List -from . import dataset +from traja import datasets import pandas as pd import traja From ed98c015a2aa34c0438f3324428fa62b1c941893 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 14 Dec 2020 17:55:27 +0100 Subject: [PATCH 060/211] traja init update datasets dir --- traja/__init__.py | 1 + traja/datasets/__init__.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/__init__.py b/traja/__init__.py index 2e884412..4e6c26d7 100644 --- a/traja/__init__.py +++ b/traja/__init__.py @@ -6,6 +6,7 @@ from .parsers import read_file, from_df from .plotting import * from .trajectory import * +from .datasets import * import logging diff --git a/traja/datasets/__init__.py b/traja/datasets/__init__.py index a668be5b..9d46f81e 100644 --- a/traja/datasets/__init__.py +++ b/traja/datasets/__init__.py @@ -2,7 +2,6 @@ import glob import os from typing import List -from traja import datasets import pandas as pd import traja From 11969c0cd5e7da3a15339daec320f2ba716b0cfd Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 14 Dec 2020 18:16:15 +0100 Subject: [PATCH 061/211] Update __init__.py --- traja/datasets/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/datasets/__init__.py b/traja/datasets/__init__.py index 9d46f81e..ccfb5a46 100644 --- a/traja/datasets/__init__.py +++ b/traja/datasets/__init__.py @@ -3,7 +3,7 @@ import os from typing import List import pandas as pd - +import .dataset import traja From 5acbc5a5f1b1c948b9bb91cb4d25da85eec740fc Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Mon, 14 Dec 2020 17:20:53 +0000 Subject: [PATCH 062/211] Resolve import issues --- traja/datasets/__init__.py | 2 +- traja/datasets/dataset.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/datasets/__init__.py b/traja/datasets/__init__.py index ccfb5a46..665235a1 100644 --- a/traja/datasets/__init__.py +++ b/traja/datasets/__init__.py @@ -3,7 +3,7 @@ import os from typing import List import pandas as pd -import .dataset +from traja.datasets import dataset import traja diff --git a/traja/datasets/dataset.py b/traja/datasets/dataset.py index 25cec13c..e891b9c9 100644 --- a/traja/datasets/dataset.py +++ b/traja/datasets/dataset.py @@ -21,7 +21,7 @@ from torch.utils.data.sampler import WeightedRandomSampler import pandas as pd from sklearn.utils import shuffle -from datasets import utils +from traja.datasets import utils logger = logging.getLogger(__name__) From 01eea3366bf7fe65a34c9f13cb8093a995444047 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 14 Dec 2020 18:45:08 +0100 Subject: [PATCH 063/211] Update __init__.py --- traja/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/traja/__init__.py b/traja/__init__.py index 4e6c26d7..09906d9f 100644 --- a/traja/__init__.py +++ b/traja/__init__.py @@ -7,6 +7,7 @@ from .plotting import * from .trajectory import * from .datasets import * +from .models import * import logging From 6a782f37f7cf9d825a05b517c08aca72ba98a44f Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 14 Dec 2020 18:48:10 +0100 Subject: [PATCH 064/211] Update __init__.py --- traja/models/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/traja/models/__init__.py b/traja/models/__init__.py index 82d110dd..9d1528c9 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1,2 +1,3 @@ -from .nn import LSTM +from traja.nn import LSTM +from traja.models import * From 04b881f51711221a9a6ce66f167db0d0a5764ffc Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 14 Dec 2020 18:49:28 +0100 Subject: [PATCH 065/211] Update __init__.py --- traja/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/__init__.py b/traja/models/__init__.py index 9d1528c9..1e70c92f 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1,3 +1,3 @@ -from traja.nn import LSTM +from .nn import LSTM from traja.models import * From 3e5c4c20b739a013e40723d0fce69c70e3170284 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 14 Dec 2020 18:51:51 +0100 Subject: [PATCH 066/211] Update train.py --- traja/models/train.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 4e56b96e..2d3f199b 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -1,12 +1,12 @@ -from models.ae import MultiModelAE -from models.vae import MultiModelVAE -from models.vaegan import MultiModelVAEGAN -from models.lstm import LSTM +from .ae import MultiModelAE +from .vae import MultiModelVAE +from .vaegan import MultiModelVAEGAN +from .lstm import LSTM import torch from functools import wraps import inspect -from models import utils -from models.losses import Criterion +from . import utils +from .losses import Criterion device = 'cuda' if torch.cuda.is_available() else 'cpu' From 19978bf5740527e1cc25727b4dda35e6967c6290 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 12:31:14 +0100 Subject: [PATCH 067/211] update init --- traja/__init__.py | 7 ++----- traja/models/__init__.py | 1 - 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/traja/__init__.py b/traja/__init__.py index 09906d9f..5a3f5355 100644 --- a/traja/__init__.py +++ b/traja/__init__.py @@ -1,13 +1,10 @@ -from . import models -from . import datasets - from .accessor import TrajaAccessor from .frame import TrajaDataFrame, TrajaCollection from .parsers import read_file, from_df from .plotting import * from .trajectory import * -from .datasets import * -from .models import * +from traja import models +from traja import datasets import logging diff --git a/traja/models/__init__.py b/traja/models/__init__.py index 1e70c92f..82d110dd 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1,3 +1,2 @@ from .nn import LSTM -from traja.models import * From 1dcad1c3aefb254c161a036bd6e9581cdc40b200 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 21:12:49 +0100 Subject: [PATCH 068/211] optimizers fix --- traja/models/__init__.py | 3 +- traja/models/ae.py | 4 +- traja/models/optimizers.py | 111 ++++++++++++++++--------------------- traja/models/train.py | 15 +++-- traja/models/utils.py | 72 +----------------------- 5 files changed, 62 insertions(+), 143 deletions(-) diff --git a/traja/models/__init__.py b/traja/models/__init__.py index 82d110dd..6def7da5 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1,2 +1 @@ -from .nn import LSTM - +from .nn import LSTM \ No newline at end of file diff --git a/traja/models/ae.py b/traja/models/ae.py index 3d4bb615..8694d9e3 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -24,8 +24,8 @@ trainer.train_latent_model(train_dataloader, test_dataloader, model_save_path=PATH)""" import torch -from .utils import TimeDistributed -from .utils import load_model +from utils import TimeDistributed +from utils import load_model device = 'cuda' if torch.cuda.is_available() else 'cpu' diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index d1e66e15..cdbad072 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -1,65 +1,49 @@ import torch +from torch.optim import optimizer from torch.optim.lr_scheduler import ReduceLROnPlateau -from traja.models.ae import MultiModelAE - +from ae import MultiModelAE class Optimizer: - - def __init__(self, model_type, model, optimizer_type, classify=False): - """ - Wrapper for setting the model optimizer and learning rate schedulers using ReduceLROnPlateau; - If the model type is 'ae' or 'vae' - var optimizers is a dict with separate optimizers for encoder, decoder, - latent and classifier. In case of 'lstm', var optimizers is an optimizer for lstm and TimeDistributed(linear layer) - :param model_type: Type of model 'ae', 'vae' or 'lstm' - :param model: Model instance - :param classify: If True, will return the Optimizer and scheduler for classifier - :param optimizer_type: Optimizer to be used; Should be one in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', - 'LBFGS', 'ASGD', 'Adamax'] - """ - - assert isinstance(model, torch.nn.Module) - assert str(optimizer_type) in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', - 'LBFGS', 'ASGD', 'Adamax'] - + + def __init__(self, model_type, model, optimizer_type): + + assert isinstance(model,torch.nn.Module) self.model_type = model_type - self.model = model + self.model = model self.optimizer_type = optimizer_type - self.classify = classify self.optimizers = {} self.schedulers = {} - + assert optimizer_type in ['Adam','Adadelta','Adagrad','AdamW','SparseAdam','RMSprop','Rprop','LBFGS','ASGD','Adamax'] + def get_optimizers(self, lr=0.0001): """Optimizers for each network in the model Args: + model_type ([type]): [description] + model ([type]): [description] lr (float, optional): [description]. Defaults to 0.0001. Returns: - dict: Optimizers + [type]: [description] """ - - if self.model_type == 'lstm': - self.optimizers = getattr(torch.optim, f'{self.optimizer_type}')(self.model.parameters(), lr=lr) - - elif self.model_type == 'ae' or 'vae': - keys = ['encoder', 'decoder', 'latent', 'classifier'] + + if self.model_type == 'ae' or 'vae': + keys = ['encoder','decoder', 'latent', 'classifier'] for network in keys: - if network != 'classifier': - self.optimizers[network] = getattr(torch.optim, f'{self.optimizer_type}')( - getattr(self.model, f'{network}').parameters(), lr=lr) - - if self.classify: - self.optimizers['classifier'] = getattr(torch.optim, f'{self.optimizer_type}')(getattr(self.model, 'classifier').parameters(), lr=lr) - else: - self.optimizers['classifier'] = None - + self.optimizers[network]=getattr(torch.optim,f'{self.optimizer_type}')(getattr(self.model,f'{network}').parameters(), lr=lr) + return self.optimizers + if self.model_type == 'lstm': + self.optimizers["lstm"] = torch.optim.Adam(self.model.parameters(), lr=lr) + return self.optimizers + elif self.model_type == 'vaegan': return NotImplementedError + + else: # LSTM + return NotImplementedError - return self.optimizers - - def get_lrschedulers(self, factor: float, patience: int): - + def get_lrschedulers(self, factor=0.1, patience=10): + """Learning rate scheduler for each network in the model NOTE: Scheduler metric should be test set loss @@ -68,32 +52,33 @@ def get_lrschedulers(self, factor: float, patience: int): patience (int, optional): [description]. Defaults to 10. Returns: - [dict]: Learning rate schedulers + [dict]: [description] """ - if self.model_type == 'lstm': - assert not isinstance(self.optimizers, dict) - self.schedulers = ReduceLROnPlateau(self.optimizers, mode='max', factor=factor, - patience=patience, verbose=True) - else: - for network in self.optimizers.keys(): - if self.optimizers[network] is not None: - self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, - patience=patience, verbose=True) - if not self.classify: - self.schedulers['classifier'] = None + for network in self.optimizers.keys(): + self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, patience=patience, verbose=True) return self.schedulers - - + if __name__ == '__main__': + # Test model_type = 'ae' - model = MultiModelAE(input_size=2, num_past=10, batch_size=5, num_future=5, lstm_hidden_size=32, num_lstm_layers=2, - classifier_hidden_size=32, num_classifier_layers=4, output_size=2, num_classes=10, - latent_size=10, batch_first=True, dropout=0.2, reset_state=True, bidirectional=True) - + model = MultiModelAE(input_size=2, + sequence_length=10, + batch_size=5, + num_future=5, + hidden_size=10, + num_layers=2, + output_size=2, + num_classes=10, + latent_size=10, + batch_first=True, + dropout = 0.2, + reset_state=True, + bidirectional=True) + # Get the optimizers opt = Optimizer(model_type, model, optimizer_type='RMSprop') model_optimizers = opt.get_optimizers(lr=0.1) - model_schedulers = opt.get_lrschedulers(factor=0.1, patience=10) - - print(model_optimizers, model_schedulers) + model_schedulers = opt.get_lrschedulers(factor=0.1,patience=10) + + print(model_optimizers,model_schedulers) \ No newline at end of file diff --git a/traja/models/train.py b/traja/models/train.py index 2d3f199b..e6c963c1 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -2,11 +2,13 @@ from .vae import MultiModelVAE from .vaegan import MultiModelVAEGAN from .lstm import LSTM + +from . import utils +from .losses import Criterion +from .optimizers import Optimizer import torch from functools import wraps import inspect -from . import utils -from .losses import Criterion device = 'cuda' if torch.cuda.is_available() else 'cpu' @@ -31,6 +33,7 @@ def __init__(self, model_type:str, bidirectional:bool =False, batch_first:bool =True, loss_type:str = 'huber', + optimizer_type = 'adam', lr_factor:float = 0.1, scheduler_patience: int=10): @@ -56,6 +59,7 @@ def __init__(self, model_type:str, self.dropout = dropout self.bidirectional= bidirectional self.loss_type = loss_type + self.optimizer_type = optimizer_type, self.lr_factor = lr_factor self.scheduler_patience = scheduler_patience self.model_hyperparameters = {'input_size':self.input_size, @@ -93,9 +97,10 @@ def __init__(self, model_type:str, if self.model_type == 'irl': return NotImplementedError - self.model_optimizers = utils.get_optimizers(self.model_type, self.model) - self.model_lrschedulers = utils.get_lrschedulers(self.model_type,self.model_optimizers, factor= self.lr_factor, patience = self.scheduler_patience) - + optimizer = Optimizer(self.model_type, self.model) + self.model_optimizers = optimizer.get_optimizers(lr=0.001) + self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor,patience=self.scheduler_patience) + def __str__(self): return "Training model type {}".format(self.model_type) diff --git a/traja/models/utils.py b/traja/models/utils.py index 800edefa..925e9260 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -1,6 +1,5 @@ import torch import matplotlib.pyplot as plt -from torch.optim.lr_scheduler import ReduceLROnPlateau import numpy as np import collections from numpy import math @@ -29,76 +28,7 @@ def forward(self, x): out = out.view(-1, x.size(1), out.size(-1)) # (timesteps, samples, output_size) return out - -class Optimizers: - - def __init__(self): - pass - - def get_optimizers(model_type, model, lr=0.0001): - """Optimizers for each network in the model - - Args: - model_type ([type]): [description] - model ([type]): [description] - lr (float, optional): [description]. Defaults to 0.0001. - - Returns: - [type]: [description] - """ - - if model_type == 'ae' or 'vae': - keys = ['encoder','decoder', 'latent', 'classifier'] - optimizers = {} - for network in keys: - optimizers[network]=torch.optim.Adam(model.keys[network].parameters(), lr=lr) - - if model_type == 'lstm': - optimizers = {} - optimizers["lstm_optimizer"] = torch.optim.Adam(model.parameters(), lr=lr) - return optimizers - - elif model_type == 'vaegan': - return NotImplementedError - - else: # LSTM - return NotImplementedError - - def get_lrschedulers(model_type, lstm_optimizer=None, encoder_optimizer=None, decoder_optimizer=None, latent_optimizer=None, classifier_optimizer=None, factor=0.1, patience=10): - - """Learning rate scheduler for each network in the model - NOTE: Scheduler metric should be test set loss - - Args: - model_type ([type]): [description] - encoder_optimizer ([type]): [description] - decoder_optimizer ([type]): [description] - latent_optimizer ([type]): [description] - classifier_optimizer ([type]): [description] - factor (float, optional): [description]. Defaults to 0.1. - patience (int, optional): [description]. Defaults to 10. - - Returns: - [type]: [description] - """ - - if model_type == 'ae' or 'vae': - keys = ['encoder_scheduler','decoder_scheduler', 'latent_scheduler', 'classifier_scheduler'] - schedulers = {} - for key in keys: - schedulers[key] = ReduceLROnPlateau(encoder_optimizer, mode='max', factor=factor, patience=patience, verbose=True) - return schedulers - if model_type == 'lstm': - scheduler = ReduceLROnPlateau(lstm_optimizer, mode='max', factor=factor, patience=patience, verbose=True) - return scheduler - - elif model_type == 'vaegan': - return NotImplementedError - - else: # irl - return NotImplementedError - - + def save_model(model, PATH): """[summary] From 3214c9664cd7c4e8de1dc1b19f83fa99011d351a Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 21:16:39 +0100 Subject: [PATCH 069/211] Update ae.py --- traja/models/ae.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index 8694d9e3..c21fc5e0 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -24,8 +24,8 @@ trainer.train_latent_model(train_dataloader, test_dataloader, model_save_path=PATH)""" import torch -from utils import TimeDistributed -from utils import load_model +from traja.utils import TimeDistributed +from traja.utils import load_model device = 'cuda' if torch.cuda.is_available() else 'cpu' From a3f3eb1d9df28c90334c4e111f82dbf525655238 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 21:17:18 +0100 Subject: [PATCH 070/211] Update ae.py --- traja/models/ae.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index c21fc5e0..c040877f 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -24,8 +24,8 @@ trainer.train_latent_model(train_dataloader, test_dataloader, model_save_path=PATH)""" import torch -from traja.utils import TimeDistributed -from traja.utils import load_model +from models.utils import TimeDistributed +from models.utils import load_model device = 'cuda' if torch.cuda.is_available() else 'cpu' From 7f9ffae1ac03733f81afc21f07f71c1acb06991d Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 21:19:03 +0100 Subject: [PATCH 071/211] Update ae.py --- traja/models/ae.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index c040877f..9e92050a 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -24,8 +24,8 @@ trainer.train_latent_model(train_dataloader, test_dataloader, model_save_path=PATH)""" import torch -from models.utils import TimeDistributed -from models.utils import load_model +from traja.models.utils import TimeDistributed +from traja.models.utils import load_model device = 'cuda' if torch.cuda.is_available() else 'cpu' From 13aaafdbe37d2e30f6a76f833fa2a677b49c4488 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 21:20:59 +0100 Subject: [PATCH 072/211] Update optimizers.py --- traja/models/optimizers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index cdbad072..d4672b33 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -1,7 +1,7 @@ import torch from torch.optim import optimizer from torch.optim.lr_scheduler import ReduceLROnPlateau -from ae import MultiModelAE +from traja.models.ae import MultiModelAE class Optimizer: From 015a580dc9c5af5cb5060774ffe5865584ae0ad2 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 21:24:11 +0100 Subject: [PATCH 073/211] Update train.py --- traja/models/train.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index e6c963c1..3e35273c 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -33,7 +33,7 @@ def __init__(self, model_type:str, bidirectional:bool =False, batch_first:bool =True, loss_type:str = 'huber', - optimizer_type = 'adam', + optimizer_type:str = 'Adam', lr_factor:float = 0.1, scheduler_patience: int=10): @@ -97,8 +97,8 @@ def __init__(self, model_type:str, if self.model_type == 'irl': return NotImplementedError - optimizer = Optimizer(self.model_type, self.model) - self.model_optimizers = optimizer.get_optimizers(lr=0.001) + optimizer = Optimizer(self.model_type, self.model,self.optimizer_type) + self.model_optimizers = optimizer.get_optimizers( lr=0.001) self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor,patience=self.scheduler_patience) def __str__(self): From a4bfabbfb58b685c69258e98b86a7adf0cca5877 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 21:28:55 +0100 Subject: [PATCH 074/211] Update train.py --- traja/models/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 3e35273c..1df49634 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -33,11 +33,11 @@ def __init__(self, model_type:str, bidirectional:bool =False, batch_first:bool =True, loss_type:str = 'huber', - optimizer_type:str = 'Adam', + optimizer_type:str = 'RMSprop', lr_factor:float = 0.1, scheduler_patience: int=10): - white_keys = ['ae','vae','lstm','vaeg an', 'irl'] + white_keys = ['ae','vae','lstm','vaegan', 'irl'] assert model_type in white_keys, "Valid models are {}".format(white_keys) self.model_type = model_type self.device = device From 5c369ee5797173beef3c4b93ebb656a7f76c94c5 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 21:39:15 +0100 Subject: [PATCH 075/211] Update optimizers.py --- traja/models/optimizers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index d4672b33..2c05a86f 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -8,12 +8,14 @@ class Optimizer: def __init__(self, model_type, model, optimizer_type): assert isinstance(model,torch.nn.Module) + print(optimizer_type) + assert optimizer_type in ['Adam','Adadelta','Adagrad','AdamW','SparseAdam','RMSprop','Rprop','LBFGS','ASGD','Adamax'] + self.model_type = model_type self.model = model self.optimizer_type = optimizer_type self.optimizers = {} self.schedulers = {} - assert optimizer_type in ['Adam','Adadelta','Adagrad','AdamW','SparseAdam','RMSprop','Rprop','LBFGS','ASGD','Adamax'] def get_optimizers(self, lr=0.0001): """Optimizers for each network in the model From 9d8430072f8d02adddec07933510539ecf7cadce Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 22:05:38 +0100 Subject: [PATCH 076/211] Update train.py --- traja/models/train.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 1df49634..b7b8ef97 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -16,6 +16,7 @@ class Trainer(object): def __init__(self, model_type:str, + optimizer_type:str, device:str, input_size:int, output_size:int, @@ -33,7 +34,7 @@ def __init__(self, model_type:str, bidirectional:bool =False, batch_first:bool =True, loss_type:str = 'huber', - optimizer_type:str = 'RMSprop', + lr_factor:float = 0.1, scheduler_patience: int=10): @@ -64,22 +65,17 @@ def __init__(self, model_type:str, self.scheduler_patience = scheduler_patience self.model_hyperparameters = {'input_size':self.input_size, 'sequence_length':self.sequence_length, - 'batch_size':self.batch_size, - 'batch_first':self.batch_first, - + 'batch_size':self.batch_size, 'hidden_size':self.lstm_hidden_size, 'num_future':self.num_future, - 'num_layers':self.lstm_num_layers, 'latent_size':self.latent_size, 'output_size':self.output_size, 'num_classes':self.num_classes, 'batch_first':self.batch_first, - 'dropout':self.dropout, 'reset_state':self.reset_state, 'bidirectional':self.bidirectional, - 'dropout':self.dropout, - + 'dropout':self.dropout } if self.model_type == 'lstm': @@ -96,7 +92,7 @@ def __init__(self, model_type:str, if self.model_type == 'irl': return NotImplementedError - + optimizer = Optimizer(self.model_type, self.model,self.optimizer_type) self.model_optimizers = optimizer.get_optimizers( lr=0.001) self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor,patience=self.scheduler_patience) From 068367ae6b7858e6db3570ba392b2ddab3a280a8 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 22:16:21 +0100 Subject: [PATCH 077/211] Update train.py --- traja/models/train.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index b7b8ef97..ad6cbfce 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -33,8 +33,7 @@ def __init__(self, model_type:str, sequence_length:int, bidirectional:bool =False, batch_first:bool =True, - loss_type:str = 'huber', - + loss_type:str = 'huber', lr_factor:float = 0.1, scheduler_patience: int=10): @@ -45,7 +44,6 @@ def __init__(self, model_type:str, self.input_size = input_size self.lstm_hidden_size = lstm_hidden_size self.lstm_num_layers = lstm_num_layers - self.num_layers = lstm_num_layers self.hidden_size = lstm_hidden_size # For classifiers too self.batch_first = batch_first self.reset_state = reset_state @@ -92,10 +90,11 @@ def __init__(self, model_type:str, if self.model_type == 'irl': return NotImplementedError - + print(self.optimizer_type) optimizer = Optimizer(self.model_type, self.model,self.optimizer_type) - self.model_optimizers = optimizer.get_optimizers( lr=0.001) - self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor,patience=self.scheduler_patience) + + self.model_optimizers = optimizer.get_optimizers(lr=0.001) + self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) def __str__(self): return "Training model type {}".format(self.model_type) From 9750aaea3f555e924f28011073aec6a60080a03d Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 22:56:39 +0100 Subject: [PATCH 078/211] optimizer fix --- traja/models/ae.py | 26 -------------------------- traja/models/optimizers.py | 1 - traja/models/train.py | 10 +++------- 3 files changed, 3 insertions(+), 34 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index 9e92050a..31f532d8 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -1,35 +1,9 @@ -""" This module implement the Auto encoder model for both forecasting -and classification of time series data. - -```USAGE``` to train AE model: -trainer = Trainer(model_type='ae', - device=device, - input_size=input_size, - output_size=output_size, - lstm_hidden_size=lstm_hidden_size, - lstm_num_layers=lstm_num_layers, - reset_state=True, - num_classes=num_classes, - latent_size=latent_size, - dropout=0.1, - num_layers=num_layers, - epochs=epochs, - batch_size=batch_size, - num_future=num_future, - sequence_length=sequence_length, - bidirectional =False, - batch_first =True, - loss_type = 'huber') - -trainer.train_latent_model(train_dataloader, test_dataloader, model_save_path=PATH)""" - import torch from traja.models.utils import TimeDistributed from traja.models.utils import load_model device = 'cuda' if torch.cuda.is_available() else 'cpu' - class LSTMEncoder(torch.nn.Module): """ Deep LSTM network. This implementation returns output_size hidden size. diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 2c05a86f..5a49c3b6 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -8,7 +8,6 @@ class Optimizer: def __init__(self, model_type, model, optimizer_type): assert isinstance(model,torch.nn.Module) - print(optimizer_type) assert optimizer_type in ['Adam','Adadelta','Adagrad','AdamW','SparseAdam','RMSprop','Rprop','LBFGS','ASGD','Adamax'] self.model_type = model_type diff --git a/traja/models/train.py b/traja/models/train.py index ad6cbfce..3f6fd2c6 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -7,16 +7,11 @@ from .losses import Criterion from .optimizers import Optimizer import torch -from functools import wraps -import inspect device = 'cuda' if torch.cuda.is_available() else 'cpu' - - class Trainer(object): - def __init__(self, model_type:str, - optimizer_type:str, + def __init__(self, model_type:str, device:str, input_size:int, output_size:int, @@ -31,6 +26,7 @@ def __init__(self, model_type:str, batch_size:int, num_future:int, sequence_length:int, + optimizer_type:str = 'Adam', bidirectional:bool =False, batch_first:bool =True, loss_type:str = 'huber', @@ -91,7 +87,7 @@ def __init__(self, model_type:str, if self.model_type == 'irl': return NotImplementedError print(self.optimizer_type) - optimizer = Optimizer(self.model_type, self.model,self.optimizer_type) + optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) self.model_optimizers = optimizer.get_optimizers(lr=0.001) self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) From b6c51221fd6a4becd82ce04f00437e3dde1b4d4f Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 23:00:47 +0100 Subject: [PATCH 079/211] optimizer fix --- traja/models/optimizers.py | 2 +- traja/models/train.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 5a49c3b6..1e7ae02d 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -8,7 +8,7 @@ class Optimizer: def __init__(self, model_type, model, optimizer_type): assert isinstance(model,torch.nn.Module) - assert optimizer_type in ['Adam','Adadelta','Adagrad','AdamW','SparseAdam','RMSprop','Rprop','LBFGS','ASGD','Adamax'] + assert str(optimizer_type) in ['Adam','Adadelta','Adagrad','AdamW','SparseAdam','RMSprop','Rprop','LBFGS','ASGD','Adamax'] self.model_type = model_type self.model = model diff --git a/traja/models/train.py b/traja/models/train.py index 3f6fd2c6..032abd04 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -86,7 +86,7 @@ def __init__(self, model_type:str, if self.model_type == 'irl': return NotImplementedError - print(self.optimizer_type) + print(str(self.optimizer_type)) optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) self.model_optimizers = optimizer.get_optimizers(lr=0.001) From 67238d26fd2bffff6b3599913b1c745ca631a09a Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 23:04:16 +0100 Subject: [PATCH 080/211] Update train.py --- traja/models/train.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 032abd04..46eda101 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -12,6 +12,7 @@ class Trainer(object): def __init__(self, model_type:str, + optimizer_type:str, device:str, input_size:int, output_size:int, @@ -26,7 +27,6 @@ def __init__(self, model_type:str, batch_size:int, num_future:int, sequence_length:int, - optimizer_type:str = 'Adam', bidirectional:bool =False, batch_first:bool =True, loss_type:str = 'huber', @@ -86,8 +86,8 @@ def __init__(self, model_type:str, if self.model_type == 'irl': return NotImplementedError - print(str(self.optimizer_type)) - optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) + print(self.optimizer_type[0]) + optimizer = Optimizer(self.model_type, self.model, self.optimizer_type[0]) self.model_optimizers = optimizer.get_optimizers(lr=0.001) self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) From a3ab3d6a27303e0d27d72dcab8152664294d3a58 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 23:24:22 +0100 Subject: [PATCH 081/211] optimizer, ae classifier loss fix --- traja/models/ae.py | 93 ++++++++++--------- traja/models/experiment.py | 175 ++++++++++++++++++----------------- traja/models/generator.py | 43 ++++----- traja/models/interpretor.py | 17 ++-- traja/models/irl.py | 17 ++-- traja/models/losses.py | 45 +++++---- traja/models/lstm.py | 29 +++--- traja/models/nn.py | 180 +++++++++++++++++++----------------- traja/models/optimizers.py | 72 ++++++++------- traja/models/train.py | 2 +- traja/models/utils.py | 19 ++-- traja/models/vae.py | 95 ++++++++++--------- traja/models/vaegan.py | 20 ++-- traja/models/visualizer.py | 99 ++++++++++---------- 14 files changed, 465 insertions(+), 441 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index 31f532d8..a4ccb6c4 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -4,6 +4,7 @@ device = 'cuda' if torch.cuda.is_available() else 'cpu' + class LSTMEncoder(torch.nn.Module): """ Deep LSTM network. This implementation returns output_size hidden size. @@ -27,7 +28,6 @@ def __init__(self, input_size: int, sequence_length: int, batch_size: int, hidden_size: int, num_layers: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): - super(LSTMEncoder, self).__init__() self.input_size = input_size @@ -45,10 +45,10 @@ def __init__(self, input_size: int, sequence_length: int, batch_size: int, bidirectional=self.bidirectional, batch_first=True) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size), torch.zeros(self.num_layers, self.batch_size, self.hidden_size)) + return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size), + torch.zeros(self.num_layers, self.batch_size, self.hidden_size)) def forward(self, x): - enc_init_hidden = self._init_hidden() enc_output, _ = self.lstm_encoder(x, enc_init_hidden) # RNNs obeys, Markovian. Consider the last state of the hidden is the markovian of the entire sequence in that batch. @@ -58,12 +58,14 @@ def forward(self, x): class DisentangledAELatent(torch.nn.Module): """Dense Dientangled Latent Layer between encoder and decoder""" - def __init__(self, hidden_size: int, latent_size: int, dropout: float): + + def __init__(self, hidden_size: int, latent_size: int, dropout: float): super(DisentangledAELatent, self).__init__() self.latent_size = latent_size self.hidden_size = hidden_size self.dropout = dropout self.latent = torch.nn.Linear(self.hidden_size, self.latent_size) + def forward(self, x): z = self.latent(x) # Shape(batch_size, latent_size*2) return z @@ -89,8 +91,9 @@ class LSTMDecoder(torch.nn.Module): reset_state: If ``True``, the hidden and cell states of the LSTM will be reset at the beginning of each batch of input """ - def __init__(self, batch_size: int, num_future: int, hidden_size: int, - num_layers: int, output_size: int, latent_size: int, + + def __init__(self, batch_size: int, num_future: int, hidden_size: int, + num_layers: int, output_size: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMDecoder, self).__init__() @@ -108,17 +111,17 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, # RNN decoder self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, hidden_size=self.hidden_size, - num_layers=self.num_layers, + num_layers=self.num_layers, dropout=self.dropout, bidirectional=self.bidirectional, batch_first=True) - self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, + self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, self.output_size)) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, - self.hidden_size).to(device), - torch.zeros(self.num_layers, self.batch_size, + return (torch.zeros(self.num_layers, self.batch_size, + self.hidden_size).to(device), + torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) def forward(self, x, num_future=None): @@ -145,7 +148,8 @@ def forward(self, x, num_future=None): class MLPClassifier(torch.nn.Module): """ MLP classifier """ - def __init__(self, hidden_size: int, num_classes: int, latent_size: int, + + def __init__(self, hidden_size: int, num_classes: int, latent_size: int, dropout: float): super(MLPClassifier, self).__init__() self.latent_size = latent_size @@ -161,7 +165,6 @@ def __init__(self, hidden_size: int, num_classes: int, latent_size: int, self.dropout = torch.nn.Dropout(p=dropout) def forward(self, x): - classifier1 = self.dropout(self.classifier1(x)) classifier2 = self.dropout(self.classifier2(classifier1)) classifier3 = self.dropout(self.classifier3(classifier2)) @@ -171,19 +174,19 @@ def forward(self, x): class MultiModelAE(torch.nn.Module): - def __init__(self, input_size: int, - sequence_length: int, - batch_size: int, - num_future: int, - hidden_size: int, + def __init__(self, input_size: int, + sequence_length: int, + batch_size: int, + num_future: int, + hidden_size: int, num_layers: int, - output_size: int, - num_classes: int, - latent_size: int, - batch_first: bool, - dropout: float, - reset_state: bool, - bidirectional: bool ): + output_size: int, + num_classes: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool): super(MultiModelAE, self).__init__() self.input_size = input_size @@ -199,35 +202,35 @@ def __init__(self, input_size: int, self.dropout = dropout self.reset_state = reset_state self.bidirectional = bidirectional - - self.encoder = LSTMEncoder(input_size=self.input_size, - sequence_length=self.sequence_length, + + self.encoder = LSTMEncoder(input_size=self.input_size, + sequence_length=self.sequence_length, batch_size=self.batch_size, - hidden_size=self.hidden_size, + hidden_size=self.hidden_size, num_layers=self.num_layers, - batch_first=self.batch_first, + batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, + reset_state=True, bidirectional=self.bidirectional) - self.latent = DisentangledAELatent(hidden_size=self.hidden_size, - latent_size=self.latent_size, + self.latent = DisentangledAELatent(hidden_size=self.hidden_size, + latent_size=self.latent_size, dropout=self.dropout) - self.decoder = LSTMDecoder(batch_size=self.batch_size, + self.decoder = LSTMDecoder(batch_size=self.batch_size, num_future=self.num_future, - hidden_size=self.hidden_size, - num_layers=self.num_layers, + hidden_size=self.hidden_size, + num_layers=self.num_layers, output_size=self.output_size, - latent_size=self.latent_size, - batch_first=self.batch_first, + latent_size=self.latent_size, + batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, + reset_state=True, bidirectional=self.bidirectional) - self.classifier = MLPClassifier(hidden_size=self.hidden_size, - num_classes=self.num_classes, - latent_size=self.latent_size, + self.classifier = MLPClassifier(hidden_size=self.hidden_size, + num_classes=self.num_classes, + latent_size=self.latent_size, dropout=self.dropout) def forward(self, data, training=True, is_classification=False): @@ -241,7 +244,7 @@ def forward(self, data, training=True, is_classification=False): param.requires_grad = True for param in self.latent.parameters(): param.requires_grad = True - + # Encoder enc_out = self.encoder(data) # Latent @@ -262,7 +265,7 @@ def forward(self, data, training=True, is_classification=False): param.requires_grad = False for param in self.latent.parameters(): param.requires_grad = False - + # Encoder enc_out = self.encoder(data) # Latent @@ -270,5 +273,3 @@ def forward(self, data, training=True, is_classification=False): # Classifier classifier_out = self.classifier(latent_out) # Deterministic return classifier_out - - diff --git a/traja/models/experiment.py b/traja/models/experiment.py index 8e800790..5e805571 100644 --- a/traja/models/experiment.py +++ b/traja/models/experiment.py @@ -24,9 +24,9 @@ from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler import torchvision.transforms as transforms - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + class Trainer: def __init__(self, model, train_loader, @@ -47,7 +47,7 @@ def __init__(self, model, self.train_loader = train_loader self.test_loader = test_loader - self.criterion =torch.nn.MSELoss() + self.criterion = torch.nn.MSELoss() print('Checking for optimizer for {}'.format(optimizer)) if optimizer == "adam": print('Using adam') @@ -78,14 +78,16 @@ def __init__(self, model, if not os.path.exists(save_dir): os.makedirs(save_dir) - self.savepath = os.path.join(save_dir, f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') + self.savepath = os.path.join(save_dir, + f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') self.experiment_done = False if os.path.exists(self.savepath): trained_epochs = len(pd.read_csv(self.savepath, sep=';')) if trained_epochs >= epochs: self.experiment_done = True - print(f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') + print( + f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') if os.path.exists((self.savepath.replace('.csv', '.pt'))): self.model.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['model_state_dict']) self.model = self.model.to(self.device) @@ -97,13 +99,12 @@ def __init__(self, model, self.start_epoch = 0 self.model = self.model.to(self.device) - def _infer_initial_epoch(self, savepath): if not os.path.exists(savepath): return 0 else: df = pd.read_csv(savepath, sep=';', index_col=0) - print(len(df)+1) + print(len(df) + 1) return len(df) def train(self): @@ -117,7 +118,7 @@ def train(self): if self.opt_name == "LRS": print('LRS step') self.lr_scheduler.step() - return self.savepath+'.csv' + return self.savepath + '.csv' def train_epoch(self): self.model.train() @@ -125,7 +126,7 @@ def train_epoch(self): running_loss = 0 old_time = time() for batch, data in enumerate(self.train_loader): - inputs, targets= data[0].to(self.device).float(), data[1].to(self.device).float() + inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() self.optimizer.zero_grad() outputs = self.model(inputs) loss = self.criterion(outputs, targets) @@ -134,12 +135,13 @@ def train_epoch(self): running_loss += loss.item() if batch % 10 == 0 and batch != 0: - print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'loss:', running_loss/total) + print(batch, 'of', len(self.train_loader), 'processing time', time() - old_time, 'loss:', + running_loss / total) old_time = time() # Increment number of batches total += 1 - return running_loss/total + return running_loss / total def test(self, epoch, save=True): self.model.eval() @@ -148,7 +150,7 @@ def test(self, epoch, save=True): with torch.no_grad(): for batch, data in enumerate(self.test_loader): if batch % 10 == 0: - print('Processing eval batch', batch,'of', len(self.test_loader)) + print('Processing eval batch', batch, 'of', len(self.test_loader)) inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() outputs = self.model(inputs) loss = self.criterion(outputs, targets) @@ -196,16 +198,17 @@ def __init__(self, input_size: int, hidden_size: int, num_layers: int, self.head = nn.Linear(hidden_size, output_size) - def forward(self, x): + def forward(self, x): x, state = self.lstm(x) # Use the last hidden state of last layer - x = state[0][-1] + x = state[0][-1] x = self.head(x) return x + class TrajectoryLSTM: def __init__( - self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() + self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() ): fig, ax = plt.subplots(2, 1) self.fig = fig @@ -224,10 +227,10 @@ def load_batch(self, batch_size=32): inds = np.random.randint(0, len(self.xy) - self.nb_steps, (self.batch_size)) for i, ind in enumerate(inds): - t_1_b[:, i] = self.xy[ind : ind + self.nb_steps] - t_b[i * nb_steps : (i + 1) * self.nb_steps] = self.xy[ - ind + 1 : ind + nb_steps + 1 - ] + t_1_b[:, i] = self.xy[ind: ind + self.nb_steps] + t_b[i * nb_steps: (i + 1) * self.nb_steps] = self.xy[ + ind + 1: ind + nb_steps + 1 + ] return torch.from_numpy(t_1_b).float(), torch.from_numpy(t_b).float() def train(self): @@ -287,6 +290,7 @@ def plot(self, interactive=True): self._plot() return self.fig + def make_mlp(dim_list, activation="relu", batch_norm=True, dropout=0): layers = [] for dim_in, dim_out in zip(dim_list[:-1], dim_list[1:]): @@ -315,7 +319,7 @@ class Encoder(nn.Module): TrajectoryDiscriminator""" def __init__( - self, embedding_dim=64, h_dim=64, mlp_dim=1024, num_layers=1, dropout=0.0 + self, embedding_dim=64, h_dim=64, mlp_dim=1024, num_layers=1, dropout=0.0 ): super(Encoder, self).__init__() @@ -355,20 +359,20 @@ class Decoder(nn.Module): """Decoder is part of TrajectoryGenerator""" def __init__( - self, - seq_len, - embedding_dim=64, - h_dim=128, - mlp_dim=1024, - num_layers=1, - pool_every_timestep=True, - dropout=0.0, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - pooling_type="pool_net", - neighborhood_size=2.0, - grid_size=8, + self, + seq_len, + embedding_dim=64, + h_dim=128, + mlp_dim=1024, + num_layers=1, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + pooling_type="pool_net", + neighborhood_size=2.0, + grid_size=8, ): super(Decoder, self).__init__() @@ -452,14 +456,14 @@ class PoolHiddenNet(nn.Module): """Pooling module as proposed in our paper""" def __init__( - self, - embedding_dim=64, - h_dim=64, - mlp_dim=1024, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - dropout=0.0, + self, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + dropout=0.0, ): super(PoolHiddenNet, self).__init__() @@ -529,14 +533,14 @@ class SocialPooling(nn.Module): http://cvgl.stanford.edu/papers/CVPR16_Social_LSTM.pdf""" def __init__( - self, - h_dim=64, - activation="relu", - batch_norm=True, - dropout=0.0, - neighborhood_size=2.0, - grid_size=8, - pool_dim=None, + self, + h_dim=64, + activation="relu", + batch_norm=True, + dropout=0.0, + neighborhood_size=2.0, + grid_size=8, + pool_dim=None, ): super(SocialPooling, self).__init__() self.h_dim = h_dim @@ -620,14 +624,14 @@ def forward(self, h_states, seq_start_end, end_pos): # Make all positions to exclude as non-zero # Find which peds to exclude x_bound = (curr_end_pos[:, 0] >= bottom_right[:, 0]) + ( - curr_end_pos[:, 0] <= top_left[:, 0] + curr_end_pos[:, 0] <= top_left[:, 0] ) y_bound = (curr_end_pos[:, 1] >= top_left[:, 1]) + ( - curr_end_pos[:, 1] <= bottom_right[:, 1] + curr_end_pos[:, 1] <= bottom_right[:, 1] ) within_bound = x_bound + y_bound - within_bound[0 :: num_ped + 1] = 1 # Don't include the ped itself + within_bound[0:: num_ped + 1] = 1 # Don't include the ped itself within_bound = within_bound.view(-1) # This is a tricky way to get scatter add to work. Helps me avoid a @@ -657,25 +661,25 @@ class TrajectoryGenerator(nn.Module): """Modified from @agrimgupta92's https://github.com/agrimgupta92/sgan/blob/master/sgan/models.py.""" def __init__( - self, - obs_len, - pred_len, - embedding_dim=64, - encoder_h_dim=64, - decoder_h_dim=128, - mlp_dim=1024, - num_layers=1, - noise_dim=(0,), - noise_type="gaussian", - noise_mix_type="ped", - pooling_type=None, - pool_every_timestep=True, - dropout=0.0, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - neighborhood_size=2.0, - grid_size=8, + self, + obs_len, + pred_len, + embedding_dim=64, + encoder_h_dim=64, + decoder_h_dim=128, + mlp_dim=1024, + num_layers=1, + noise_dim=(0,), + noise_type="gaussian", + noise_mix_type="ped", + pooling_type=None, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + neighborhood_size=2.0, + grid_size=8, ): super(TrajectoryGenerator, self).__init__() @@ -805,9 +809,9 @@ def add_noise(self, _input, seq_start_end, user_noise=None): def mlp_decoder_needed(self): if ( - self.noise_dim - or self.pooling_type - or self.encoder_h_dim != self.decoder_h_dim + self.noise_dim + or self.pooling_type + or self.encoder_h_dim != self.decoder_h_dim ): return True else: @@ -858,19 +862,20 @@ def forward(self, obs_traj, obs_traj_rel, seq_start_end, user_noise=None): return pred_traj_fake_rel + class TrajectoryDiscriminator(nn.Module): def __init__( - self, - obs_len, - pred_len, - embedding_dim=64, - h_dim=64, - mlp_dim=1024, - num_layers=1, - activation="relu", - batch_norm=True, - dropout=0.0, - d_type="local", + self, + obs_len, + pred_len, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + num_layers=1, + activation="relu", + batch_norm=True, + dropout=0.0, + d_type="local", ): super(TrajectoryDiscriminator, self).__init__() diff --git a/traja/models/generator.py b/traja/models/generator.py index 6df87234..b0624da8 100644 --- a/traja/models/generator.py +++ b/traja/models/generator.py @@ -11,52 +11,53 @@ device = 'cuda' if torch.cuda.is_available() else 'cpu' -def timeseries(model_type:str, model_hyperparameters:dict, model_path:str, batch_size:int, num_future:int, ): + +def timeseries(model_type: str, model_hyperparameters: dict, model_path: str, batch_size: int, num_future: int, ): # Generating few samples - batch_size = model_hyperparameters.batch_size # Number of samples - num_future = model_hyperparameters.num_future # Number of time steps in each sample + batch_size = model_hyperparameters.batch_size # Number of samples + num_future = model_hyperparameters.num_future # Number of time steps in each sample if model_type == 'ae': model = MultiModelAE(**model_hyperparameters) - + if model_type == 'vae': model = MultiModelVAE(**model_hyperparameters) - + if model_type == 'vaegan': model = MultiModelVAEGAN(**model_hyperparameters) return NotImplementedError - + if model_type == 'irl': model = MultiModelIRL(**model_hyperparameters) return NotImplementedError - + # Load the model from model path: model = load_model(model, model_hyperparameters, model_path) # z = torch.randn((batch_size, latent_size)).to(device) - z = torch.empty(10, model_hyperparameters.latent_size).normal_(mean=0,std=.1).to(device) + z = torch.empty(10, model_hyperparameters.latent_size).normal_(mean=0, std=.1).to(device) # Category from the noise cat = model.classifier(z) # Generate trajectories from the noise - out = model.decoder(z,num_future).cpu().detach().numpy() - out = out.reshape(out.shape[0]*out.shape[1],out.shape[2]) + out = model.decoder(z, num_future).cpu().detach().numpy() + out = out.reshape(out.shape[0] * out.shape[1], out.shape[2]) # for index, i in enumerate(train_df.columns): # scaler = scalers['scaler_'+i] # out[:,index] = scaler.inverse_transform(out[:,index].reshape(1, -1)) - print('IDs in this batch of synthetic data',torch.max(cat,1).indices+1) - plt.figure(figsize=(12,4)) - plt.plot(out[:,0], label='Generated x: Longitude') - plt.plot(out[:,1], label='Generated y: Latitude') + print('IDs in this batch of synthetic data', torch.max(cat, 1).indices + 1) + plt.figure(figsize=(12, 4)) + plt.plot(out[:, 0], label='Generated x: Longitude') + plt.plot(out[:, 1], label='Generated y: Latitude') plt.legend() - fig, ax = plt.subplots(nrows=2, ncols= 5, figsize=(16, 5), sharey=True) + fig, ax = plt.subplots(nrows=2, ncols=5, figsize=(16, 5), sharey=True) # plt.ticklabel_format(useOffset=False) - fig.set_size_inches(20,5) + fig.set_size_inches(20, 5) for i in range(2): for j in range(5): - ax[i,j].plot(out[:,0][(i+j)*num_future:(i+j)*num_future + num_future],out[:,1][(i+j)*num_future:(i+j)*num_future+ num_future],label = 'Animal ID {}'.format((torch.max(cat,1).indices+1).detach()[i+j]),color='g') - ax[i,j].legend() + ax[i, j].plot(out[:, 0][(i + j) * num_future:(i + j) * num_future + num_future], + out[:, 1][(i + j) * num_future:(i + j) * num_future + num_future], + label='Animal ID {}'.format((torch.max(cat, 1).indices + 1).detach()[i + j]), color='g') + ax[i, j].legend() plt.show() - - return out - + return out diff --git a/traja/models/interpretor.py b/traja/models/interpretor.py index 7ac0eb91..b4c3a1b9 100644 --- a/traja/models/interpretor.py +++ b/traja/models/interpretor.py @@ -6,6 +6,7 @@ from .vaegan import MultiModelVAEGAN from .irl import MultiModelIRL + def DisplayLatentDynamics(latent): r"""Visualize the dynamics of combination of latents Args: @@ -13,14 +14,14 @@ def DisplayLatentDynamics(latent): Latent shape (batch_size, latent_dim) Usage: DisplayLatentDynamics(latent)""" - + latents = {} - latents.fromkeys(list(range(latent.shape[1]))) + latents.fromkeys(list(range(latent.shape[1]))) for i in range(latent.shape[1]): - latents[f'{i}']=latent[:,i].cpu().detach().numpy() - fig= px.scatter_matrix(latents) + latents[f'{i}'] = latent[:, i].cpu().detach().numpy() + fig = px.scatter_matrix(latents) fig.update_layout( - autosize=False, - width=1600, - height=1000,) - return fig.show() \ No newline at end of file + autosize=False, + width=1600, + height=1000, ) + return fig.show() diff --git a/traja/models/irl.py b/traja/models/irl.py index 2fac5848..f7d5b593 100644 --- a/traja/models/irl.py +++ b/traja/models/irl.py @@ -1,20 +1,19 @@ """ Implementation of Inverse Reinforcement Learning algorithm for Time series""" import torch + class MultiModelIRL(torch.nn.Module): - def __init__(self,*model_hyperparameters, **kwargs): - super(MultiModelIRL,self).__init__() - + def __init__(self, *model_hyperparameters, **kwargs): + super(MultiModelIRL, self).__init__() + for dictionary in model_hyperparameters: for key in dictionary: - setattr(self, key, dictionary[key]) + setattr(self, key, dictionary[key]) for key in kwargs: setattr(self, key, kwargs[key]) - + def __new__(cls): pass - - def forward(self, *input:None, **kwargs: None): + + def forward(self, *input: None, **kwargs: None): return NotImplementedError - - \ No newline at end of file diff --git a/traja/models/losses.py b/traja/models/losses.py index b8378e13..16c47915 100644 --- a/traja/models/losses.py +++ b/traja/models/losses.py @@ -1,10 +1,11 @@ import torch + class Criterion(object): - + def __init__(self, model_type): self.model_type = model_type - + @staticmethod def ae_criterion(recon_x, x, loss_type='huber'): """[summary] @@ -18,13 +19,13 @@ def ae_criterion(recon_x, x, loss_type='huber'): [type]: [description] """ if loss_type == 'huber': - + huber_loss = torch.nn.SmoothL1Loss(reduction='sum') - dist_x = huber_loss(recon_x,x) + dist_x = huber_loss(recon_x, x) return dist_x - else: # RMSE - return torch.sqrt(torch.mean((recon_x-x)**2)) - + else: # RMSE + return torch.sqrt(torch.mean((recon_x - x) ** 2)) + @staticmethod def vae_criterion(recon_x, x, mu, logvar, loss_type='huber'): """Time series generative model loss function @@ -38,32 +39,30 @@ def vae_criterion(recon_x, x, mu, logvar, loss_type='huber'): Returns: [type]: [description] """ - if loss_type=='huber': + if loss_type == 'huber': huber_loss = torch.nn.SmoothL1Loss(reduction='sum') - dist_x = huber_loss(recon_x,x) - else: - dist_x = torch.sqrt(torch.mean((recon_x-x)**2)) - - KLD = -0.5 * torch.sum(1 + logvar - mu**2 - logvar.exp()) - + dist_x = huber_loss(recon_x, x) + else: + dist_x = torch.sqrt(torch.mean((recon_x - x) ** 2)) + + KLD = -0.5 * torch.sum(1 + logvar - mu ** 2 - logvar.exp()) + return dist_x + KLD - + @staticmethod - def classifier_criterion(): + def classifier_criterion(predicted,target): """Classifier loss function""" - classifier_criterion = torch.nn.CrossEntropyLoss() - return classifier_criterion - + loss = torch.nn.CrossEntropyLoss(predicted,target) + return loss + def vaegan_criterion(): return NotImplementedError - + def lstm_criterion(): return NotImplementedError - - # VAE loss # VAE-GAN loss -# LSTM \ No newline at end of file +# LSTM diff --git a/traja/models/lstm.py b/traja/models/lstm.py index 584b512c..558f0204 100644 --- a/traja/models/lstm.py +++ b/traja/models/lstm.py @@ -1,6 +1,7 @@ """Implementation of Multimodel LSTM""" -import torch +import torch + class LSTM(torch.nn.Module): """ Deep LSTM network. This implementation @@ -21,11 +22,11 @@ class LSTM(torch.nn.Module): bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` """ - def __init__(self, batch_size:int, num_future:int, hidden_size: int, num_layers: int, - output_size: int, input_size:int, batch_first: bool, dropout: float, - reset_state: bool, bidirectional: bool): + def __init__(self, batch_size: int, num_future: int, hidden_size: int, num_layers: int, + output_size: int, input_size: int, batch_first: bool, dropout: float, + reset_state: bool, bidirectional: bool): super(LSTM, self).__init__() - + self.batch_size = batch_size self.input_size = input_size self.num_future = num_future @@ -39,21 +40,21 @@ def __init__(self, batch_size:int, num_future:int, hidden_size: int, num_layers: # RNN decoder self.lstm_decoder = torch.nn.LSTM(input_size=self.input_size, hidden_size=self.hidden_size, - num_layers=self.num_layers, dropout=self.dropout, - bidirectional=self.bidirectional, batch_first=True) - self.output = TimeDistributed(torch.nn.Linear(self.hidden_size , self.output_size)) + num_layers=self.num_layers, dropout=self.dropout, + bidirectional=self.bidirectional, batch_first=True) + self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, self.output_size)) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device), torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) + return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device), + torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) def forward(self, x): - # To feed the latent states into lstm decoder, repeat the tensor n_future times at second dim _init_hidden = self._init_hidden() - + # Decoder input Shape(batch_size, num_futures, latent_size) - dec,(dec_hidden,dec_cell) = self.lstm_decoder(x,_init_hidden) - + dec, (dec_hidden, dec_cell) = self.lstm_decoder(x, _init_hidden) + # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) to Time Dsitributed Linear Layer output = self.output(dec) - return output \ No newline at end of file + return output diff --git a/traja/models/nn.py b/traja/models/nn.py index 817e437a..d2f26e36 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -25,7 +25,8 @@ nb_steps = 10 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - + + class LossMse: """ Calculate the Mean Squared Error between y_true and y_pred @@ -33,15 +34,17 @@ class LossMse: y_true is the desired output. y_pred is the model's output. """ + def __init__(self) -> None: pass - def __call__(self, y_pred, y_true): + def __call__(self, y_pred, y_true): # Calculate the Mean Squared Error and use it as loss. mse = torch.mean(torch.square(y_true - y_pred)) return mse + class Trainer: def __init__(self, model, train_loader, @@ -93,14 +96,16 @@ def __init__(self, model, if not os.path.exists(save_dir): os.makedirs(save_dir) - self.savepath = os.path.join(save_dir, f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') + self.savepath = os.path.join(save_dir, + f'{model.name}_bs{batch_size}_e{epochs}_dspl{downsampling}_id{run_id}.csv') self.experiment_done = False if os.path.exists(self.savepath): trained_epochs = len(pd.read_csv(self.savepath, sep=';')) if trained_epochs >= epochs: self.experiment_done = True - print(f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') + print( + f'Experiment Logs for the exact same experiment with identical run_id was detected, training will be skipped, consider using another run_id') if os.path.exists((self.savepath.replace('.csv', '.pt'))): self.model.load_state_dict(torch.load(self.savepath.replace('.csv', '.pt'))['model_state_dict']) self.model = self.model.to(self.device) @@ -112,13 +117,12 @@ def __init__(self, model, self.start_epoch = 0 self.model = self.model.to(self.device) - def _infer_initial_epoch(self, savepath): if not os.path.exists(savepath): return 0 else: df = pd.read_csv(savepath, sep=';', index_col=0) - print(len(df)+1) + print(len(df) + 1) return len(df) def train(self): @@ -132,7 +136,7 @@ def train(self): if self.opt_name == "LRS": print('LRS step') self.lr_scheduler.step() - return self.savepath+'.csv' + return self.savepath + '.csv' def train_epoch(self): self.model.train() @@ -140,8 +144,8 @@ def train_epoch(self): running_loss = 0 old_time = time() for batch, data in enumerate(self.train_loader): - - inputs, targets= data[0].to(self.device).float(), data[1].to(self.device).float() + + inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() self.optimizer.zero_grad() outputs = self.model(inputs) loss = self.criterion(outputs, targets) @@ -150,12 +154,13 @@ def train_epoch(self): running_loss += loss.item() if batch % 10 == 0 and batch != 0: - print(batch, 'of', len(self.train_loader), 'processing time', time()-old_time, 'loss:', running_loss/total) + print(batch, 'of', len(self.train_loader), 'processing time', time() - old_time, 'loss:', + running_loss / total) old_time = time() # Increment number of batches total += 1 - return running_loss/total + return running_loss / total def test(self, epoch, save=True): self.model.eval() @@ -164,7 +169,7 @@ def test(self, epoch, save=True): with torch.no_grad(): for batch, data in enumerate(self.test_loader): if batch % 10 == 0: - print('Processing eval batch', batch,'of', len(self.test_loader)) + print('Processing eval batch', batch, 'of', len(self.test_loader)) inputs, targets = data[0].to(self.device).float(), data[1].to(self.device).float() outputs = self.model(inputs) loss = self.criterion(outputs, targets) @@ -212,16 +217,17 @@ def __init__(self, input_size: int, hidden_size: int, num_layers: int, self.head = nn.Linear(hidden_size, output_size) - def forward(self, x): + def forward(self, x): x, state = self.lstm(x) # Use the last hidden state of last layer - x = state[0][-1] + x = state[0][-1] x = self.head(x) return x + class TrajectoryLSTM: def __init__( - self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() + self, xy, nb_steps=10, epochs=1000, batch_size=1, criterion=nn.MSELoss() ): fig, ax = plt.subplots(2, 1) self.fig = fig @@ -240,10 +246,10 @@ def load_batch(self, batch_size=32): inds = np.random.randint(0, len(self.xy) - self.nb_steps, (self.batch_size)) for i, ind in enumerate(inds): - t_1_b[:, i] = self.xy[ind : ind + self.nb_steps] - t_b[i * nb_steps : (i + 1) * self.nb_steps] = self.xy[ - ind + 1 : ind + nb_steps + 1 - ] + t_1_b[:, i] = self.xy[ind: ind + self.nb_steps] + t_b[i * nb_steps: (i + 1) * self.nb_steps] = self.xy[ + ind + 1: ind + nb_steps + 1 + ] return torch.from_numpy(t_1_b).float(), torch.from_numpy(t_b).float() def train(self): @@ -303,6 +309,7 @@ def plot(self, interactive=True): self._plot() return self.fig + def make_mlp(dim_list, activation="relu", batch_norm=True, dropout=0): layers = [] for dim_in, dim_out in zip(dim_list[:-1], dim_list[1:]): @@ -331,7 +338,7 @@ class Encoder(nn.Module): TrajectoryDiscriminator""" def __init__( - self, embedding_dim=64, h_dim=64, mlp_dim=1024, num_layers=1, dropout=0.0 + self, embedding_dim=64, h_dim=64, mlp_dim=1024, num_layers=1, dropout=0.0 ): super(Encoder, self).__init__() @@ -371,20 +378,20 @@ class Decoder(nn.Module): """Decoder is part of TrajectoryGenerator""" def __init__( - self, - seq_len, - embedding_dim=64, - h_dim=128, - mlp_dim=1024, - num_layers=1, - pool_every_timestep=True, - dropout=0.0, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - pooling_type="pool_net", - neighborhood_size=2.0, - grid_size=8, + self, + seq_len, + embedding_dim=64, + h_dim=128, + mlp_dim=1024, + num_layers=1, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + pooling_type="pool_net", + neighborhood_size=2.0, + grid_size=8, ): super(Decoder, self).__init__() @@ -468,14 +475,14 @@ class PoolHiddenNet(nn.Module): """Pooling module as proposed in our paper""" def __init__( - self, - embedding_dim=64, - h_dim=64, - mlp_dim=1024, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - dropout=0.0, + self, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + dropout=0.0, ): super(PoolHiddenNet, self).__init__() @@ -545,14 +552,14 @@ class SocialPooling(nn.Module): http://cvgl.stanford.edu/papers/CVPR16_Social_LSTM.pdf""" def __init__( - self, - h_dim=64, - activation="relu", - batch_norm=True, - dropout=0.0, - neighborhood_size=2.0, - grid_size=8, - pool_dim=None, + self, + h_dim=64, + activation="relu", + batch_norm=True, + dropout=0.0, + neighborhood_size=2.0, + grid_size=8, + pool_dim=None, ): super(SocialPooling, self).__init__() self.h_dim = h_dim @@ -636,14 +643,14 @@ def forward(self, h_states, seq_start_end, end_pos): # Make all positions to exclude as non-zero # Find which peds to exclude x_bound = (curr_end_pos[:, 0] >= bottom_right[:, 0]) + ( - curr_end_pos[:, 0] <= top_left[:, 0] + curr_end_pos[:, 0] <= top_left[:, 0] ) y_bound = (curr_end_pos[:, 1] >= top_left[:, 1]) + ( - curr_end_pos[:, 1] <= bottom_right[:, 1] + curr_end_pos[:, 1] <= bottom_right[:, 1] ) within_bound = x_bound + y_bound - within_bound[0 :: num_ped + 1] = 1 # Don't include the ped itself + within_bound[0:: num_ped + 1] = 1 # Don't include the ped itself within_bound = within_bound.view(-1) # This is a tricky way to get scatter add to work. Helps me avoid a @@ -673,25 +680,25 @@ class TrajectoryGenerator(nn.Module): """Modified from @agrimgupta92's https://github.com/agrimgupta92/sgan/blob/master/sgan/models.py.""" def __init__( - self, - obs_len, - pred_len, - embedding_dim=64, - encoder_h_dim=64, - decoder_h_dim=128, - mlp_dim=1024, - num_layers=1, - noise_dim=(0,), - noise_type="gaussian", - noise_mix_type="ped", - pooling_type=None, - pool_every_timestep=True, - dropout=0.0, - bottleneck_dim=1024, - activation="relu", - batch_norm=True, - neighborhood_size=2.0, - grid_size=8, + self, + obs_len, + pred_len, + embedding_dim=64, + encoder_h_dim=64, + decoder_h_dim=128, + mlp_dim=1024, + num_layers=1, + noise_dim=(0,), + noise_type="gaussian", + noise_mix_type="ped", + pooling_type=None, + pool_every_timestep=True, + dropout=0.0, + bottleneck_dim=1024, + activation="relu", + batch_norm=True, + neighborhood_size=2.0, + grid_size=8, ): super(TrajectoryGenerator, self).__init__() @@ -821,9 +828,9 @@ def add_noise(self, _input, seq_start_end, user_noise=None): def mlp_decoder_needed(self): if ( - self.noise_dim - or self.pooling_type - or self.encoder_h_dim != self.decoder_h_dim + self.noise_dim + or self.pooling_type + or self.encoder_h_dim != self.decoder_h_dim ): return True else: @@ -874,19 +881,20 @@ def forward(self, obs_traj, obs_traj_rel, seq_start_end, user_noise=None): return pred_traj_fake_rel + class TrajectoryDiscriminator(nn.Module): def __init__( - self, - obs_len, - pred_len, - embedding_dim=64, - h_dim=64, - mlp_dim=1024, - num_layers=1, - activation="relu", - batch_norm=True, - dropout=0.0, - d_type="local", + self, + obs_len, + pred_len, + embedding_dim=64, + h_dim=64, + mlp_dim=1024, + num_layers=1, + activation="relu", + batch_norm=True, + dropout=0.0, + d_type="local", ): super(TrajectoryDiscriminator, self).__init__() diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 1e7ae02d..c321fbd6 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -3,19 +3,21 @@ from torch.optim.lr_scheduler import ReduceLROnPlateau from traja.models.ae import MultiModelAE + class Optimizer: - + def __init__(self, model_type, model, optimizer_type): - - assert isinstance(model,torch.nn.Module) - assert str(optimizer_type) in ['Adam','Adadelta','Adagrad','AdamW','SparseAdam','RMSprop','Rprop','LBFGS','ASGD','Adamax'] - + + assert isinstance(model, torch.nn.Module) + assert str(optimizer_type) in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', + 'LBFGS', 'ASGD', 'Adamax'] + self.model_type = model_type - self.model = model + self.model = model self.optimizer_type = optimizer_type self.optimizers = {} self.schedulers = {} - + def get_optimizers(self, lr=0.0001): """Optimizers for each network in the model @@ -27,24 +29,25 @@ def get_optimizers(self, lr=0.0001): Returns: [type]: [description] """ - + if self.model_type == 'ae' or 'vae': - keys = ['encoder','decoder', 'latent', 'classifier'] + keys = ['encoder', 'decoder', 'latent', 'classifier'] for network in keys: - self.optimizers[network]=getattr(torch.optim,f'{self.optimizer_type}')(getattr(self.model,f'{network}').parameters(), lr=lr) + self.optimizers[network] = getattr(torch.optim, f'{self.optimizer_type}')( + getattr(self.model, f'{network}').parameters(), lr=lr) return self.optimizers if self.model_type == 'lstm': self.optimizers["lstm"] = torch.optim.Adam(self.model.parameters(), lr=lr) return self.optimizers - + elif self.model_type == 'vaegan': return NotImplementedError - - else: # LSTM + + else: # LSTM return NotImplementedError def get_lrschedulers(self, factor=0.1, patience=10): - + """Learning rate scheduler for each network in the model NOTE: Scheduler metric should be test set loss @@ -56,30 +59,31 @@ def get_lrschedulers(self, factor=0.1, patience=10): [dict]: [description] """ for network in self.optimizers.keys(): - self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, patience=patience, verbose=True) + self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, + patience=patience, verbose=True) return self.schedulers - + + if __name__ == '__main__': - # Test model_type = 'ae' - model = MultiModelAE(input_size=2, - sequence_length=10, - batch_size=5, - num_future=5, - hidden_size=10, - num_layers=2, - output_size=2, - num_classes=10, - latent_size=10, - batch_first=True, - dropout = 0.2, - reset_state=True, - bidirectional=True) - + model = MultiModelAE(input_size=2, + sequence_length=10, + batch_size=5, + num_future=5, + hidden_size=10, + num_layers=2, + output_size=2, + num_classes=10, + latent_size=10, + batch_first=True, + dropout=0.2, + reset_state=True, + bidirectional=True) + # Get the optimizers opt = Optimizer(model_type, model, optimizer_type='RMSprop') model_optimizers = opt.get_optimizers(lr=0.1) - model_schedulers = opt.get_lrschedulers(factor=0.1,patience=10) - - print(model_optimizers,model_schedulers) \ No newline at end of file + model_schedulers = opt.get_lrschedulers(factor=0.1, patience=10) + + print(model_optimizers, model_schedulers) diff --git a/traja/models/train.py b/traja/models/train.py index 46eda101..99927e3a 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -86,7 +86,7 @@ def __init__(self, model_type:str, if self.model_type == 'irl': return NotImplementedError - print(self.optimizer_type[0]) + optimizer = Optimizer(self.model_type, self.model, self.optimizer_type[0]) self.model_optimizers = optimizer.get_optimizers(lr=0.001) diff --git a/traja/models/utils.py b/traja/models/utils.py index 925e9260..5b670334 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -4,8 +4,10 @@ import collections from numpy import math + class TimeDistributed(torch.nn.Module): """ Time distributed wrapper compatible with linear/dense pytorch layer modules""" + def __init__(self, module, batch_first=True): super(TimeDistributed, self).__init__() self.module = module @@ -29,7 +31,7 @@ def forward(self, x): return out - + def save_model(model, PATH): """[summary] @@ -37,13 +39,14 @@ def save_model(model, PATH): model ([type]): [description] PATH ([type]): [description] """ - + # PATH = "state_dict_model.pt" # Save torch.save(model.state_dict(), PATH) print('Model saved at {}'.format(PATH)) - -def load_model(model,model_hyperparameters, PATH): + + +def load_model(model, model_hyperparameters, PATH): """[summary] Args: @@ -57,11 +60,5 @@ def load_model(model,model_hyperparameters, PATH): # Load model = model(model_hyperparameters) model.load_state_dict(torch.load(PATH)) - - return model - - - - - + return model diff --git a/traja/models/vae.py b/traja/models/vae.py index 02d9bc97..bb3641a5 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -53,7 +53,6 @@ def __init__(self, input_size: int, sequence_length: int, batch_size: int, hidden_size: int, num_layers: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): - super(LSTMEncoder, self).__init__() self.input_size = input_size @@ -71,10 +70,10 @@ def __init__(self, input_size: int, sequence_length: int, batch_size: int, bidirectional=self.bidirectional, batch_first=True) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size), torch.zeros(self.num_layers, self.batch_size, self.hidden_size)) + return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size), + torch.zeros(self.num_layers, self.batch_size, self.hidden_size)) def forward(self, x): - enc_init_hidden = self._init_hidden() enc_output, _ = self.lstm_encoder(x, enc_init_hidden) # RNNs obeys, Markovian. Consider the last state of the hidden is the markovian of the entire sequence in that batch. @@ -84,14 +83,16 @@ def forward(self, x): class DisentangledAELatent(torch.nn.Module): """Dense Dientangled Latent Layer between encoder and decoder""" - def __init__(self, hidden_size: int, latent_size: int, dropout: float): + + def __init__(self, hidden_size: int, latent_size: int, dropout: float): super(DisentangledAELatent, self).__init__() self.latent_size = latent_size self.hidden_size = hidden_size self.dropout = dropout self.latent = torch.nn.Linear(self.hidden_size, self.latent_size) - - def reparameterize(self, mu, logvar, training= True): + + @staticmethod + def reparameterize(mu, logvar, training=True): if training: std = logvar.mul(0.5).exp_() eps = std.data.new(std.size()).normal_() @@ -126,8 +127,9 @@ class LSTMDecoder(torch.nn.Module): reset_state: If ``True``, the hidden and cell states of the LSTM will be reset at the beginning of each batch of input """ - def __init__(self, batch_size: int, num_future: int, hidden_size: int, - num_layers: int, output_size: int, latent_size: int, + + def __init__(self, batch_size: int, num_future: int, hidden_size: int, + num_layers: int, output_size: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMDecoder, self).__init__() @@ -145,17 +147,17 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, # RNN decoder self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, hidden_size=self.hidden_size, - num_layers=self.num_layers, + num_layers=self.num_layers, dropout=self.dropout, bidirectional=self.bidirectional, batch_first=True) - self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, + self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, self.output_size)) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, - self.hidden_size).to(device), - torch.zeros(self.num_layers, self.batch_size, + return (torch.zeros(self.num_layers, self.batch_size, + self.hidden_size).to(device), + torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) def forward(self, x, num_future=None): @@ -182,7 +184,8 @@ def forward(self, x, num_future=None): class MLPClassifier(torch.nn.Module): """ MLP classifier """ - def __init__(self, hidden_size: int, num_classes: int, latent_size: int, + + def __init__(self, hidden_size: int, num_classes: int, latent_size: int, dropout: float): super(MLPClassifier, self).__init__() self.latent_size = latent_size @@ -198,29 +201,30 @@ def __init__(self, hidden_size: int, num_classes: int, latent_size: int, self.dropout = torch.nn.Dropout(p=dropout) def forward(self, x): - classifier1 = self.dropout(self.classifier1(x)) classifier2 = self.dropout(self.classifier2(classifier1)) classifier3 = self.dropout(self.classifier3(classifier2)) classifier4 = self.classifier4(classifier3) return classifier4 + class MultiModelVAE(torch.nn.Module): """Implementation of Multimodel Variational autoencoders; """ - def __init__(self, input_size: int, - sequence_length: int, - batch_size: int, - num_future: int, - hidden_size: int, + + def __init__(self, input_size: int, + sequence_length: int, + batch_size: int, + num_future: int, + hidden_size: int, num_layers: int, - output_size: int, - num_classes: int, - latent_size: int, - batch_first: bool, - dropout: float, - reset_state: bool, - bidirectional: bool ): + output_size: int, + num_classes: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool): super(MultiModelVAE, self).__init__() self.input_size = input_size @@ -237,37 +241,36 @@ def __init__(self, input_size: int, self.reset_state = reset_state self.bidirectional = bidirectional - self.encoder = LSTMEncoder(input_size=self.input_size, - sequence_length=self.sequence_length, + self.encoder = LSTMEncoder(input_size=self.input_size, + sequence_length=self.sequence_length, batch_size=self.batch_size, - hidden_size=self.hidden_size, + hidden_size=self.hidden_size, num_layers=self.num_layers, - batch_first=self.batch_first, + batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, + reset_state=True, bidirectional=self.bidirectional) - self.latent = DisentangledAELatent(hidden_size=self.hidden_size, - latent_size=self.latent_size, + self.latent = DisentangledAELatent(hidden_size=self.hidden_size, + latent_size=self.latent_size, dropout=self.dropout) - self.decoder = LSTMDecoder(batch_size=self.batch_size, + self.decoder = LSTMDecoder(batch_size=self.batch_size, num_future=self.num_future, - hidden_size=self.hidden_size, - num_layers=self.num_layers, + hidden_size=self.hidden_size, + num_layers=self.num_layers, output_size=self.output_size, - latent_size=self.latent_size, - batch_first=self.batch_first, + latent_size=self.latent_size, + batch_first=self.batch_first, dropout=self.dropout, - reset_state=True, + reset_state=True, bidirectional=self.bidirectional) - self.classifier = MLPClassifier(hidden_size=self.hidden_size, - num_classes=self.num_classes, - latent_size=self.latent_size, + self.classifier = MLPClassifier(hidden_size=self.hidden_size, + num_classes=self.num_classes, + latent_size=self.latent_size, dropout=self.dropout) - def forward(self, data, training=True, is_classification=False): if not is_classification: @@ -280,7 +283,7 @@ def forward(self, data, training=True, is_classification=False): param.requires_grad = True for param in self.latent.parameters(): param.requires_grad = True - + # Encoder enc_out = self.encoder(data) # Latent @@ -300,7 +303,7 @@ def forward(self, data, training=True, is_classification=False): param.requires_grad = False for param in self.latent.parameters(): param.requires_grad = False - + # Encoder enc_out = self.encoder(data) # Latent diff --git a/traja/models/vaegan.py b/traja/models/vaegan.py index ce47a12f..5fb817bc 100644 --- a/traja/models/vaegan.py +++ b/traja/models/vaegan.py @@ -6,22 +6,22 @@ 1. MSE 2. Huber Loss""" - import torch + class MultiModelVAEGAN(torch.nn.Module): - - def __init__(self,*model_hyperparameters, **kwargs): - super(MultiModelVAEGAN,self).__init__() - + + def __init__(self, *model_hyperparameters, **kwargs): + super(MultiModelVAEGAN, self).__init__() + for dictionary in model_hyperparameters: for key in dictionary: - setattr(self, key, dictionary[key]) + setattr(self, key, dictionary[key]) for key in kwargs: setattr(self, key, kwargs[key]) - + def __new__(cls): pass - - def forward(self, *input:None, **kwargs: None): - return NotImplementedError \ No newline at end of file + + def forward(self, *input: None, **kwargs: None): + return NotImplementedError diff --git a/traja/models/visualizer.py b/traja/models/visualizer.py index 8b6b2bd1..9e1462bd 100644 --- a/traja/models/visualizer.py +++ b/traja/models/visualizer.py @@ -19,28 +19,30 @@ import matplotlib from matplotlib import style from scipy.sparse import csgraph -import argparse,copy,h5py, os,sys,time,socket +import argparse, copy, h5py, os, sys, time, socket import tensorflow as tf -import torch,torchvision,torch.nn as nn +import torch, torchvision, torch.nn as nn import torch.optim as optim import torchvision.transforms as transforms # from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot -from matplotlib import ticker,colors +from matplotlib import ticker, colors # print(matplotlib.get_backend()) # matplotlib.rcParams["backend"] = "Gtk3Agg" # print(matplotlib.get_backend()) # matplotlib.use('Gtk3Agg') import matplotlib.pyplot as plt + # plt.switch_backend('Qt4Agg') # print(matplotlib.get_backend()) plt.switch_backend("TkAgg") # seed value and plotly # init_notebook_mode(connected=True) -np.set_printoptions(suppress=True,precision=3,) +np.set_printoptions(suppress=True, precision=3, ) style.use('ggplot') + class DirectedNetwork(object): def __init__(self): @@ -78,35 +80,36 @@ def show(self, states, weight, fig): vmax = np.max(states) cmap = plt.cm.coolwarm edge_cmap = plt.cm.Spectral - nx.draw(G, with_labels=True, - cmap=cmap, node_color = neuron_color, - node_size=200, linewidths=5, - edge_color = edge_colors_, - edge_cmap = edge_cmap, font_size=10, + nx.draw(G, with_labels=True, + cmap=cmap, node_color=neuron_color, + node_size=200, linewidths=5, + edge_color=edge_colors_, + edge_cmap=edge_cmap, font_size=10, connectionstyle='arc3, rad=0.3') sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=vmin, vmax=vmax)) sm.set_array([]) cbar = plt.colorbar(sm, orientation="vertical", pad=0.1) - + # State of streaming plot if plt.fignum_exists(fig.number): fig.canvas.draw() fig.canvas.flush_events() fig.clear() - + # Plot is not closed return False else: return True + class LocalLinearEmbedding(object): def __init__(self): super(LocalLinearEmbedding, self).__init__() pass - def local_linear_embedding(self,X,d,k,alpha = 0.1): + def local_linear_embedding(self, X, d, k, alpha=0.1): """ Local Linear Embeddings @@ -134,7 +137,7 @@ def local_linear_embedding(self,X,d,k,alpha = 0.1): # Calculate the matrix G G_i = Z_i @ Z_i.T - + # Weights between neigbors w_i = scipy.linalg.pinv(G_i + alpha * np.eye(k)) @ np.ones(k) W[i, k_indices] = w_i / w_i.sum() @@ -149,44 +152,43 @@ def local_linear_embedding(self,X,d,k,alpha = 0.1): # Return the vectors and discard the first column of the matrix return vectors[:, 1:] - def show(self,pc,fig2): + def show(self, pc, fig2): """[summary] Args: pc ([type]): [description] fig2 ([type]): [description] """ - + ax = Axes3D(fig2) - f = ax.scatter(pc[:, 0], pc[:, 1], pc[:, 2], s = 40,c = pc[:, 2]) -# ax.set_xlim(-1,1) -# ax.set_ylim(-1,1) -# ax.set_zlim(-1,1) + f = ax.scatter(pc[:, 0], pc[:, 1], pc[:, 2], s=40, c=pc[:, 2]) + # ax.set_xlim(-1,1) + # ax.set_ylim(-1,1) + # ax.set_zlim(-1,1) for i in range(len(pc)): - - ax.plot3D(pc[i:, 0], pc[i:, 1], pc[i:, 2], - alpha = i/len(pc), color = 'red', linewidth=1 ) + ax.plot3D(pc[i:, 0], pc[i:, 1], pc[i:, 2], + alpha=i / len(pc), color='red', linewidth=1) fig2.colorbar(f) -# plt.pause(0.0001) + # plt.pause(0.0001) # State of streaming plot if plt.fignum_exists(fig2.number): fig2.canvas.draw() fig2.canvas.flush_events() fig2.clear() - + # Plot is not closed return False else: return True - - + + class SpectralEmbedding(object): def __init__(self): super(SpectralEmbedding, self).__init__() pass - def spectral_embedding(self,X,rad): + def spectral_embedding(self, X, rad): """ Spectral Clustering @@ -196,11 +198,11 @@ def spectral_embedding(self,X,rad): :return Y: numpy.ndarray - matrix m row, d attributes are reduced dimensional """ # Get the adjacency matrix/nearest neighbor graph; neighbors within the radius of 0.4 - A = radius_neighbors_graph(X.T,rad,mode='distance', - metric='minkowski', p=2, - metric_params=None, include_self=False) + A = radius_neighbors_graph(X.T, rad, mode='distance', + metric='minkowski', p=2, + metric_params=None, include_self=False) A = A.toarray() - + # Find the laplacian of the neighbour graph # L = D - A ; where D is the diagonal degree matrix L = csgraph.laplacian(A, normed=False) @@ -210,12 +212,12 @@ def spectral_embedding(self,X,rad): # the second smallest eigenvalue represents sparsest cut of the graph. np.where(eigval == np.partition(eigval, 1)[1]) # Partition the graph using the smallest eigen value - y_spec =eigvec[:,1].copy() + y_spec = eigvec[:, 1].copy() y_spec[y_spec < 0] = 0 y_spec[y_spec > 0] = 1 return y_spec - def show(self,X, spec_embed,fig3): + def show(self, X, spec_embed, fig3): """[summary] Args: @@ -229,33 +231,36 @@ def show(self,X, spec_embed,fig3): ax3 = fig3.add_subplot() X = X.T - fi = ax3.scatter(x = X[:, 0], y = X[:, 1],c=spec_embed ,s=30, cmap=plt.cm.Spectral) - for i in range(len(X[:,0])): + fi = ax3.scatter(x=X[:, 0], y=X[:, 1], c=spec_embed, s=30, cmap=plt.cm.Spectral) + for i in range(len(X[:, 0])): ax3.annotate(i, (X[:, 0][i], X[:, 1][i])) fig3.colorbar(fi) - + # State of streaming plot if plt.fignum_exists(fig3.number): fig3.canvas.draw() fig3.canvas.flush_events() fig3.clear() - + # Plot is not closed return False else: return True - + + if __name__ == '__main__': - # create the coordinates - numebr_of_points = 21 ; small_range = -1.0 ; large_range = 1.0 + numebr_of_points = 21; + small_range = -1.0; + large_range = 1.0 - xcoordinates = np.linspace(small_range, large_range, num=numebr_of_points) - ycoordinates = np.linspace(small_range, large_range, num=numebr_of_points) + xcoordinates = np.linspace(small_range, large_range, num=numebr_of_points) + ycoordinates = np.linspace(small_range, large_range, num=numebr_of_points) xcoord_mesh, ycoord_mesh = np.meshgrid(xcoordinates, ycoordinates) - inds = np.array(range(numebr_of_points**2)) - s1 = xcoord_mesh.ravel()[inds] - s2 = ycoord_mesh.ravel()[inds] - coordinate = np.c_[s1,s2] - print('From ',small_range,' to ',large_range,' with ',numebr_of_points,' total number of coordinate: ', numebr_of_points**2) \ No newline at end of file + inds = np.array(range(numebr_of_points ** 2)) + s1 = xcoord_mesh.ravel()[inds] + s2 = ycoord_mesh.ravel()[inds] + coordinate = np.c_[s1, s2] + print('From ', small_range, ' to ', large_range, ' with ', numebr_of_points, ' total number of coordinate: ', + numebr_of_points ** 2) From f77c2e3725c3580c09b9e1dcd4c656cf8b7e6a0e Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 23:26:29 +0100 Subject: [PATCH 082/211] Update losses.py --- traja/models/losses.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/traja/models/losses.py b/traja/models/losses.py index 16c47915..7644138c 100644 --- a/traja/models/losses.py +++ b/traja/models/losses.py @@ -52,7 +52,8 @@ def vae_criterion(recon_x, x, mu, logvar, loss_type='huber'): @staticmethod def classifier_criterion(predicted,target): """Classifier loss function""" - loss = torch.nn.CrossEntropyLoss(predicted,target) + loss = torch.nn.CrossEntropyLoss() + loss = loss(predicted, target) return loss def vaegan_criterion(): From 90640466d690db4984bb6b9e43f2d3ba510134c6 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 23:44:06 +0100 Subject: [PATCH 083/211] vae fix --- traja/models/losses.py | 4 ++-- traja/models/vae.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/traja/models/losses.py b/traja/models/losses.py index 7644138c..028ade21 100644 --- a/traja/models/losses.py +++ b/traja/models/losses.py @@ -56,10 +56,10 @@ def classifier_criterion(predicted,target): loss = loss(predicted, target) return loss - def vaegan_criterion(): + def vaegan_criterion(self): return NotImplementedError - def lstm_criterion(): + def lstm_criterion(self): return NotImplementedError # VAE loss diff --git a/traja/models/vae.py b/traja/models/vae.py index bb3641a5..4e439de8 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -287,7 +287,7 @@ def forward(self, data, training=True, is_classification=False): # Encoder enc_out = self.encoder(data) # Latent - latent_out = self.latent(enc_out) + latent_out, mu, logvar = self.latent(enc_out) # Decoder decoder_out = self.decoder(latent_out) return decoder_out, latent_out From c31dfd0fae3fb628f6208f1381495871d6168fd1 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 16 Dec 2020 23:59:30 +0100 Subject: [PATCH 084/211] Update vae.py --- traja/models/vae.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/traja/models/vae.py b/traja/models/vae.py index 4e439de8..3ec2adb9 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -100,8 +100,11 @@ def reparameterize(mu, logvar, training=True): return mu def forward(self, x, training=True): + z_variables = self.latent(x) # [batch_size, latent_size*2] mu, logvar = torch.chunk(z_variables, 2, dim=1) # [batch_size,latent_size] + if not training: + return mu # Reparameterize z = self.reparameterize(mu, logvar, training=training) # [batch_size,latent_size] return z, mu, logvar @@ -173,6 +176,7 @@ def forward(self, x, num_future=None): decoder_inputs = decoder_inputs.repeat(1, num_future, 1) # Decoder input Shape(batch_size, num_futures, latent_size) + print(decoder_inputs.shape) dec, _ = self.lstm_decoder(decoder_inputs, _init_hidden) # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) From 92aa569a9b447e5aca104882d7b891da524a8030 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 00:21:39 +0100 Subject: [PATCH 085/211] Update vae.py --- traja/models/vae.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/vae.py b/traja/models/vae.py index 3ec2adb9..a264bd5b 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -89,7 +89,7 @@ def __init__(self, hidden_size: int, latent_size: int, dropout: float): self.latent_size = latent_size self.hidden_size = hidden_size self.dropout = dropout - self.latent = torch.nn.Linear(self.hidden_size, self.latent_size) + self.latent = torch.nn.Linear(self.hidden_size, self.latent_size*2) @staticmethod def reparameterize(mu, logvar, training=True): @@ -176,7 +176,7 @@ def forward(self, x, num_future=None): decoder_inputs = decoder_inputs.repeat(1, num_future, 1) # Decoder input Shape(batch_size, num_futures, latent_size) - print(decoder_inputs.shape) + dec, _ = self.lstm_decoder(decoder_inputs, _init_hidden) # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) From 1d97c72c1cab3ee6fc8ce10ba041d3e3bdc377a2 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 00:32:24 +0100 Subject: [PATCH 086/211] Update vae.py --- traja/models/vae.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/traja/models/vae.py b/traja/models/vae.py index a264bd5b..24a3c52b 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -103,8 +103,6 @@ def forward(self, x, training=True): z_variables = self.latent(x) # [batch_size, latent_size*2] mu, logvar = torch.chunk(z_variables, 2, dim=1) # [batch_size,latent_size] - if not training: - return mu # Reparameterize z = self.reparameterize(mu, logvar, training=training) # [batch_size,latent_size] return z, mu, logvar @@ -176,7 +174,6 @@ def forward(self, x, num_future=None): decoder_inputs = decoder_inputs.repeat(1, num_future, 1) # Decoder input Shape(batch_size, num_futures, latent_size) - dec, _ = self.lstm_decoder(decoder_inputs, _init_hidden) # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) @@ -294,7 +291,7 @@ def forward(self, data, training=True, is_classification=False): latent_out, mu, logvar = self.latent(enc_out) # Decoder decoder_out = self.decoder(latent_out) - return decoder_out, latent_out + return decoder_out, latent_out, mu,logvar else: # training_mode = 'classification' # Unfreeze classifier parameters and freeze all other From 3d4fdedcea75c7a3a0029b70b614969698cdd410 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 00:38:57 +0100 Subject: [PATCH 087/211] vae fix --- traja/models/train.py | 4 ++-- traja/models/vae.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 99927e3a..cc8f0b34 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -164,10 +164,10 @@ def train_latent_model(self, train_loader, test_loader, model_save_path): out, latent = self.model(data, training=False, is_classification=False) test_loss_forecasting += Criterion.ae_criterion(out,target).item() else: - decoder_out,latent_out,mu,logvar= self.model(data,training=False, is_classification=False) + decoder_out,latent_out, mu, logvar= self.model(data,training=False, is_classification=False) test_loss_forecasting += Criterion.vae_criterion(decoder_out, target,mu,logvar) # Classification test - classifier_out= self.model(data,training=False, is_classification=True) + classifier_out, latent_out, mu, logvar= self.model(data,training=False, is_classification=True) test_loss_classification += Criterion.classifier_criterion(classifier_out, category-1).item() test_loss_forecasting /= len(test_loader.dataset) diff --git a/traja/models/vae.py b/traja/models/vae.py index 24a3c52b..b690a30a 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -291,7 +291,7 @@ def forward(self, data, training=True, is_classification=False): latent_out, mu, logvar = self.latent(enc_out) # Decoder decoder_out = self.decoder(latent_out) - return decoder_out, latent_out, mu,logvar + return decoder_out, latent_out, mu, logvar else: # training_mode = 'classification' # Unfreeze classifier parameters and freeze all other @@ -310,5 +310,5 @@ def forward(self, data, training=True, is_classification=False): # Latent latent_out, mu, logvar = self.latent(enc_out, training=training) # Classifier - classifier_out = self.classifier(latent_out) # Deterministic + classifier_out = self.classifier(mu) # Deterministic return classifier_out, latent_out, mu, logvar From 3fc2fdb5b37d259a57cd75c6bee396c7a80009eb Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 00:45:51 +0100 Subject: [PATCH 088/211] Update train.py --- traja/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index cc8f0b34..9ed54ad5 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -142,7 +142,7 @@ def train_latent_model(self, train_loader, test_loader, model_save_path): else: # training_mode == 'classification' - classifier_out = self.model(data, training=True, is_classification=True) + classifier_out, latent_out, mu, logvar = self.model(data, training=True, is_classification=True) loss = Criterion.classifier_criterion(classifier_out, category-1) loss.backward() classifier_optimizer.step() From ba939b8e4a3b84c8fc29ad38486bf2147bbe8a3e Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 12:49:45 +0100 Subject: [PATCH 089/211] lstm fix --- traja/models/losses.py | 33 ++-- traja/models/train.py | 351 +++++++++++++++++++++-------------------- 2 files changed, 196 insertions(+), 188 deletions(-) diff --git a/traja/models/losses.py b/traja/models/losses.py index 028ade21..d1a8a074 100644 --- a/traja/models/losses.py +++ b/traja/models/losses.py @@ -5,9 +5,10 @@ class Criterion(object): def __init__(self, model_type): self.model_type = model_type + self.huber_loss = torch.nn.SmoothL1Loss(reduction='sum') + self.crossentropy_loss = torch.nn.CrossEntropyLoss() - @staticmethod - def ae_criterion(recon_x, x, loss_type='huber'): + def ae_criterion(self, recon_x, x, loss_type='huber'): """[summary] Args: @@ -19,15 +20,12 @@ def ae_criterion(recon_x, x, loss_type='huber'): [type]: [description] """ if loss_type == 'huber': - - huber_loss = torch.nn.SmoothL1Loss(reduction='sum') - dist_x = huber_loss(recon_x, x) + dist_x = self.huber_loss(recon_x, x) return dist_x - else: # RMSE + else: # Root MSE return torch.sqrt(torch.mean((recon_x - x) ** 2)) - @staticmethod - def vae_criterion(recon_x, x, mu, logvar, loss_type='huber'): + def vae_criterion(self, recon_x, x, mu, logvar, loss_type='huber'): """Time series generative model loss function Args: @@ -40,26 +38,23 @@ def vae_criterion(recon_x, x, mu, logvar, loss_type='huber'): [type]: [description] """ if loss_type == 'huber': - huber_loss = torch.nn.SmoothL1Loss(reduction='sum') - dist_x = huber_loss(recon_x, x) + dist_x = self.huber_loss(recon_x, x) else: dist_x = torch.sqrt(torch.mean((recon_x - x) ** 2)) - KLD = -0.5 * torch.sum(1 + logvar - mu ** 2 - logvar.exp()) - return dist_x + KLD - @staticmethod - def classifier_criterion(predicted,target): + def classifier_criterion(self, predicted, target): """Classifier loss function""" - loss = torch.nn.CrossEntropyLoss() - loss = loss(predicted, target) + loss = self.crossentropy_loss(predicted, target) return loss - def vaegan_criterion(self): - return NotImplementedError + def lstm_criterion(self, predicted, target): + + loss = self.huber_loss(predicted, target) + return loss - def lstm_criterion(self): + def vaegan_criterion(self): return NotImplementedError # VAE loss diff --git a/traja/models/train.py b/traja/models/train.py index 9ed54ad5..8e410e62 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -9,38 +9,40 @@ import torch device = 'cuda' if torch.cuda.is_available() else 'cpu' -class Trainer(object): - - def __init__(self, model_type:str, - optimizer_type:str, - device:str, - input_size:int, - output_size:int, - lstm_hidden_size:int, - lstm_num_layers:int, - reset_state:bool, - num_classes:int, - latent_size:int, - dropout:float, - num_layers:int, - epochs:int, - batch_size:int, - num_future:int, - sequence_length:int, - bidirectional:bool =False, - batch_first:bool =True, - loss_type:str = 'huber', - lr_factor:float = 0.1, - scheduler_patience: int=10): - - white_keys = ['ae','vae','lstm','vaegan', 'irl'] - assert model_type in white_keys, "Valid models are {}".format(white_keys) + + +class LatentModelTrainer(object): + + def __init__(self, model_type: str, + optimizer_type: str, + device: str, + input_size: int, + output_size: int, + lstm_hidden_size: int, + lstm_num_layers: int, + reset_state: bool, + num_classes: int, + latent_size: int, + dropout: float, + num_layers: int, + epochs: int, + batch_size: int, + num_future: int, + sequence_length: int, + bidirectional: bool = False, + batch_first: bool = True, + loss_type: str = 'huber', + lr_factor: float = 0.1, + scheduler_patience: int = 10): + + white_keys = ['ae', 'vae'] + assert model_type in white_keys, "Valid models are {}".format(white_keys) self.model_type = model_type self.device = device self.input_size = input_size self.lstm_hidden_size = lstm_hidden_size self.lstm_num_layers = lstm_num_layers - self.hidden_size = lstm_hidden_size # For classifiers too + self.hidden_size = lstm_hidden_size # For classifiers too self.batch_first = batch_first self.reset_state = reset_state self.output_size = output_size @@ -52,123 +54,114 @@ def __init__(self, model_type:str, self.batch_size = batch_size self.sequence_length = sequence_length self.dropout = dropout - self.bidirectional= bidirectional + self.bidirectional = bidirectional self.loss_type = loss_type self.optimizer_type = optimizer_type, self.lr_factor = lr_factor self.scheduler_patience = scheduler_patience - self.model_hyperparameters = {'input_size':self.input_size, - 'sequence_length':self.sequence_length, - 'batch_size':self.batch_size, - 'hidden_size':self.lstm_hidden_size, - 'num_future':self.num_future, - 'num_layers':self.lstm_num_layers, - 'latent_size':self.latent_size, - 'output_size':self.output_size, - 'num_classes':self.num_classes, - 'batch_first':self.batch_first, - 'reset_state':self.reset_state, - 'bidirectional':self.bidirectional, - 'dropout':self.dropout - } - - if self.model_type == 'lstm': - self.model = LSTM(self.model_hyperparameters) - + self.model_hyperparameters = {'input_size': self.input_size, + 'sequence_length': self.sequence_length, + 'batch_size': self.batch_size, + 'hidden_size': self.lstm_hidden_size, + 'num_future': self.num_future, + 'num_layers': self.lstm_num_layers, + 'latent_size': self.latent_size, + 'output_size': self.output_size, + 'num_classes': self.num_classes, + 'batch_first': self.batch_first, + 'reset_state': self.reset_state, + 'bidirectional': self.bidirectional, + 'dropout': self.dropout + } + if self.model_type == 'ae': self.model = MultiModelAE(**self.model_hyperparameters) - + if self.model_type == 'vae': self.model = MultiModelVAE(**self.model_hyperparameters) - - if self.model_type == 'vaegan': - self.model = MultiModelVAEGAN(self.model_hyperparameters) - - if self.model_type == 'irl': - return NotImplementedError - + optimizer = Optimizer(self.model_type, self.model, self.optimizer_type[0]) - + self.model_optimizers = optimizer.get_optimizers(lr=0.001) self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) - + def __str__(self): return "Training model type {}".format(self.model_type) - - # TRAIN AUTOENCODERS/VARIATIONAL AUTOENCODERS - def train_latent_model(self, train_loader, test_loader, model_save_path): - + + def train(self, train_loader, test_loader, model_save_path): + assert self.model_type == 'ae' or 'vae' - # Move the model to target device self.model.to(device) - + encoder_optimizer, latent_optimizer, decoder_optimizer, classifier_optimizer = self.model_optimizers.values() encoder_scheduler, latent_scheduler, decoder_scheduler, classifier_scheduler = self.model_lrschedulers.values() - + # Training mode: Switch from Generative to classifier training mode training_mode = 'forecasting' # Training - for epoch in range(self.epochs*2): # First half for generative model and next for classifier + for epoch in range(self.epochs * 2): # First half for generative model and next for classifier test_loss_forecasting = 0 test_loss_classification = 0 - if epoch>0: # Initial step is to test and set LR schduler + if epoch > 0: # Initial step is to test and set LR schduler # Training self.model.train() - total_loss = 0 - for idx, (data, target,category) in enumerate(train_loader): + total_loss = 0 + for idx, (data, target, category) in enumerate(train_loader): # Reset optimizer states encoder_optimizer.zero_grad() latent_optimizer.zero_grad() decoder_optimizer.zero_grad() classifier_optimizer.zero_grad() - - data, target,category = data.float().to(device), target.float().to(device), category.to(device) - - if training_mode =='forecasting': + + data, target, category = data.float().to(device), target.float().to(device), category.to(device) + + if training_mode == 'forecasting': if self.model_type == 'ae': decoder_out, latent_out = self.model(data, training=True, is_classification=False) loss = Criterion.ae_criterion(decoder_out, target) - - else: # vae - decoder_out, latent_out, mu, logvar= self.model(data, training=True, is_classification=False) + + else: # vae + decoder_out, latent_out, mu, logvar = self.model(data, training=True, + is_classification=False) loss = Criterion.vae_criterion(decoder_out, target, mu, logvar) - - loss.backward() + loss.backward() encoder_optimizer.step() decoder_optimizer.step() latent_optimizer.step() - else: # training_mode == 'classification' - + else: # training_mode == 'classification' + classifier_out, latent_out, mu, logvar = self.model(data, training=True, is_classification=True) - loss = Criterion.classifier_criterion(classifier_out, category-1) + loss = Criterion.classifier_criterion(classifier_out, category - 1) loss.backward() classifier_optimizer.step() - total_loss+=loss - - print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss/(idx+1))) - - if epoch+1 == self.epochs: # + total_loss += loss + + print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss / (idx + 1))) + + if epoch + 1 == self.epochs: training_mode = 'classification' # Testing - if epoch%10==0: + if epoch % 10 == 0: with torch.no_grad(): self.model.eval() - for idx, (data, target,category) in enumerate(list(test_loader)): + for idx, (data, target, category) in enumerate(list(test_loader)): data, target, category = data.float().to(device), target.float().to(device), category.to(device) - # Time seriesforecasting test - if self.model_type=='ae': + # Time series forecasting test + if self.model_type == 'ae': out, latent = self.model(data, training=False, is_classification=False) - test_loss_forecasting += Criterion.ae_criterion(out,target).item() + test_loss_forecasting += Criterion.ae_criterion(out, target).item() else: - decoder_out,latent_out, mu, logvar= self.model(data,training=False, is_classification=False) - test_loss_forecasting += Criterion.vae_criterion(decoder_out, target,mu,logvar) + decoder_out, latent_out, mu, logvar = self.model(data, training=False, + is_classification=False) + test_loss_forecasting += Criterion.vae_criterion(decoder_out, target, mu, logvar) # Classification test - classifier_out, latent_out, mu, logvar= self.model(data,training=False, is_classification=True) - test_loss_classification += Criterion.classifier_criterion(classifier_out, category-1).item() + classifier_out, latent_out, mu, logvar = self.model(data, training=False, + is_classification=True) + test_loss_classification += Criterion.classifier_criterion(classifier_out, category - 1).item() test_loss_forecasting /= len(test_loader.dataset) print(f'====> Mean test set generator loss: {test_loss_forecasting:.4f}') @@ -176,7 +169,7 @@ def train_latent_model(self, train_loader, test_loader, model_save_path): print(f'====> Mean test set classifier loss: {test_loss_classification:.4f}') # Scheduler metric is test set loss - if training_mode =='forecasting': + if training_mode == 'forecasting': encoder_scheduler.step(test_loss_forecasting) decoder_scheduler.step(test_loss_forecasting) latent_scheduler.step(test_loss_forecasting) @@ -184,97 +177,117 @@ def train_latent_model(self, train_loader, test_loader, model_save_path): classifier_scheduler.step(test_loss_classification) # Save the model at target path - utils.save_model(self.model,PATH = model_save_path) - - - # TRAIN VARIATIONAL AUTOENCODERS-GAN - def train_vaegan(self): - assert self.model_type == 'vaegan' - return NotImplementedError - - # TRAIN INVERSE RL - def train_irl(self): - assert self.model_type == 'irl' - return NotImplementedError - - # TRAIN LSTM - def train_lstm(self): + utils.save_model(self.model, PATH=model_save_path) + + +class LSTMTrainer: + + def __init__(self, + model_type: str, + optimizer_type: str, + device: str, + epochs: int, + input_size: int, + batch_size: int, + hidden_size: int, + num_future: int, + num_layers: int, + output_size: int, + lr_factor:float, + scheduler_patience:int, + batch_first: True, + dropout: float, + reset_state: bool, + bidirectional: bool): + + self.model_type = model_type + self.optimizer_type = optimizer_type + self.device = device + self.epochs = epochs + self.input_size = input_size + self.batch_size = batch_size + self.hidden_size = hidden_size + self.num_future = num_future + self.num_layers = num_layers + self.output_size = output_size + self.batch_first = batch_first + self.lr_factor = lr_factor, + self.scheduler_patience=scheduler_patience, + self.dropout = dropout + self.reset_state = reset_state + self.bidirectional = bidirectional + + self.model_hyperparameters = {'input_size': self.input_size, + 'batch_size': self.batch_size, + 'hidden_size': self.hidden_size, + 'num_future': self.num_future, + 'num_layers': self.num_layers, + 'output_size': self.output_size, + 'batch_first': self.batch_first, + 'reset_state': self.reset_state, + 'bidirectional': self.bidirectional, + 'dropout': self.dropout + } + + self.model = LSTM(**self.model_hyperparameters) + optimizer = Optimizer(self.model_type, self.model, self.optimizer_type[0]) + self.optimizer = optimizer.get_optimizers(lr=0.001).values() + self.scheduler= optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience).values() + + def train(self, train_loader, test_loader, model_save_path): + assert self.model_type == 'lstm' - - # Move the model to target device self.model.to(device) - # Training - for epoch in range(self.epochs): # First half for generative model and next for classifier - if epoch>0: # Initial step is to test and set LR schduler + for epoch in range(self.epochs): # First half for generative model and next for classifier + if epoch > 0: # Initial step is to test and set LR schduler # Training self.model.train() total_loss = 0 - for idx, (data, target,category) in enumerate(self.train_loader): + for idx, (data, target, _) in enumerate(train_loader): # Reset optimizer states - self.encoder_optimizer.zero_grad() - - data, target,category = data.float().to(device), target.float().to(device), category.to(device) - - if self.training_mode =='forecasting': - if self.model_type == 'ae': - decoder_out, latent_out = self.model(data, training=True, is_classification=False) - loss = Criterion.ae_criterion(decoder_out, target) - - else: # vae - decoder_out, latent_out, mu, logvar= self.model(data, training=True, is_classification=False) - loss = Criterion.vae_criterion(decoder_out, target, mu, logvar) - - loss.backward() + self.optimizer.zero_grad() + data, target = data.float().to(device), target.float().to(device) - self.encoder_optimizer.step() - self.decoder_optimizer.step() - self.latent_optimizer.step() + output = self.model(data) + loss = Criterion.lstm_criterion(output, target) + loss.backward() + self.optimizer.step() + total_loss += loss - else: # training_mode == 'classification' - - classifier_out = self.model(data, training=True, is_classification=True) - loss = Criterion.classifier_criterion(classifier_out, category-1) - loss.backward() - self.classifier_optimizer.step() - total_loss+=loss - - print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss/(idx+1))) - - if epoch+1 == self.epochs: # - training_mode = 'classification' + print('Epoch {} | loss {}'.format(epoch, total_loss / (idx + 1))) # Testing - if epoch%10==0: + if epoch % 10 == 0: with torch.no_grad(): self.model.eval() test_loss_forecasting = 0 - test_loss_classification = 0 - for idx, (data, target,category) in enumerate(list(test_loader)): - data, target, category = data.float().to(device), target.float().to(device), category.to(device) - # Time seriesforecasting test - if self.model_type=='ae': - out, latent = self.model(data, training=False, is_classification=False) - test_loss_forecasting += Criterion.ae_criterion(out,target).item() - else: - decoder_out,latent_out,mu,logvar= self.model(data,training=False, is_classification=False) - test_loss_forecasting += Criterion.vae_criterion(decoder_out, target,mu,logvar) - # Classification test - classifier_out= self.model(data,training=False, is_classification=True) - test_loss_classification += Criterion.classifier_criterion(classifier_out, category-1).item() + for idx, (data, target, _) in enumerate(list(test_loader)): + data, target = data.float().to(device), target.float().to(device) + out = self.model(data) + test_loss_forecasting += Criterion.lstm_criterion(out, target).item() test_loss_forecasting /= len(test_loader.dataset) - print(f'====> Mean test set generator loss: {test_loss_forecasting:.4f}') - test_loss_classification /= len(test_loader.dataset) - print(f'====> Mean test set classifier loss: {test_loss_classification:.4f}') + print(f'====> Test set generator loss: {test_loss_forecasting:.4f}') - # Scheduler metric is test set loss - if training_mode =='forecasting': - self.encoder_scheduler.step(self.test_loss_forecasting) - self.decoder_scheduler.step(self.test_loss_forecasting) - self.latent_scheduler.step(self.test_loss_forecasting) - else: - self.classifier_scheduler.step(self.test_loss_classification) + # Scheduler metric is test set loss + self.scheduler.step(test_loss_forecasting) # Save the model at target path - utils.save_model(self.model,PATH = model_save_path) \ No newline at end of file + utils.save_model(self.model, PATH=model_save_path) + + +class VAEGANTrainer: + def __init__(self): + pass + + def train(self): + return NotImplementedError + + +class IRLTrainer: + def __init__(self): + pass + + def train(self): + return NotImplementedError From fa7a57661a1d5805b5619dc75a3e36b0fa2edfa4 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 12:58:19 +0100 Subject: [PATCH 090/211] lstm fix --- traja/models/ae.py | 1 - traja/models/lstm.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index a4ccb6c4..57e380b2 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -1,6 +1,5 @@ import torch from traja.models.utils import TimeDistributed -from traja.models.utils import load_model device = 'cuda' if torch.cuda.is_available() else 'cpu' diff --git a/traja/models/lstm.py b/traja/models/lstm.py index 558f0204..d6b7a894 100644 --- a/traja/models/lstm.py +++ b/traja/models/lstm.py @@ -1,6 +1,8 @@ """Implementation of Multimodel LSTM""" - import torch +from traja.models.utils import TimeDistributed + +device = 'cuda' if torch.cuda.is_available() else 'cpu' class LSTM(torch.nn.Module): From 32dc6b7cd79fd5cbd9936e7e9b34eaac2893c0d1 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 13:00:15 +0100 Subject: [PATCH 091/211] Update optimizers.py --- traja/models/optimizers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index c321fbd6..2ba6877f 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -9,6 +9,7 @@ class Optimizer: def __init__(self, model_type, model, optimizer_type): assert isinstance(model, torch.nn.Module) + print(optimizer_type) assert str(optimizer_type) in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', 'LBFGS', 'ASGD', 'Adamax'] From f8ddc94477bf8298c4f32d44f897e19031c4f26f Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 13:03:13 +0100 Subject: [PATCH 092/211] lstm opt fix --- traja/models/optimizers.py | 2 +- traja/models/train.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 2ba6877f..92be7e41 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -9,7 +9,7 @@ class Optimizer: def __init__(self, model_type, model, optimizer_type): assert isinstance(model, torch.nn.Module) - print(optimizer_type) + assert str(optimizer_type) in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', 'LBFGS', 'ASGD', 'Adamax'] diff --git a/traja/models/train.py b/traja/models/train.py index 8e410e62..53cdcde3 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -80,7 +80,7 @@ def __init__(self, model_type: str, if self.model_type == 'vae': self.model = MultiModelVAE(**self.model_hyperparameters) - optimizer = Optimizer(self.model_type, self.model, self.optimizer_type[0]) + optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) self.model_optimizers = optimizer.get_optimizers(lr=0.001) self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) @@ -230,7 +230,7 @@ def __init__(self, } self.model = LSTM(**self.model_hyperparameters) - optimizer = Optimizer(self.model_type, self.model, self.optimizer_type[0]) + optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) self.optimizer = optimizer.get_optimizers(lr=0.001).values() self.scheduler= optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience).values() From b6e792358daeb13e80a4c0cf8a455a104f2bc1e2 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 13:06:31 +0100 Subject: [PATCH 093/211] Update optimizers.py --- traja/models/optimizers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 92be7e41..c8bec64c 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -30,7 +30,7 @@ def get_optimizers(self, lr=0.0001): Returns: [type]: [description] """ - + print(self.model_type) if self.model_type == 'ae' or 'vae': keys = ['encoder', 'decoder', 'latent', 'classifier'] for network in keys: @@ -44,8 +44,6 @@ def get_optimizers(self, lr=0.0001): elif self.model_type == 'vaegan': return NotImplementedError - else: # LSTM - return NotImplementedError def get_lrschedulers(self, factor=0.1, patience=10): From a72cc645ccb425e75d0f80f7791a6c2eb119b583 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 13:10:21 +0100 Subject: [PATCH 094/211] Update optimizers.py --- traja/models/optimizers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index c8bec64c..72ee2165 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -30,20 +30,20 @@ def get_optimizers(self, lr=0.0001): Returns: [type]: [description] """ - print(self.model_type) + if self.model_type == 'ae' or 'vae': keys = ['encoder', 'decoder', 'latent', 'classifier'] for network in keys: self.optimizers[network] = getattr(torch.optim, f'{self.optimizer_type}')( getattr(self.model, f'{network}').parameters(), lr=lr) - return self.optimizers - if self.model_type == 'lstm': + + elif self.model_type == 'lstm': self.optimizers["lstm"] = torch.optim.Adam(self.model.parameters(), lr=lr) - return self.optimizers elif self.model_type == 'vaegan': return NotImplementedError + return self.optimizers def get_lrschedulers(self, factor=0.1, patience=10): From dbf9713cb15d6cce3123b119f9ebb2cc974c91bf Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 13:20:20 +0100 Subject: [PATCH 095/211] lstm fix --- traja/models/lstm.py | 12 ++++++------ traja/models/optimizers.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/traja/models/lstm.py b/traja/models/lstm.py index d6b7a894..d5f01d51 100644 --- a/traja/models/lstm.py +++ b/traja/models/lstm.py @@ -41,9 +41,9 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, num_layer self.bidirectional = bidirectional # RNN decoder - self.lstm_decoder = torch.nn.LSTM(input_size=self.input_size, hidden_size=self.hidden_size, - num_layers=self.num_layers, dropout=self.dropout, - bidirectional=self.bidirectional, batch_first=True) + self.lstm = torch.nn.LSTM(input_size=self.input_size, hidden_size=self.hidden_size, + num_layers=self.num_layers, dropout=self.dropout, + bidirectional=self.bidirectional, batch_first=True) self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, self.output_size)) def _init_hidden(self): @@ -55,8 +55,8 @@ def forward(self, x): _init_hidden = self._init_hidden() # Decoder input Shape(batch_size, num_futures, latent_size) - dec, (dec_hidden, dec_cell) = self.lstm_decoder(x, _init_hidden) + out, (dec_hidden, dec_cell) = self.lstm(x, _init_hidden) # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) to Time Dsitributed Linear Layer - output = self.output(dec) - return output + out = self.output(out) + return out diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 72ee2165..08ae3aa6 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -38,7 +38,7 @@ def get_optimizers(self, lr=0.0001): getattr(self.model, f'{network}').parameters(), lr=lr) elif self.model_type == 'lstm': - self.optimizers["lstm"] = torch.optim.Adam(self.model.parameters(), lr=lr) + self.optimizers["lstm"] = getattr(torch.optim, f'{self.optimizer_type}')(self.model.parameters(), lr=lr) elif self.model_type == 'vaegan': return NotImplementedError From bcb93d549d4703d42f9626ed7ee17e2ed24594d0 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 13:26:41 +0100 Subject: [PATCH 096/211] Update optimizers.py --- traja/models/optimizers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 08ae3aa6..7aee4147 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -31,15 +31,15 @@ def get_optimizers(self, lr=0.0001): [type]: [description] """ - if self.model_type == 'ae' or 'vae': + if self.model_type == 'lstm': + self.optimizers["lstm"] = getattr(torch.optim, f'{self.optimizer_type}')(self.model.parameters(), lr=lr) + + elif self.model_type == 'ae' or 'vae': keys = ['encoder', 'decoder', 'latent', 'classifier'] for network in keys: self.optimizers[network] = getattr(torch.optim, f'{self.optimizer_type}')( getattr(self.model, f'{network}').parameters(), lr=lr) - elif self.model_type == 'lstm': - self.optimizers["lstm"] = getattr(torch.optim, f'{self.optimizer_type}')(self.model.parameters(), lr=lr) - elif self.model_type == 'vaegan': return NotImplementedError From 6e77b0ee3efc1ff7201e163caf706fbeef72ba3d Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 18:22:29 +0100 Subject: [PATCH 097/211] lstm optim fix --- traja/models/optimizers.py | 1 + traja/models/train.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 7aee4147..ab250a3c 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -57,6 +57,7 @@ def get_lrschedulers(self, factor=0.1, patience=10): Returns: [dict]: [description] """ + for network in self.optimizers.keys(): self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, patience=patience, verbose=True) diff --git a/traja/models/train.py b/traja/models/train.py index 53cdcde3..694f1587 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -56,7 +56,7 @@ def __init__(self, model_type: str, self.dropout = dropout self.bidirectional = bidirectional self.loss_type = loss_type - self.optimizer_type = optimizer_type, + self.optimizer_type = optimizer_type self.lr_factor = lr_factor self.scheduler_patience = scheduler_patience self.model_hyperparameters = {'input_size': self.input_size, From e1cc8bd5cb4c057434881bca162789455fc2038a Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 18:54:37 +0100 Subject: [PATCH 098/211] lstm optim fix --- traja/models/losses.py | 37 +++++++++---------------------------- traja/models/train.py | 7 +++---- 2 files changed, 12 insertions(+), 32 deletions(-) diff --git a/traja/models/losses.py b/traja/models/losses.py index d1a8a074..cb9db996 100644 --- a/traja/models/losses.py +++ b/traja/models/losses.py @@ -1,46 +1,27 @@ import torch -class Criterion(object): +class Criterion: - def __init__(self, model_type): - self.model_type = model_type + def __init__(self): self.huber_loss = torch.nn.SmoothL1Loss(reduction='sum') self.crossentropy_loss = torch.nn.CrossEntropyLoss() - def ae_criterion(self, recon_x, x, loss_type='huber'): - """[summary] + def ae_criterion(self, predicted, target, loss_type='huber'): - Args: - recon_x ([type]): [description] - x ([type]): [description] - loss_type(str): Type of Loss; huber or RMSE - - Returns: - [type]: [description] - """ if loss_type == 'huber': - dist_x = self.huber_loss(recon_x, x) - return dist_x + loss = self.huber_loss(predicted, target) + return loss else: # Root MSE - return torch.sqrt(torch.mean((recon_x - x) ** 2)) + return torch.sqrt(torch.mean((predicted - target) ** 2)) - def vae_criterion(self, recon_x, x, mu, logvar, loss_type='huber'): + def vae_criterion(self, predicted, target, mu, logvar, loss_type='huber'): """Time series generative model loss function - - Args: - recon_x ([type]): [description] - x ([type]): [description] - mu ([type]): [description] - logvar ([type]): [description] - - Returns: - [type]: [description] """ if loss_type == 'huber': - dist_x = self.huber_loss(recon_x, x) + dist_x = self.huber_loss(predicted, target) else: - dist_x = torch.sqrt(torch.mean((recon_x - x) ** 2)) + dist_x = torch.sqrt(torch.mean((predicted - target) ** 2)) KLD = -0.5 * torch.sum(1 + logvar - mu ** 2 - logvar.exp()) return dist_x + KLD diff --git a/traja/models/train.py b/traja/models/train.py index 694f1587..4489ae10 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -1,6 +1,5 @@ from .ae import MultiModelAE from .vae import MultiModelVAE -from .vaegan import MultiModelVAEGAN from .lstm import LSTM from . import utils @@ -153,15 +152,15 @@ def train(self, train_loader, test_loader, model_save_path): # Time series forecasting test if self.model_type == 'ae': out, latent = self.model(data, training=False, is_classification=False) - test_loss_forecasting += Criterion.ae_criterion(out, target).item() + test_loss_forecasting += Criterion().ae_criterion(out, target).item() else: decoder_out, latent_out, mu, logvar = self.model(data, training=False, is_classification=False) - test_loss_forecasting += Criterion.vae_criterion(decoder_out, target, mu, logvar) + test_loss_forecasting += Criterion().vae_criterion(decoder_out, target, mu, logvar) # Classification test classifier_out, latent_out, mu, logvar = self.model(data, training=False, is_classification=True) - test_loss_classification += Criterion.classifier_criterion(classifier_out, category - 1).item() + test_loss_classification += Criterion().classifier_criterion(classifier_out, category - 1).item() test_loss_forecasting /= len(test_loader.dataset) print(f'====> Mean test set generator loss: {test_loss_forecasting:.4f}') From b2e9277ead3b2a46284c7b3ca7ad820111132e52 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 19:26:06 +0100 Subject: [PATCH 099/211] Update train.py --- traja/models/train.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 4489ae10..dd360e57 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -131,8 +131,11 @@ def train(self, train_loader, test_loader, model_save_path): latent_optimizer.step() else: # training_mode == 'classification' - - classifier_out, latent_out, mu, logvar = self.model(data, training=True, is_classification=True) + if self.model_type == 'vae': + classifier_out, latent_out, mu, logvar = self.model(data, training=True, is_classification=True) + else: + classifier_out = self.model(data, training=True, + is_classification=True) loss = Criterion.classifier_criterion(classifier_out, category - 1) loss.backward() classifier_optimizer.step() @@ -158,9 +161,15 @@ def train(self, train_loader, test_loader, model_save_path): is_classification=False) test_loss_forecasting += Criterion().vae_criterion(decoder_out, target, mu, logvar) # Classification test - classifier_out, latent_out, mu, logvar = self.model(data, training=False, - is_classification=True) - test_loss_classification += Criterion().classifier_criterion(classifier_out, category - 1).item() + if self.model_type == 'ae': + classifier_out = self.model(data, training=False, + is_classification=True) + else: + classifier_out, latent_out, mu, logvar = self.model(data, training=False, + is_classification=True) + + test_loss_classification += Criterion().classifier_criterion(classifier_out, + category - 1).item() test_loss_forecasting /= len(test_loader.dataset) print(f'====> Mean test set generator loss: {test_loss_forecasting:.4f}') @@ -192,8 +201,8 @@ def __init__(self, num_future: int, num_layers: int, output_size: int, - lr_factor:float, - scheduler_patience:int, + lr_factor: float, + scheduler_patience: int, batch_first: True, dropout: float, reset_state: bool, @@ -211,7 +220,7 @@ def __init__(self, self.output_size = output_size self.batch_first = batch_first self.lr_factor = lr_factor, - self.scheduler_patience=scheduler_patience, + self.scheduler_patience = scheduler_patience, self.dropout = dropout self.reset_state = reset_state self.bidirectional = bidirectional @@ -231,7 +240,7 @@ def __init__(self, self.model = LSTM(**self.model_hyperparameters) optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) self.optimizer = optimizer.get_optimizers(lr=0.001).values() - self.scheduler= optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience).values() + self.scheduler = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience).values() def train(self, train_loader, test_loader, model_save_path): From b8ec374262fa337b51fe15fbeac23a222cca96ff Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 19:28:29 +0100 Subject: [PATCH 100/211] Update train.py --- traja/models/train.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index dd360e57..218f90eb 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -118,12 +118,12 @@ def train(self, train_loader, test_loader, model_save_path): if training_mode == 'forecasting': if self.model_type == 'ae': decoder_out, latent_out = self.model(data, training=True, is_classification=False) - loss = Criterion.ae_criterion(decoder_out, target) + loss = Criterion().ae_criterion(decoder_out, target) else: # vae decoder_out, latent_out, mu, logvar = self.model(data, training=True, is_classification=False) - loss = Criterion.vae_criterion(decoder_out, target, mu, logvar) + loss = Criterion().vae_criterion(decoder_out, target, mu, logvar) loss.backward() encoder_optimizer.step() @@ -136,7 +136,7 @@ def train(self, train_loader, test_loader, model_save_path): else: classifier_out = self.model(data, training=True, is_classification=True) - loss = Criterion.classifier_criterion(classifier_out, category - 1) + loss = Criterion().classifier_criterion(classifier_out, category - 1) loss.backward() classifier_optimizer.step() total_loss += loss From 06ed630e75233d48e24ac624fb77b29cd77c57da Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 20:34:28 +0100 Subject: [PATCH 101/211] Update optimizers.py --- traja/models/optimizers.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index ab250a3c..cc5b485f 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -42,7 +42,6 @@ def get_optimizers(self, lr=0.0001): elif self.model_type == 'vaegan': return NotImplementedError - return self.optimizers def get_lrschedulers(self, factor=0.1, patience=10): @@ -57,10 +56,13 @@ def get_lrschedulers(self, factor=0.1, patience=10): Returns: [dict]: [description] """ - - for network in self.optimizers.keys(): - self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, - patience=patience, verbose=True) + if self.model_type == 'lstm': + self.schedulers["lstm"] = ReduceLROnPlateau(self.optimizers["lstm"], mode='max', factor=factor, + patience=patience, verbose=True) + else: + for network in self.optimizers.keys(): + self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, + patience=patience, verbose=True) return self.schedulers From b17d883e9c70e41be5add415ea9d6ff51c4ab1a7 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 20:53:41 +0100 Subject: [PATCH 102/211] Update optimizers.py --- traja/models/optimizers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index cc5b485f..645be421 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -57,8 +57,8 @@ def get_lrschedulers(self, factor=0.1, patience=10): [dict]: [description] """ if self.model_type == 'lstm': - self.schedulers["lstm"] = ReduceLROnPlateau(self.optimizers["lstm"], mode='max', factor=factor, - patience=patience, verbose=True) + self.schedulers["lstm"] = ReduceLROnPlateau(self.optimizers["lstm"], mode='max', + patience=patience, verbose=True) else: for network in self.optimizers.keys(): self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, From c5aa956239bef07e3dfd113b60f805ae0c990de4 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 21:02:12 +0100 Subject: [PATCH 103/211] Update optimizers.py --- traja/models/optimizers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 645be421..e6858f37 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -57,7 +57,7 @@ def get_lrschedulers(self, factor=0.1, patience=10): [dict]: [description] """ if self.model_type == 'lstm': - self.schedulers["lstm"] = ReduceLROnPlateau(self.optimizers["lstm"], mode='max', + self.schedulers["lstm"] = ReduceLROnPlateau(self.optimizers["lstm"], mode='max', factor= factor, patience=patience, verbose=True) else: for network in self.optimizers.keys(): From 624f6efb52adf1dbf2a1f6a5f2376410dac3dcf0 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 21:12:22 +0100 Subject: [PATCH 104/211] lstm optim fix --- traja/models/optimizers.py | 2 +- traja/models/train.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index e6858f37..5e0f165e 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -44,7 +44,7 @@ def get_optimizers(self, lr=0.0001): return NotImplementedError return self.optimizers - def get_lrschedulers(self, factor=0.1, patience=10): + def get_lrschedulers(self, factor:float, patience:int): """Learning rate scheduler for each network in the model NOTE: Scheduler metric should be test set loss diff --git a/traja/models/train.py b/traja/models/train.py index 218f90eb..01af35d1 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -219,8 +219,8 @@ def __init__(self, self.num_layers = num_layers self.output_size = output_size self.batch_first = batch_first - self.lr_factor = lr_factor, - self.scheduler_patience = scheduler_patience, + self.lr_factor = lr_factor + self.scheduler_patience = scheduler_patience self.dropout = dropout self.reset_state = reset_state self.bidirectional = bidirectional From 1e497ed60e6efca83047fd456ab8999977e008aa Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 21:17:54 +0100 Subject: [PATCH 105/211] Update train.py --- traja/models/train.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 01af35d1..31c91e0b 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -247,8 +247,8 @@ def train(self, train_loader, test_loader, model_save_path): assert self.model_type == 'lstm' self.model.to(device) - for epoch in range(self.epochs): # First half for generative model and next for classifier - if epoch > 0: # Initial step is to test and set LR schduler + for epoch in range(self.epochs): + if epoch > 0: # Training self.model.train() total_loss = 0 @@ -258,7 +258,7 @@ def train(self, train_loader, test_loader, model_save_path): data, target = data.float().to(device), target.float().to(device) output = self.model(data) - loss = Criterion.lstm_criterion(output, target) + loss = Criterion().lstm_criterion(output, target) loss.backward() self.optimizer.step() total_loss += loss @@ -273,7 +273,7 @@ def train(self, train_loader, test_loader, model_save_path): for idx, (data, target, _) in enumerate(list(test_loader)): data, target = data.float().to(device), target.float().to(device) out = self.model(data) - test_loss_forecasting += Criterion.lstm_criterion(out, target).item() + test_loss_forecasting += Criterion().lstm_criterion(out, target).item() test_loss_forecasting /= len(test_loader.dataset) print(f'====> Test set generator loss: {test_loss_forecasting:.4f}') From 826c201b22b108994e7776397ecdc57cf1fc6d8c Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 21:31:41 +0100 Subject: [PATCH 106/211] lstm optim fix --- traja/models/optimizers.py | 2 +- traja/models/train.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 5e0f165e..cd1471c3 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -57,7 +57,7 @@ def get_lrschedulers(self, factor:float, patience:int): [dict]: [description] """ if self.model_type == 'lstm': - self.schedulers["lstm"] = ReduceLROnPlateau(self.optimizers["lstm"], mode='max', factor= factor, + self.schedulers= ReduceLROnPlateau(self.optimizers["lstm"], mode='max', factor= factor, patience=patience, verbose=True) else: for network in self.optimizers.keys(): diff --git a/traja/models/train.py b/traja/models/train.py index 31c91e0b..ed4b4e31 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -240,7 +240,7 @@ def __init__(self, self.model = LSTM(**self.model_hyperparameters) optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) self.optimizer = optimizer.get_optimizers(lr=0.001).values() - self.scheduler = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience).values() + self.scheduler = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) def train(self, train_loader, test_loader, model_save_path): @@ -249,14 +249,11 @@ def train(self, train_loader, test_loader, model_save_path): for epoch in range(self.epochs): if epoch > 0: - # Training self.model.train() total_loss = 0 for idx, (data, target, _) in enumerate(train_loader): - # Reset optimizer states self.optimizer.zero_grad() data, target = data.float().to(device), target.float().to(device) - output = self.model(data) loss = Criterion().lstm_criterion(output, target) loss.backward() From 40a27418d46b5a515fd1b133c2e48d7f635b7c04 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 21:35:38 +0100 Subject: [PATCH 107/211] Update optimizers.py --- traja/models/optimizers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index cd1471c3..2b472a58 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -32,7 +32,7 @@ def get_optimizers(self, lr=0.0001): """ if self.model_type == 'lstm': - self.optimizers["lstm"] = getattr(torch.optim, f'{self.optimizer_type}')(self.model.parameters(), lr=lr) + self.optimizers = getattr(torch.optim, f'{self.optimizer_type}')(self.model.parameters(), lr=lr) elif self.model_type == 'ae' or 'vae': keys = ['encoder', 'decoder', 'latent', 'classifier'] @@ -57,7 +57,8 @@ def get_lrschedulers(self, factor:float, patience:int): [dict]: [description] """ if self.model_type == 'lstm': - self.schedulers= ReduceLROnPlateau(self.optimizers["lstm"], mode='max', factor= factor, + assert not isinstance(self.optimizers, dict) + self.schedulers= ReduceLROnPlateau(self.optimizers, mode='max', factor= factor, patience=patience, verbose=True) else: for network in self.optimizers.keys(): From eb88374d04812293d1cc7670f0934a715a41ec1f Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Thu, 17 Dec 2020 21:36:29 +0100 Subject: [PATCH 108/211] Update train.py --- traja/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index ed4b4e31..ccacfe7d 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -239,7 +239,7 @@ def __init__(self, self.model = LSTM(**self.model_hyperparameters) optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) - self.optimizer = optimizer.get_optimizers(lr=0.001).values() + self.optimizer = optimizer.get_optimizers(lr=0.001) self.scheduler = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) def train(self, train_loader, test_loader, model_save_path): From a6e9a93a44d71477520c63dac0e473fa7c3919ee Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 18 Dec 2020 16:41:52 +0100 Subject: [PATCH 109/211] documentation, lstm,ae,vae wrappers, trainers --- traja/models/ae.py | 60 ++++++++++++++------------ traja/models/train.py | 99 +++++++++++++++++++++++++++++++++++++++---- traja/models/vae.py | 64 +++++++++++++++------------- 3 files changed, 158 insertions(+), 65 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index 57e380b2..5756ba3f 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -24,7 +24,7 @@ class LSTMEncoder(torch.nn.Module): """ def __init__(self, input_size: int, sequence_length: int, batch_size: int, - hidden_size: int, num_layers: int, + hidden_size: int, num_lstm_layers: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMEncoder, self).__init__() @@ -33,14 +33,14 @@ def __init__(self, input_size: int, sequence_length: int, batch_size: int, self.sequence_length = sequence_length self.batch_size = batch_size self.hidden_size = hidden_size - self.num_layers = num_layers + self.num_lstm_layers = num_lstm_layers self.batch_first = batch_first self.dropout = dropout self.reset_state = reset_state self.bidirectional = bidirectional self.lstm_encoder = torch.nn.LSTM(input_size=input_size, hidden_size=self.hidden_size, - num_layers=num_layers, dropout=dropout, + num_layers=num_lstm_layers, dropout=dropout, bidirectional=self.bidirectional, batch_first=True) def _init_hidden(self): @@ -50,7 +50,8 @@ def _init_hidden(self): def forward(self, x): enc_init_hidden = self._init_hidden() enc_output, _ = self.lstm_encoder(x, enc_init_hidden) - # RNNs obeys, Markovian. Consider the last state of the hidden is the markovian of the entire sequence in that batch. + # RNNs obeys, Markovian. So, the last state of the hidden is the markovian state for the entire + # sequence in that batch. enc_output = enc_output[:, -1, :] # Shape(batch_size,hidden_dim) return enc_output @@ -92,7 +93,7 @@ class LSTMDecoder(torch.nn.Module): """ def __init__(self, batch_size: int, num_future: int, hidden_size: int, - num_layers: int, output_size: int, latent_size: int, + num_lstm_layers: int, output_size: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMDecoder, self).__init__() @@ -100,7 +101,7 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, self.latent_size = latent_size self.num_future = num_future self.hidden_size = hidden_size - self.num_layers = num_layers + self.num_lstm_layers = num_lstm_layers self.output_size = output_size self.batch_first = batch_first self.dropout = dropout @@ -110,7 +111,7 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, # RNN decoder self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, hidden_size=self.hidden_size, - num_layers=self.num_layers, + num_layers=self.num_lstm_layers, dropout=self.dropout, bidirectional=self.bidirectional, batch_first=True) @@ -148,27 +149,27 @@ class MLPClassifier(torch.nn.Module): """ MLP classifier """ - def __init__(self, hidden_size: int, num_classes: int, latent_size: int, + def __init__(self, hidden_size: int, num_classes: int, latent_size: int, num_classifier_layers: int, dropout: float): super(MLPClassifier, self).__init__() self.latent_size = latent_size self.hidden_size = hidden_size self.num_classes = num_classes + self.num_classifier_layers = num_classifier_layers self.dropout = dropout # Classifier layers - self.classifier1 = torch.nn.Linear(self.latent_size, self.hidden_size) - self.classifier2 = torch.nn.Linear(self.hidden_size, self.hidden_size) - self.classifier3 = torch.nn.Linear(self.hidden_size, self.hidden_size) - self.classifier4 = torch.nn.Linear(self.hidden_size, self.num_classes) + self.inp = torch.nn.Linear(self.latent_size, self.hidden_size) + self.hidden = torch.nn.ModuleList([torch.nn.Linear(hidden_size, hidden_size) + for _ in range(self.num_classifier_layers)]) + self.out = torch.nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) def forward(self, x): - classifier1 = self.dropout(self.classifier1(x)) - classifier2 = self.dropout(self.classifier2(classifier1)) - classifier3 = self.dropout(self.classifier3(classifier2)) - classifier4 = self.classifier4(classifier3) - return classifier4 + x = self.dropout(self.inp(x)) + x = self.dropout(self.hidden(x)) + out = self.out(x) + return out class MultiModelAE(torch.nn.Module): @@ -177,8 +178,10 @@ def __init__(self, input_size: int, sequence_length: int, batch_size: int, num_future: int, - hidden_size: int, - num_layers: int, + lstm_hidden_size: int, + num_lstm_layers: int, + classifier_hidden_size: int, + num_classifier_layers: int, output_size: int, num_classes: int, latent_size: int, @@ -193,8 +196,10 @@ def __init__(self, input_size: int, self.batch_size = batch_size self.latent_size = latent_size self.num_future = num_future - self.hidden_size = hidden_size - self.num_layers = num_layers + self.lstm_hidden_size = lstm_hidden_size + self.num_lstm_layers = num_lstm_layers + self.num_classifier_layers = num_classifier_layers + self.classifier_hidden_size = classifier_hidden_size self.output_size = output_size self.num_classes = num_classes self.batch_first = batch_first @@ -205,21 +210,21 @@ def __init__(self, input_size: int, self.encoder = LSTMEncoder(input_size=self.input_size, sequence_length=self.sequence_length, batch_size=self.batch_size, - hidden_size=self.hidden_size, - num_layers=self.num_layers, + hidden_size=self.lstm_hidden_size, + num_lstm_layers=self.num_lstm_layers, batch_first=self.batch_first, dropout=self.dropout, reset_state=True, bidirectional=self.bidirectional) - self.latent = DisentangledAELatent(hidden_size=self.hidden_size, + self.latent = DisentangledAELatent(hidden_size=self.lstm_hidden_size, latent_size=self.latent_size, dropout=self.dropout) self.decoder = LSTMDecoder(batch_size=self.batch_size, num_future=self.num_future, - hidden_size=self.hidden_size, - num_layers=self.num_layers, + hidden_size=self.lstm_hidden_size, + num_lstm_layers=self.num_lstm_layers, output_size=self.output_size, latent_size=self.latent_size, batch_first=self.batch_first, @@ -227,9 +232,10 @@ def __init__(self, input_size: int, reset_state=True, bidirectional=self.bidirectional) - self.classifier = MLPClassifier(hidden_size=self.hidden_size, + self.classifier = MLPClassifier(hidden_size=self.classifier_hidden_size, num_classes=self.num_classes, latent_size=self.latent_size, + num_classifier_layers=self.num_classifier_layers, dropout=self.dropout) def forward(self, data, training=True, is_classification=False): diff --git a/traja/models/train.py b/traja/models/train.py index ccacfe7d..4a7fb1a5 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -11,6 +11,38 @@ class LatentModelTrainer(object): + """ + Wrapper for training and testing the LSTM model + + :param model_type: Type of model should be "LSTM" + :param optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', + 'AdamW', 'SparseAdam', 'RMSprop', ' + Rprop', 'LBFGS', 'ASGD', 'Adamax'] + :param device: Selected device; 'cuda' or 'cpu' + :param input_size: The number of expected features in the input x + :param output_size: Output feature dimension + :param lstm_hidden_size: The number of features in the hidden state h + :param num_layers: Number of layers in the LSTM model + :param reset_state: If True, will reset the hidden and cell state for each batch of data + :param num_classes: + :param latent_size: + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param num_classifier_layers: Number of layers in the classifier + :param epochs: Number of epochs to train the network + :param batch_size: Number of samples in a batch + :param num_future: Number of time steps to be predicted forward + :param sequence_length: Number of past time steps otherwise, length of sequences in each batch of data. + :param bidirectional: If True, becomes a bidirectional LSTM + :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + :param loss_type: + :param lr_factor: Factor by which the learning rate will be reduced + :param scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. + For example, if patience = 2, then we will ignore the first 2 epochs with no + improvement, and will only decrease the LR after the 3rd epoch if the loss still + hasn’t improved then. + + """ def __init__(self, model_type: str, optimizer_type: str, @@ -18,12 +50,13 @@ def __init__(self, model_type: str, input_size: int, output_size: int, lstm_hidden_size: int, - lstm_num_layers: int, + num_lstm_layers: int, + classifier_hidden_size: int, + num_classifier_layers: int, reset_state: bool, num_classes: int, latent_size: int, dropout: float, - num_layers: int, epochs: int, batch_size: int, num_future: int, @@ -36,18 +69,21 @@ def __init__(self, model_type: str, white_keys = ['ae', 'vae'] assert model_type in white_keys, "Valid models are {}".format(white_keys) + assert sequence_length == num_future, "For Autoencoders and Variational Autoencoders, " \ + "sequence length==num_past== or != num_future" self.model_type = model_type self.device = device self.input_size = input_size self.lstm_hidden_size = lstm_hidden_size - self.lstm_num_layers = lstm_num_layers - self.hidden_size = lstm_hidden_size # For classifiers too + self.num_lstm_layers = num_lstm_layers + self.classifier_hidden_size = classifier_hidden_size + self.num_classifier_layers = num_classifier_layers self.batch_first = batch_first self.reset_state = reset_state self.output_size = output_size self.num_classes = num_classes self.latent_size = latent_size - self.num_layers = num_layers + self.num_classifier_layers = num_classifier_layers self.num_future = num_future self.epochs = epochs self.batch_size = batch_size @@ -61,9 +97,11 @@ def __init__(self, model_type: str, self.model_hyperparameters = {'input_size': self.input_size, 'sequence_length': self.sequence_length, 'batch_size': self.batch_size, - 'hidden_size': self.lstm_hidden_size, + 'lstm_hidden_size': self.lstm_hidden_size, + 'num_lstm_layers':self.num_lstm_layers, + 'clasifier_hidden_size':self.classifier_hidden_size, + 'num_classifier_layers':self.num_classifier_layers, 'num_future': self.num_future, - 'num_layers': self.lstm_num_layers, 'latent_size': self.latent_size, 'output_size': self.output_size, 'num_classes': self.num_classes, @@ -73,14 +111,16 @@ def __init__(self, model_type: str, 'dropout': self.dropout } + # Instantiate model instance based on model_type if self.model_type == 'ae': self.model = MultiModelAE(**self.model_hyperparameters) if self.model_type == 'vae': self.model = MultiModelVAE(**self.model_hyperparameters) + # Model optimizer and the learning rate scheduler based on user defined optimizer_type + # and learning rate parameters optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) - self.model_optimizers = optimizer.get_optimizers(lr=0.001) self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) @@ -88,6 +128,15 @@ def __str__(self): return "Training model type {}".format(self.model_type) def train(self, train_loader, test_loader, model_save_path): + """ + This method implements the batch- wise training and testing protocol for both time series forecasting and + classification of the timeseries + + :param train_loader: Dataloader object of train dataset with batch data [data,target,category] + :param test_loader: Dataloader object of test dataset with [data,target,category] + :param model_save_path: Directory path to save the model + :return: None + """ assert self.model_type == 'ae' or 'vae' self.model.to(device) @@ -189,6 +238,33 @@ def train(self, train_loader, test_loader, model_save_path): class LSTMTrainer: + """ + Wrapper for training and testing the LSTM model + + :param model_type: Type of model should be "LSTM" + :param optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', + 'AdamW', 'SparseAdam', 'RMSprop', ' + Rprop', 'LBFGS', 'ASGD', 'Adamax'] + :param device: Selected device; 'cuda' or 'cpu' + :param epochs: Number of epochs to train the network + :param input_size: The number of expected features in the input x + :param batch_size: Number of samples in a batch + :param hidden_size: The number of features in the hidden state h + :param num_future: Number of time steps to be predicted forward + :param num_layers: Number of layers in the LSTM model + :param output_size: Output feature dimension + :param lr_factor: Factor by which the learning rate will be reduced + :param scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. + For example, if patience = 2, then we will ignore the first 2 epochs with no + improvement, and will only decrease the LR after the 3rd epoch if the loss still + hasn’t improved then. + :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param reset_state: If True, will reset the hidden and cell state for each batch of data + :param bidirectional: If True, becomes a bidirectional LSTM + + """ def __init__(self, model_type: str, @@ -207,7 +283,6 @@ def __init__(self, dropout: float, reset_state: bool, bidirectional: bool): - self.model_type = model_type self.optimizer_type = optimizer_type self.device = device @@ -244,6 +319,12 @@ def __init__(self, def train(self, train_loader, test_loader, model_save_path): + """ Implements the batch wise training and testing for time series forecasting + :param train_loader: Dataloader object of train dataset with batch data [data,target,category] + :param test_loader: Dataloader object of test dataset with [data,target,category] + :param model_save_path: Directory path to save the model + :return: None""" + assert self.model_type == 'lstm' self.model.to(device) diff --git a/traja/models/vae.py b/traja/models/vae.py index b690a30a..3f64aa92 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -50,7 +50,7 @@ class LSTMEncoder(torch.nn.Module): """ def __init__(self, input_size: int, sequence_length: int, batch_size: int, - hidden_size: int, num_layers: int, + hidden_size: int, num_lstm_layers: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMEncoder, self).__init__() @@ -59,14 +59,14 @@ def __init__(self, input_size: int, sequence_length: int, batch_size: int, self.sequence_length = sequence_length self.batch_size = batch_size self.hidden_size = hidden_size - self.num_layers = num_layers + self.num_lstm_layers = num_lstm_layers self.batch_first = batch_first self.dropout = dropout self.reset_state = reset_state self.bidirectional = bidirectional self.lstm_encoder = torch.nn.LSTM(input_size=input_size, hidden_size=self.hidden_size, - num_layers=num_layers, dropout=dropout, + num_layers=num_lstm_layers, dropout=dropout, bidirectional=self.bidirectional, batch_first=True) def _init_hidden(self): @@ -76,7 +76,8 @@ def _init_hidden(self): def forward(self, x): enc_init_hidden = self._init_hidden() enc_output, _ = self.lstm_encoder(x, enc_init_hidden) - # RNNs obeys, Markovian. Consider the last state of the hidden is the markovian of the entire sequence in that batch. + # RNNs obeys, Markovian. So, the last state of the hidden is the markovian state for the entire + # sequence in that batch. enc_output = enc_output[:, -1, :] # Shape(batch_size,hidden_dim) return enc_output @@ -130,7 +131,7 @@ class LSTMDecoder(torch.nn.Module): """ def __init__(self, batch_size: int, num_future: int, hidden_size: int, - num_layers: int, output_size: int, latent_size: int, + num_lstm_layers: int, output_size: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMDecoder, self).__init__() @@ -138,7 +139,7 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, self.latent_size = latent_size self.num_future = num_future self.hidden_size = hidden_size - self.num_layers = num_layers + self.num_lstm_layers = num_lstm_layers self.output_size = output_size self.batch_first = batch_first self.dropout = dropout @@ -148,7 +149,7 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, # RNN decoder self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, hidden_size=self.hidden_size, - num_layers=self.num_layers, + num_layers=self.num_lstm_layers, dropout=self.dropout, bidirectional=self.bidirectional, batch_first=True) @@ -163,8 +164,8 @@ def _init_hidden(self): def forward(self, x, num_future=None): - # To feed the latent states into lstm decoder, repeat the - # tensor n_future times at second dim + # To feed the latent states into lstm decoder, + # repeat the tensor n_future times at second dim _init_hidden = self._init_hidden() decoder_inputs = x.unsqueeze(1) @@ -186,27 +187,27 @@ class MLPClassifier(torch.nn.Module): """ MLP classifier """ - def __init__(self, hidden_size: int, num_classes: int, latent_size: int, + def __init__(self, hidden_size: int, num_classes: int, latent_size: int, num_classifier_layers: int, dropout: float): super(MLPClassifier, self).__init__() self.latent_size = latent_size self.hidden_size = hidden_size self.num_classes = num_classes + self.num_classifier_layers = num_classifier_layers self.dropout = dropout # Classifier layers - self.classifier1 = torch.nn.Linear(self.latent_size, self.hidden_size) - self.classifier2 = torch.nn.Linear(self.hidden_size, self.hidden_size) - self.classifier3 = torch.nn.Linear(self.hidden_size, self.hidden_size) - self.classifier4 = torch.nn.Linear(self.hidden_size, self.num_classes) + self.inp = torch.nn.Linear(self.latent_size, self.hidden_size) + self.hidden = torch.nn.ModuleList([torch.nn.Linear(hidden_size, hidden_size) + for _ in range(self.num_classifier_layers)]) + self.out = torch.nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) def forward(self, x): - classifier1 = self.dropout(self.classifier1(x)) - classifier2 = self.dropout(self.classifier2(classifier1)) - classifier3 = self.dropout(self.classifier3(classifier2)) - classifier4 = self.classifier4(classifier3) - return classifier4 + x = self.dropout(self.inp(x)) + x = self.dropout(self.hidden(x)) + out = self.out(x) + return out class MultiModelVAE(torch.nn.Module): @@ -217,8 +218,10 @@ def __init__(self, input_size: int, sequence_length: int, batch_size: int, num_future: int, - hidden_size: int, - num_layers: int, + lstm_hidden_size: int, + num_lstm_layers: int, + classifier_hidden_size: int, + num_classifier_layers: int, output_size: int, num_classes: int, latent_size: int, @@ -233,8 +236,10 @@ def __init__(self, input_size: int, self.batch_size = batch_size self.latent_size = latent_size self.num_future = num_future - self.hidden_size = hidden_size - self.num_layers = num_layers + self.lstm_hidden_size = lstm_hidden_size + self.num_lstm_layers = num_lstm_layers + self.classifier_hidden_size = classifier_hidden_size + self.num_classifier_layers = num_classifier_layers self.output_size = output_size self.num_classes = num_classes self.batch_first = batch_first @@ -245,21 +250,21 @@ def __init__(self, input_size: int, self.encoder = LSTMEncoder(input_size=self.input_size, sequence_length=self.sequence_length, batch_size=self.batch_size, - hidden_size=self.hidden_size, - num_layers=self.num_layers, + hidden_size=self.lstm_hidden_size, + num_layers=self.num_lstm_layers, batch_first=self.batch_first, dropout=self.dropout, reset_state=True, bidirectional=self.bidirectional) - self.latent = DisentangledAELatent(hidden_size=self.hidden_size, + self.latent = DisentangledAELatent(hidden_size=self.lstm_hidden_size, latent_size=self.latent_size, dropout=self.dropout) self.decoder = LSTMDecoder(batch_size=self.batch_size, num_future=self.num_future, - hidden_size=self.hidden_size, - num_layers=self.num_layers, + hidden_size=self.lstm_hidden_size, + num_layers=self.num_lstm_layers, output_size=self.output_size, latent_size=self.latent_size, batch_first=self.batch_first, @@ -267,9 +272,10 @@ def __init__(self, input_size: int, reset_state=True, bidirectional=self.bidirectional) - self.classifier = MLPClassifier(hidden_size=self.hidden_size, + self.classifier = MLPClassifier(hidden_size=self.classifier_hidden_size, num_classes=self.num_classes, latent_size=self.latent_size, + num_classifier_layers=self.num_classifier_layers, dropout=self.dropout) def forward(self, data, training=True, is_classification=False): From eea1f5ae1f9b90314963ee4e86520fb3a90bc90e Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Fri, 18 Dec 2020 11:02:25 +0000 Subject: [PATCH 110/211] Send vae encoder tensors to device --- traja/models/vae.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/traja/models/vae.py b/traja/models/vae.py index 3f64aa92..91167b9f 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -70,8 +70,11 @@ def __init__(self, input_size: int, sequence_length: int, batch_size: int, bidirectional=self.bidirectional, batch_first=True) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size), - torch.zeros(self.num_layers, self.batch_size, self.hidden_size)) + return (torch.zeros(self.num_layers, self.batch_size, + self.hidden_size).to(device), + torch.zeros(self.num_layers, self.batch_size, + self.hidden_size).to(device)) + def forward(self, x): enc_init_hidden = self._init_hidden() From a02d0c449990ad44d304e135a2f3345c4185601a Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Fri, 18 Dec 2020 11:02:39 +0000 Subject: [PATCH 111/211] Import MultiModalVAE --- traja/models/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/traja/models/__init__.py b/traja/models/__init__.py index 6def7da5..54b2acdf 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1 +1,3 @@ -from .nn import LSTM \ No newline at end of file +from .nn import LSTM +from .vae import MultiModelVAE + From 2598f973cdf2ae65cc2b1db6c23d5c0435e2037b Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Fri, 18 Dec 2020 11:20:29 +0000 Subject: [PATCH 112/211] Pass only latent output to decoder --- traja/models/vae.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/vae.py b/traja/models/vae.py index 91167b9f..0ddeb2af 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -297,7 +297,7 @@ def forward(self, data, training=True, is_classification=False): # Encoder enc_out = self.encoder(data) # Latent - latent_out, mu, logvar = self.latent(enc_out) + latent_out, mu, logvar = self.latent(enc_out, training=training) # Decoder decoder_out = self.decoder(latent_out) return decoder_out, latent_out, mu, logvar From 38827987b20d92b654dc2145b88beaa505286743 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Fri, 18 Dec 2020 11:46:19 +0000 Subject: [PATCH 113/211] Multiply latent size by 2 in vae --- traja/models/vae.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/traja/models/vae.py b/traja/models/vae.py index 0ddeb2af..09eca807 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -93,10 +93,9 @@ def __init__(self, hidden_size: int, latent_size: int, dropout: float): self.latent_size = latent_size self.hidden_size = hidden_size self.dropout = dropout - self.latent = torch.nn.Linear(self.hidden_size, self.latent_size*2) - - @staticmethod - def reparameterize(mu, logvar, training=True): + self.latent = torch.nn.Linear(self.hidden_size, self.latent_size * 2) + + def reparameterize(self, mu, logvar, training= True): if training: std = logvar.mul(0.5).exp_() eps = std.data.new(std.size()).normal_() From 0d69790a07acc70c54a6baee10e8cdc97cf5e29f Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 18 Dec 2020 19:50:32 +0100 Subject: [PATCH 114/211] Update train.py --- traja/models/train.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 4a7fb1a5..0b0afc9c 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -69,8 +69,7 @@ def __init__(self, model_type: str, white_keys = ['ae', 'vae'] assert model_type in white_keys, "Valid models are {}".format(white_keys) - assert sequence_length == num_future, "For Autoencoders and Variational Autoencoders, " \ - "sequence length==num_past== or != num_future" + self.model_type = model_type self.device = device self.input_size = input_size From 93cec7f25273c562830e7f7916c169da8f7bb038 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 18 Dec 2020 19:54:30 +0100 Subject: [PATCH 115/211] Update train.py --- traja/models/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 0b0afc9c..c322366e 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -69,7 +69,7 @@ def __init__(self, model_type: str, white_keys = ['ae', 'vae'] assert model_type in white_keys, "Valid models are {}".format(white_keys) - + self.model_type = model_type self.device = device self.input_size = input_size @@ -98,7 +98,7 @@ def __init__(self, model_type: str, 'batch_size': self.batch_size, 'lstm_hidden_size': self.lstm_hidden_size, 'num_lstm_layers':self.num_lstm_layers, - 'clasifier_hidden_size':self.classifier_hidden_size, + 'classifier_hidden_size':self.classifier_hidden_size, 'num_classifier_layers':self.num_classifier_layers, 'num_future': self.num_future, 'latent_size': self.latent_size, From 74a0c5352d04ed62532a5c1dbe2089ed49fed1c7 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 18 Dec 2020 19:59:56 +0100 Subject: [PATCH 116/211] lstm update --- traja/models/ae.py | 8 ++++---- traja/models/vae.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index 5756ba3f..80e0dc51 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -44,8 +44,8 @@ def __init__(self, input_size: int, sequence_length: int, batch_size: int, bidirectional=self.bidirectional, batch_first=True) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size), - torch.zeros(self.num_layers, self.batch_size, self.hidden_size)) + return (torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size), + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size)) def forward(self, x): enc_init_hidden = self._init_hidden() @@ -119,9 +119,9 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, self.output_size)) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, + return (torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device), - torch.zeros(self.num_layers, self.batch_size, + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device)) def forward(self, x, num_future=None): diff --git a/traja/models/vae.py b/traja/models/vae.py index 09eca807..25abfa44 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -253,7 +253,7 @@ def __init__(self, input_size: int, sequence_length=self.sequence_length, batch_size=self.batch_size, hidden_size=self.lstm_hidden_size, - num_layers=self.num_lstm_layers, + num_lstm_layers=self.num_lstm_layers, batch_first=self.batch_first, dropout=self.dropout, reset_state=True, @@ -266,7 +266,7 @@ def __init__(self, input_size: int, self.decoder = LSTMDecoder(batch_size=self.batch_size, num_future=self.num_future, hidden_size=self.lstm_hidden_size, - num_layers=self.num_lstm_layers, + num_lstm_layers=self.num_lstm_layers, output_size=self.output_size, latent_size=self.latent_size, batch_first=self.batch_first, From 77da4ba41baf9494868fcb1affd2d013bf146fb9 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 18 Dec 2020 20:01:41 +0100 Subject: [PATCH 117/211] Update vae.py --- traja/models/vae.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/vae.py b/traja/models/vae.py index 25abfa44..85ba6971 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -70,7 +70,7 @@ def __init__(self, input_size: int, sequence_length: int, batch_size: int, bidirectional=self.bidirectional, batch_first=True) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, + return (torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device), torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) @@ -159,7 +159,7 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, self.output_size)) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, + return (torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device), torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) From 610a3ee05896dec2d0b81cd2a474b23115602bb0 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 18 Dec 2020 20:03:13 +0100 Subject: [PATCH 118/211] Update vae.py --- traja/models/vae.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/vae.py b/traja/models/vae.py index 85ba6971..6ea80117 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -72,7 +72,7 @@ def __init__(self, input_size: int, sequence_length: int, batch_size: int, def _init_hidden(self): return (torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device), - torch.zeros(self.num_layers, self.batch_size, + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device)) @@ -161,7 +161,7 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, def _init_hidden(self): return (torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device), - torch.zeros(self.num_layers, self.batch_size, + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to(device)) def forward(self, x, num_future=None): From 42516bce164ce0c154c0773ed9dfa09c050c7a8f Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 18 Dec 2020 21:20:54 +0100 Subject: [PATCH 119/211] classifier update --- traja/models/ae.py | 15 +++++++-------- traja/models/vae.py | 13 ++++++------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index 80e0dc51..a5d14a62 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -1,6 +1,6 @@ import torch from traja.models.utils import TimeDistributed - +from torch import nn device = 'cuda' if torch.cuda.is_available() else 'cpu' @@ -17,7 +17,7 @@ class LSTMEncoder(torch.nn.Module): with the second LSTM taking in outputs of the first LSTM and computing the final results. Default: 1 output_size: The number of output dimensions - dropout: If non-zero, introduces a `Dropout` layer on the outputs of each + dropout : If non-zero, introduces a `Dropout` layer on the outputs of each LSTM layer except the last layer, with dropout probability equal to :attr:`dropout`. Default: 0 bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` @@ -149,24 +149,23 @@ class MLPClassifier(torch.nn.Module): """ MLP classifier """ - def __init__(self, hidden_size: int, num_classes: int, latent_size: int, num_classifier_layers: int, + def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_size: int, num_classifier_layers: int, dropout: float): super(MLPClassifier, self).__init__() self.latent_size = latent_size + self.input_size = input_size self.hidden_size = hidden_size self.num_classes = num_classes self.num_classifier_layers = num_classifier_layers self.dropout = dropout # Classifier layers - self.inp = torch.nn.Linear(self.latent_size, self.hidden_size) - self.hidden = torch.nn.ModuleList([torch.nn.Linear(hidden_size, hidden_size) - for _ in range(self.num_classifier_layers)]) - self.out = torch.nn.Linear(self.hidden_size, self.num_classes) + self.hidden = nn.ModuleList([nn.Linear(self.input_size, self.hidden_size)]) + self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_layers - 1)]) + self.out = nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) def forward(self, x): - x = self.dropout(self.inp(x)) x = self.dropout(self.hidden(x)) out = self.out(x) return out diff --git a/traja/models/vae.py b/traja/models/vae.py index 6ea80117..fb5fe749 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -25,7 +25,7 @@ import torch from .utils import TimeDistributed -from .utils import load_model +from torch import nn device = 'cuda' if torch.cuda.is_available() else 'cpu' @@ -189,24 +189,23 @@ class MLPClassifier(torch.nn.Module): """ MLP classifier """ - def __init__(self, hidden_size: int, num_classes: int, latent_size: int, num_classifier_layers: int, + def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_size: int, num_classifier_layers: int, dropout: float): super(MLPClassifier, self).__init__() self.latent_size = latent_size + self.input_size = input_size self.hidden_size = hidden_size self.num_classes = num_classes self.num_classifier_layers = num_classifier_layers self.dropout = dropout # Classifier layers - self.inp = torch.nn.Linear(self.latent_size, self.hidden_size) - self.hidden = torch.nn.ModuleList([torch.nn.Linear(hidden_size, hidden_size) - for _ in range(self.num_classifier_layers)]) - self.out = torch.nn.Linear(self.hidden_size, self.num_classes) + self.hidden = nn.ModuleList([nn.Linear(self.input_size, self.hidden_size)]) + self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_layers - 1)]) + self.out = nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) def forward(self, x): - x = self.dropout(self.inp(x)) x = self.dropout(self.hidden(x)) out = self.out(x) return out From 7870a456b8f0c4572fc6521bf337802c1c4d4540 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 18 Dec 2020 21:25:57 +0100 Subject: [PATCH 120/211] classifier update --- traja/models/ae.py | 3 ++- traja/models/vae.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index a5d14a62..8e694e19 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -231,7 +231,8 @@ def __init__(self, input_size: int, reset_state=True, bidirectional=self.bidirectional) - self.classifier = MLPClassifier(hidden_size=self.classifier_hidden_size, + self.classifier = MLPClassifier(input_size=self.latent_size, + hidden_size=self.classifier_hidden_size, num_classes=self.num_classes, latent_size=self.latent_size, num_classifier_layers=self.num_classifier_layers, diff --git a/traja/models/vae.py b/traja/models/vae.py index fb5fe749..770ba139 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -273,7 +273,8 @@ def __init__(self, input_size: int, reset_state=True, bidirectional=self.bidirectional) - self.classifier = MLPClassifier(hidden_size=self.classifier_hidden_size, + self.classifier = MLPClassifier(input_size=self.latent_size, + hidden_size = self.classifier_hidden_size, num_classes=self.num_classes, latent_size=self.latent_size, num_classifier_layers=self.num_classifier_layers, From 597420389d5b1c345695e5941d1b1d161f95652b Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 18 Dec 2020 21:28:39 +0100 Subject: [PATCH 121/211] classifier fix --- traja/models/ae.py | 2 +- traja/models/vae.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index 8e694e19..4461ba94 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -161,7 +161,7 @@ def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_si # Classifier layers self.hidden = nn.ModuleList([nn.Linear(self.input_size, self.hidden_size)]) - self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_layers - 1)]) + self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_classifier_layers - 1)]) self.out = nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) diff --git a/traja/models/vae.py b/traja/models/vae.py index 770ba139..d7544e1a 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -201,7 +201,7 @@ def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_si # Classifier layers self.hidden = nn.ModuleList([nn.Linear(self.input_size, self.hidden_size)]) - self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_layers - 1)]) + self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_classifier_layers - 1)]) self.out = nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) From 3d3c01058ade9730de350f0b706d6d022900b65c Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 18 Dec 2020 21:33:01 +0100 Subject: [PATCH 122/211] classifier fix --- traja/models/ae.py | 1 + traja/models/vae.py | 1 + 2 files changed, 2 insertions(+) diff --git a/traja/models/ae.py b/traja/models/ae.py index 4461ba94..b25f584f 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -162,6 +162,7 @@ def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_si # Classifier layers self.hidden = nn.ModuleList([nn.Linear(self.input_size, self.hidden_size)]) self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_classifier_layers - 1)]) + self.hidden = nn.Sequential(*self.hidden) self.out = nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) diff --git a/traja/models/vae.py b/traja/models/vae.py index d7544e1a..c787b799 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -202,6 +202,7 @@ def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_si # Classifier layers self.hidden = nn.ModuleList([nn.Linear(self.input_size, self.hidden_size)]) self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_classifier_layers - 1)]) + self.hidden = nn.Sequential(*self.hidden) self.out = nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) From b8365e02445f7b49a16f20acf41ab59df43a6b86 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 18 Dec 2020 22:27:07 +0100 Subject: [PATCH 123/211] docs, lstm,ae,vae fix --- traja/models/ae.py | 55 +++++++++++++++----------------------- traja/models/generator.py | 2 +- traja/models/losses.py | 33 +++++++++++++++++------ traja/models/optimizers.py | 33 +++++++++++------------ traja/models/train.py | 18 ++++++------- traja/models/vae.py | 43 +++++++++++++---------------- 6 files changed, 89 insertions(+), 95 deletions(-) diff --git a/traja/models/ae.py b/traja/models/ae.py index b25f584f..648e6b82 100644 --- a/traja/models/ae.py +++ b/traja/models/ae.py @@ -5,32 +5,30 @@ class LSTMEncoder(torch.nn.Module): - """ Deep LSTM network. This implementation - returns output_size hidden size. - Args: - input_size: The number of expected features in the input `x` - batch_size: - sequence_length: The number of in each sample - hidden_size: The number of features in the hidden state `h` - num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` - would mean stacking two LSTMs together to form a `stacked LSTM`, - with the second LSTM taking in outputs of the first LSTM and - computing the final results. Default: 1 - output_size: The number of output dimensions - dropout : If non-zero, introduces a `Dropout` layer on the outputs of each - LSTM layer except the last layer, with dropout probability equal to - :attr:`dropout`. Default: 0 - bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` + + """ Implementation of Encoder network using LSTM layers + :param input_size: The number of expected features in the input x + :param num_past: Number of time steps to look backwards to predict num_future steps forward + :param batch_size: Number of samples in a batch + :param hidden_size: The number of features in the hidden state h + :param num_lstm_layers: Number of layers in the LSTM model + + :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param reset_state: If True, will reset the hidden and cell state for each batch of data + :param bidirectional: If True, becomes a bidirectional LSTM """ - def __init__(self, input_size: int, sequence_length: int, batch_size: int, + def __init__(self, input_size: int, num_past: int, batch_size: int, hidden_size: int, num_lstm_layers: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): + super(LSTMEncoder, self).__init__() self.input_size = input_size - self.sequence_length = sequence_length + self.num_past = num_past self.batch_size = batch_size self.hidden_size = hidden_size self.num_lstm_layers = num_lstm_layers @@ -174,25 +172,14 @@ def forward(self, x): class MultiModelAE(torch.nn.Module): - def __init__(self, input_size: int, - sequence_length: int, - batch_size: int, - num_future: int, - lstm_hidden_size: int, - num_lstm_layers: int, - classifier_hidden_size: int, - num_classifier_layers: int, - output_size: int, - num_classes: int, - latent_size: int, - batch_first: bool, - dropout: float, - reset_state: bool, + def __init__(self, input_size: int, num_past: int, batch_size: int, num_future: int, lstm_hidden_size: int, + num_lstm_layers: int, classifier_hidden_size: int, num_classifier_layers: int, output_size: int, + num_classes: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(MultiModelAE, self).__init__() self.input_size = input_size - self.sequence_length = sequence_length + self.num_past = num_past self.batch_size = batch_size self.latent_size = latent_size self.num_future = num_future @@ -208,7 +195,7 @@ def __init__(self, input_size: int, self.bidirectional = bidirectional self.encoder = LSTMEncoder(input_size=self.input_size, - sequence_length=self.sequence_length, + num_past=self.num_past, batch_size=self.batch_size, hidden_size=self.lstm_hidden_size, num_lstm_layers=self.num_lstm_layers, diff --git a/traja/models/generator.py b/traja/models/generator.py index b0624da8..32949852 100644 --- a/traja/models/generator.py +++ b/traja/models/generator.py @@ -17,7 +17,7 @@ def timeseries(model_type: str, model_hyperparameters: dict, model_path: str, ba batch_size = model_hyperparameters.batch_size # Number of samples num_future = model_hyperparameters.num_future # Number of time steps in each sample if model_type == 'ae': - model = MultiModelAE(**model_hyperparameters) + model = MultiModelAE(, if model_type == 'vae': model = MultiModelVAE(**model_hyperparameters) diff --git a/traja/models/losses.py b/traja/models/losses.py index cb9db996..90060e70 100644 --- a/traja/models/losses.py +++ b/traja/models/losses.py @@ -2,12 +2,23 @@ class Criterion: + """Implements the loss functions of Autoencoders, Variational Autoencoders and LSTM models + Huber loss is set as default for reconstruction loss, alternative is to use rmse, + Cross entropy loss used for classification + Variational loss used huber loss and unweighted KL Divergence loss""" def __init__(self): + self.huber_loss = torch.nn.SmoothL1Loss(reduction='sum') self.crossentropy_loss = torch.nn.CrossEntropyLoss() def ae_criterion(self, predicted, target, loss_type='huber'): + """ Implements the Autoencoder loss for time series forecasting + :param predicted: Predicted time series by the model + :param target: Target time series + :param loss_type: Type of criterion; Defaults: 'huber' + :return: + """ if loss_type == 'huber': loss = self.huber_loss(predicted, target) @@ -16,7 +27,13 @@ def ae_criterion(self, predicted, target, loss_type='huber'): return torch.sqrt(torch.mean((predicted - target) ** 2)) def vae_criterion(self, predicted, target, mu, logvar, loss_type='huber'): - """Time series generative model loss function + """ Time series generative model loss function + :param predicted: Predicted time series by the model + :param target: Target time series + :param mu: Latent variable, Mean + :param logvar: Latent variable, Log(Variance) + :param loss_type: Type of criterion; Defaults: 'huber' + :return: Reconstruction loss + KLD loss """ if loss_type == 'huber': dist_x = self.huber_loss(predicted, target) @@ -26,7 +43,13 @@ def vae_criterion(self, predicted, target, mu, logvar, loss_type='huber'): return dist_x + KLD def classifier_criterion(self, predicted, target): - """Classifier loss function""" + """ + Classifier loss function + :param predicted: Predicted label + :param target: Target label + :return: Cross entropy loss + """ + loss = self.crossentropy_loss(predicted, target) return loss @@ -37,9 +60,3 @@ def lstm_criterion(self, predicted, target): def vaegan_criterion(self): return NotImplementedError - -# VAE loss - -# VAE-GAN loss - -# LSTM diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 2b472a58..5bd3f902 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -1,5 +1,4 @@ import torch -from torch.optim import optimizer from torch.optim.lr_scheduler import ReduceLROnPlateau from traja.models.ae import MultiModelAE @@ -7,9 +6,17 @@ class Optimizer: def __init__(self, model_type, model, optimizer_type): + """ + Wrapper for setting the model optimizer and learning rate schedulers using ReduceLROnPlateau; + If the model type is 'ae' or 'vae' - var optimizers is a dict with separate optimizers for encoder, decoder, + latent and classifier. In case of 'lstm', var optimizers is an optimizer for lstm and TimeDistributed(linear layer) + :param model_type: Type of model 'ae', 'vae' or 'lstm' + :param model: Model instance + :param optimizer_type: Optimizer to be used; Should be one in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', + 'LBFGS', 'ASGD', 'Adamax'] + """ assert isinstance(model, torch.nn.Module) - assert str(optimizer_type) in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', 'LBFGS', 'ASGD', 'Adamax'] @@ -44,7 +51,7 @@ def get_optimizers(self, lr=0.0001): return NotImplementedError return self.optimizers - def get_lrschedulers(self, factor:float, patience:int): + def get_lrschedulers(self, factor: float, patience: int): """Learning rate scheduler for each network in the model NOTE: Scheduler metric should be test set loss @@ -58,8 +65,8 @@ def get_lrschedulers(self, factor:float, patience:int): """ if self.model_type == 'lstm': assert not isinstance(self.optimizers, dict) - self.schedulers= ReduceLROnPlateau(self.optimizers, mode='max', factor= factor, - patience=patience, verbose=True) + self.schedulers = ReduceLROnPlateau(self.optimizers, mode='max', factor=factor, + patience=patience, verbose=True) else: for network in self.optimizers.keys(): self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, @@ -70,19 +77,9 @@ def get_lrschedulers(self, factor:float, patience:int): if __name__ == '__main__': # Test model_type = 'ae' - model = MultiModelAE(input_size=2, - sequence_length=10, - batch_size=5, - num_future=5, - hidden_size=10, - num_layers=2, - output_size=2, - num_classes=10, - latent_size=10, - batch_first=True, - dropout=0.2, - reset_state=True, - bidirectional=True) + model = MultiModelAE(input_size=2, num_past=10, batch_size=5, num_future=5, lstm_hidden_size=32, num_lstm_layers=2, + classifier_hidden_size=32, num_classifier_layers=4, output_size=2, num_classes=10, + latent_size=10, batch_first=True, dropout=0.2, reset_state=True, bidirectional=True) # Get the optimizers opt = Optimizer(model_type, model, optimizer_type='RMSprop') diff --git a/traja/models/train.py b/traja/models/train.py index c322366e..5f7ccda6 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -1,7 +1,6 @@ from .ae import MultiModelAE from .vae import MultiModelVAE from .lstm import LSTM - from . import utils from .losses import Criterion from .optimizers import Optimizer @@ -9,7 +8,6 @@ device = 'cuda' if torch.cuda.is_available() else 'cpu' - class LatentModelTrainer(object): """ Wrapper for training and testing the LSTM model @@ -22,20 +20,20 @@ class LatentModelTrainer(object): :param input_size: The number of expected features in the input x :param output_size: Output feature dimension :param lstm_hidden_size: The number of features in the hidden state h - :param num_layers: Number of layers in the LSTM model + :param num_lstm_layers: Number of layers in the LSTM model :param reset_state: If True, will reset the hidden and cell state for each batch of data - :param num_classes: - :param latent_size: + :param num_classes: Number of categories/labels + :param latent_size: Latent space dimension :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, with dropout probability equal to dropout :param num_classifier_layers: Number of layers in the classifier :param epochs: Number of epochs to train the network :param batch_size: Number of samples in a batch :param num_future: Number of time steps to be predicted forward - :param sequence_length: Number of past time steps otherwise, length of sequences in each batch of data. + :param num_past: Number of past time steps otherwise, length of sequences in each batch of data. :param bidirectional: If True, becomes a bidirectional LSTM :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - :param loss_type: + :param loss_type: Type of reconstruction loss to apply, 'huber' or 'rmse'. Default:'huber' :param lr_factor: Factor by which the learning rate will be reduced :param scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. For example, if patience = 2, then we will ignore the first 2 epochs with no @@ -60,7 +58,7 @@ def __init__(self, model_type: str, epochs: int, batch_size: int, num_future: int, - sequence_length: int, + num_past: int, bidirectional: bool = False, batch_first: bool = True, loss_type: str = 'huber', @@ -86,7 +84,7 @@ def __init__(self, model_type: str, self.num_future = num_future self.epochs = epochs self.batch_size = batch_size - self.sequence_length = sequence_length + self.num_past = num_past self.dropout = dropout self.bidirectional = bidirectional self.loss_type = loss_type @@ -94,7 +92,7 @@ def __init__(self, model_type: str, self.lr_factor = lr_factor self.scheduler_patience = scheduler_patience self.model_hyperparameters = {'input_size': self.input_size, - 'sequence_length': self.sequence_length, + 'num_past': self.num_past, 'batch_size': self.batch_size, 'lstm_hidden_size': self.lstm_hidden_size, 'num_lstm_layers':self.num_lstm_layers, diff --git a/traja/models/vae.py b/traja/models/vae.py index c787b799..d0075b8d 100644 --- a/traja/models/vae.py +++ b/traja/models/vae.py @@ -16,7 +16,7 @@ epochs=epochs, batch_size=batch_size, num_future=num_future, - sequence_length=sequence_length, + num_past=num_past, bidirectional =False, batch_first =True, loss_type = 'huber') @@ -31,32 +31,27 @@ class LSTMEncoder(torch.nn.Module): - """ Deep LSTM network. This implementation - returns output_size hidden size. - Args: - input_size: The number of expected features in the input `x` - batch_size: - sequence_length: The number of in each sample - hidden_size: The number of features in the hidden state `h` - num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` - would mean stacking two LSTMs together to form a `stacked LSTM`, - with the second LSTM taking in outputs of the first LSTM and - computing the final results. Default: 1 - output_size: The number of output dimensions - dropout: If non-zero, introduces a `Dropout` layer on the outputs of each - LSTM layer except the last layer, with dropout probability equal to - :attr:`dropout`. Default: 0 - bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` - """ - - def __init__(self, input_size: int, sequence_length: int, batch_size: int, + """ Implementation of Encoder network using LSTM layers + :param input_size: The number of expected features in the input x + :param num_past: Number of time steps to look backwards to predict num_future steps forward + :param batch_size: Number of samples in a batch + :param hidden_size: The number of features in the hidden state h + :param num_lstm_layers: Number of layers in the LSTM model + + :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param reset_state: If True, will reset the hidden and cell state for each batch of data + :param bidirectional: If True, becomes a bidirectional LSTM + """ + def __init__(self, input_size: int, num_past: int, batch_size: int, hidden_size: int, num_lstm_layers: int, batch_first: bool, dropout: float, reset_state: bool, bidirectional: bool): super(LSTMEncoder, self).__init__() self.input_size = input_size - self.sequence_length = sequence_length + self.num_past = num_past self.batch_size = batch_size self.hidden_size = hidden_size self.num_lstm_layers = num_lstm_layers @@ -217,7 +212,7 @@ class MultiModelVAE(torch.nn.Module): """ def __init__(self, input_size: int, - sequence_length: int, + num_past: int, batch_size: int, num_future: int, lstm_hidden_size: int, @@ -234,7 +229,7 @@ def __init__(self, input_size: int, super(MultiModelVAE, self).__init__() self.input_size = input_size - self.sequence_length = sequence_length + self.num_past = num_past self.batch_size = batch_size self.latent_size = latent_size self.num_future = num_future @@ -250,7 +245,7 @@ def __init__(self, input_size: int, self.bidirectional = bidirectional self.encoder = LSTMEncoder(input_size=self.input_size, - sequence_length=self.sequence_length, + num_past=self.num_past, batch_size=self.batch_size, hidden_size=self.lstm_hidden_size, num_lstm_layers=self.num_lstm_layers, From 15eb2078ea169ea120f13f5b093683484443aa7a Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sat, 19 Dec 2020 09:04:32 +0100 Subject: [PATCH 124/211] Update train.py --- traja/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index 5f7ccda6..898989c3 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -8,7 +8,7 @@ device = 'cuda' if torch.cuda.is_available() else 'cpu' -class LatentModelTrainer(object): +class HybridTrainer(object): """ Wrapper for training and testing the LSTM model From 21f436435d82f84c6d1769a7c9a3ab758502fcd8 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Sat, 26 Dec 2020 21:19:28 +0100 Subject: [PATCH 125/211] initial update custom model support --- .../visualizer.py | 95 ++++++---- traja/models/__init__.py | 8 +- traja/models/{ => generative_models}/vae.py | 0 .../models/{ => generative_models}/vaegan.py | 0 traja/models/generator.py | 177 +++++++++++++----- traja/models/{ => predictive_models}/ae.py | 0 traja/models/{ => predictive_models}/irl.py | 0 traja/models/{ => predictive_models}/lstm.py | 0 traja/models/test.py | 44 ----- traja/models/train.py | 41 ++-- traja/models/utils.py | 14 +- 11 files changed, 225 insertions(+), 154 deletions(-) rename traja/{models => dimensionality_reduction}/visualizer.py (80%) rename traja/models/{ => generative_models}/vae.py (100%) rename traja/models/{ => generative_models}/vaegan.py (100%) rename traja/models/{ => predictive_models}/ae.py (100%) rename traja/models/{ => predictive_models}/irl.py (100%) rename traja/models/{ => predictive_models}/lstm.py (100%) delete mode 100644 traja/models/test.py diff --git a/traja/models/visualizer.py b/traja/dimensionality_reduction/visualizer.py similarity index 80% rename from traja/models/visualizer.py rename to traja/dimensionality_reduction/visualizer.py index 9e1462bd..ed416a7e 100644 --- a/traja/models/visualizer.py +++ b/traja/dimensionality_reduction/visualizer.py @@ -1,3 +1,4 @@ +import os, sys import networkx as nx import pandas as pd import numpy as np @@ -11,40 +12,27 @@ from mpl_toolkits.mplot3d import Axes3D from matplotlib.axes import Axes from matplotlib import cm -# import plotly -# import plotly.graph_objs as go -# import plotly.io as pio -import os, sys import seaborn as sns import matplotlib from matplotlib import style -from scipy.sparse import csgraph import argparse, copy, h5py, os, sys, time, socket import tensorflow as tf import torch, torchvision, torch.nn as nn import torch.optim as optim import torchvision.transforms as transforms -# from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot from matplotlib import ticker, colors -# print(matplotlib.get_backend()) -# matplotlib.rcParams["backend"] = "Gtk3Agg" -# print(matplotlib.get_backend()) -# matplotlib.use('Gtk3Agg') -import matplotlib.pyplot as plt -# plt.switch_backend('Qt4Agg') -# print(matplotlib.get_backend()) plt.switch_backend("TkAgg") -# seed value and plotly -# init_notebook_mode(connected=True) -np.set_printoptions(suppress=True, precision=3, ) -style.use('ggplot') +np.set_printoptions( + suppress=True, precision=3, +) +style.use("ggplot") -class DirectedNetwork(object): +class DirectedNetwork(object): def __init__(self): super().__init__() pass @@ -67,10 +55,12 @@ def show(self, states, weight, fig): self_connections = [weight[i][i] for i in range(len(weight))] # Intialize graph - G = nx.from_numpy_matrix(weight, create_using=nx.MultiDiGraph, parallel_edges=True) + G = nx.from_numpy_matrix( + weight, create_using=nx.MultiDiGraph, parallel_edges=True + ) edge_colors = weight.tolist() - edge_colors_ = [float('%.8f' % j) for i in edge_colors for j in i] + edge_colors_ = [float("%.8f" % j) for i in edge_colors for j in i] # Set up nodes neuron_color = [state_dict.get(node, 0.25) for node in G.nodes()] @@ -80,12 +70,18 @@ def show(self, states, weight, fig): vmax = np.max(states) cmap = plt.cm.coolwarm edge_cmap = plt.cm.Spectral - nx.draw(G, with_labels=True, - cmap=cmap, node_color=neuron_color, - node_size=200, linewidths=5, - edge_color=edge_colors_, - edge_cmap=edge_cmap, font_size=10, - connectionstyle='arc3, rad=0.3') + nx.draw( + G, + with_labels=True, + cmap=cmap, + node_color=neuron_color, + node_size=200, + linewidths=5, + edge_color=edge_colors_, + edge_cmap=edge_cmap, + font_size=10, + connectionstyle="arc3, rad=0.3", + ) sm = plt.cm.ScalarMappable(cmap=cmap, norm=plt.Normalize(vmin=vmin, vmax=vmax)) sm.set_array([]) @@ -104,7 +100,6 @@ def show(self, states, weight, fig): class LocalLinearEmbedding(object): - def __init__(self): super(LocalLinearEmbedding, self).__init__() pass @@ -166,8 +161,14 @@ def show(self, pc, fig2): # ax.set_ylim(-1,1) # ax.set_zlim(-1,1) for i in range(len(pc)): - ax.plot3D(pc[i:, 0], pc[i:, 1], pc[i:, 2], - alpha=i / len(pc), color='red', linewidth=1) + ax.plot3D( + pc[i:, 0], + pc[i:, 1], + pc[i:, 2], + alpha=i / len(pc), + color="red", + linewidth=1, + ) fig2.colorbar(f) # plt.pause(0.0001) # State of streaming plot @@ -183,7 +184,6 @@ def show(self, pc, fig2): class SpectralEmbedding(object): - def __init__(self): super(SpectralEmbedding, self).__init__() pass @@ -198,15 +198,21 @@ def spectral_embedding(self, X, rad): :return Y: numpy.ndarray - matrix m row, d attributes are reduced dimensional """ # Get the adjacency matrix/nearest neighbor graph; neighbors within the radius of 0.4 - A = radius_neighbors_graph(X.T, rad, mode='distance', - metric='minkowski', p=2, - metric_params=None, include_self=False) + A = radius_neighbors_graph( + X.T, + rad, + mode="distance", + metric="minkowski", + p=2, + metric_params=None, + include_self=False, + ) A = A.toarray() # Find the laplacian of the neighbour graph - # L = D - A ; where D is the diagonal degree matrix + # L = D - A ; where D is the diagonal degree matrix L = csgraph.laplacian(A, normed=False) - # Embedd the data points i low dimension using the Eigen values/vectos + # Embedd the data points i low dimension using the Eigen values/vectos # of the laplacian graph to get the most optimal partition of the graph eigval, eigvec = np.linalg.eig(L) # the second smallest eigenvalue represents sparsest cut of the graph. @@ -248,10 +254,10 @@ def show(self, X, spec_embed, fig3): return True -if __name__ == '__main__': +if __name__ == "__main__": # create the coordinates - numebr_of_points = 21; - small_range = -1.0; + numebr_of_points = 21 + small_range = -1.0 large_range = 1.0 xcoordinates = np.linspace(small_range, large_range, num=numebr_of_points) @@ -262,5 +268,14 @@ def show(self, X, spec_embed, fig3): s1 = xcoord_mesh.ravel()[inds] s2 = ycoord_mesh.ravel()[inds] coordinate = np.c_[s1, s2] - print('From ', small_range, ' to ', large_range, ' with ', numebr_of_points, ' total number of coordinate: ', - numebr_of_points ** 2) + print( + "From ", + small_range, + " to ", + large_range, + " with ", + numebr_of_points, + " total number of coordinate: ", + numebr_of_points ** 2, + ) + diff --git a/traja/models/__init__.py b/traja/models/__init__.py index 54b2acdf..4e16eb50 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1,3 +1,7 @@ -from .nn import LSTM -from .vae import MultiModelVAE +# from .nn import LSTM +from traja.models.generative_models.vae import MultiModelVAE +from traja.models.predictive_models.ae import MultiModelAE +from traja.models.predictive_models.lstm import LSTM +from traja.models.predictive_models.irl import MultiModelIRL +from traja.models.generative_models.vaegan import MultiModelVAEGAN diff --git a/traja/models/vae.py b/traja/models/generative_models/vae.py similarity index 100% rename from traja/models/vae.py rename to traja/models/generative_models/vae.py diff --git a/traja/models/vaegan.py b/traja/models/generative_models/vaegan.py similarity index 100% rename from traja/models/vaegan.py rename to traja/models/generative_models/vaegan.py diff --git a/traja/models/generator.py b/traja/models/generator.py index 32949852..0c23e333 100644 --- a/traja/models/generator.py +++ b/traja/models/generator.py @@ -6,58 +6,145 @@ from .vae import MultiModelVAE from .vaegan import MultiModelVAEGAN from .irl import MultiModelIRL +from .lstm import LSTM from .utils import load_model import matplotlib.pyplot as plt -device = 'cuda' if torch.cuda.is_available() else 'cpu' +device = "cuda" if torch.cuda.is_available() else "cpu" -def timeseries(model_type: str, model_hyperparameters: dict, model_path: str, batch_size: int, num_future: int, ): - # Generating few samples - batch_size = model_hyperparameters.batch_size # Number of samples - num_future = model_hyperparameters.num_future # Number of time steps in each sample - if model_type == 'ae': - model = MultiModelAE(, +class Generate: + def __init__( + self, + model_type: str = None, + model_path: str = None, + model_hyperparameters: dict = None, + model: torch.nn.Module = None, + ): + """Generate a batch of future steps from a random latent state of Multi variate multi label models - if model_type == 'vae': - model = MultiModelVAE(**model_hyperparameters) + Args: + model_type (str, optional): Type of model ['vae','vaegan','custom']. Defaults to None. + model_path (str, optional): [description]. Defaults to None. + model_hyperparameters (dict, optional): [description]. Defaults to None. + model (torch.nn.Module, optional): Custom model from user. Defaults to None + """ - if model_type == 'vaegan': - model = MultiModelVAEGAN(**model_hyperparameters) - return NotImplementedError + self.model_type = model_type + self.model_path = model_path + self.model_hyperparameters = model_hyperparameters + + # Batch size and time step size + self.batch_size = self.model_hyperparameters.batch_size + self.num_future = self.model_hyperparameters.num_future + + if self.model_type == "vae": + self.model = MultiModelVAE(**self.model_hyperparameters) + + if self.model_type == "vaegan": + self.model = MultiModelVAEGAN(**self.model_hyperparameters) + + if self.model_type == "custom": + assert model is not None + self.model = model(**self.model_hyperparameters) + + def generate_batch(self): + + # Load the model + model = load_model(self.model, self.model_hyperparameters, self.model_path) + + if self.model_type == "vae": + # Random noise + z = ( + torch.empty(self.batch_size, self.model_hyperparameters.latent_size) + .normal_(mean=0, std=0.1) + .to(device) + ) + # Generate trajectories from the noise + out = model.decoder(z, self.num_future).cpu().detach().numpy() + out = out.reshape(out.shape[0] * out.shape[1], out.shape[2]) + try: + cat = model.classifier(z) + print( + "IDs in this batch of synthetic data", torch.max(cat, 1).indices + 1 + ) + except: + pass + + plt.figure(figsize=(12, 4)) + plt.plot(out[:, 0], label="Generated x: Longitude") + plt.plot(out[:, 1], label="Generated y: Latitude") + plt.legend() + + fig, ax = plt.subplots(nrows=2, ncols=5, figsize=(16, 5), sharey=True) + fig.set_size_inches(20, 5) + + for i in range(2): + for j in range(5): + ax[i, j].plot( + out[:, 0][ + (i + j) * self.num_future : (i + j) * self.num_future + + self.num_future + ], + out[:, 1][ + (i + j) * self.num_future : (i + j) * self.num_future + + self.num_future + ], + label="Animal ID {}".format( + (torch.max(cat, 1).indices + 1).detach()[i + j] + ), + color="g", + ) + ax[i, j].legend() + plt.show() - if model_type == 'irl': - model = MultiModelIRL(**model_hyperparameters) + return out + + elif self.model_type == "vaegan" or "custom": + return NotImplementedError + + def generate_timeseries(num_steps): return NotImplementedError - # Load the model from model path: - model = load_model(model, model_hyperparameters, model_path) - # z = torch.randn((batch_size, latent_size)).to(device) - z = torch.empty(10, model_hyperparameters.latent_size).normal_(mean=0, std=.1).to(device) - # Category from the noise - cat = model.classifier(z) - # Generate trajectories from the noise - out = model.decoder(z, num_future).cpu().detach().numpy() - out = out.reshape(out.shape[0] * out.shape[1], out.shape[2]) - - # for index, i in enumerate(train_df.columns): - # scaler = scalers['scaler_'+i] - # out[:,index] = scaler.inverse_transform(out[:,index].reshape(1, -1)) - print('IDs in this batch of synthetic data', torch.max(cat, 1).indices + 1) - plt.figure(figsize=(12, 4)) - plt.plot(out[:, 0], label='Generated x: Longitude') - plt.plot(out[:, 1], label='Generated y: Latitude') - plt.legend() - - fig, ax = plt.subplots(nrows=2, ncols=5, figsize=(16, 5), sharey=True) - # plt.ticklabel_format(useOffset=False) - fig.set_size_inches(20, 5) - for i in range(2): - for j in range(5): - ax[i, j].plot(out[:, 0][(i + j) * num_future:(i + j) * num_future + num_future], - out[:, 1][(i + j) * num_future:(i + j) * num_future + num_future], - label='Animal ID {}'.format((torch.max(cat, 1).indices + 1).detach()[i + j]), color='g') - ax[i, j].legend() - plt.show() - - return out + +class Predict: + def __init__( + self, + model_type: str = None, + model_path: str = None, + model_hyperparameters: dict = None, + model: torch.nn.Module = None, + ): + """Generate a batch of future steps from a random latent state of Multi variate multi label models + + Args: + model_type (str, optional): Type of model ['ae','irl','lstm','custom']. Defaults to None. + model_path (str, optional): [description]. Defaults to None. + model_hyperparameters (dict, optional): [description]. Defaults to None. + model (torch.nn.Module, optional): Custom model from user. Defaults to None + """ + + self.model_type = model_type + self.model_path = model_path + self.model_hyperparameters = model_hyperparameters + + # Batch size and time step size + self.batch_size = self.model_hyperparameters.batch_size + self.num_future = self.model_hyperparameters.num_future + + if self.model_type == "ae": + self.model = MultiModelAE(**self.model_hyperparameters) + + if self.model_type == "lstm": + self.model = LSTM(**self.model_hyperparameters) + + if self.model_type == "irl": + self.model = MultiModelIRL(**self.model_hyperparameters) + + if self.model_type == "custom": + assert model is not None + self.model = model(**self.model_hyperparameters) + + + + diff --git a/traja/models/ae.py b/traja/models/predictive_models/ae.py similarity index 100% rename from traja/models/ae.py rename to traja/models/predictive_models/ae.py diff --git a/traja/models/irl.py b/traja/models/predictive_models/irl.py similarity index 100% rename from traja/models/irl.py rename to traja/models/predictive_models/irl.py diff --git a/traja/models/lstm.py b/traja/models/predictive_models/lstm.py similarity index 100% rename from traja/models/lstm.py rename to traja/models/predictive_models/lstm.py diff --git a/traja/models/test.py b/traja/models/test.py deleted file mode 100644 index 14d38fa2..00000000 --- a/traja/models/test.py +++ /dev/null @@ -1,44 +0,0 @@ -import pandas as pd -from models.train import Trainer -from datasets.dataset import MultiModalDataLoader - -# Downlaod dataset (9 jaguars) from github -data_url = "https://raw.githubusercontent.com/traja-team/traja-research/dataset_und_notebooks/dataset_analysis/jaguar5.csv" -df = pd.read_csv(data_url, error_bad_lines=False) - -# Hyperparameters -batch_size = 10 -num_past = 10 -num_future = 5 -model_save_path = './model.pt' - -# Prepare the dataloader -train_loader, test_loader = MultiModalDataLoader(df, - batch_size=batch_size, - n_past=num_past, - n_future=num_future, - num_workers=1) - -# Initialize the models -trainer = Trainer(model_type='vae', - device='cpu', - input_size=2, - output_size=2, - lstm_hidden_size=512, - lstm_num_layers=4, - reset_state=True, - num_classes=9, - latent_size=10, - dropout=0.1, - num_layers=4, - epochs=10, - batch_size=batch_size, - num_future=num_future, - sequence_length=num_past, - bidirectional=False, - batch_first=True, - loss_type='huber') - -# Train the model -trainer.train_latent_model(train_loader, test_loader, model_save_path) - diff --git a/traja/models/train.py b/traja/models/train.py index 898989c3..979751ba 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -1,6 +1,6 @@ -from .ae import MultiModelAE -from .vae import MultiModelVAE -from .lstm import LSTM +from models import MultiModelAE +from models import MultiModelVAE +from models import LSTM from . import utils from .losses import Criterion from .optimizers import Optimizer @@ -8,6 +8,7 @@ device = 'cuda' if torch.cuda.is_available() else 'cpu' + class HybridTrainer(object): """ Wrapper for training and testing the LSTM model @@ -91,22 +92,24 @@ def __init__(self, model_type: str, self.optimizer_type = optimizer_type self.lr_factor = lr_factor self.scheduler_patience = scheduler_patience - self.model_hyperparameters = {'input_size': self.input_size, - 'num_past': self.num_past, - 'batch_size': self.batch_size, - 'lstm_hidden_size': self.lstm_hidden_size, - 'num_lstm_layers':self.num_lstm_layers, - 'classifier_hidden_size':self.classifier_hidden_size, - 'num_classifier_layers':self.num_classifier_layers, - 'num_future': self.num_future, - 'latent_size': self.latent_size, - 'output_size': self.output_size, - 'num_classes': self.num_classes, - 'batch_first': self.batch_first, - 'reset_state': self.reset_state, - 'bidirectional': self.bidirectional, - 'dropout': self.dropout - } + + self.model_hyperparameters = { + "input_size": self.input_size, + "num_past": self.num_past, + "batch_size": self.batch_size, + "lstm_hidden_size": self.lstm_hidden_size, + "num_lstm_layers": self.num_lstm_layers, + "classifier_hidden_size": self.classifier_hidden_size, + "num_classifier_layers": self.num_classifier_layers, + "num_future": self.num_future, + "latent_size": self.latent_size, + "output_size": self.output_size, + "num_classes": self.num_classes, + "batch_first": self.batch_first, + "reset_state": self.reset_state, + "bidirectional": self.bidirectional, + "dropout": self.dropout, + } # Instantiate model instance based on model_type if self.model_type == 'ae': diff --git a/traja/models/utils.py b/traja/models/utils.py index 5b670334..9501302b 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -20,14 +20,20 @@ def forward(self, x): return self.module(x) # Squash samples and timesteps into a single axis - x_reshape = x.contiguous().view(-1, x.size(-1)) # (samples * timesteps, input_size) + x_reshape = x.contiguous().view( + -1, x.size(-1) + ) # (samples * timesteps, input_size) out = self.module(x_reshape) # We have to reshape Y back to the target shape if self.batch_first: - out = out.contiguous().view(x.size(0), -1, out.size(-1)) # (samples, timesteps, output_size) + out = out.contiguous().view( + x.size(0), -1, out.size(-1) + ) # (samples, timesteps, output_size) else: - out = out.view(-1, x.size(1), out.size(-1)) # (timesteps, samples, output_size) + out = out.view( + -1, x.size(1), out.size(-1) + ) # (timesteps, samples, output_size) return out @@ -43,7 +49,7 @@ def save_model(model, PATH): # PATH = "state_dict_model.pt" # Save torch.save(model.state_dict(), PATH) - print('Model saved at {}'.format(PATH)) + print("Model saved at {}".format(PATH)) def load_model(model, model_hyperparameters, PATH): From 3cdbaa66e519729fb4e0e3ef52b82de1c959e69e Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 28 Dec 2020 14:50:38 +0100 Subject: [PATCH 126/211] custom trainer, user defined models, traja trainer, visualizer, manifold intial update --- traja/models/interpretor.py | 27 -- traja/models/manifolds.py | 14 + traja/models/optimizers.py | 93 +++-- traja/models/train.py | 334 +++++++++++++++--- .../visualizer.py | 27 +- 5 files changed, 390 insertions(+), 105 deletions(-) delete mode 100644 traja/models/interpretor.py create mode 100644 traja/models/manifolds.py rename traja/{dimensionality_reduction => models}/visualizer.py (91%) diff --git a/traja/models/interpretor.py b/traja/models/interpretor.py deleted file mode 100644 index b4c3a1b9..00000000 --- a/traja/models/interpretor.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Model interpretion and Visualization""" -import plotly.express as px - -from .ae import MultiModelAE -from .vae import MultiModelVAE -from .vaegan import MultiModelVAEGAN -from .irl import MultiModelIRL - - -def DisplayLatentDynamics(latent): - r"""Visualize the dynamics of combination of latents - Args: - latent(tensor): Each point in the list is latent's state at the end of a sequence of each batch. - Latent shape (batch_size, latent_dim) - Usage: - DisplayLatentDynamics(latent)""" - - latents = {} - latents.fromkeys(list(range(latent.shape[1]))) - for i in range(latent.shape[1]): - latents[f'{i}'] = latent[:, i].cpu().detach().numpy() - fig = px.scatter_matrix(latents) - fig.update_layout( - autosize=False, - width=1600, - height=1000, ) - return fig.show() diff --git a/traja/models/manifolds.py b/traja/models/manifolds.py new file mode 100644 index 00000000..f1a472be --- /dev/null +++ b/traja/models/manifolds.py @@ -0,0 +1,14 @@ +from sklearn.manifold import * + + +class Manifold: + """Wrap all the manifold functionalities provided by scikit learn. Provide interface to apply non-linear dimensionality reduction techniques, + visualize and infer the strucure of the data using neural networks + """ + + def __init__(self, manifold_type): + + pass + + def __new__(cls): + pass diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 5bd3f902..07a5b2ab 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -4,8 +4,8 @@ class Optimizer: + def __init__(self, model_type, model, optimizer_type, classify=False): - def __init__(self, model_type, model, optimizer_type): """ Wrapper for setting the model optimizer and learning rate schedulers using ReduceLROnPlateau; If the model type is 'ae' or 'vae' - var optimizers is a dict with separate optimizers for encoder, decoder, @@ -17,8 +17,18 @@ def __init__(self, model_type, model, optimizer_type): """ assert isinstance(model, torch.nn.Module) - assert str(optimizer_type) in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', - 'LBFGS', 'ASGD', 'Adamax'] + assert str(optimizer_type) in [ + "Adam", + "Adadelta", + "Adagrad", + "AdamW", + "SparseAdam", + "RMSprop", + "Rprop", + "LBFGS", + "ASGD", + "Adamax", + ] self.model_type = model_type self.model = model @@ -38,16 +48,27 @@ def get_optimizers(self, lr=0.0001): [type]: [description] """ - if self.model_type == 'lstm': - self.optimizers = getattr(torch.optim, f'{self.optimizer_type}')(self.model.parameters(), lr=lr) + if self.model_type == "lstm" or "custom": + self.optimizers = getattr(torch.optim, f"{self.optimizer_type}")( + self.model.parameters(), lr=lr + ) - elif self.model_type == 'ae' or 'vae': - keys = ['encoder', 'decoder', 'latent', 'classifier'] + elif self.model_type == "ae" or "vae": + keys = ["encoder", "decoder", "latent", "classifier"] for network in keys: - self.optimizers[network] = getattr(torch.optim, f'{self.optimizer_type}')( - getattr(self.model, f'{network}').parameters(), lr=lr) - - elif self.model_type == 'vaegan': + if network != "classifier": + self.optimizers[network] = getattr( + torch.optim, f"{self.optimizer_type}" + )(getattr(self.model, f"{network}").parameters(), lr=lr) + + if self.classify: + self.optimizers["classifier"] = getattr( + torch.optim, f"{self.optimizer_type}" + )(getattr(self.model, "classifier").parameters(), lr=lr) + else: + self.optimizers["classifier"] = None + + elif self.model_type == "vaegan": return NotImplementedError return self.optimizers @@ -63,26 +84,54 @@ def get_lrschedulers(self, factor: float, patience: int): Returns: [dict]: [description] """ - if self.model_type == 'lstm': + if self.model_type == "lstm" or "custom": assert not isinstance(self.optimizers, dict) - self.schedulers = ReduceLROnPlateau(self.optimizers, mode='max', factor=factor, - patience=patience, verbose=True) + self.schedulers = ReduceLROnPlateau( + self.optimizers, + mode="max", + factor=factor, + patience=patience, + verbose=True, + ) else: for network in self.optimizers.keys(): - self.schedulers[network] = ReduceLROnPlateau(self.optimizers[network], mode='max', factor=factor, - patience=patience, verbose=True) + if self.optimizers[network] is not None: + self.schedulers[network] = ReduceLROnPlateau( + self.optimizers[network], + mode="max", + factor=factor, + patience=patience, + verbose=True, + ) + if not self.classify: + self.schedulers["classifier"] = None + return self.schedulers -if __name__ == '__main__': +if __name__ == "__main__": # Test - model_type = 'ae' - model = MultiModelAE(input_size=2, num_past=10, batch_size=5, num_future=5, lstm_hidden_size=32, num_lstm_layers=2, - classifier_hidden_size=32, num_classifier_layers=4, output_size=2, num_classes=10, - latent_size=10, batch_first=True, dropout=0.2, reset_state=True, bidirectional=True) + model_type = "custom" + model = MultiModelAE( + input_size=2, + num_past=10, + batch_size=5, + num_future=5, + lstm_hidden_size=32, + num_lstm_layers=2, + classifier_hidden_size=32, + num_classifier_layers=4, + output_size=2, + num_classes=10, + latent_size=10, + batch_first=True, + dropout=0.2, + reset_state=True, + bidirectional=True, + ) # Get the optimizers - opt = Optimizer(model_type, model, optimizer_type='RMSprop') + opt = Optimizer(model_type, model, optimizer_type="RMSprop") model_optimizers = opt.get_optimizers(lr=0.1) model_schedulers = opt.get_lrschedulers(factor=0.1, patience=10) diff --git a/traja/models/train.py b/traja/models/train.py index 979751ba..ce8164c3 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -12,6 +12,65 @@ class HybridTrainer(object): """ Wrapper for training and testing the LSTM model + Args: + model_type: Type of model should be "LSTM" + optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', + 'AdamW', 'SparseAdam', 'RMSprop', ' + Rprop', 'LBFGS', 'ASGD', 'Adamax'] + device: Selected device; 'cuda' or 'cpu' + input_size: The number of expected features in the input x + output_size: Output feature dimension + lstm_hidden_size: The number of features in the hidden state h + num_lstm_layers: Number of layers in the LSTM model + reset_state: If True, will reset the hidden and cell state for each batch of data + num_classes: Number of categories/labels + latent_size: Latent space dimension + dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + num_classifier_layers: Number of layers in the classifier + epochs: Number of epochs to train the network + batch_size: Number of samples in a batch + num_future: Number of time steps to be predicted forward + num_past: Number of past time steps otherwise, length of sequences in each batch of data. + bidirectional: If True, becomes a bidirectional LSTM + batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + loss_type: Type of reconstruction loss to apply, 'huber' or 'rmse'. Default:'huber' + lr_factor: Factor by which the learning rate will be reduced + scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. + For example, if patience = 2, then we will ignore the first 2 epochs with no + improvement, and will only decrease the LR after the 3rd epoch if the loss still + hasn’t improved then. + + """ + + def __init__( + self, + model_type: str, + optimizer_type: str, + device: str, + input_size: int, + output_size: int, + lstm_hidden_size: int, + num_lstm_layers: int, + reset_state: bool, + latent_size: int, + dropout: float, + epochs: int, + batch_size: int, + num_future: int, + num_past: int, + num_classes: int = None, + classifier_hidden_size: int = None, + num_classifier_layers: int = None, + bidirectional: bool = False, + batch_first: bool = True, + loss_type: str = "huber", + lr: float = 0.001, + lr_factor: float = 0.1, + scheduler_patience: int = 10, + ): + + white_keys = ["ae", "vae"] :param model_type: Type of model should be "LSTM" :param optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', @@ -90,6 +149,7 @@ def __init__(self, model_type: str, self.bidirectional = bidirectional self.loss_type = loss_type self.optimizer_type = optimizer_type + self.lr = lr self.lr_factor = lr_factor self.scheduler_patience = scheduler_patience @@ -118,16 +178,23 @@ def __init__(self, model_type: str, if self.model_type == 'vae': self.model = MultiModelVAE(**self.model_hyperparameters) - # Model optimizer and the learning rate scheduler based on user defined optimizer_type - # and learning rate parameters - optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) - self.model_optimizers = optimizer.get_optimizers(lr=0.001) - self.model_lrschedulers = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) + # Classification task check + self.classify = True if self.classifier_hidden_size is not None else False + + # Model optimizer and the learning rate scheduler + optimizer = Optimizer( + self.model_type, self.model, self.optimizer_type, classify=self.classify + ) + + self.model_optimizers = optimizer.get_optimizers(lr=self.lr) + self.model_lrschedulers = optimizer.get_lrschedulers( + factor=self.lr_factor, patience=self.scheduler_patience + ) def __str__(self): return "Training model type {}".format(self.model_type) - def train(self, train_loader, test_loader, model_save_path): + def fit(self, train_loader, test_loader, model_save_path=None): """ This method implements the batch- wise training and testing protocol for both time series forecasting and classification of the timeseries @@ -240,49 +307,68 @@ def train(self, train_loader, test_loader, model_save_path): class LSTMTrainer: """ Wrapper for training and testing the LSTM model - - :param model_type: Type of model should be "LSTM" - :param optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', - 'AdamW', 'SparseAdam', 'RMSprop', ' - Rprop', 'LBFGS', 'ASGD', 'Adamax'] - :param device: Selected device; 'cuda' or 'cpu' - :param epochs: Number of epochs to train the network - :param input_size: The number of expected features in the input x - :param batch_size: Number of samples in a batch - :param hidden_size: The number of features in the hidden state h - :param num_future: Number of time steps to be predicted forward - :param num_layers: Number of layers in the LSTM model - :param output_size: Output feature dimension - :param lr_factor: Factor by which the learning rate will be reduced - :param scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. - For example, if patience = 2, then we will ignore the first 2 epochs with no - improvement, and will only decrease the LR after the 3rd epoch if the loss still - hasn’t improved then. - :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - :param reset_state: If True, will reset the hidden and cell state for each batch of data - :param bidirectional: If True, becomes a bidirectional LSTM + Parameters: + ----------- + model_type: {'lstm'} + Type of model should be "LSTM" + optimizer_type: {'Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', 'LBFGS', 'ASGD', 'Adamax'} + Type of optimizer to use for training. + device: {'cuda', 'cpu'} + Target device to use for training the model + epochs: int, default=100 + Number of epochs to train the network + input_size: + The number of expected features in the input x + batch_size: + Number of samples in a batch + hidden_size: + The number of features in the hidden state h + num_future: + Number of time steps to be predicted forward + num_layers: + Number of layers in the LSTM model + output_size: + Output feature dimension + lr: + Optimizer learning rate + lr_factor: + Factor by which the learning rate will be reduced + scheduler_patience: + Number of epochs with no improvement after which learning rate will be reduced. + For example, if patience = 2, then we will ignore the first 2 epochs with no + improvement, and will only decrease the LR after the 3rd epoch if the loss still + hasn’t improved then. + batch_first: + If True, then the input and output tensors are provided as (batch, seq, feature) + dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + reset_state: + If True, will reset the hidden and cell state for each batch of data + bidirectional: + If True, becomes a bidirectional LSTM """ - def __init__(self, - model_type: str, - optimizer_type: str, - device: str, - epochs: int, - input_size: int, - batch_size: int, - hidden_size: int, - num_future: int, - num_layers: int, - output_size: int, - lr_factor: float, - scheduler_patience: int, - batch_first: True, - dropout: float, - reset_state: bool, - bidirectional: bool): + def __init__( + self, + model_type: str, + optimizer_type: str, + device: str, + epochs: int, + input_size: int, + batch_size: int, + hidden_size: int, + num_future: int, + num_layers: int, + output_size: int, + lr: float, + lr_factor: float, + scheduler_patience: int, + batch_first: True, + dropout: float, + reset_state: bool, + bidirectional: bool, + ): self.model_type = model_type self.optimizer_type = optimizer_type self.device = device @@ -294,6 +380,7 @@ def __init__(self, self.num_layers = num_layers self.output_size = output_size self.batch_first = batch_first + self.lr = lr self.lr_factor = lr_factor self.scheduler_patience = scheduler_patience self.dropout = dropout @@ -317,13 +404,14 @@ def __init__(self, self.optimizer = optimizer.get_optimizers(lr=0.001) self.scheduler = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) - def train(self, train_loader, test_loader, model_save_path): + def fit(self, train_loader, test_loader, model_save_path): - """ Implements the batch wise training and testing for time series forecasting - :param train_loader: Dataloader object of train dataset with batch data [data,target,category] - :param test_loader: Dataloader object of test dataset with [data,target,category] - :param model_save_path: Directory path to save the model - :return: None""" + """ Implements the batch wise training and testing for time series forecasting. + Args: + train_loader: Dataloader object of train dataset with batch data [data,target,category] + test_loader: Dataloader object of test dataset with [data,target,category] + model_save_path: Directory path to save the model + Return: None""" assert self.model_type == 'lstm' self.model.to(device) @@ -363,6 +451,147 @@ def train(self, train_loader, test_loader, model_save_path): utils.save_model(self.model, PATH=model_save_path) +class CustomTrainer: + """ + Wrapper for training and testing user defined models + Args: + model: Custom/User-defined model + optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', + 'AdamW', 'SparseAdam', 'RMSprop', ' + Rprop', 'LBFGS', 'ASGD', 'Adamax'] + device: Selected device; 'cuda' or 'cpu' + epochs: Number of epochs to train the network + lr:Optimizer learning rate + lr_factor: Factor by which the learning rate will be reduced + scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. + For example, if patience = 2, then we will ignore the first 2 epochs with no + improvement, and will only decrease the LR after the 3rd epoch if the loss still + hasn’t improved then. + """ + + def __init__( + self, + model: torch.nn.Module, + optimizer_type: None, + criterion: None, + device: str, + epochs: int, + lr: float = 0.001, + lr_factor: float = 0.001, + scheduler_patience: int = 10, + ): + self.model = model + self.optimizer_type = optimizer_type + self.criterion = criterion + self.device = device + self.epochs = epochs + self.lr = lr + self.lr_factor = lr_factor + self.scheduler_patience = scheduler_patience + self.model_type == "custom" + optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) + self.optimizer = optimizer.get_optimizers(lr=self.lr) + self.scheduler = optimizer.get_lrschedulers( + factor=self.lr_factor, patience=self.scheduler_patience + ) + + def fit(self, train_loader, test_loader, model_save_path): + + """ Implements the batch wise training and testing for time series forecasting + Save train, test and validation performance in forecasting/classification tasks as a performance.csv + Args: + train_loader: Dataloader object of train dataset with batch data [data,target,category] + test_loader: Dataloader object of test dataset with [data,target,category] + model_save_path: Directory path to save the model + Return: None + """ + + self.model.to(device) + + for epoch in range(self.epochs): + if epoch > 0: + self.model.train() + total_loss = 0 + for idx, (data, target, _) in enumerate(train_loader): + self.optimizer.zero_grad() + data, target = data.float().to(device), target.float().to(device) + output = self.model(data) + loss = self.criterion(output, target) + loss.backward() + self.optimizer.step() + total_loss += loss + + print("Epoch {} | loss {}".format(epoch, total_loss / (idx + 1))) + + # Testing + if epoch % 10 == 0: + with torch.no_grad(): + self.model.eval() + test_loss_forecasting = 0 + for idx, (data, target, _) in enumerate(list(test_loader)): + data, target = ( + data.float().to(device), + target.float().to(device), + ) + out = self.model(data) + test_loss_forecasting += self.criterion(out, target).item() + + test_loss_forecasting /= len(test_loader.dataset) + print(f"====> Test set generator loss: {test_loss_forecasting:.4f}") + + # Scheduler metric is test set loss + self.scheduler.step(test_loss_forecasting) + + # Save the model at target path + utils.save_model(self.model, PATH=model_save_path) + + +class Trainer: + + """Wraps all the Trainers. Instantiate and return the Trainer of model type + + Usage: + ====== + + trainer = Trainer(model_type='vae', # "ae" or "vae" + optimizer_type='Adam', # ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop','LBFGS', 'ASGD', 'Adamax'] + device='cuda', # 'cpu', 'cuda' + input_size=2, + output_size=2, + lstm_hidden_size=32, + num_lstm_layers=2, + reset_state=True, + latent_size=10, + dropout=0.1, + num_classes=9, # Uncomment to create and train classifier network + num_classifier_layers=4, + classifier_hidden_size= 32, + epochs=10, + batch_size=batch_size, + num_future=num_future, + num_past=num_past, + bidirectional=False, + batch_first=True, + loss_type='huber') # 'rmse' or 'huber' + + trainer.train(train_loader, test_loader, model_save_path) + """ + + def __init__(self): + pass + + # Check model type and instantiate corresponding trainer class: + def __new__(cls): + # Generative model trainer(model_type) + + # Predictive model trainer(model_type) + + # Custom trainer(classify=False) + + # Return the instance of the trainer + return NotImplementedError + + class VAEGANTrainer: def __init__(self): pass @@ -377,3 +606,4 @@ def __init__(self): def train(self): return NotImplementedError + diff --git a/traja/dimensionality_reduction/visualizer.py b/traja/models/visualizer.py similarity index 91% rename from traja/dimensionality_reduction/visualizer.py rename to traja/models/visualizer.py index ed416a7e..e6a9bb4d 100644 --- a/traja/dimensionality_reduction/visualizer.py +++ b/traja/models/visualizer.py @@ -21,7 +21,7 @@ import torch.optim as optim import torchvision.transforms as transforms from matplotlib import ticker, colors - +import plotly.express as px plt.switch_backend("TkAgg") @@ -32,6 +32,28 @@ style.use("ggplot") +def DisplayLatentDynamics(latent): + """Visualize the dynamics of combination of latents + Args: + latent(tensor): Each point in the list is latent's state at the end of a sequence of each batch. + Latent shape (batch_size, latent_dim) + Return: Relative plots of latent unit activations + Usage: + ====== + DisplayLatentDynamics(latent) + """ + + latents = {} + latents.fromkeys(list(range(latent.shape[1]))) + for i in range(latent.shape[1]): + latents[f"{i}"] = latent[:, i].cpu().detach().numpy() + fig = px.scatter_matrix(latents) + fig.update_layout( + autosize=False, width=1600, height=1000, + ) + return fig.show() + + class DirectedNetwork(object): def __init__(self): super().__init__() @@ -157,9 +179,6 @@ def show(self, pc, fig2): ax = Axes3D(fig2) f = ax.scatter(pc[:, 0], pc[:, 1], pc[:, 2], s=40, c=pc[:, 2]) - # ax.set_xlim(-1,1) - # ax.set_ylim(-1,1) - # ax.set_zlim(-1,1) for i in range(len(pc)): ax.plot3D( pc[i:, 0], From 5f75d704b1faa662442d79e56e0be3f4255ad871 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 28 Dec 2020 15:22:45 +0100 Subject: [PATCH 127/211] CustomTrainer test --- traja/models/generative_models/vae.py | 236 ++++++++++++++++---------- traja/models/train.py | 6 - 2 files changed, 151 insertions(+), 91 deletions(-) diff --git a/traja/models/generative_models/vae.py b/traja/models/generative_models/vae.py index d0075b8d..ae237f08 100644 --- a/traja/models/generative_models/vae.py +++ b/traja/models/generative_models/vae.py @@ -24,10 +24,10 @@ trainer.train_latent_model(train_dataloader, test_dataloader, model_save_path=PATH)""" import torch -from .utils import TimeDistributed +from traja.models.utils import TimeDistributed from torch import nn -device = 'cuda' if torch.cuda.is_available() else 'cpu' +device = "cuda" if torch.cuda.is_available() else "cpu" class LSTMEncoder(torch.nn.Module): @@ -44,10 +44,19 @@ class LSTMEncoder(torch.nn.Module): :param reset_state: If True, will reset the hidden and cell state for each batch of data :param bidirectional: If True, becomes a bidirectional LSTM """ - def __init__(self, input_size: int, num_past: int, batch_size: int, - hidden_size: int, num_lstm_layers: int, - batch_first: bool, dropout: float, - reset_state: bool, bidirectional: bool): + + def __init__( + self, + input_size: int, + num_past: int, + batch_size: int, + hidden_size: int, + num_lstm_layers: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool, + ): super(LSTMEncoder, self).__init__() self.input_size = input_size @@ -60,15 +69,24 @@ def __init__(self, input_size: int, num_past: int, batch_size: int, self.reset_state = reset_state self.bidirectional = bidirectional - self.lstm_encoder = torch.nn.LSTM(input_size=input_size, hidden_size=self.hidden_size, - num_layers=num_lstm_layers, dropout=dropout, - bidirectional=self.bidirectional, batch_first=True) + self.lstm_encoder = torch.nn.LSTM( + input_size=input_size, + hidden_size=self.hidden_size, + num_layers=num_lstm_layers, + dropout=dropout, + bidirectional=self.bidirectional, + batch_first=True, + ) def _init_hidden(self): - return (torch.zeros(self.num_lstm_layers, self.batch_size, - self.hidden_size).to(device), - torch.zeros(self.num_lstm_layers, self.batch_size, - self.hidden_size).to(device)) + return ( + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to( + device + ), + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to( + device + ), + ) def forward(self, x): @@ -98,11 +116,13 @@ def reparameterize(self, mu, logvar, training= True): return mu def forward(self, x, training=True): - + z_variables = self.latent(x) # [batch_size, latent_size*2] mu, logvar = torch.chunk(z_variables, 2, dim=1) # [batch_size,latent_size] - # Reparameterize - z = self.reparameterize(mu, logvar, training=training) # [batch_size,latent_size] + # Reparameterize + z = self.reparameterize( + mu, logvar, training=training + ) # [batch_size,latent_size] return z, mu, logvar @@ -127,10 +147,19 @@ class LSTMDecoder(torch.nn.Module): be reset at the beginning of each batch of input """ - def __init__(self, batch_size: int, num_future: int, hidden_size: int, - num_lstm_layers: int, output_size: int, latent_size: int, - batch_first: bool, dropout: float, - reset_state: bool, bidirectional: bool): + def __init__( + self, + batch_size: int, + num_future: int, + hidden_size: int, + num_lstm_layers: int, + output_size: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool, + ): super(LSTMDecoder, self).__init__() self.batch_size = batch_size self.latent_size = latent_size @@ -144,20 +173,27 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, self.bidirectional = bidirectional # RNN decoder - self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, - hidden_size=self.hidden_size, - num_layers=self.num_lstm_layers, - dropout=self.dropout, - bidirectional=self.bidirectional, - batch_first=True) - self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, - self.output_size)) + self.lstm_decoder = torch.nn.LSTM( + input_size=self.latent_size, + hidden_size=self.hidden_size, + num_layers=self.num_lstm_layers, + dropout=self.dropout, + bidirectional=self.bidirectional, + batch_first=True, + ) + self.output = TimeDistributed( + torch.nn.Linear(self.hidden_size, self.output_size) + ) def _init_hidden(self): - return (torch.zeros(self.num_lstm_layers, self.batch_size, - self.hidden_size).to(device), - torch.zeros(self.num_lstm_layers, self.batch_size, - self.hidden_size).to(device)) + return ( + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to( + device + ), + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to( + device + ), + ) def forward(self, x, num_future=None): @@ -174,18 +210,31 @@ def forward(self, x, num_future=None): # Decoder input Shape(batch_size, num_futures, latent_size) dec, _ = self.lstm_decoder(decoder_inputs, _init_hidden) - # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) + # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) # to Time Dsitributed Linear Layer output = self.output(dec) return output class MLPClassifier(torch.nn.Module): - """ MLP classifier - """ - - def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_size: int, num_classifier_layers: int, - dropout: float): + """ MLP classifier: Classify the input data using the latent embeddings + :param input_size: The number of expected latent size + :param hidden_size: The number of features in the hidden state h + :param num_classes: Size of labels or the number of categories in the data + :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + :param num_classifier_layers: Number of hidden layers in the classifier + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_classes: int, + latent_size: int, + num_classifier_layers: int, + dropout: float, + ): super(MLPClassifier, self).__init__() self.latent_size = latent_size self.input_size = input_size @@ -196,7 +245,12 @@ def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_si # Classifier layers self.hidden = nn.ModuleList([nn.Linear(self.input_size, self.hidden_size)]) - self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_classifier_layers - 1)]) + self.hidden.extend( + [ + nn.Linear(self.hidden_size, self.hidden_size) + for _ in range(1, self.num_classifier_layers - 1) + ] + ) self.hidden = nn.Sequential(*self.hidden) self.out = nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) @@ -211,21 +265,24 @@ class MultiModelVAE(torch.nn.Module): """Implementation of Multimodel Variational autoencoders; """ - def __init__(self, input_size: int, - num_past: int, - batch_size: int, - num_future: int, - lstm_hidden_size: int, - num_lstm_layers: int, - classifier_hidden_size: int, - num_classifier_layers: int, - output_size: int, - num_classes: int, - latent_size: int, - batch_first: bool, - dropout: float, - reset_state: bool, - bidirectional: bool): + def __init__( + self, + input_size: int, + num_past: int, + batch_size: int, + num_future: int, + lstm_hidden_size: int, + num_lstm_layers: int, + output_size: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool = False, + num_classifier_layers: int = None, + classifier_hidden_size: int = None, + num_classes: int = None, + ): super(MultiModelVAE, self).__init__() self.input_size = input_size @@ -244,37 +301,46 @@ def __init__(self, input_size: int, self.reset_state = reset_state self.bidirectional = bidirectional - self.encoder = LSTMEncoder(input_size=self.input_size, - num_past=self.num_past, - batch_size=self.batch_size, - hidden_size=self.lstm_hidden_size, - num_lstm_layers=self.num_lstm_layers, - batch_first=self.batch_first, - dropout=self.dropout, - reset_state=True, - bidirectional=self.bidirectional) - - self.latent = DisentangledAELatent(hidden_size=self.lstm_hidden_size, - latent_size=self.latent_size, - dropout=self.dropout) - - self.decoder = LSTMDecoder(batch_size=self.batch_size, - num_future=self.num_future, - hidden_size=self.lstm_hidden_size, - num_lstm_layers=self.num_lstm_layers, - output_size=self.output_size, - latent_size=self.latent_size, - batch_first=self.batch_first, - dropout=self.dropout, - reset_state=True, - bidirectional=self.bidirectional) - - self.classifier = MLPClassifier(input_size=self.latent_size, - hidden_size = self.classifier_hidden_size, - num_classes=self.num_classes, - latent_size=self.latent_size, - num_classifier_layers=self.num_classifier_layers, - dropout=self.dropout) + self.encoder = LSTMEncoder( + input_size=self.input_size, + num_past=self.num_past, + batch_size=self.batch_size, + hidden_size=self.lstm_hidden_size, + num_lstm_layers=self.num_lstm_layers, + batch_first=self.batch_first, + dropout=self.dropout, + reset_state=True, + bidirectional=self.bidirectional, + ) + + self.latent = DisentangledAELatent( + hidden_size=self.lstm_hidden_size, + latent_size=self.latent_size, + dropout=self.dropout, + ) + + self.decoder = LSTMDecoder( + batch_size=self.batch_size, + num_future=self.num_future, + hidden_size=self.lstm_hidden_size, + num_lstm_layers=self.num_lstm_layers, + output_size=self.output_size, + latent_size=self.latent_size, + batch_first=self.batch_first, + dropout=self.dropout, + reset_state=True, + bidirectional=self.bidirectional, + ) + + if self.num_classes is not None: + self.classifier = MLPClassifier( + input_size=self.latent_size, + hidden_size=self.classifier_hidden_size, + num_classes=self.num_classes, + latent_size=self.latent_size, + num_classifier_layers=self.num_classifier_layers, + dropout=self.dropout, + ) def forward(self, data, training=True, is_classification=False): diff --git a/traja/models/train.py b/traja/models/train.py index ce8164c3..e8027af0 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -47,7 +47,6 @@ def __init__( self, model_type: str, optimizer_type: str, - device: str, input_size: int, output_size: int, lstm_hidden_size: int, @@ -129,7 +128,6 @@ def __init__(self, model_type: str, assert model_type in white_keys, "Valid models are {}".format(white_keys) self.model_type = model_type - self.device = device self.input_size = input_size self.lstm_hidden_size = lstm_hidden_size self.num_lstm_layers = num_lstm_layers @@ -353,7 +351,6 @@ def __init__( self, model_type: str, optimizer_type: str, - device: str, epochs: int, input_size: int, batch_size: int, @@ -371,7 +368,6 @@ def __init__( ): self.model_type = model_type self.optimizer_type = optimizer_type - self.device = device self.epochs = epochs self.input_size = input_size self.batch_size = batch_size @@ -474,7 +470,6 @@ def __init__( model: torch.nn.Module, optimizer_type: None, criterion: None, - device: str, epochs: int, lr: float = 0.001, lr_factor: float = 0.001, @@ -483,7 +478,6 @@ def __init__( self.model = model self.optimizer_type = optimizer_type self.criterion = criterion - self.device = device self.epochs = epochs self.lr = lr self.lr_factor = lr_factor From 12fbf2e6da00dedb7f38b449f9b3aafe0400212f Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 28 Dec 2020 15:24:48 +0100 Subject: [PATCH 128/211] Update train.py --- traja/models/train.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index e8027af0..1efb8a47 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -1,6 +1,6 @@ -from models import MultiModelAE -from models import MultiModelVAE -from models import LSTM +from traja.models.predictive_models import MultiModelAE +from traja.models.generative_models import MultiModelVAE +from traja.models.predictive_models import LSTM from . import utils from .losses import Criterion from .optimizers import Optimizer From 1fa9b37c8ab9de6e97606d1798f7dae41f48dcf5 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 28 Dec 2020 15:26:21 +0100 Subject: [PATCH 129/211] Update train.py --- traja/models/train.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 1efb8a47..4c14a810 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -1,6 +1,8 @@ -from traja.models.predictive_models import MultiModelAE -from traja.models.generative_models import MultiModelVAE -from traja.models.predictive_models import LSTM +from traja.models.generative_models.vae import MultiModelVAE +from traja.models.predictive_models.ae import MultiModelAE +from traja.models.predictive_models.lstm import LSTM +from traja.models.predictive_models.irl import MultiModelIRL +from traja.models.generative_models.vaegan import MultiModelVAEGAN from . import utils from .losses import Criterion from .optimizers import Optimizer From bf1ffa79fb4b496cc8444375ee2012f3572abff4 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 28 Dec 2020 15:27:32 +0100 Subject: [PATCH 130/211] Update optimizers.py --- traja/models/optimizers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 07a5b2ab..fc488166 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -1,6 +1,10 @@ import torch from torch.optim.lr_scheduler import ReduceLROnPlateau -from traja.models.ae import MultiModelAE +from traja.models.generative_models.vae import MultiModelVAE +from traja.models.predictive_models.ae import MultiModelAE +from traja.models.predictive_models.lstm import LSTM +from traja.models.predictive_models.irl import MultiModelIRL +from traja.models.generative_models.vaegan import MultiModelVAEGAN class Optimizer: From fac26166631229a872f0c40cd24db320434b1084 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 28 Dec 2020 15:58:30 +0100 Subject: [PATCH 131/211] optimizer custom model update --- traja/models/optimizers.py | 4 ++++ traja/models/train.py | 16 ++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index fc488166..8ecf58ab 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -74,6 +74,10 @@ def get_optimizers(self, lr=0.0001): elif self.model_type == "vaegan": return NotImplementedError + + else: # self.model_type == "irl": + return NotImplementedError + return self.optimizers def get_lrschedulers(self, factor: float, patience: int): diff --git a/traja/models/train.py b/traja/models/train.py index 4c14a810..4b7d8b0e 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -197,12 +197,16 @@ def __str__(self): def fit(self, train_loader, test_loader, model_save_path=None): """ This method implements the batch- wise training and testing protocol for both time series forecasting and - classification of the timeseries - - :param train_loader: Dataloader object of train dataset with batch data [data,target,category] - :param test_loader: Dataloader object of test dataset with [data,target,category] - :param model_save_path: Directory path to save the model - :return: None + classification of the timeseriesis_classification + + Parameters: + ----------- + train_loader: Dataloader object of train dataset with batch [data,target,category] as a tuple + test_loader: Dataloader object of test dataset with [data,target,category] as a tuple + model_save_path: Directory path to save the model + Return: + -------- + None """ assert self.model_type == 'ae' or 'vae' From 2bd9bb5445688225912b145bc5233275ff5e81b0 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 28 Dec 2020 16:14:20 +0100 Subject: [PATCH 132/211] Update optimizers.py --- traja/models/optimizers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 8ecf58ab..0e1a0f1d 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -44,15 +44,15 @@ def get_optimizers(self, lr=0.0001): """Optimizers for each network in the model Args: - model_type ([type]): [description] - model ([type]): [description] - lr (float, optional): [description]. Defaults to 0.0001. + + lr (float, optional): Optimizer learning rate. Defaults to 0.0001. Returns: [type]: [description] """ if self.model_type == "lstm" or "custom": + print("Im running") self.optimizers = getattr(torch.optim, f"{self.optimizer_type}")( self.model.parameters(), lr=lr ) From bac3f37a65a6825d01f9b2797138962464a1a22a Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 28 Dec 2020 16:18:04 +0100 Subject: [PATCH 133/211] custom model opt update --- traja/models/optimizers.py | 5 +- ...rs.py.09b1d3e7d7591b567ec56016b6fae274.tmp | 150 ++++++++++++++++++ traja/models/train.py | 56 +------ 3 files changed, 153 insertions(+), 58 deletions(-) create mode 100644 traja/models/optimizers.py.09b1d3e7d7591b567ec56016b6fae274.tmp diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 0e1a0f1d..66844041 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -51,13 +51,12 @@ def get_optimizers(self, lr=0.0001): [type]: [description] """ - if self.model_type == "lstm" or "custom": - print("Im running") + if self.model_type in ["lstm", "custom"]: self.optimizers = getattr(torch.optim, f"{self.optimizer_type}")( self.model.parameters(), lr=lr ) - elif self.model_type == "ae" or "vae": + elif self.model_type in ["ae","vae"]: keys = ["encoder", "decoder", "latent", "classifier"] for network in keys: if network != "classifier": diff --git a/traja/models/optimizers.py.09b1d3e7d7591b567ec56016b6fae274.tmp b/traja/models/optimizers.py.09b1d3e7d7591b567ec56016b6fae274.tmp new file mode 100644 index 00000000..eca2f038 --- /dev/null +++ b/traja/models/optimizers.py.09b1d3e7d7591b567ec56016b6fae274.tmp @@ -0,0 +1,150 @@ +import torch +from torch.optim.lr_scheduler import ReduceLROnPlateau +from traja.models.generative_models.vae import MultiModelVAE +from traja.models.predictive_models.ae import MultiModelAE +from traja.models.predictive_models.lstm import LSTM +from traja.models.predictive_models.irl import MultiModelIRL +from traja.models.generative_models.vaegan import MultiModelVAEGAN + + +class Optimizer: + def __init__(self, model_type, model, optimizer_type, classify=False): + + """ + Wrapper for setting the model optimizer and learning rate schedulers using ReduceLROnPlateau; + If the model type is 'ae' or 'vae' - var optimizers is a dict with separate optimizers for encoder, decoder, + latent and classifier. In case of 'lstm', var optimizers is an optimizer for lstm and TimeDistributed(linear layer) + :param model_type: Type of model 'ae', 'vae' or 'lstm' + :param model: Model instance + :param classify: If True, will return the Optimizer and scheduler for classifier + + :param optimizer_type: Optimizer to be used; Should be one in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', + 'LBFGS', 'ASGD', 'Adamax'] + """ + + assert isinstance(model, torch.nn.Module) + assert str(optimizer_type) in [ + "Adam", + "Adadelta", + "Adagrad", + "AdamW", + "SparseAdam", + "RMSprop", + "Rprop", + "LBFGS", + "ASGD", + "Adamax", + ] + + self.model_type = model_type + self.model = model + self.optimizer_type = optimizer_type + self.classify = classify + self.optimizers = {} + self.schedulers = {} + + def get_optimizers(self, lr=0.0001): + """Optimizers for each network in the model + + Args: + + lr (float, optional): Optimizer learning rate. Defaults to 0.0001. + + Returns: + dict: Optimizers + + """ + + if self.model_type in ["lstm", "custom"]: + self.optimizers = getattr(torch.optim, f"{self.optimizer_type}")( + self.model.parameters(), lr=lr + ) + + elif self.model_type in ["ae","vae"]: + keys = ["encoder", "decoder", "latent", "classifier"] + for network in keys: + if network != "classifier": + self.optimizers[network] = getattr( + torch.optim, f"{self.optimizer_type}" + )(getattr(self.model, f"{network}").parameters(), lr=lr) + + if self.classify: + self.optimizers["classifier"] = getattr( + torch.optim, f"{self.optimizer_type}" + )(getattr(self.model, "classifier").parameters(), lr=lr) + else: + self.optimizers["classifier"] = None + + elif self.model_type == "vaegan": + return NotImplementedError + + else: # self.model_type == "irl": + return NotImplementedError + + return self.optimizers + + def get_lrschedulers(self, factor: float, patience: int): + + """Learning rate scheduler for each network in the model + NOTE: Scheduler metric should be test set loss + + Args: + factor (float, optional): [description]. Defaults to 0.1. + patience (int, optional): [description]. Defaults to 10. + + Returns: + [dict]: Learning rate schedulers + + """ + if self.model_type == "lstm" or "custom": + assert not isinstance(self.optimizers, dict) + self.schedulers = ReduceLROnPlateau( + self.optimizers, + mode="max", + factor=factor, + patience=patience, + verbose=True, + ) + else: + for network in self.optimizers.keys(): + if self.optimizers[network] is not None: + self.schedulers[network] = ReduceLROnPlateau( + self.optimizers[network], + mode="max", + factor=factor, + patience=patience, + verbose=True, + ) + if not self.classify: + self.schedulers["classifier"] = None + + return self.schedulers + + +if __name__ == "__main__": + # Test + model_type = "custom" + model = MultiModelAE( + input_size=2, + num_past=10, + batch_size=5, + num_future=5, + lstm_hidden_size=32, + num_lstm_layers=2, + classifier_hidden_size=32, + num_classifier_layers=4, + output_size=2, + num_classes=10, + latent_size=10, + batch_first=True, + dropout=0.2, + reset_state=True, + bidirectional=True, + ) + + # Get the optimizers + opt = Optimizer(model_type, model, optimizer_type="RMSprop") + model_optimizers = opt.get_optimizers(lr=0.1) + model_schedulers = opt.get_lrschedulers(factor=0.1, patience=10) + + print(model_optimizers, model_schedulers) diff --git a/traja/models/train.py b/traja/models/train.py index 4b7d8b0e..ada13d34 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -73,61 +73,7 @@ def __init__( white_keys = ["ae", "vae"] - :param model_type: Type of model should be "LSTM" - :param optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', - 'AdamW', 'SparseAdam', 'RMSprop', ' - Rprop', 'LBFGS', 'ASGD', 'Adamax'] - :param device: Selected device; 'cuda' or 'cpu' - :param input_size: The number of expected features in the input x - :param output_size: Output feature dimension - :param lstm_hidden_size: The number of features in the hidden state h - :param num_lstm_layers: Number of layers in the LSTM model - :param reset_state: If True, will reset the hidden and cell state for each batch of data - :param num_classes: Number of categories/labels - :param latent_size: Latent space dimension - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - :param num_classifier_layers: Number of layers in the classifier - :param epochs: Number of epochs to train the network - :param batch_size: Number of samples in a batch - :param num_future: Number of time steps to be predicted forward - :param num_past: Number of past time steps otherwise, length of sequences in each batch of data. - :param bidirectional: If True, becomes a bidirectional LSTM - :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - :param loss_type: Type of reconstruction loss to apply, 'huber' or 'rmse'. Default:'huber' - :param lr_factor: Factor by which the learning rate will be reduced - :param scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. - For example, if patience = 2, then we will ignore the first 2 epochs with no - improvement, and will only decrease the LR after the 3rd epoch if the loss still - hasn’t improved then. - - """ - - def __init__(self, model_type: str, - optimizer_type: str, - device: str, - input_size: int, - output_size: int, - lstm_hidden_size: int, - num_lstm_layers: int, - classifier_hidden_size: int, - num_classifier_layers: int, - reset_state: bool, - num_classes: int, - latent_size: int, - dropout: float, - epochs: int, - batch_size: int, - num_future: int, - num_past: int, - bidirectional: bool = False, - batch_first: bool = True, - loss_type: str = 'huber', - lr_factor: float = 0.1, - scheduler_patience: int = 10): - - white_keys = ['ae', 'vae'] - assert model_type in white_keys, "Valid models are {}".format(white_keys) + assert self.model_type in white_keys, "Valid models are {}".format(white_keys) self.model_type = model_type self.input_size = input_size From a52e0e0a3a7d882d8f8e2ee519cdc64f1f8dce77 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 28 Dec 2020 16:19:33 +0100 Subject: [PATCH 134/211] Update train.py --- traja/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index ada13d34..b311c5e6 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -73,7 +73,7 @@ def __init__( white_keys = ["ae", "vae"] - assert self.model_type in white_keys, "Valid models are {}".format(white_keys) + assert model_type in white_keys, "Valid models are {}".format(white_keys) self.model_type = model_type self.input_size = input_size From 13ab5dc8c28795b423c224e8968f516d8677d9b5 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 28 Dec 2020 16:23:38 +0100 Subject: [PATCH 135/211] custom model opt, scheduler fix --- traja/models/optimizers.py | 12 +- ...rs.py.09b1d3e7d7591b567ec56016b6fae274.tmp | 150 ------------------ 2 files changed, 9 insertions(+), 153 deletions(-) delete mode 100644 traja/models/optimizers.py.09b1d3e7d7591b567ec56016b6fae274.tmp diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 66844041..a25a2f41 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -56,7 +56,7 @@ def get_optimizers(self, lr=0.0001): self.model.parameters(), lr=lr ) - elif self.model_type in ["ae","vae"]: + elif self.model_type in ["ae", "vae"]: keys = ["encoder", "decoder", "latent", "classifier"] for network in keys: if network != "classifier": @@ -91,7 +91,7 @@ def get_lrschedulers(self, factor: float, patience: int): Returns: [dict]: [description] """ - if self.model_type == "lstm" or "custom": + if self.model_type in ["lstm", "custom"]: assert not isinstance(self.optimizers, dict) self.schedulers = ReduceLROnPlateau( self.optimizers, @@ -100,7 +100,7 @@ def get_lrschedulers(self, factor: float, patience: int): patience=patience, verbose=True, ) - else: + elif self.model_type in ["ae", "vae"]: for network in self.optimizers.keys(): if self.optimizers[network] is not None: self.schedulers[network] = ReduceLROnPlateau( @@ -113,6 +113,12 @@ def get_lrschedulers(self, factor: float, patience: int): if not self.classify: self.schedulers["classifier"] = None + elif self.model_type == "irl": + return NotImplementedError + + else: # self.model_type == 'vaegan': + return NotImplementedError + return self.schedulers diff --git a/traja/models/optimizers.py.09b1d3e7d7591b567ec56016b6fae274.tmp b/traja/models/optimizers.py.09b1d3e7d7591b567ec56016b6fae274.tmp deleted file mode 100644 index eca2f038..00000000 --- a/traja/models/optimizers.py.09b1d3e7d7591b567ec56016b6fae274.tmp +++ /dev/null @@ -1,150 +0,0 @@ -import torch -from torch.optim.lr_scheduler import ReduceLROnPlateau -from traja.models.generative_models.vae import MultiModelVAE -from traja.models.predictive_models.ae import MultiModelAE -from traja.models.predictive_models.lstm import LSTM -from traja.models.predictive_models.irl import MultiModelIRL -from traja.models.generative_models.vaegan import MultiModelVAEGAN - - -class Optimizer: - def __init__(self, model_type, model, optimizer_type, classify=False): - - """ - Wrapper for setting the model optimizer and learning rate schedulers using ReduceLROnPlateau; - If the model type is 'ae' or 'vae' - var optimizers is a dict with separate optimizers for encoder, decoder, - latent and classifier. In case of 'lstm', var optimizers is an optimizer for lstm and TimeDistributed(linear layer) - :param model_type: Type of model 'ae', 'vae' or 'lstm' - :param model: Model instance - :param classify: If True, will return the Optimizer and scheduler for classifier - - :param optimizer_type: Optimizer to be used; Should be one in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', - 'LBFGS', 'ASGD', 'Adamax'] - """ - - assert isinstance(model, torch.nn.Module) - assert str(optimizer_type) in [ - "Adam", - "Adadelta", - "Adagrad", - "AdamW", - "SparseAdam", - "RMSprop", - "Rprop", - "LBFGS", - "ASGD", - "Adamax", - ] - - self.model_type = model_type - self.model = model - self.optimizer_type = optimizer_type - self.classify = classify - self.optimizers = {} - self.schedulers = {} - - def get_optimizers(self, lr=0.0001): - """Optimizers for each network in the model - - Args: - - lr (float, optional): Optimizer learning rate. Defaults to 0.0001. - - Returns: - dict: Optimizers - - """ - - if self.model_type in ["lstm", "custom"]: - self.optimizers = getattr(torch.optim, f"{self.optimizer_type}")( - self.model.parameters(), lr=lr - ) - - elif self.model_type in ["ae","vae"]: - keys = ["encoder", "decoder", "latent", "classifier"] - for network in keys: - if network != "classifier": - self.optimizers[network] = getattr( - torch.optim, f"{self.optimizer_type}" - )(getattr(self.model, f"{network}").parameters(), lr=lr) - - if self.classify: - self.optimizers["classifier"] = getattr( - torch.optim, f"{self.optimizer_type}" - )(getattr(self.model, "classifier").parameters(), lr=lr) - else: - self.optimizers["classifier"] = None - - elif self.model_type == "vaegan": - return NotImplementedError - - else: # self.model_type == "irl": - return NotImplementedError - - return self.optimizers - - def get_lrschedulers(self, factor: float, patience: int): - - """Learning rate scheduler for each network in the model - NOTE: Scheduler metric should be test set loss - - Args: - factor (float, optional): [description]. Defaults to 0.1. - patience (int, optional): [description]. Defaults to 10. - - Returns: - [dict]: Learning rate schedulers - - """ - if self.model_type == "lstm" or "custom": - assert not isinstance(self.optimizers, dict) - self.schedulers = ReduceLROnPlateau( - self.optimizers, - mode="max", - factor=factor, - patience=patience, - verbose=True, - ) - else: - for network in self.optimizers.keys(): - if self.optimizers[network] is not None: - self.schedulers[network] = ReduceLROnPlateau( - self.optimizers[network], - mode="max", - factor=factor, - patience=patience, - verbose=True, - ) - if not self.classify: - self.schedulers["classifier"] = None - - return self.schedulers - - -if __name__ == "__main__": - # Test - model_type = "custom" - model = MultiModelAE( - input_size=2, - num_past=10, - batch_size=5, - num_future=5, - lstm_hidden_size=32, - num_lstm_layers=2, - classifier_hidden_size=32, - num_classifier_layers=4, - output_size=2, - num_classes=10, - latent_size=10, - batch_first=True, - dropout=0.2, - reset_state=True, - bidirectional=True, - ) - - # Get the optimizers - opt = Optimizer(model_type, model, optimizer_type="RMSprop") - model_optimizers = opt.get_optimizers(lr=0.1) - model_schedulers = opt.get_lrschedulers(factor=0.1, patience=10) - - print(model_optimizers, model_schedulers) From ebb2b9f554309f7a9bfc798160d58eedd3c178bb Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 28 Dec 2020 19:07:47 +0100 Subject: [PATCH 136/211] Update losses.py --- traja/models/losses.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/traja/models/losses.py b/traja/models/losses.py index 90060e70..15730a3d 100644 --- a/traja/models/losses.py +++ b/traja/models/losses.py @@ -9,10 +9,14 @@ class Criterion: def __init__(self): - self.huber_loss = torch.nn.SmoothL1Loss(reduction='sum') + self.huber_loss = torch.nn.SmoothL1Loss(reduction="sum") + self.mse_loss = torch.nn.MSELoss() self.crossentropy_loss = torch.nn.CrossEntropyLoss() - def ae_criterion(self, predicted, target, loss_type='huber'): + def RMSELoss(self, predicted, target): + return torch.sqrt(self.mse_loss(predicted, target)) + + def ae_criterion(self, predicted, target, loss_type="huber"): """ Implements the Autoencoder loss for time series forecasting :param predicted: Predicted time series by the model :param target: Target time series @@ -20,13 +24,13 @@ def ae_criterion(self, predicted, target, loss_type='huber'): :return: """ - if loss_type == 'huber': + if loss_type == "huber": loss = self.huber_loss(predicted, target) return loss else: # Root MSE return torch.sqrt(torch.mean((predicted - target) ** 2)) - def vae_criterion(self, predicted, target, mu, logvar, loss_type='huber'): + def vae_criterion(self, predicted, target, mu, logvar, loss_type="huber"): """ Time series generative model loss function :param predicted: Predicted time series by the model :param target: Target time series @@ -35,7 +39,7 @@ def vae_criterion(self, predicted, target, mu, logvar, loss_type='huber'): :param loss_type: Type of criterion; Defaults: 'huber' :return: Reconstruction loss + KLD loss """ - if loss_type == 'huber': + if loss_type == "huber": dist_x = self.huber_loss(predicted, target) else: dist_x = torch.sqrt(torch.mean((predicted - target) ** 2)) From 6eae8095a7289ac6681e6a1a3c7fdae3d91bad29 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Mon, 28 Dec 2020 19:34:31 +0100 Subject: [PATCH 137/211] Update train.py --- traja/models/train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index b311c5e6..c27a3a2f 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -434,7 +434,8 @@ def __init__( self.lr = lr self.lr_factor = lr_factor self.scheduler_patience = scheduler_patience - self.model_type == "custom" + + self.model_type = "custom" optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) self.optimizer = optimizer.get_optimizers(lr=self.lr) self.scheduler = optimizer.get_lrschedulers( From 2ae1155ae49c5588365b3f403a9b09767155b999 Mon Sep 17 00:00:00 2001 From: Justin Shenk Date: Mon, 28 Dec 2020 20:30:23 +0100 Subject: [PATCH 138/211] Fix assertion, formatting, train switching --- traja/models/train.py | 100 ++++++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 52 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index c27a3a2f..d55e65fa 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -14,37 +14,39 @@ class HybridTrainer(object): """ Wrapper for training and testing the LSTM model - Args: - model_type: Type of model should be "LSTM" - optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', - 'AdamW', 'SparseAdam', 'RMSprop', ' - Rprop', 'LBFGS', 'ASGD', 'Adamax'] - device: Selected device; 'cuda' or 'cpu' - input_size: The number of expected features in the input x - output_size: Output feature dimension - lstm_hidden_size: The number of features in the hidden state h - num_lstm_layers: Number of layers in the LSTM model - reset_state: If True, will reset the hidden and cell state for each batch of data - num_classes: Number of categories/labels - latent_size: Latent space dimension - dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - num_classifier_layers: Number of layers in the classifier - epochs: Number of epochs to train the network - batch_size: Number of samples in a batch - num_future: Number of time steps to be predicted forward - num_past: Number of past time steps otherwise, length of sequences in each batch of data. - bidirectional: If True, becomes a bidirectional LSTM - batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - loss_type: Type of reconstruction loss to apply, 'huber' or 'rmse'. Default:'huber' - lr_factor: Factor by which the learning rate will be reduced - scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. - For example, if patience = 2, then we will ignore the first 2 epochs with no - improvement, and will only decrease the LR after the 3rd epoch if the loss still - hasn’t improved then. - - """ + Args: + model_type: Type of model should be "LSTM" + optimizer_type: Type of optimizer to use for training.Should be from ['Adam', 'Adadelta', 'Adagrad', + 'AdamW', 'SparseAdam', 'RMSprop', ' + Rprop', 'LBFGS', 'ASGD', 'Adamax'] + device: Selected device; 'cuda' or 'cpu' + input_size: The number of expected features in the input x + output_size: Output feature dimension + lstm_hidden_size: The number of features in the hidden state h + num_lstm_layers: Number of layers in the LSTM model + reset_state: If True, will reset the hidden and cell state for each batch of data + num_classes: Number of categories/labels + latent_size: Latent space dimension + dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + num_classifier_layers: Number of layers in the classifier + epochs: Number of epochs to train the network + batch_size: Number of samples in a batch + num_future: Number of time steps to be predicted forward + num_past: Number of past time steps otherwise, length of sequences in each batch of data. + bidirectional: If True, becomes a bidirectional LSTM + batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + loss_type: Type of reconstruction loss to apply, 'huber' or 'rmse'. Default:'huber' + lr_factor: Factor by which the learning rate will be reduced + scheduler_patience: Number of epochs with no improvement after which learning rate will be reduced. + For example, if patience = 2, then we will ignore the first 2 epochs with no + improvement, and will only decrease the LR after the 3rd epoch if the loss still + hasn’t improved then. + """ + + valid_models = ["ae", "vae"] + def __init__( self, model_type: str, @@ -69,11 +71,9 @@ def __init__( lr: float = 0.001, lr_factor: float = 0.1, scheduler_patience: int = 10, - ): - - white_keys = ["ae", "vae"] + ): - assert model_type in white_keys, "Valid models are {}".format(white_keys) + assert model_type in HybridTrainer.valid_models, "Model type is {model_type}, valid models are {}".format(HybridTrainer.valid_models) self.model_type = model_type self.input_size = input_size @@ -120,8 +120,7 @@ def __init__( # Instantiate model instance based on model_type if self.model_type == 'ae': self.model = MultiModelAE(**self.model_hyperparameters) - - if self.model_type == 'vae': + elif self.model_type == "vae": self.model = MultiModelVAE(**self.model_hyperparameters) # Classification task check @@ -138,7 +137,7 @@ def __init__( ) def __str__(self): - return "Training model type {}".format(self.model_type) + return f"Training model type {self.model_type}" def fit(self, train_loader, test_loader, model_save_path=None): """ @@ -150,12 +149,10 @@ def fit(self, train_loader, test_loader, model_save_path=None): train_loader: Dataloader object of train dataset with batch [data,target,category] as a tuple test_loader: Dataloader object of test dataset with [data,target,category] as a tuple model_save_path: Directory path to save the model - Return: - -------- - None """ - assert self.model_type == 'ae' or 'vae' + assert model_save_path is not None, f"Model path {model_save_path} unknown" + self.model.to(device) encoder_optimizer, latent_optimizer, decoder_optimizer, classifier_optimizer = self.model_optimizers.values() @@ -165,7 +162,7 @@ def fit(self, train_loader, test_loader, model_save_path=None): training_mode = 'forecasting' # Training - for epoch in range(self.epochs * 2): # First half for generative model and next for classifier + for epoch in range(self.epochs): test_loss_forecasting = 0 test_loss_classification = 0 if epoch > 0: # Initial step is to test and set LR schduler @@ -179,13 +176,12 @@ def fit(self, train_loader, test_loader, model_save_path=None): decoder_optimizer.zero_grad() classifier_optimizer.zero_grad() - data, target, category = data.float().to(device), target.float().to(device), category.to(device) - - if training_mode == 'forecasting': - if self.model_type == 'ae': - decoder_out, latent_out = self.model(data, training=True, is_classification=False) + if training_mode == "forecasting": + if self.model_type == "ae": + decoder_out, latent_out = self.model( + data, training=True, classify=False + ) loss = Criterion().ae_criterion(decoder_out, target) - else: # vae decoder_out, latent_out, mu, logvar = self.model(data, training=True, is_classification=False) @@ -209,8 +205,8 @@ def fit(self, train_loader, test_loader, model_save_path=None): print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss / (idx + 1))) - if epoch + 1 == self.epochs: - training_mode = 'classification' + if (epoch + 1) * 2 == self.epochs and self.classify: + training_mode = "classification" # Testing if epoch % 10 == 0: @@ -296,7 +292,6 @@ class LSTMTrainer: If True, will reset the hidden and cell state for each batch of data bidirectional: If True, becomes a bidirectional LSTM - """ def __init__( @@ -450,7 +445,8 @@ def fit(self, train_loader, test_loader, model_save_path): train_loader: Dataloader object of train dataset with batch data [data,target,category] test_loader: Dataloader object of test dataset with [data,target,category] model_save_path: Directory path to save the model - Return: None + Return: + None """ self.model.to(device) From a462cf6c8bbde33008b68efcd3512b0afaf3cf00 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 11:20:10 +0100 Subject: [PATCH 139/211] generator, predictor, model,hyper save,load and infer update --- traja/models/__init__.py | 2 +- traja/models/generative_models/vae.py | 24 +- traja/models/generator.py | 3 +- traja/models/predictive_models/ae.py | 335 ++++++++++++++++++-------- traja/models/train.py | 101 ++++---- traja/models/utils.py | 68 ++++-- 6 files changed, 354 insertions(+), 179 deletions(-) diff --git a/traja/models/__init__.py b/traja/models/__init__.py index 4e16eb50..d8a302e1 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -4,4 +4,4 @@ from traja.models.predictive_models.lstm import LSTM from traja.models.predictive_models.irl import MultiModelIRL from traja.models.generative_models.vaegan import MultiModelVAEGAN - +from .utils import TimeDistributed, read_hyperparameters, save, load diff --git a/traja/models/generative_models/vae.py b/traja/models/generative_models/vae.py index ae237f08..7ba4f6bc 100644 --- a/traja/models/generative_models/vae.py +++ b/traja/models/generative_models/vae.py @@ -80,12 +80,12 @@ def __init__( def _init_hidden(self): return ( - torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to( - device - ), - torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to( - device - ), + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) + .requires_grad() + .to(device), + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) + .requires_grad() + .to(device), ) @@ -187,12 +187,12 @@ def __init__( def _init_hidden(self): return ( - torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to( - device - ), - torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size).to( - device - ), + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) + .requires_grad() + .to(device), + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) + .requires_grad() + .to(device), ) def forward(self, x, num_future=None): diff --git a/traja/models/generator.py b/traja/models/generator.py index 0c23e333..a5cd8fdc 100644 --- a/traja/models/generator.py +++ b/traja/models/generator.py @@ -9,6 +9,7 @@ from .lstm import LSTM from .utils import load_model import matplotlib.pyplot as plt +import os device = "cuda" if torch.cuda.is_available() else "cpu" @@ -25,7 +26,7 @@ def __init__( Args: model_type (str, optional): Type of model ['vae','vaegan','custom']. Defaults to None. - model_path (str, optional): [description]. Defaults to None. + model_path (str, optional): Path to trained model (model.pt). Defaults to None. model_hyperparameters (dict, optional): [description]. Defaults to None. model (torch.nn.Module, optional): Custom model from user. Defaults to None """ diff --git a/traja/models/predictive_models/ae.py b/traja/models/predictive_models/ae.py index 648e6b82..fa84e670 100644 --- a/traja/models/predictive_models/ae.py +++ b/traja/models/predictive_models/ae.py @@ -1,29 +1,40 @@ import torch from traja.models.utils import TimeDistributed from torch import nn -device = 'cuda' if torch.cuda.is_available() else 'cpu' + +device = "cuda" if torch.cuda.is_available() else "cpu" class LSTMEncoder(torch.nn.Module): """ Implementation of Encoder network using LSTM layers - :param input_size: The number of expected features in the input x - :param num_past: Number of time steps to look backwards to predict num_future steps forward - :param batch_size: Number of samples in a batch - :param hidden_size: The number of features in the hidden state h - :param num_lstm_layers: Number of layers in the LSTM model - - :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - :param reset_state: If True, will reset the hidden and cell state for each batch of data - :param bidirectional: If True, becomes a bidirectional LSTM + Parameters: + ----------- + input_size: The number of expected features in the input x + num_past: Number of time steps to look backwards to predict num_future steps forward + batch_size: Number of samples in a batch + hidden_size: The number of features in the hidden state h + num_lstm_layers: Number of layers in the LSTM model + + batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + reset_state: If True, will reset the hidden and cell state for each batch of data + bidirectional: If True, becomes a bidirectional LSTM """ - def __init__(self, input_size: int, num_past: int, batch_size: int, - hidden_size: int, num_lstm_layers: int, - batch_first: bool, dropout: float, - reset_state: bool, bidirectional: bool): + def __init__( + self, + input_size: int, + num_past: int, + batch_size: int, + hidden_size: int, + num_lstm_layers: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool, + ): super(LSTMEncoder, self).__init__() @@ -37,13 +48,24 @@ def __init__(self, input_size: int, num_past: int, batch_size: int, self.reset_state = reset_state self.bidirectional = bidirectional - self.lstm_encoder = torch.nn.LSTM(input_size=input_size, hidden_size=self.hidden_size, - num_layers=num_lstm_layers, dropout=dropout, - bidirectional=self.bidirectional, batch_first=True) + self.lstm_encoder = torch.nn.LSTM( + input_size=input_size, + hidden_size=self.hidden_size, + num_layers=num_lstm_layers, + dropout=dropout, + bidirectional=self.bidirectional, + batch_first=True, + ) def _init_hidden(self): - return (torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size), - torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size)) + return ( + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) + .requires_grad() + .to(device), + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) + .requires_grad() + .to(device), + ) def forward(self, x): enc_init_hidden = self._init_hidden() @@ -70,30 +92,37 @@ def forward(self, x): class LSTMDecoder(torch.nn.Module): - """ Deep LSTM network. This implementation - returns output_size outputs. - Args: - latent_size: The number of dimensions of the latent layer - batch_size: Number of samples in each batch of training data - hidden_size: The number of features in the hidden state `h` - num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` - would mean stacking two LSTMs together to form a `stacked LSTM`, - with the second LSTM taking in outputs of the first LSTM and - computing the final results. Default: 1 - output_size: The number of output/input dimensions - num_future: The number of time steps in future predictions - dropout: If non-zero, introduces a `Dropout` layer on the outputs of each - LSTM layer except the last layer, with dropout probability equal to - :attr:`dropout`. Default: 0 - bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` - reset_state: If ``True``, the hidden and cell states of the LSTM will - be reset at the beginning of each batch of input + + """ Implementation of Decoder network using LSTM layers + Parameters: + ------------ + input_size: The number of expected features in the input x + num_future: Number of time steps to be predicted given the num_past steps + batch_size: Number of samples in a batch + hidden_size: The number of features in the hidden state h + num_lstm_layers: Number of layers in the LSTM model + output_size: Number of expectd features in the output x_ + batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + reset_state: If True, will reset the hidden and cell state for each batch of data + bidirectional: If True, becomes a bidirectional LSTM + """ - def __init__(self, batch_size: int, num_future: int, hidden_size: int, - num_lstm_layers: int, output_size: int, latent_size: int, - batch_first: bool, dropout: float, - reset_state: bool, bidirectional: bool): + def __init__( + self, + batch_size: int, + num_future: int, + hidden_size: int, + num_lstm_layers: int, + output_size: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool, + ): super(LSTMDecoder, self).__init__() self.batch_size = batch_size self.latent_size = latent_size @@ -107,24 +136,31 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, self.bidirectional = bidirectional # RNN decoder - self.lstm_decoder = torch.nn.LSTM(input_size=self.latent_size, - hidden_size=self.hidden_size, - num_layers=self.num_lstm_layers, - dropout=self.dropout, - bidirectional=self.bidirectional, - batch_first=True) - self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, - self.output_size)) + self.lstm_decoder = torch.nn.LSTM( + input_size=self.latent_size, + hidden_size=self.hidden_size, + num_layers=self.num_lstm_layers, + dropout=self.dropout, + bidirectional=self.bidirectional, + batch_first=True, + ) + self.output = TimeDistributed( + torch.nn.Linear(self.hidden_size, self.output_size) + ) def _init_hidden(self): - return (torch.zeros(self.num_lstm_layers, self.batch_size, - self.hidden_size).to(device), - torch.zeros(self.num_lstm_layers, self.batch_size, - self.hidden_size).to(device)) + return ( + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) + .requires_grad() + .to(device), + torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) + .requires_grad() + .to(device), + ) def forward(self, x, num_future=None): - # To feed the latent states into lstm decoder, + # To feed the latent states into lstm decoder, # repeat the tensor n_future times at second dim _init_hidden = self._init_hidden() decoder_inputs = x.unsqueeze(1) @@ -137,7 +173,7 @@ def forward(self, x, num_future=None): # Decoder input Shape(batch_size, num_futures, latent_size) dec, _ = self.lstm_decoder(decoder_inputs, _init_hidden) - # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) + # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) # to Time Dsitributed Linear Layer output = self.output(dec) return output @@ -147,8 +183,26 @@ class MLPClassifier(torch.nn.Module): """ MLP classifier """ - def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_size: int, num_classifier_layers: int, - dropout: float): + """ MLP classifier: Classify the input data using the latent embeddings + Parameters: + ----------- + input_size: The number of expected latent size + hidden_size: The number of features in the hidden state h + num_classes: Size of labels or the number of categories in the data + dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + num_classifier_layers: Number of hidden layers in the classifier + """ + + def __init__( + self, + input_size: int, + hidden_size: int, + num_classes: int, + latent_size: int, + num_classifier_layers: int, + dropout: float, + ): super(MLPClassifier, self).__init__() self.latent_size = latent_size self.input_size = input_size @@ -159,7 +213,12 @@ def __init__(self, input_size: int, hidden_size:int, num_classes: int, latent_si # Classifier layers self.hidden = nn.ModuleList([nn.Linear(self.input_size, self.hidden_size)]) - self.hidden.extend([nn.Linear(self.hidden_size, self.hidden_size) for _ in range(1, self.num_classifier_layers - 1)]) + self.hidden.extend( + [ + nn.Linear(self.hidden_size, self.hidden_size) + for _ in range(1, self.num_classifier_layers - 1) + ] + ) self.hidden = nn.Sequential(*self.hidden) self.out = nn.Linear(self.hidden_size, self.num_classes) self.dropout = torch.nn.Dropout(p=dropout) @@ -172,10 +231,43 @@ def forward(self, x): class MultiModelAE(torch.nn.Module): - def __init__(self, input_size: int, num_past: int, batch_size: int, num_future: int, lstm_hidden_size: int, - num_lstm_layers: int, classifier_hidden_size: int, num_classifier_layers: int, output_size: int, - num_classes: int, latent_size: int, batch_first: bool, dropout: float, reset_state: bool, - bidirectional: bool): + """Implementation of Multimodel autoencoders; This Module wraps the Autoencoder + models [Encoder,Latent,Decoder]. If classify=True, then the wrapper also include classification layers + + Parameters: + ----------- + input_size: The number of expected features in the input x + num_future: Number of time steps to be predicted given the num_past steps + batch_size: Number of samples in a batch + hidden_size: The number of features in the hidden state h + num_lstm_layers: Number of layers in the LSTM model + output_size: Number of expectd features in the output x_ + batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + reset_state: If True, will reset the hidden and cell state for each batch of data + bidirectional: If True, becomes a bidirectional LSTM + + """ + + def __init__( + self, + input_size: int, + num_past: int, + batch_size: int, + num_future: int, + lstm_hidden_size: int, + num_lstm_layers: int, + output_size: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool = False, + num_classifier_layers: int = None, + classifier_hidden_size: int = None, + num_classes: int = None, + ): super(MultiModelAE, self).__init__() self.input_size = input_size @@ -194,37 +286,84 @@ def __init__(self, input_size: int, num_past: int, batch_size: int, num_future: self.reset_state = reset_state self.bidirectional = bidirectional - self.encoder = LSTMEncoder(input_size=self.input_size, - num_past=self.num_past, - batch_size=self.batch_size, - hidden_size=self.lstm_hidden_size, - num_lstm_layers=self.num_lstm_layers, - batch_first=self.batch_first, - dropout=self.dropout, - reset_state=True, - bidirectional=self.bidirectional) - - self.latent = DisentangledAELatent(hidden_size=self.lstm_hidden_size, - latent_size=self.latent_size, - dropout=self.dropout) - - self.decoder = LSTMDecoder(batch_size=self.batch_size, - num_future=self.num_future, - hidden_size=self.lstm_hidden_size, - num_lstm_layers=self.num_lstm_layers, - output_size=self.output_size, - latent_size=self.latent_size, - batch_first=self.batch_first, - dropout=self.dropout, - reset_state=True, - bidirectional=self.bidirectional) - - self.classifier = MLPClassifier(input_size=self.latent_size, - hidden_size=self.classifier_hidden_size, - num_classes=self.num_classes, - latent_size=self.latent_size, - num_classifier_layers=self.num_classifier_layers, - dropout=self.dropout) + self.encoder = LSTMEncoder( + input_size=self.input_size, + num_past=self.num_past, + batch_size=self.batch_size, + hidden_size=self.lstm_hidden_size, + num_lstm_layers=self.num_lstm_layers, + batch_first=self.batch_first, + dropout=self.dropout, + reset_state=True, + bidirectional=self.bidirectional, + ) + + self.latent = DisentangledAELatent( + hidden_size=self.lstm_hidden_size, + latent_size=self.latent_size, + dropout=self.dropout, + ) + + self.decoder = LSTMDecoder( + batch_size=self.batch_size, + num_future=self.num_future, + hidden_size=self.lstm_hidden_size, + num_lstm_layers=self.num_lstm_layers, + output_size=self.output_size, + latent_size=self.latent_size, + batch_first=self.batch_first, + dropout=self.dropout, + reset_state=True, + bidirectional=self.bidirectional, + ) + + if self.num_classes is not None: + self.classifier = MLPClassifier( + input_size=self.latent_size, + hidden_size=self.classifier_hidden_size, + num_classes=self.num_classes, + latent_size=self.latent_size, + num_classifier_layers=self.num_classifier_layers, + dropout=self.dropout, + ) + + def get_ae_parameters(self): + """ + Return: + ------- + Tuple of parameters of the encoder, latent and decoder networks + """ + return [ + self.encoder.parameters(), + self.latent.parameters(), + self.decoder.parameters(), + ] + + def get_classifier_parameters(self): + """ + Return: + ------- + Tuple of parameters of classifier network + """ + assert self.classifier_hidden_size is not None, "Classifier not found" + return [self.classifier.parameters()] + + def forward(self, data, classify=False, training=True): + """ + Parameters: + ----------- + data: Train or test data + training: If Training= False, latents are deterministic; This arg is unused; + classify: If True, perform classification of input data using the latent embeddings + Return: + ------- + decoder_out,latent_out or classifier out + """ + if not classify: + # Set the classifier grad off + if self.num_classes is not None: + for param in self.classifier.parameters(): + param.requires_grad = False def forward(self, data, training=True, is_classification=False): if not is_classification: @@ -238,14 +377,16 @@ def forward(self, data, training=True, is_classification=False): for param in self.latent.parameters(): param.requires_grad = True - # Encoder + # Encoder -->Latent --> Decoder enc_out = self.encoder(data) # Latent latent_out = self.latent(enc_out) # Decoder decoder_out = self.decoder(latent_out) - return decoder_out, latent_out + else: # Classify + # Unfreeze classifier and freeze the rest + assert self.num_classifier_layers is not None, "Classifier not found" else: # training_mode = 'classification' # Unfreeze classifier parameters and freeze all other diff --git a/traja/models/train.py b/traja/models/train.py index d55e65fa..91493fa6 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -5,7 +5,7 @@ from traja.models.generative_models.vaegan import MultiModelVAEGAN from . import utils from .losses import Criterion -from .optimizers import Optimizer +from .optimizers import Optimizer, model_type import torch device = 'cuda' if torch.cuda.is_available() else 'cpu' @@ -46,7 +46,7 @@ class HybridTrainer(object): """ valid_models = ["ae", "vae"] - + def __init__( self, model_type: str, @@ -71,9 +71,13 @@ def __init__( lr: float = 0.001, lr_factor: float = 0.1, scheduler_patience: int = 10, - ): + ): - assert model_type in HybridTrainer.valid_models, "Model type is {model_type}, valid models are {}".format(HybridTrainer.valid_models) + assert ( + model_type in HybridTrainer.valid_models + ), "Model type is {model_type}, valid models are {}".format( + HybridTrainer.valid_models + ) self.model_type = model_type self.input_size = input_size @@ -247,7 +251,7 @@ def fit(self, train_loader, test_loader, model_save_path=None): classifier_scheduler.step(test_loss_classification) # Save the model at target path - utils.save_model(self.model, PATH=model_save_path) + utils.save_model(self.model, self.model_hyperparameters, PATH=model_save_path) class LSTMTrainer: @@ -391,7 +395,7 @@ def fit(self, train_loader, test_loader, model_save_path): self.scheduler.step(test_loss_forecasting) # Save the model at target path - utils.save_model(self.model, PATH=model_save_path) + utils.save_model(self.model, self.model_hyperparameters, PATH=model_save_path) class CustomTrainer: @@ -486,56 +490,18 @@ def fit(self, train_loader, test_loader, model_save_path): self.scheduler.step(test_loss_forecasting) # Save the model at target path - utils.save_model(self.model, PATH=model_save_path) + utils.save_model(self.model, self.model_hyperparameters, PATH=model_save_path) -class Trainer: - - """Wraps all the Trainers. Instantiate and return the Trainer of model type - - Usage: - ====== - - trainer = Trainer(model_type='vae', # "ae" or "vae" - optimizer_type='Adam', # ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop','LBFGS', 'ASGD', 'Adamax'] - device='cuda', # 'cpu', 'cuda' - input_size=2, - output_size=2, - lstm_hidden_size=32, - num_lstm_layers=2, - reset_state=True, - latent_size=10, - dropout=0.1, - num_classes=9, # Uncomment to create and train classifier network - num_classifier_layers=4, - classifier_hidden_size= 32, - epochs=10, - batch_size=batch_size, - num_future=num_future, - num_past=num_past, - bidirectional=False, - batch_first=True, - loss_type='huber') # 'rmse' or 'huber' - - trainer.train(train_loader, test_loader, model_save_path) - """ - +class VAEGANTrainer: def __init__(self): pass - # Check model type and instantiate corresponding trainer class: - def __new__(cls): - # Generative model trainer(model_type) - - # Predictive model trainer(model_type) - - # Custom trainer(classify=False) - - # Return the instance of the trainer + def train(self): return NotImplementedError -class VAEGANTrainer: +class IRLTrainer: def __init__(self): pass @@ -543,10 +509,41 @@ def train(self): return NotImplementedError -class IRLTrainer: - def __init__(self): - pass +# TODO +class Trainer: - def train(self): + """Wraps all above Trainers. Instantiate and return the Trainer of model type """ + + def __init__(self, *model_hyperparameters, **kwargs): + self.model_type = model_hyperparameters["model_type"] + self.TrainerType = None + + @property + def TrainerType(self): + return self.__TrainerType + + @TrainerType.setter + def TrainerType(self, model_type): + """[summary] + + Args: + model_type ([type]): [description] + """ + if model_type in ["ae", "vae"]: + self.__TrainerType = HybridTrainer + elif model_type in ["lstm"]: + self.__TrainerType = LSTMTrainer + else: + self.__TrainerType = CustomTrainer + + # Check model type and instantiate corresponding trainer class: + def __new__(cls): + # Generative model trainer(model_type) + + # Predictive model trainer(model_type) + + # Custom trainer(classify=False) + + # Return the instance of the trainer return NotImplementedError diff --git a/traja/models/utils.py b/traja/models/utils.py index 9501302b..0cb0c946 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -3,6 +3,7 @@ import numpy as np import collections from numpy import math +import json class TimeDistributed(torch.nn.Module): @@ -38,33 +39,68 @@ def forward(self, x): return out -def save_model(model, PATH): - """[summary] - +def save(model, hyperparameters, PATH=None): + """Save the trained model(.pth) along with its hyperparameters as a json (hyper.json) at the user defined Path Args: - model ([type]): [description] - PATH ([type]): [description] + model (torch.nn.Module): Trained Model + hyperparameters(dict): Hyperparameters of the model + PATH (str): Directory path to save the trained model and its hyperparameters + Returns: + None """ - - # PATH = "state_dict_model.pt" + if not isinstance(hyperparameters, dict): + raise Exception("Invalid argument, hyperparameters must be dict") # Save + if PATH is None: + PATH = os.getcwd() + "model.pt" torch.save(model.state_dict(), PATH) - print("Model saved at {}".format(PATH)) + with open(PATH + "/hypers.json", "w") as fp: + json.dump(hyperparameters, fp, sort_keys=False) + print("Model and hyperparameters saved at {} ".format(PATH)) -def load_model(model, model_hyperparameters, PATH): - """[summary] +def load(model, PATH=None): + """Load trained model from PATH using the model_hyperparameters saved in the Args: - model ([type]): [description] - model_hyperparameters ([type]): [description] - PATH ([type]): [description] - + model (torch.nn.Module): Type of the model ['ae','vae','vaegan','irl','lstm','custom'] + model_hyperparameters (dict): Dictionary of hyperparameters used to initiate model + PATH (str): Directory path of the model Returns: - [type]: [description] + model(torch.nn.module): Model """ + # Hyperparameters + if PATH is None: + PATH = os.getcwd() + "/model.pt" + print(f"Model loaded from {PATH}") + else: + raise Exception("Model not found at " f"{PATH}") + + # Get hyperparameters from the model path + PATH, _ = os.path.split(PATH) + try: + with open(PATH + "/hypers.json") as f: + hypers = json.load(f) + except: + raise Exception("Hyper parameters not found at " f"{PATH}") + # Load - model = model(model_hyperparameters) + model = model(**hypers) + # Load state of the model model.load_state_dict(torch.load(PATH)) return model + + +def read_hyperparameters(hyperparameter_json): + """Read the json file and return the hyperparameters as dict + + Args: + hyperparameter_json (json): Json file containing the hyperparameters of the trained model + + Returns: + [dict]: Python dictionary of the hyperparameters + """ + with open(hyperparameter_json) as f_in: + return json.load(f_in) + From d0b3c94571d0198fa2826a4f507113b3ffd0384f Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 11:37:04 +0100 Subject: [PATCH 140/211] Update train.py --- traja/models/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 91493fa6..c1d58f46 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -5,7 +5,7 @@ from traja.models.generative_models.vaegan import MultiModelVAEGAN from . import utils from .losses import Criterion -from .optimizers import Optimizer, model_type +from .optimizers import Optimizer import torch device = 'cuda' if torch.cuda.is_available() else 'cpu' @@ -536,7 +536,7 @@ def TrainerType(self, model_type): else: self.__TrainerType = CustomTrainer - # Check model type and instantiate corresponding trainer class: + # Check model type, instantiate and set corresponding trainer as traja trainer: def __new__(cls): # Generative model trainer(model_type) From 3882a9a3509e09a5d6018fcb9eaca36d3072727a Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 11:52:04 +0100 Subject: [PATCH 141/211] lstms,ae,vae init requires grad set --- traja/models/generative_models/vae.py | 8 ++--- traja/models/predictive_models/ae.py | 8 ++--- traja/models/predictive_models/lstm.py | 43 ++++++++++++++++++++------ traja/models/train.py | 6 ++-- 4 files changed, 44 insertions(+), 21 deletions(-) diff --git a/traja/models/generative_models/vae.py b/traja/models/generative_models/vae.py index 7ba4f6bc..37924458 100644 --- a/traja/models/generative_models/vae.py +++ b/traja/models/generative_models/vae.py @@ -81,10 +81,10 @@ def __init__( def _init_hidden(self): return ( torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad() + .requires_grad_() .to(device), torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad() + .requires_grad_() .to(device), ) @@ -188,10 +188,10 @@ def __init__( def _init_hidden(self): return ( torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad() + .requires_grad_() .to(device), torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad() + .requires_grad_() .to(device), ) diff --git a/traja/models/predictive_models/ae.py b/traja/models/predictive_models/ae.py index fa84e670..b6ccf063 100644 --- a/traja/models/predictive_models/ae.py +++ b/traja/models/predictive_models/ae.py @@ -60,10 +60,10 @@ def __init__( def _init_hidden(self): return ( torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad() + .requires_grad_() .to(device), torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad() + .requires_grad_() .to(device), ) @@ -151,10 +151,10 @@ def __init__( def _init_hidden(self): return ( torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad() + .requires_grad_() .to(device), torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad() + .requires_grad_() .to(device), ) diff --git a/traja/models/predictive_models/lstm.py b/traja/models/predictive_models/lstm.py index d5f01d51..d8f97cdb 100644 --- a/traja/models/predictive_models/lstm.py +++ b/traja/models/predictive_models/lstm.py @@ -2,7 +2,7 @@ import torch from traja.models.utils import TimeDistributed -device = 'cuda' if torch.cuda.is_available() else 'cpu' +device = "cuda" if torch.cuda.is_available() else "cpu" class LSTM(torch.nn.Module): @@ -24,9 +24,19 @@ class LSTM(torch.nn.Module): bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` """ - def __init__(self, batch_size: int, num_future: int, hidden_size: int, num_layers: int, - output_size: int, input_size: int, batch_first: bool, dropout: float, - reset_state: bool, bidirectional: bool): + def __init__( + self, + batch_size: int, + num_future: int, + hidden_size: int, + num_layers: int, + output_size: int, + input_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool, + ): super(LSTM, self).__init__() self.batch_size = batch_size @@ -41,14 +51,27 @@ def __init__(self, batch_size: int, num_future: int, hidden_size: int, num_layer self.bidirectional = bidirectional # RNN decoder - self.lstm = torch.nn.LSTM(input_size=self.input_size, hidden_size=self.hidden_size, - num_layers=self.num_layers, dropout=self.dropout, - bidirectional=self.bidirectional, batch_first=True) - self.output = TimeDistributed(torch.nn.Linear(self.hidden_size, self.output_size)) + self.lstm = torch.nn.LSTM( + input_size=self.input_size, + hidden_size=self.hidden_size, + num_layers=self.num_layers, + dropout=self.dropout, + bidirectional=self.bidirectional, + batch_first=True, + ) + self.output = TimeDistributed( + torch.nn.Linear(self.hidden_size, self.output_size) + ) def _init_hidden(self): - return (torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device), - torch.zeros(self.num_layers, self.batch_size, self.hidden_size).to(device)) + return ( + torch.zeros(self.num_layers, self.batch_size, self.hidden_size) + .requires_grad_() + .to(device), + torch.zeros(self.num_layers, self.batch_size, self.hidden_size) + .requires_grad_() + .to(device), + ) def forward(self, x): # To feed the latent states into lstm decoder, repeat the tensor n_future times at second dim diff --git a/traja/models/train.py b/traja/models/train.py index c1d58f46..9fbb5185 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -251,7 +251,7 @@ def fit(self, train_loader, test_loader, model_save_path=None): classifier_scheduler.step(test_loss_classification) # Save the model at target path - utils.save_model(self.model, self.model_hyperparameters, PATH=model_save_path) + utils.save(self.model, self.model_hyperparameters, PATH=model_save_path) class LSTMTrainer: @@ -395,7 +395,7 @@ def fit(self, train_loader, test_loader, model_save_path): self.scheduler.step(test_loss_forecasting) # Save the model at target path - utils.save_model(self.model, self.model_hyperparameters, PATH=model_save_path) + utils.save(self.model, self.model_hyperparameters, PATH=model_save_path) class CustomTrainer: @@ -490,7 +490,7 @@ def fit(self, train_loader, test_loader, model_save_path): self.scheduler.step(test_loss_forecasting) # Save the model at target path - utils.save_model(self.model, self.model_hyperparameters, PATH=model_save_path) + utils.save(self.model, self.model_hyperparameters, PATH=model_save_path) class VAEGANTrainer: From 06b9d7838f642d73a8bd8e9bb9a63cb67165dc58 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 12:05:08 +0100 Subject: [PATCH 142/211] Update utils.py --- traja/models/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/traja/models/utils.py b/traja/models/utils.py index 0cb0c946..c4b20660 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -3,7 +3,7 @@ import numpy as np import collections from numpy import math -import json +import os, json class TimeDistributed(torch.nn.Module): @@ -54,10 +54,10 @@ def save(model, hyperparameters, PATH=None): if PATH is None: PATH = os.getcwd() + "model.pt" torch.save(model.state_dict(), PATH) - - with open(PATH + "/hypers.json", "w") as fp: + hyperdir = os.path.split(PATH) + with open(hyperdir + "/hypers.json", "w") as fp: json.dump(hyperparameters, fp, sort_keys=False) - print("Model and hyperparameters saved at {} ".format(PATH)) + print("Model and hyperparameters saved at {} ".format(hyperdir)) def load(model, PATH=None): From ab1cea9490790463cbe0ee752035d9d334851ff8 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 19:01:02 +0100 Subject: [PATCH 143/211] network and embedding visualizer update --- traja/models/generative_models/vae.py | 4 +- traja/models/predictive_models/ae.py | 8 +-- traja/models/predictive_models/lstm.py | 4 +- traja/models/train.py | 88 +++++++++++++++++++++++--- traja/models/visualizer.py | 20 ++++++ 5 files changed, 108 insertions(+), 16 deletions(-) diff --git a/traja/models/generative_models/vae.py b/traja/models/generative_models/vae.py index 37924458..5bbd9ec9 100644 --- a/traja/models/generative_models/vae.py +++ b/traja/models/generative_models/vae.py @@ -199,7 +199,7 @@ def forward(self, x, num_future=None): # To feed the latent states into lstm decoder, # repeat the tensor n_future times at second dim - _init_hidden = self._init_hidden() + (h0, c0) = self._init_hidden() decoder_inputs = x.unsqueeze(1) if num_future is None: @@ -208,7 +208,7 @@ def forward(self, x, num_future=None): decoder_inputs = decoder_inputs.repeat(1, num_future, 1) # Decoder input Shape(batch_size, num_futures, latent_size) - dec, _ = self.lstm_decoder(decoder_inputs, _init_hidden) + dec, _ = self.lstm_decoder(decoder_inputs, (h0.detach(), c0.detach())) # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) # to Time Dsitributed Linear Layer diff --git a/traja/models/predictive_models/ae.py b/traja/models/predictive_models/ae.py index b6ccf063..dd1cffc1 100644 --- a/traja/models/predictive_models/ae.py +++ b/traja/models/predictive_models/ae.py @@ -68,8 +68,8 @@ def _init_hidden(self): ) def forward(self, x): - enc_init_hidden = self._init_hidden() - enc_output, _ = self.lstm_encoder(x, enc_init_hidden) + (h0, c0) = self._init_hidden() + enc_output, _ = self.lstm_encoder(x, (h0.detach(), c0.detach())) # RNNs obeys, Markovian. So, the last state of the hidden is the markovian state for the entire # sequence in that batch. enc_output = enc_output[:, -1, :] # Shape(batch_size,hidden_dim) @@ -162,7 +162,7 @@ def forward(self, x, num_future=None): # To feed the latent states into lstm decoder, # repeat the tensor n_future times at second dim - _init_hidden = self._init_hidden() + (h0, c0) = self._init_hidden() decoder_inputs = x.unsqueeze(1) if num_future is None: @@ -171,7 +171,7 @@ def forward(self, x, num_future=None): decoder_inputs = decoder_inputs.repeat(1, num_future, 1) # Decoder input Shape(batch_size, num_futures, latent_size) - dec, _ = self.lstm_decoder(decoder_inputs, _init_hidden) + dec, _ = self.lstm_decoder(decoder_inputs, (h0.detach(), c0.detach())) # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) # to Time Dsitributed Linear Layer diff --git a/traja/models/predictive_models/lstm.py b/traja/models/predictive_models/lstm.py index d8f97cdb..519904e8 100644 --- a/traja/models/predictive_models/lstm.py +++ b/traja/models/predictive_models/lstm.py @@ -75,10 +75,10 @@ def _init_hidden(self): def forward(self, x): # To feed the latent states into lstm decoder, repeat the tensor n_future times at second dim - _init_hidden = self._init_hidden() + (h0, c0) = self._init_hidden() # Decoder input Shape(batch_size, num_futures, latent_size) - out, (dec_hidden, dec_cell) = self.lstm(x, _init_hidden) + out, (dec_hidden, dec_cell) = self.lstm(x, (h0.detach(), c0.detach())) # Map the decoder output: Shape(batch_size, sequence_len, hidden_dim) to Time Dsitributed Linear Layer out = self.output(out) diff --git a/traja/models/train.py b/traja/models/train.py index 9fbb5185..e206f7f7 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -4,9 +4,11 @@ from traja.models.predictive_models.irl import MultiModelIRL from traja.models.generative_models.vaegan import MultiModelVAEGAN from . import utils +from . import visualizer from .losses import Criterion from .optimizers import Optimizer import torch +import matplotlib.pyplot as plt device = 'cuda' if torch.cuda.is_available() else 'cpu' @@ -453,6 +455,23 @@ def fit(self, train_loader, test_loader, model_save_path): None """ + # Init Visualization + if self.viz == "True": + self.fig = plt.figure(num="Latent Network Activity") + self.plt_close = False + self.directednetwork = visualizer.DirectedNetwork() + + self.fig2 = plt.figure(num="Local Linear Embedded Trajectory") + self.plt2_close = False + self.lle = visualizer.LocalLinearEmbedding() + + self.fig3 = plt.figure(num="Spectral Embedded Latent") + self.plt3_close = False + self.spectral_clustering = visualizer.SpectralEmbedding() + + plt.pause(0.00001) + + # Training loop self.model.to(device) for epoch in range(self.epochs): @@ -462,32 +481,85 @@ def fit(self, train_loader, test_loader, model_save_path): for idx, (data, target, _) in enumerate(train_loader): self.optimizer.zero_grad() data, target = data.float().to(device), target.float().to(device) - output = self.model(data) + activations, output = self.model(data) loss = self.criterion(output, target) loss.backward() self.optimizer.step() total_loss += loss + # TODO: Wrapper for visualization at visualizer. + if self.viz == "True": + # Visualize the network during training + if not self.plt_close: + # Get the hidden to hidden weights in the network and plot the connections + # TODO: Visualization of multiple layer activations in a window + hidden_weights = ( + dict(self.model.lstm.w_hhl0 + .clone() + .detach() + .numpy() + ) + + # Hidden activativations + hidden_activ = list(activations.clone().detach().numpy()[0]) + + try: + plt_close = self.directednetwork.show( + hidden_activ, hidden_weights, self.fig4 + ) + except Exception: + plt_close = True + pass + + plt_close = self.directednetwork.show( + hidden_activ, hidden_weights, self.fig + ) + + # # Visualize the network during training + if not self.plt2_close: + # Get the principle components + pc = self.lle.local_linear_embedding( + X=activations.clone().detach().numpy(), + d=3, + k=20, + alpha=0.1, + ) + plt2_close = self.lle.show(pc, self.fig2) + + # Visualize the graph embedding using spectral clusters + if not self.plt3_close: + # Get the principle components + embeddings = self.spectral_clustering.spectral_embedding( + X=activations.clone().detach().numpy(), rad=0.8 + ) + plt3_close = self.spectral_clustering.show( + activations.clone().detach().numpy(), + embeddings, + self.fig3, + ) + print("Epoch {} | loss {}".format(epoch, total_loss / (idx + 1))) - # Testing + # Testing loop if epoch % 10 == 0: with torch.no_grad(): self.model.eval() - test_loss_forecasting = 0 + self.test_loss_forecasting = 0 for idx, (data, target, _) in enumerate(list(test_loader)): data, target = ( data.float().to(device), target.float().to(device), ) out = self.model(data) - test_loss_forecasting += self.criterion(out, target).item() + self.test_loss_forecasting += self.criterion(out, target).item() - test_loss_forecasting /= len(test_loader.dataset) - print(f"====> Test set generator loss: {test_loss_forecasting:.4f}") + self.test_loss_forecasting /= len(self.test_loader.dataset) + print( + f"====> Test set generator loss: {self.test_loss_forecasting:.4f}" + ) - # Scheduler metric is test set loss - self.scheduler.step(test_loss_forecasting) + # Scheduler metric is test set loss + self.scheduler.step(self.test_loss_forecasting) # Save the model at target path utils.save(self.model, self.model_hyperparameters, PATH=model_save_path) diff --git a/traja/models/visualizer.py b/traja/models/visualizer.py index e6a9bb4d..2c864764 100644 --- a/traja/models/visualizer.py +++ b/traja/models/visualizer.py @@ -1,4 +1,5 @@ import os, sys +from matplotlib.pyplot import figimage import networkx as nx import pandas as pd import numpy as np @@ -298,3 +299,22 @@ def show(self, X, spec_embed, fig3): numebr_of_points ** 2, ) + +class Network: + def __init__(self, activity, weights): + + pass + + def show(self): + fig = None + return fig + + +class ShowManifold: + def __init__(self, inputs, manifold): + pass + + def show(self): + fig = None + return fig + From f7b7ff17d37449f4f272230769f0738639b19687 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Tue, 29 Dec 2020 15:36:57 +0000 Subject: [PATCH 144/211] Add manhattan, mse and ae losses --- traja/models/losses.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/traja/models/losses.py b/traja/models/losses.py index 15730a3d..93bcc1b0 100644 --- a/traja/models/losses.py +++ b/traja/models/losses.py @@ -10,6 +10,7 @@ class Criterion: def __init__(self): self.huber_loss = torch.nn.SmoothL1Loss(reduction="sum") + self.manhattan_loss = torch.nn.L1Loss(reduction="sum") self.mse_loss = torch.nn.MSELoss() self.crossentropy_loss = torch.nn.CrossEntropyLoss() @@ -32,19 +33,33 @@ def ae_criterion(self, predicted, target, loss_type="huber"): def vae_criterion(self, predicted, target, mu, logvar, loss_type="huber"): """ Time series generative model loss function + Provides both vae loss functions (huber, manhattan, mse) + and ae loss functions (huber_ae, manhattan_ae, mse_ae). :param predicted: Predicted time series by the model :param target: Target time series :param mu: Latent variable, Mean :param logvar: Latent variable, Log(Variance) - :param loss_type: Type of criterion; Defaults: 'huber' - :return: Reconstruction loss + KLD loss + :param loss_type: Type of criterion (huber, manhattan, mse, huber_ae, manhattan_ae, mse_ae); Defaults: 'huber' + :return: Reconstruction loss + KLD loss (if not ae) """ + + KLD = -0.5 * torch.sum(1 + logvar - mu ** 2 - logvar.exp()) + if loss_type == "huber": - dist_x = self.huber_loss(predicted, target) + loss = self.huber_loss(predicted, target) + KLD + elif loss_type == "manhattan": + loss = self.manhattan_loss(predicted, target) + KLD + elif loss_type == "mse": + loss = self.mse_loss(predicted, target) + KLD + elif loss_type == "huber_ae": + loss = self.huber_loss(predicted, target) + elif loss_type == "manhattan_ae": + loss = self.manhattan_loss(predicted, target) + elif loss_type == "mse_ae": + loss = self.mse_loss(predicted, target) else: - dist_x = torch.sqrt(torch.mean((predicted - target) ** 2)) - KLD = -0.5 * torch.sum(1 + logvar - mu ** 2 - logvar.exp()) - return dist_x + KLD + raise Exception("Loss type '{}' is unknown!".format(loss_type)) + return loss def classifier_criterion(self, predicted, target): """ From 5b3d6412608a12a6fe8c960c740d5ed783220450 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Tue, 29 Dec 2020 15:37:13 +0000 Subject: [PATCH 145/211] Add Neural Network tests --- traja/tests/test_nn.py | 92 +++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/traja/tests/test_nn.py b/traja/tests/test_nn.py index 3ddb4570..0f132a82 100644 --- a/traja/tests/test_nn.py +++ b/traja/tests/test_nn.py @@ -1,49 +1,47 @@ - -import sys -sys.path.append('../../delve') - -import delve - - import pandas as pd - import traja - - - - - - -def test_from_df(): - df = traja.generate(n=599581) - - df = df.filter(items=['x', 'y']) - - save_path = 'temp/test' - - warmup_steps = 50 - - # Run 1 - timeseries_method = 'last_timestep' - - model = traja.models.LSTM(input_size=2, hidden_size=2, num_layers=3, output_size=2, dropout=0, bidirectional=False) - writer = delve.writers.CSVandPlottingWriter(save_path, fontsize=16, primary_metric='test_accuracy') - saturation = delve.CheckLayerSat(save_path, - [writer], model, - stats=['embed'], - timeseries_method=timeseries_method) - - sequence_length = 2000 - train_fraction = .25 - batch_size = 50 - shift = 2 - - train_loader, test_loader = traja.models.get_timeseries_data_loaders(df, sequence_length, - train_fraction, batch_size, shift) - - trainer = traja.models.Trainer(model, train_loader, test_loader, epochs=6, optimizer="adam", - warmup_steps=warmup_steps) - - trainer.train() - - pass \ No newline at end of file +from traja import models +from traja import datasets +from traja.datasets import dataset +from traja.models.train import LSTMTrainer, HybridTrainer, CustomTrainer + +data_url = "https://raw.githubusercontent.com/traja-team/traja-research/dataset_und_notebooks/dataset_analysis/jaguar5.csv" +df = pd.read_csv(data_url, error_bad_lines=False) +model_save_path = './model.pt' + +# Hyperparameters +batch_size = 10 +num_past = 10 +num_future = 5 + +if __name__ == '__main__': + # Prepare the dataloader + train_loader, test_loader = dataset.MultiModalDataLoader(df, + batch_size=batch_size, + n_past=num_past, + n_future=num_future, + num_workers=2) + + trainer = HybridTrainer(model_type='vae', # "ae" or "vae" + optimizer_type='Adam', # ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop','LBFGS', 'ASGD', 'Adamax'] + input_size=2, + output_size=2, + lstm_hidden_size=32, + num_lstm_layers=2, + reset_state=True, + latent_size=10, + dropout=0.1, + num_classes=9, # Uncomment to create and train classifier network + num_classifier_layers=4, + classifier_hidden_size= 32, + epochs=10, + batch_size=batch_size, + num_future=num_future, + num_past=num_past, + bidirectional=False, + batch_first=True, + loss_type='huber') # 'rmse' or 'huber' + + + # Train the model + trainer.fit(train_loader, test_loader, model_save_path) From be5537ca46702b13a5bb4876fc5ad481f1c2074b Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Tue, 29 Dec 2020 16:26:08 +0000 Subject: [PATCH 146/211] Add classifier accuracy --- traja/models/train.py | 57 +++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index e206f7f7..c1e14dd0 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -198,13 +198,19 @@ def fit(self, train_loader, test_loader, model_save_path=None): decoder_optimizer.step() latent_optimizer.step() - else: # training_mode == 'classification' - if self.model_type == 'vae': - classifier_out, latent_out, mu, logvar = self.model(data, training=True, is_classification=True) - else: - classifier_out = self.model(data, training=True, - is_classification=True) - loss = Criterion().classifier_criterion(classifier_out, category - 1) + elif self.classify and training_mode != "forecasting": + if self.model_type == "vae": + classifier_out, latent_out, mu, logvar = self.model( + data, training=True, classify=True + ) + else: # "ae" + classifier_out = self.model( + data, training=True, classify=True + ) + loss = Criterion().classifier_criterion( + classifier_out, (category - 1).long() + ) + loss.backward() classifier_optimizer.step() total_loss += loss @@ -217,6 +223,9 @@ def fit(self, train_loader, test_loader, model_save_path=None): # Testing if epoch % 10 == 0: with torch.no_grad(): + if self.classify: + total = 0.0 + correct = 0.0 self.model.eval() for idx, (data, target, category) in enumerate(list(test_loader)): data, target, category = data.float().to(device), target.float().to(device), category.to(device) @@ -229,20 +238,36 @@ def fit(self, train_loader, test_loader, model_save_path=None): is_classification=False) test_loss_forecasting += Criterion().vae_criterion(decoder_out, target, mu, logvar) # Classification test - if self.model_type == 'ae': - classifier_out = self.model(data, training=False, - is_classification=True) - else: - classifier_out, latent_out, mu, logvar = self.model(data, training=False, - is_classification=True) + if self.classify: + category = category.long() + if self.model_type == "ae": + classifier_out = self.model( + data, training=False, classify=True + ) + else: + classifier_out, latent_out, mu, logvar = self.model( + data, training=False, classify=True + ) test_loss_classification += Criterion().classifier_criterion(classifier_out, category - 1).item() + # Compute number of correct samples + total += category.size(0) + _, predicted = torch.max(classifier_out.data, 1) + correct += (predicted == (category - 1)).sum().item() + test_loss_forecasting /= len(test_loader.dataset) - print(f'====> Mean test set generator loss: {test_loss_forecasting:.4f}') - test_loss_classification /= len(test_loader.dataset) - print(f'====> Mean test set classifier loss: {test_loss_classification:.4f}') + print( + f"====> Mean test set generator loss: {test_loss_forecasting:.4f}" + ) + if self.classify: + accuracy = correct / total + if test_loss_classification != 0: + test_loss_classification /= len(test_loader.dataset) + print( + f"====> Mean test set classifier loss: {test_loss_classification:.4f}; accuracy: {accuracy:.2f}" + ) # Scheduler metric is test set loss if training_mode == 'forecasting': From c981954e27f9aacdcbc9764baf96b71c79f36150 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 19:07:01 +0100 Subject: [PATCH 147/211] tbptt for lstm sequences --- traja/models/generative_models/vae.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/generative_models/vae.py b/traja/models/generative_models/vae.py index 5bbd9ec9..3f5cf90d 100644 --- a/traja/models/generative_models/vae.py +++ b/traja/models/generative_models/vae.py @@ -90,8 +90,8 @@ def _init_hidden(self): def forward(self, x): - enc_init_hidden = self._init_hidden() - enc_output, _ = self.lstm_encoder(x, enc_init_hidden) + (h0, c0) = self._init_hidden() + enc_output, _ = self.lstm_encoder(x, (h0.detach(), c0.detach())) # RNNs obeys, Markovian. So, the last state of the hidden is the markovian state for the entire # sequence in that batch. enc_output = enc_output[:, -1, :] # Shape(batch_size,hidden_dim) From 8411f1824e4193d971e00cc4e462995843fa32ca Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 19:22:40 +0100 Subject: [PATCH 148/211] Update utils.py --- traja/models/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/utils.py b/traja/models/utils.py index c4b20660..2788296d 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -54,8 +54,8 @@ def save(model, hyperparameters, PATH=None): if PATH is None: PATH = os.getcwd() + "model.pt" torch.save(model.state_dict(), PATH) - hyperdir = os.path.split(PATH) - with open(hyperdir + "/hypers.json", "w") as fp: + hyperdir, _ = os.path.split(PATH) + with open(hyperdir + "hypers.json", "w") as fp: json.dump(hyperparameters, fp, sort_keys=False) print("Model and hyperparameters saved at {} ".format(hyperdir)) From c6bd319aed7771f2690eb5dd9b9b4ac25c168278 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 19:23:36 +0100 Subject: [PATCH 149/211] Update train.py --- traja/models/train.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index c1e14dd0..6dd35e16 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -518,11 +518,8 @@ def fit(self, train_loader, test_loader, model_save_path): if not self.plt_close: # Get the hidden to hidden weights in the network and plot the connections # TODO: Visualization of multiple layer activations in a window - hidden_weights = ( - dict(self.model.lstm.w_hhl0 - .clone() - .detach() - .numpy() + hidden_weights = dict( + self.model.lstm.w_hhl0.clone().detach().numpy() ) # Hidden activativations From 4fb3f14fc89f589e5dc79766ae261868a6cf3f3d Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 19:24:33 +0100 Subject: [PATCH 150/211] Update train.py --- traja/models/train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/traja/models/train.py b/traja/models/train.py index 6dd35e16..1db03d5b 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -467,6 +467,7 @@ def __init__( self.scheduler = optimizer.get_lrschedulers( factor=self.lr_factor, patience=self.scheduler_patience ) + self.viz = True def fit(self, train_loader, test_loader, model_save_path): From 422c67820b904e57439cfebca42523dc8d08b13c Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 19:29:29 +0100 Subject: [PATCH 151/211] Update visualizer.py --- traja/models/visualizer.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/traja/models/visualizer.py b/traja/models/visualizer.py index 2c864764..9339d582 100644 --- a/traja/models/visualizer.py +++ b/traja/models/visualizer.py @@ -3,7 +3,6 @@ import networkx as nx import pandas as pd import numpy as np -import matplotlib.pyplot as plt import scipy import sklearn from sklearn import neighbors @@ -11,11 +10,7 @@ from sklearn.neighbors import radius_neighbors_graph from sklearn.neighbors import kneighbors_graph from mpl_toolkits.mplot3d import Axes3D -from matplotlib.axes import Axes -from matplotlib import cm import seaborn as sns -import matplotlib -from matplotlib import style import argparse, copy, h5py, os, sys, time, socket import tensorflow as tf import torch, torchvision, torch.nn as nn @@ -24,6 +19,15 @@ from matplotlib import ticker, colors import plotly.express as px + +import matplotlib + +matplotlib.use("TKAgg") +import matplotlib.pyplot as plt +from matplotlib.axes import Axes +from matplotlib import cm +from matplotlib import style + plt.switch_backend("TkAgg") From e48b1540b7b817d3ce4f21b4bc02ebe46fbc0103 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 19:32:12 +0100 Subject: [PATCH 152/211] Update visualizer.py --- traja/models/visualizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/visualizer.py b/traja/models/visualizer.py index 9339d582..bfc78d92 100644 --- a/traja/models/visualizer.py +++ b/traja/models/visualizer.py @@ -22,13 +22,13 @@ import matplotlib -matplotlib.use("TKAgg") +# matplotlib.use("TKAgg") import matplotlib.pyplot as plt from matplotlib.axes import Axes from matplotlib import cm from matplotlib import style -plt.switch_backend("TkAgg") +# plt.switch_backend("TkAgg") np.set_printoptions( From dc18953dbfd30bb64609a97876fed42670b0cbc8 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 19:41:49 +0100 Subject: [PATCH 153/211] viz test --- traja/models/train.py | 2 +- traja/models/utils.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 1db03d5b..8bd2a30e 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -573,7 +573,7 @@ def fit(self, train_loader, test_loader, model_save_path): data.float().to(device), target.float().to(device), ) - out = self.model(data) + activations, out = self.model(data) self.test_loss_forecasting += self.criterion(out, target).item() self.test_loss_forecasting /= len(self.test_loader.dataset) diff --git a/traja/models/utils.py b/traja/models/utils.py index 2788296d..5a1988ea 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -54,10 +54,10 @@ def save(model, hyperparameters, PATH=None): if PATH is None: PATH = os.getcwd() + "model.pt" torch.save(model.state_dict(), PATH) - hyperdir, _ = os.path.split(PATH) - with open(hyperdir + "hypers.json", "w") as fp: + dir, _ = os.path.split(PATH) + with open("./hypers.json", "w") as fp: json.dump(hyperparameters, fp, sort_keys=False) - print("Model and hyperparameters saved at {} ".format(hyperdir)) + print("Model and hyperparameters saved at {} ".format(dir)) def load(model, PATH=None): From 0f822789331a81b1aab5de7e7b36e53bf8a755ae Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 19:56:41 +0100 Subject: [PATCH 154/211] viz test --- traja/models/train.py | 2 +- traja/models/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 8bd2a30e..5df1c8c4 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -576,7 +576,7 @@ def fit(self, train_loader, test_loader, model_save_path): activations, out = self.model(data) self.test_loss_forecasting += self.criterion(out, target).item() - self.test_loss_forecasting /= len(self.test_loader.dataset) + self.test_loss_forecasting /= len(test_loader.dataset) print( f"====> Test set generator loss: {self.test_loss_forecasting:.4f}" ) diff --git a/traja/models/utils.py b/traja/models/utils.py index 5a1988ea..acad76d1 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -54,10 +54,10 @@ def save(model, hyperparameters, PATH=None): if PATH is None: PATH = os.getcwd() + "model.pt" torch.save(model.state_dict(), PATH) - dir, _ = os.path.split(PATH) + _dir, _ = os.path.split(PATH) with open("./hypers.json", "w") as fp: json.dump(hyperparameters, fp, sort_keys=False) - print("Model and hyperparameters saved at {} ".format(dir)) + print("Model and hyperparameters saved at {} ".format(_dir)) def load(model, PATH=None): From bc9401d4a452ad0dd60193b46c6af8b3d36a93bf Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 19:58:22 +0100 Subject: [PATCH 155/211] Update utils.py --- traja/models/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/traja/models/utils.py b/traja/models/utils.py index acad76d1..31f98dda 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -41,11 +41,13 @@ def forward(self, x): def save(model, hyperparameters, PATH=None): """Save the trained model(.pth) along with its hyperparameters as a json (hyper.json) at the user defined Path - Args: + Parameters: + ----------- model (torch.nn.Module): Trained Model hyperparameters(dict): Hyperparameters of the model PATH (str): Directory path to save the trained model and its hyperparameters Returns: + --------- None """ if not isinstance(hyperparameters, dict): @@ -62,11 +64,13 @@ def save(model, hyperparameters, PATH=None): def load(model, PATH=None): """Load trained model from PATH using the model_hyperparameters saved in the - Args: + Parameters: + ----------- model (torch.nn.Module): Type of the model ['ae','vae','vaegan','irl','lstm','custom'] model_hyperparameters (dict): Dictionary of hyperparameters used to initiate model PATH (str): Directory path of the model Returns: + --------- model(torch.nn.module): Model """ # Hyperparameters From b574e9ba75c1c49c16ab5cde836742bb8683c63a Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 20:07:44 +0100 Subject: [PATCH 156/211] Custom trainer viz test --- traja/models/train.py | 2 +- traja/models/utils.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 5df1c8c4..84ef0a47 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -585,7 +585,7 @@ def fit(self, train_loader, test_loader, model_save_path): self.scheduler.step(self.test_loss_forecasting) # Save the model at target path - utils.save(self.model, self.model_hyperparameters, PATH=model_save_path) + utils.save(self.model, hyperparameters=None, PATH=model_save_path) class VAEGANTrainer: diff --git a/traja/models/utils.py b/traja/models/utils.py index 31f98dda..0b177076 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -57,9 +57,10 @@ def save(model, hyperparameters, PATH=None): PATH = os.getcwd() + "model.pt" torch.save(model.state_dict(), PATH) _dir, _ = os.path.split(PATH) - with open("./hypers.json", "w") as fp: - json.dump(hyperparameters, fp, sort_keys=False) - print("Model and hyperparameters saved at {} ".format(_dir)) + if hyperparameters is not None: + with open("./hypers.json", "w") as fp: + json.dump(hyperparameters, fp, sort_keys=False) + print("Model saved at {} ".format(_dir)) def load(model, PATH=None): From bfa7191f6173deb3b5304ce246fe6e4bb9ee6e2a Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 20:11:32 +0100 Subject: [PATCH 157/211] Update utils.py --- traja/models/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/traja/models/utils.py b/traja/models/utils.py index 0b177076..43c66af9 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -50,7 +50,8 @@ def save(model, hyperparameters, PATH=None): --------- None """ - if not isinstance(hyperparameters, dict): + + if hyperparameters is not None and not isinstance(hyperparameters, dict): raise Exception("Invalid argument, hyperparameters must be dict") # Save if PATH is None: From 33056de3e1916023acfc2e500bc3ddc78854e918 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 20:56:21 +0100 Subject: [PATCH 158/211] Update __init__.py --- traja/models/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/traja/models/__init__.py b/traja/models/__init__.py index d8a302e1..cdeda8f6 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -5,3 +5,4 @@ from traja.models.predictive_models.irl import MultiModelIRL from traja.models.generative_models.vaegan import MultiModelVAEGAN from .utils import TimeDistributed, read_hyperparameters, save, load +from .generator import * From 6dfde0489f20056c809d52a637f2867dd9b9196e Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 21:37:47 +0100 Subject: [PATCH 159/211] generator and predictor update --- traja/models/generator.py | 14 +++++--------- traja/models/{manifolds.py => manifold.py} | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) rename traja/models/{manifolds.py => manifold.py} (97%) diff --git a/traja/models/generator.py b/traja/models/generator.py index a5cd8fdc..33b53b62 100644 --- a/traja/models/generator.py +++ b/traja/models/generator.py @@ -2,11 +2,11 @@ import plotly.express as px import torch -from .ae import MultiModelAE -from .vae import MultiModelVAE -from .vaegan import MultiModelVAEGAN -from .irl import MultiModelIRL -from .lstm import LSTM +from traja.models.predictive_models.ae import MultiModelAE +from traja.models.generative_models.vae import MultiModelVAE +from traja.models.generative_models.vaegan import MultiModelVAEGAN +from traja.models.predictive_models.irl import MultiModelIRL +from traja.models.predictive_models.lstm import LSTM from .utils import load_model import matplotlib.pyplot as plt import os @@ -145,7 +145,3 @@ def __init__( if self.model_type == "custom": assert model is not None self.model = model(**self.model_hyperparameters) - - - - diff --git a/traja/models/manifolds.py b/traja/models/manifold.py similarity index 97% rename from traja/models/manifolds.py rename to traja/models/manifold.py index f1a472be..90244697 100644 --- a/traja/models/manifolds.py +++ b/traja/models/manifold.py @@ -7,7 +7,7 @@ class Manifold: """ def __init__(self, manifold_type): - + pass def __new__(cls): From e2210464d1203b486cfa47f831769d6f7df12a4b Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 21:40:12 +0100 Subject: [PATCH 160/211] Update generator.py --- traja/models/generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/generator.py b/traja/models/generator.py index 33b53b62..f400fce2 100644 --- a/traja/models/generator.py +++ b/traja/models/generator.py @@ -7,7 +7,7 @@ from traja.models.generative_models.vaegan import MultiModelVAEGAN from traja.models.predictive_models.irl import MultiModelIRL from traja.models.predictive_models.lstm import LSTM -from .utils import load_model +from traja.models.utils import load import matplotlib.pyplot as plt import os @@ -52,7 +52,7 @@ def __init__( def generate_batch(self): # Load the model - model = load_model(self.model, self.model_hyperparameters, self.model_path) + model = load(self.model, self.model_hyperparameters, self.model_path) if self.model_type == "vae": # Random noise From 47da90ef3038e30a65a01bff5372672433a5301b Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 21:46:53 +0100 Subject: [PATCH 161/211] Update generator.py --- traja/models/generator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/traja/models/generator.py b/traja/models/generator.py index f400fce2..15517bc9 100644 --- a/traja/models/generator.py +++ b/traja/models/generator.py @@ -36,8 +36,8 @@ def __init__( self.model_hyperparameters = model_hyperparameters # Batch size and time step size - self.batch_size = self.model_hyperparameters.batch_size - self.num_future = self.model_hyperparameters.num_future + self.batch_size = self.model_hyperparameters["batch_size"] + self.num_future = self.model_hyperparameters["num_future"] if self.model_type == "vae": self.model = MultiModelVAE(**self.model_hyperparameters) @@ -130,8 +130,8 @@ def __init__( self.model_hyperparameters = model_hyperparameters # Batch size and time step size - self.batch_size = self.model_hyperparameters.batch_size - self.num_future = self.model_hyperparameters.num_future + self.batch_size = self.model_hyperparameters["batch_size"] + self.num_future = self.model_hyperparameters["num_future"] if self.model_type == "ae": self.model = MultiModelAE(**self.model_hyperparameters) From 7905399345e900eaac7b27b790a02d3b8cc9334d Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Tue, 29 Dec 2020 22:10:44 +0100 Subject: [PATCH 162/211] trained model load fix --- traja/models/generator.py | 48 ++++++++++++++++++++------------------- traja/models/utils.py | 6 ++--- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/traja/models/generator.py b/traja/models/generator.py index 15517bc9..28b86fef 100644 --- a/traja/models/generator.py +++ b/traja/models/generator.py @@ -35,10 +35,6 @@ def __init__( self.model_path = model_path self.model_hyperparameters = model_hyperparameters - # Batch size and time step size - self.batch_size = self.model_hyperparameters["batch_size"] - self.num_future = self.model_hyperparameters["num_future"] - if self.model_type == "vae": self.model = MultiModelVAE(**self.model_hyperparameters) @@ -49,28 +45,30 @@ def __init__( assert model is not None self.model = model(**self.model_hyperparameters) - def generate_batch(self): + def generate_batch(self, batch_size, num_future, classify=True): # Load the model - model = load(self.model, self.model_hyperparameters, self.model_path) + model = load(self.model) if self.model_type == "vae": # Random noise z = ( - torch.empty(self.batch_size, self.model_hyperparameters.latent_size) + torch.empty(batch_size, self.model_hyperparameters.latent_size) .normal_(mean=0, std=0.1) .to(device) ) # Generate trajectories from the noise - out = model.decoder(z, self.num_future).cpu().detach().numpy() + out = model.decoder(z, num_future).cpu().detach().numpy() out = out.reshape(out.shape[0] * out.shape[1], out.shape[2]) - try: - cat = model.classifier(z) - print( - "IDs in this batch of synthetic data", torch.max(cat, 1).indices + 1 - ) - except: - pass + if classify: + try: + cat = model.classifier(z) + print( + "IDs in this batch of synthetic data", + torch.max(cat, 1).indices + 1, + ) + except Exception as error: + print("Classifier not found: " + repr(error)) plt.figure(figsize=(12, 4)) plt.plot(out[:, 0], label="Generated x: Longitude") @@ -82,18 +80,22 @@ def generate_batch(self): for i in range(2): for j in range(5): + if classify: + try: + label = "Animal ID {}".format( + (torch.max(cat, 1).indices + 1).detach()[i + j] + ) + except Exception as error: + print("Classifier not found:" + repr(error)) + ax[i, j].plot( out[:, 0][ - (i + j) * self.num_future : (i + j) * self.num_future - + self.num_future + (i + j) * num_future : (i + j) * num_future + num_future ], out[:, 1][ - (i + j) * self.num_future : (i + j) * self.num_future - + self.num_future + (i + j) * num_future : (i + j) * num_future + num_future ], - label="Animal ID {}".format( - (torch.max(cat, 1).indices + 1).detach()[i + j] - ), + label=label, color="g", ) ax[i, j].legend() @@ -131,7 +133,7 @@ def __init__( # Batch size and time step size self.batch_size = self.model_hyperparameters["batch_size"] - self.num_future = self.model_hyperparameters["num_future"] + num_future = self.model_hyperparameters["num_future"] if self.model_type == "ae": self.model = MultiModelAE(**self.model_hyperparameters) diff --git a/traja/models/utils.py b/traja/models/utils.py index 43c66af9..5d9a5b6d 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -61,7 +61,7 @@ def save(model, hyperparameters, PATH=None): if hyperparameters is not None: with open("./hypers.json", "w") as fp: json.dump(hyperparameters, fp, sort_keys=False) - print("Model saved at {} ".format(_dir)) + print(f"Model saved at {_dir}") def load(model, PATH=None): @@ -80,7 +80,7 @@ def load(model, PATH=None): PATH = os.getcwd() + "/model.pt" print(f"Model loaded from {PATH}") else: - raise Exception("Model not found at " f"{PATH}") + raise Exception(f"Model not found at {PATH}") # Get hyperparameters from the model path PATH, _ = os.path.split(PATH) @@ -88,7 +88,7 @@ def load(model, PATH=None): with open(PATH + "/hypers.json") as f: hypers = json.load(f) except: - raise Exception("Hyper parameters not found at " f"{PATH}") + raise Exception(f"Hyper parameters not found at {PATH}") # Load model = model(**hypers) From 737e20dae30889dc3ff98a752b5c9374219c8018 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 30 Dec 2020 14:12:56 +0100 Subject: [PATCH 163/211] model reload and infer fix --- traja/models/__init__.py | 2 +- traja/models/generative_models/vae.py | 92 +++++++++++++-------- traja/models/{generator.py => inference.py} | 20 +++-- traja/models/utils.py | 17 +--- 4 files changed, 72 insertions(+), 59 deletions(-) rename traja/models/{generator.py => inference.py} (93%) diff --git a/traja/models/__init__.py b/traja/models/__init__.py index cdeda8f6..f6629e51 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -5,4 +5,4 @@ from traja.models.predictive_models.irl import MultiModelIRL from traja.models.generative_models.vaegan import MultiModelVAEGAN from .utils import TimeDistributed, read_hyperparameters, save, load -from .generator import * +from .inference import * diff --git a/traja/models/generative_models/vae.py b/traja/models/generative_models/vae.py index 3f5cf90d..17de371b 100644 --- a/traja/models/generative_models/vae.py +++ b/traja/models/generative_models/vae.py @@ -32,17 +32,17 @@ class LSTMEncoder(torch.nn.Module): """ Implementation of Encoder network using LSTM layers - :param input_size: The number of expected features in the input x - :param num_past: Number of time steps to look backwards to predict num_future steps forward - :param batch_size: Number of samples in a batch - :param hidden_size: The number of features in the hidden state h - :param num_lstm_layers: Number of layers in the LSTM model - - :param batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + input_size: The number of expected features in the input x + num_past: Number of time steps to look backwards to predict num_future steps forward + batch_size: Number of samples in a batch + hidden_size: The number of features in the hidden state h + num_lstm_layers: Number of layers in the LSTM model + + batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, with dropout probability equal to dropout - :param reset_state: If True, will reset the hidden and cell state for each batch of data - :param bidirectional: If True, becomes a bidirectional LSTM + reset_state: If True, will reset the hidden and cell state for each batch of data + bidirectional: If True, becomes a bidirectional LSTM """ def __init__( @@ -127,25 +127,20 @@ def forward(self, x, training=True): class LSTMDecoder(torch.nn.Module): - """ Deep LSTM network. This implementation - returns output_size outputs. - Args: - latent_size: The number of dimensions of the latent layer - batch_size: Number of samples in each batch of training data - hidden_size: The number of features in the hidden state `h` - num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` - would mean stacking two LSTMs together to form a `stacked LSTM`, - with the second LSTM taking in outputs of the first LSTM and - computing the final results. Default: 1 - output_size: The number of output/input dimensions - num_future: The number of time steps in future predictions - dropout: If non-zero, introduces a `Dropout` layer on the outputs of each - LSTM layer except the last layer, with dropout probability equal to - :attr:`dropout`. Default: 0 - bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` - reset_state: If ``True``, the hidden and cell states of the LSTM will - be reset at the beginning of each batch of input - """ + + """ Implementation of Decoder network using LSTM layers + input_size: The number of expected features in the input x + num_future: Number of time steps to be predicted given the num_past steps + batch_size: Number of samples in a batch + hidden_size: The number of features in the hidden state h + num_lstm_layers: Number of layers in the LSTM model + output_size: Number of expectd features in the output x_ + batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + reset_state: If True, will reset the hidden and cell state for each batch of data + bidirectional: If True, becomes a bidirectional LSTM + """ def __init__( self, @@ -218,12 +213,12 @@ def forward(self, x, num_future=None): class MLPClassifier(torch.nn.Module): """ MLP classifier: Classify the input data using the latent embeddings - :param input_size: The number of expected latent size - :param hidden_size: The number of features in the hidden state h - :param num_classes: Size of labels or the number of categories in the data - :param dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + input_size: The number of expected latent size + hidden_size: The number of features in the hidden state h + num_classes: Size of labels or the number of categories in the data + dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, with dropout probability equal to dropout - :param num_classifier_layers: Number of hidden layers in the classifier + num_classifier_layers: Number of hidden layers in the classifier """ def __init__( @@ -262,7 +257,21 @@ def forward(self, x): class MultiModelVAE(torch.nn.Module): - """Implementation of Multimodel Variational autoencoders; + + """Implementation of Multimodel Variational autoencoders; This Module wraps the Variational Autoencoder + models [Encoder,Latent[Sampler],Decoder]. If classify=True, then the wrapper also include classification layers + + input_size: The number of expected features in the input x + num_future: Number of time steps to be predicted given the num_past steps + batch_size: Number of samples in a batch + hidden_size: The number of features in the hidden state h + num_lstm_layers: Number of layers in the LSTM model + output_size: Number of expectd features in the output x_ + batch_first: If True, then the input and output tensors are provided as (batch, seq, feature) + dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, + with dropout probability equal to dropout + reset_state: If True, will reset the hidden and cell state for each batch of data + bidirectional: If True, becomes a bidirectional LSTM """ def __init__( @@ -342,7 +351,18 @@ def __init__( dropout=self.dropout, ) - def forward(self, data, training=True, is_classification=False): + def forward(self, data, training=True, classify=False): + """ + Parameters: + ----------- + data: Train or test data + training: If Training= False, latents are deterministic + classify: If True, perform classification of input data using the latent embeddings + Return: + ------- + decoder_out,latent_out or classifier out + """ + if not classify: if not is_classification: # Set the classifier grad off diff --git a/traja/models/generator.py b/traja/models/inference.py similarity index 93% rename from traja/models/generator.py rename to traja/models/inference.py index 28b86fef..7e8a0496 100644 --- a/traja/models/generator.py +++ b/traja/models/inference.py @@ -14,7 +14,7 @@ device = "cuda" if torch.cuda.is_available() else "cpu" -class Generate: +class Generator: def __init__( self, model_type: str = None, @@ -47,9 +47,6 @@ def __init__( def generate_batch(self, batch_size, num_future, classify=True): - # Load the model - model = load(self.model) - if self.model_type == "vae": # Random noise z = ( @@ -58,11 +55,11 @@ def generate_batch(self, batch_size, num_future, classify=True): .to(device) ) # Generate trajectories from the noise - out = model.decoder(z, num_future).cpu().detach().numpy() + out = self.model.decoder(z, num_future).cpu().detach().numpy() out = out.reshape(out.shape[0] * out.shape[1], out.shape[2]) if classify: try: - cat = model.classifier(z) + cat = self.model.classifier(z) print( "IDs in this batch of synthetic data", torch.max(cat, 1).indices + 1, @@ -106,11 +103,20 @@ def generate_batch(self, batch_size, num_future, classify=True): elif self.model_type == "vaegan" or "custom": return NotImplementedError + # TODO: State space models def generate_timeseries(num_steps): + """Recurrently generate time series for infinite time steps. + + Args: + num_steps ([type]): [description] + + Returns: + [type]: [description] + """ return NotImplementedError -class Predict: +class Predictor: def __init__( self, model_type: str = None, diff --git a/traja/models/utils.py b/traja/models/utils.py index 5d9a5b6d..7159d0d7 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -69,8 +69,7 @@ def load(model, PATH=None): Parameters: ----------- model (torch.nn.Module): Type of the model ['ae','vae','vaegan','irl','lstm','custom'] - model_hyperparameters (dict): Dictionary of hyperparameters used to initiate model - PATH (str): Directory path of the model + PATH (str): Directory path of the model: Defaults to None: Means Current working directory Returns: --------- model(torch.nn.module): Model @@ -80,21 +79,9 @@ def load(model, PATH=None): PATH = os.getcwd() + "/model.pt" print(f"Model loaded from {PATH}") else: - raise Exception(f"Model not found at {PATH}") - - # Get hyperparameters from the model path - PATH, _ = os.path.split(PATH) - try: - with open(PATH + "/hypers.json") as f: - hypers = json.load(f) - except: - raise Exception(f"Hyper parameters not found at {PATH}") - - # Load - model = model(**hypers) + raise Exception(f"Model state dict not found at {PATH}") # Load state of the model model.load_state_dict(torch.load(PATH)) - return model From b42af7f661b78016c37f9692c99b3f94d46e1a2d Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 30 Dec 2020 14:34:33 +0100 Subject: [PATCH 164/211] generator update --- traja/models/inference.py | 2 +- traja/models/utils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/traja/models/inference.py b/traja/models/inference.py index 7e8a0496..5e677c06 100644 --- a/traja/models/inference.py +++ b/traja/models/inference.py @@ -50,7 +50,7 @@ def generate_batch(self, batch_size, num_future, classify=True): if self.model_type == "vae": # Random noise z = ( - torch.empty(batch_size, self.model_hyperparameters.latent_size) + torch.empty(batch_size, self.model_hyperparameters["latent_size"]) .normal_(mean=0, std=0.1) .to(device) ) diff --git a/traja/models/utils.py b/traja/models/utils.py index 7159d0d7..7bfaddaa 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -80,6 +80,7 @@ def load(model, PATH=None): print(f"Model loaded from {PATH}") else: raise Exception(f"Model state dict not found at {PATH}") + # Load state of the model model.load_state_dict(torch.load(PATH)) return model From 2377776365754d76f1a47312b837b24721dc6889 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 30 Dec 2020 15:38:31 +0100 Subject: [PATCH 165/211] Update inference.py --- traja/models/inference.py | 1 + 1 file changed, 1 insertion(+) diff --git a/traja/models/inference.py b/traja/models/inference.py index 5e677c06..378ab2eb 100644 --- a/traja/models/inference.py +++ b/traja/models/inference.py @@ -47,6 +47,7 @@ def __init__( def generate_batch(self, batch_size, num_future, classify=True): + self.model.to(device) if self.model_type == "vae": # Random noise z = ( From 5e4d3384bba9065865b2019558789e9aa8b02cf0 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 30 Dec 2020 15:49:53 +0100 Subject: [PATCH 166/211] Update inference.py --- traja/models/inference.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/traja/models/inference.py b/traja/models/inference.py index 378ab2eb..16ce4dd5 100644 --- a/traja/models/inference.py +++ b/traja/models/inference.py @@ -63,7 +63,7 @@ def generate_batch(self, batch_size, num_future, classify=True): cat = self.model.classifier(z) print( "IDs in this batch of synthetic data", - torch.max(cat, 1).indices + 1, + torch.max(cat, 1).indices.detach() + 1, ) except Exception as error: print("Classifier not found: " + repr(error)) @@ -85,7 +85,8 @@ def generate_batch(self, batch_size, num_future, classify=True): ) except Exception as error: print("Classifier not found:" + repr(error)) - + else: + label = "" ax[i, j].plot( out[:, 0][ (i + j) * num_future : (i + j) * num_future + num_future From b68bd56d61d55416f64a29a439adf86e0cf73e22 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 30 Dec 2020 22:04:16 +0100 Subject: [PATCH 167/211] predictor update --- traja/datasets/dataset.py | 117 ++++++++++++++++++--------- traja/models/inference.py | 110 +++++++++++++++++++++++++ traja/models/predictive_models/ae.py | 87 ++++++++++---------- 3 files changed, 232 insertions(+), 82 deletions(-) diff --git a/traja/datasets/dataset.py b/traja/datasets/dataset.py index e891b9c9..7cbd7b1f 100644 --- a/traja/datasets/dataset.py +++ b/traja/datasets/dataset.py @@ -222,14 +222,16 @@ def __getitem__(self, index): self.loss_mask[start:end, :], ] return out - + + class TimeSeriesDataset(Dataset): r"""Pytorch Dataset object Args: Dataset (torch.utils.data.Dataset): Pyptorch dataset object """ - def __init__(self,data, target, category): + + def __init__(self, data, target, category): r""" Args: data (array): Data @@ -239,16 +241,17 @@ def __init__(self,data, target, category): self.data = data self.target = target self.category = category - + def __getitem__(self, index): x = self.data[index] y = self.target[index] z = self.category[index] - return x, y,z + return x, y, z def __len__(self): return len(self.data) + class MultiModalDataLoader: """ MultiModalDataLoader wraps the following data preparation steps, @@ -272,50 +275,90 @@ class MultiModalDataLoader: num_workers (int): Number of cpu subprocess occupied during data loading process Usage: - train_dataloader, test_dataloader = MultiModalDataLoader(df = data_frame, batch_size=32, - n_past = 20, n_future = 10, num_workers=4) + [train_dataloader, test_dataloader], + [train_x_scaler, + train_y_scaler, + test_x_scaler, + test_y_scaler] = MultiModalDataLoader(df = data_frame, batch_size=32, + n_past = 20, n_future = 10, num_workers=4) """ - - def __init__(self, df:pd.DataFrame, batch_size:int, n_past:int, n_future:int, num_workers: int): - + + def __init__( + self, + df: pd.DataFrame, + batch_size: int, + n_past: int, + n_future: int, + num_workers: int, + ): + # Extract/generate data from the pandas df - train_data, target_data, target_category = utils.generate_dataset(df, n_past, n_future) - + train_data, target_data, target_category = utils.generate_dataset( + df, n_past, n_future + ) + # Shuffle and split the data - [train_x,train_y,train_z],[test_x,test_y,test_z] = utils.shuffle_split(train_data, target_data, target_category, train_ratio = 0.75) - + [train_x, train_y, train_z], [test_x, test_y, test_z] = utils.shuffle_split( + train_data, target_data, target_category, train_ratio=0.75 + ) + # Scale data - (train_x, train_x_scaler),(train_y,train_y_scaler) = utils.scale_data(train_x,sequence_length=n_past),utils.scale_data(train_y,sequence_length=n_future) - (test_x,test_x_scaler),(test_y,test_y_scaler) = utils.scale_data(test_x,sequence_length=n_past),utils.scale_data(test_y,sequence_length=n_future) + (train_x, self.train_x_scaler), (train_y, self.train_y_scaler) = ( + utils.scale_data(train_x, sequence_length=n_past), + utils.scale_data(train_y, sequence_length=n_future), + ) + (test_x, self.test_x_scaler), (test_y, self.test_y_scaler) = ( + utils.scale_data(test_x, sequence_length=n_past), + utils.scale_data(test_y, sequence_length=n_future), + ) # Weighted Random Sampler - train_weighted_sampler, test_weighted_sampler = utils.weighted_random_samplers(train_z,test_z) - + train_weighted_sampler, test_weighted_sampler = utils.weighted_random_samplers( + train_z, test_z + ) + # Dataset - train_dataset = TimeSeriesDataset(train_x,train_y,train_z) - test_dataset = TimeSeriesDataset(test_x,test_y,test_z) + train_dataset = TimeSeriesDataset(train_x, train_y, train_z) + test_dataset = TimeSeriesDataset(test_x, test_y, test_z) # Dataloader with weighted samplers - self.train_loader = torch.utils.data.DataLoader(dataset=train_dataset, shuffle=False, - batch_size=batch_size, sampler=train_weighted_sampler, - drop_last=True, num_workers = num_workers) - self.test_loader = torch.utils.data.DataLoader(dataset=test_dataset, shuffle=False, - batch_size=batch_size, sampler=test_weighted_sampler, - drop_last=True, num_workers=num_workers) - - def __new__(cls, df:pd.DataFrame, batch_size:int, n_past:int, n_future:int, num_workers:int): + self.train_loader = torch.utils.data.DataLoader( + dataset=train_dataset, + shuffle=False, + batch_size=batch_size, + sampler=train_weighted_sampler, + drop_last=True, + num_workers=num_workers, + ) + self.test_loader = torch.utils.data.DataLoader( + dataset=test_dataset, + shuffle=False, + batch_size=batch_size, + sampler=test_weighted_sampler, + drop_last=True, + num_workers=num_workers, + ) + + def __new__( + cls, + df: pd.DataFrame, + batch_size: int, + n_past: int, + n_future: int, + num_workers: int, + ): """Constructor of MultiModalDataLoader""" - # Loader instance + # Loader instance loader_instance = super(MultiModalDataLoader, cls).__new__(cls) loader_instance.__init__(df, batch_size, n_past, n_future, num_workers) # Return train and test loader attributes - return loader_instance.train_loader, loader_instance.test_loader - - - - - - - - + return ( + [loader_instance.train_loader, loader_instance.test_loader], + [ + loader_instance.train_x_scaler, + loader_instance.train_y_scaler, + loader_instance.test_x_scaler, + loader_instance.test_y_scaler, + ], + ) diff --git a/traja/models/inference.py b/traja/models/inference.py index 16ce4dd5..bfa60014 100644 --- a/traja/models/inference.py +++ b/traja/models/inference.py @@ -10,6 +10,7 @@ from traja.models.utils import load import matplotlib.pyplot as plt import os +import numpy as np device = "cuda" if torch.cuda.is_available() else "cpu" @@ -152,6 +153,115 @@ def __init__( if self.model_type == "irl": self.model = MultiModelIRL(**self.model_hyperparameters) + if self.model_type == "vaegan": + self.model = MultiModelVAEGAN(**self.model_hyperparameters) + if self.model_type == "custom": assert model is not None self.model = model(**self.model_hyperparameters) + + def predict_batch(self, data_loader, num_future, scaler, classify=True): + """[summary] + + Args: + data_loader ([type]): [description] + num_future ([type]): [description] + scaler (dict): Scalers of the target data. This scale the model predictions to the scale of the target (future steps). + : This scaler will be returned by the traja data preprocessing and loading helper function. + classify (bool, optional): [description]. Defaults to True. + + Returns: + [type]: [description] + """ + + self.model.to(device) + if self.model_type == "ae": + for data, target, category in data_loader: + predicted_data, predicted_category = self.model( + data.float().to(device), training=False, classify=classify + ) + target = target.cpu().detach().numpy() + target = target.reshape( + target.shape[0] * target.shape[1], target.shape[2] + ) + predicted_data = predicted_data.cpu().detach().numpy() + predicted_data = predicted_data.reshape( + predicted_data.shape[0] * predicted_data.shape[1], + predicted_data.shape[2], + ) + + # Rescaling predicted data + for i in range(predicted_data.shape[1]): + s_s = scaler[f"scaler_{i}"].inverse_transform( + predicted_data[:, i].reshape(-1, 1) + ) + s_s = np.reshape(s_s, len(s_s)) + predicted_data[:, i] = s_s + + # TODO:Deprecated;Slicing the data into batches + predicted_data = np.array( + [ + predicted_data[i : i + num_future] + for i in range(0, len(predicted_data), num_future) + ] + ) + # Rescaling target data + target_data = target.copy() + for i in range(target_data.shape[1]): + + s_s = scaler["scaler_{}".format(i)].inverse_transform( + target_data[:, i].reshape(-1, 1) + ) + s_s = np.reshape(s_s, len(s_s)) + target_data[:, i] = s_s + # TODO:Deprecated;Slicing the data into batches + target_data = np.array( + [ + target_data[i : i + num_future] + for i in range(0, len(target_data), num_future) + ] + ) + + # Reshape [batch_size*num_future,input_dim] + predicted_data_ = predicted_data.reshape( + predicted_data.shape[0] * predicted_data.shape[1], + predicted_data.shape[2], + ) + target_data_ = target_data.reshape( + target_data.shape[0] * target_data.shape[1], target_data.shape[2] + ) + + fig, ax = plt.subplots(nrows=2, ncols=5, figsize=(16, 5), sharey=False) + fig.set_size_inches(40, 20) + for i in range(2): + for j in range(5): + ax[i, j].plot( + predicted_data_[:, 0][ + (i + j) * num_future : (i + j) * num_future + num_future + ], + predicted_data_[:, 1][ + (i + j) * num_future : (i + j) * num_future + num_future + ], + label=f"Predicted ID {predicted_category[i+j]}", + ) + + ax[i, j].plot( + target_data_[:, 0][ + (i + j) * num_future : (i + j) * num_future + num_future + ], + target_data_[:, 1][ + (i + j) * num_future : (i + j) * num_future + num_future + ], + label=f"Target ID {category[i+j]}", + color="g", + ) + ax[i, j].legend() + + plt.autoscale(True, axis="y", tight=False) + plt.show() + + # TODO: Convert predicted_data Tensor into Traja dataframe + return predicted_data + + elif self.model_type == "vaegan" or "custom": + return NotImplementedError diff --git a/traja/models/predictive_models/ae.py b/traja/models/predictive_models/ae.py index dd1cffc1..3aa781ba 100644 --- a/traja/models/predictive_models/ae.py +++ b/traja/models/predictive_models/ae.py @@ -359,51 +359,48 @@ def forward(self, data, classify=False, training=True): ------- decoder_out,latent_out or classifier out """ - if not classify: - # Set the classifier grad off - if self.num_classes is not None: + + if not classify: + # Set the classifier grad off + if self.num_classes is not None: + for param in self.classifier.parameters(): + param.requires_grad = False + + for param in self.encoder.parameters(): + param.requires_grad = True + for param in self.decoder.parameters(): + param.requires_grad = True + for param in self.latent.parameters(): + param.requires_grad = True + + # Encoder -->Latent --> Decoder + enc_out = self.encoder(data) + latent_out = self.latent(enc_out) + decoder_out = self.decoder(latent_out) + + else: # Classify + # Unfreeze classifier and freeze the rest + assert self.num_classifier_layers is not None, "Classifier not found" + for param in self.classifier.parameters(): + param.requires_grad = True + for param in self.encoder.parameters(): + param.requires_grad = False + for param in self.decoder.parameters(): param.requires_grad = False + for param in self.latent.parameters(): + param.requires_grad = False + + # Encoder-->Latent-->Classifier + enc_out = self.encoder(data) + latent_out = self.latent(enc_out) + + classifier_out = self.classifier(latent_out) # Deterministic - def forward(self, data, training=True, is_classification=False): - if not is_classification: - # Set the classifier grad off - for param in self.classifier.parameters(): - param.requires_grad = False - for param in self.encoder.parameters(): - param.requires_grad = True - for param in self.decoder.parameters(): - param.requires_grad = True - for param in self.latent.parameters(): - param.requires_grad = True - - # Encoder -->Latent --> Decoder - enc_out = self.encoder(data) - # Latent - latent_out = self.latent(enc_out) - # Decoder - decoder_out = self.decoder(latent_out) - - else: # Classify - # Unfreeze classifier and freeze the rest - assert self.num_classifier_layers is not None, "Classifier not found" - - else: # training_mode = 'classification' - # Unfreeze classifier parameters and freeze all other - # network parameters - for param in self.classifier.parameters(): - param.requires_grad = True - for param in self.encoder.parameters(): - param.requires_grad = False - for param in self.decoder.parameters(): - param.requires_grad = False - for param in self.latent.parameters(): - param.requires_grad = False - - # Encoder - enc_out = self.encoder(data) - # Latent - latent_out = self.latent(enc_out) - # Classifier - classifier_out = self.classifier(latent_out) # Deterministic - return classifier_out + if training and not classify: # Only forecasting network is trained + return decoder_out, latent_out + if classify and training: # Classifier is trained + return classifier_out + if classify and not training: # Inference + return decoder_out,classifier_out + From 956cea3c53d6a363d23e1118e24a2b249fa0c80e Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 30 Dec 2020 22:09:43 +0100 Subject: [PATCH 168/211] Update ae.py --- traja/models/predictive_models/ae.py | 80 ++++++++++++++-------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/traja/models/predictive_models/ae.py b/traja/models/predictive_models/ae.py index 3aa781ba..47bf1130 100644 --- a/traja/models/predictive_models/ae.py +++ b/traja/models/predictive_models/ae.py @@ -360,47 +360,47 @@ def forward(self, data, classify=False, training=True): decoder_out,latent_out or classifier out """ - if not classify: - # Set the classifier grad off - if self.num_classes is not None: - for param in self.classifier.parameters(): - param.requires_grad = False - - for param in self.encoder.parameters(): - param.requires_grad = True - for param in self.decoder.parameters(): - param.requires_grad = True - for param in self.latent.parameters(): - param.requires_grad = True - - # Encoder -->Latent --> Decoder - enc_out = self.encoder(data) - latent_out = self.latent(enc_out) - decoder_out = self.decoder(latent_out) - - else: # Classify - # Unfreeze classifier and freeze the rest - assert self.num_classifier_layers is not None, "Classifier not found" - + if not classify: + # Set the classifier grad off + if self.num_classes is not None: for param in self.classifier.parameters(): - param.requires_grad = True - for param in self.encoder.parameters(): - param.requires_grad = False - for param in self.decoder.parameters(): param.requires_grad = False - for param in self.latent.parameters(): - param.requires_grad = False - - # Encoder-->Latent-->Classifier - enc_out = self.encoder(data) - latent_out = self.latent(enc_out) - classifier_out = self.classifier(latent_out) # Deterministic + for param in self.encoder.parameters(): + param.requires_grad = True + for param in self.decoder.parameters(): + param.requires_grad = True + for param in self.latent.parameters(): + param.requires_grad = True + + # Encoder -->Latent --> Decoder + enc_out = self.encoder(data) + latent_out = self.latent(enc_out) + decoder_out = self.decoder(latent_out) + + else: # Classify + # Unfreeze classifier and freeze the rest + assert self.num_classifier_layers is not None, "Classifier not found" + + for param in self.classifier.parameters(): + param.requires_grad = True + for param in self.encoder.parameters(): + param.requires_grad = False + for param in self.decoder.parameters(): + param.requires_grad = False + for param in self.latent.parameters(): + param.requires_grad = False + + # Encoder-->Latent-->Classifier + enc_out = self.encoder(data) + latent_out = self.latent(enc_out) + + classifier_out = self.classifier(latent_out) # Deterministic + + if training and not classify: # Only forecasting network is trained + return decoder_out, latent_out + if classify and training: # Classifier is trained + return classifier_out + if classify and not training: # Inference + return decoder_out, classifier_out - if training and not classify: # Only forecasting network is trained - return decoder_out, latent_out - if classify and training: # Classifier is trained - return classifier_out - if classify and not training: # Inference - return decoder_out,classifier_out - From af2c00ccdfc12e93aca4c08396d6bf90664b03ff Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 30 Dec 2020 22:18:07 +0100 Subject: [PATCH 169/211] Update ae.py --- traja/models/predictive_models/ae.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/traja/models/predictive_models/ae.py b/traja/models/predictive_models/ae.py index 47bf1130..a5170a69 100644 --- a/traja/models/predictive_models/ae.py +++ b/traja/models/predictive_models/ae.py @@ -359,6 +359,14 @@ def forward(self, data, classify=False, training=True): ------- decoder_out,latent_out or classifier out """ + if not training: + # Encoder -->Latent --> Decoder + # |--> Classifier + enc_out = self.encoder(data) + latent_out = self.latent(enc_out) + decoder_out = self.decoder(latent_out) + classifier_out = self.classifier(latent_out) + return decoder_out, classifier_out if not classify: # Set the classifier grad off @@ -377,6 +385,7 @@ def forward(self, data, classify=False, training=True): enc_out = self.encoder(data) latent_out = self.latent(enc_out) decoder_out = self.decoder(latent_out) + return decoder_out, latent_out else: # Classify # Unfreeze classifier and freeze the rest @@ -396,11 +405,4 @@ def forward(self, data, classify=False, training=True): latent_out = self.latent(enc_out) classifier_out = self.classifier(latent_out) # Deterministic - - if training and not classify: # Only forecasting network is trained - return decoder_out, latent_out - if classify and training: # Classifier is trained return classifier_out - if classify and not training: # Inference - return decoder_out, classifier_out - From 9752f3eff2b4065174a1e885cebe01ca3d9090c3 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 30 Dec 2020 22:36:16 +0100 Subject: [PATCH 170/211] dataloader and scalers as dict --- traja/datasets/dataset.py | 31 +++++++++++++++---------------- traja/models/train.py | 25 +++++++++++++++---------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/traja/datasets/dataset.py b/traja/datasets/dataset.py index 7cbd7b1f..7dd2a919 100644 --- a/traja/datasets/dataset.py +++ b/traja/datasets/dataset.py @@ -265,7 +265,7 @@ class MultiModalDataLoader: 4. Create train test split: Split the shuffled batches into train (data, target, category) and test(data, target, category) 5. Weighted Random sampling: Apply weights with respect to category counts in the dataset: category_sample_weight = 1/num_category_samples; This avoid model overfit to category appear often in the dataset 6. Create pytorch Dataset instances - 7. Returns the train and test data loader instances given the dataset instances and batch size + 7. Returns the train and test data loader instances along with their scalers as a dictionaries given the dataset instances and batch size Args: df (pd.DataFrame): Dataset @@ -275,12 +275,8 @@ class MultiModalDataLoader: num_workers (int): Number of cpu subprocess occupied during data loading process Usage: - [train_dataloader, test_dataloader], - [train_x_scaler, - train_y_scaler, - test_x_scaler, - test_y_scaler] = MultiModalDataLoader(df = data_frame, batch_size=32, - n_past = 20, n_future = 10, num_workers=4) + ------ + dataloaders, scalers = MultiModalDataLoader(df = data_frame, batch_size=32, n_past = 20, n_future = 10, num_workers=4) """ def __init__( @@ -339,6 +335,17 @@ def __init__( num_workers=num_workers, ) + self.dataloaders = { + "train_loader": self.train_loader, + "test_loader": self.test_loader, + } + self.scalers = { + "train_data_scaler": self.train_x_scaler, + "train_target_scaler": self.train_y_scaler, + "test_data_scaler": self.test_x_scaler, + "test_target_scaler": self.test_y_scaler, + } + def __new__( cls, df: pd.DataFrame, @@ -352,13 +359,5 @@ def __new__( loader_instance = super(MultiModalDataLoader, cls).__new__(cls) loader_instance.__init__(df, batch_size, n_past, n_future, num_workers) # Return train and test loader attributes - return ( - [loader_instance.train_loader, loader_instance.test_loader], - [ - loader_instance.train_x_scaler, - loader_instance.train_y_scaler, - loader_instance.test_x_scaler, - loader_instance.test_y_scaler, - ], - ) + return loader_instance.datalaoders, loader_instance.scalers diff --git a/traja/models/train.py b/traja/models/train.py index 84ef0a47..892b5e3a 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -145,15 +145,16 @@ def __init__( def __str__(self): return f"Training model type {self.model_type}" - def fit(self, train_loader, test_loader, model_save_path=None): + def fit(self, dataloaders, model_save_path=None): """ This method implements the batch- wise training and testing protocol for both time series forecasting and classification of the timeseriesis_classification Parameters: ----------- - train_loader: Dataloader object of train dataset with batch [data,target,category] as a tuple - test_loader: Dataloader object of test dataset with [data,target,category] as a tuple + dataloaders: Dictionary containing train and test dataloaders + train_loader: Dataloader object of train dataset with batch data [data,target,category] + test_loader: Dataloader object of test dataset with [data,target,category] model_save_path: Directory path to save the model """ @@ -167,6 +168,7 @@ def fit(self, train_loader, test_loader, model_save_path=None): # Training mode: Switch from Generative to classifier training mode training_mode = 'forecasting' + train_loader, test_loader = dataloaders.values() # Training for epoch in range(self.epochs): test_loss_forecasting = 0 @@ -378,17 +380,19 @@ def __init__( self.optimizer = optimizer.get_optimizers(lr=0.001) self.scheduler = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) - def fit(self, train_loader, test_loader, model_save_path): + def fit(self, dataloaders, model_save_path): """ Implements the batch wise training and testing for time series forecasting. Args: - train_loader: Dataloader object of train dataset with batch data [data,target,category] - test_loader: Dataloader object of test dataset with [data,target,category] + dataloaders: Dictionary containing train and test dataloaders + train_loader: Dataloader object of train dataset with batch data [data,target,category] + test_loader: Dataloader object of test dataset with [data,target,category] model_save_path: Directory path to save the model Return: None""" assert self.model_type == 'lstm' self.model.to(device) + train_loader, test_loader = dataloaders.values() for epoch in range(self.epochs): if epoch > 0: @@ -469,13 +473,14 @@ def __init__( ) self.viz = True - def fit(self, train_loader, test_loader, model_save_path): + def fit(self, dataloaders, model_save_path): """ Implements the batch wise training and testing for time series forecasting Save train, test and validation performance in forecasting/classification tasks as a performance.csv Args: - train_loader: Dataloader object of train dataset with batch data [data,target,category] - test_loader: Dataloader object of test dataset with [data,target,category] + dataloaders: Dictionary containing train and test dataloaders + train_loader: Dataloader object of train dataset with batch data [data,target,category] + test_loader: Dataloader object of test dataset with [data,target,category] model_save_path: Directory path to save the model Return: None @@ -499,7 +504,7 @@ def fit(self, train_loader, test_loader, model_save_path): # Training loop self.model.to(device) - + train_loader, test_loader = dataloaders.values() for epoch in range(self.epochs): if epoch > 0: self.model.train() From c1d99a579f4826b29326eec9b1c90d16876e50a3 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 30 Dec 2020 22:37:35 +0100 Subject: [PATCH 171/211] Update dataset.py --- traja/datasets/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/datasets/dataset.py b/traja/datasets/dataset.py index 7dd2a919..aa8019bd 100644 --- a/traja/datasets/dataset.py +++ b/traja/datasets/dataset.py @@ -359,5 +359,5 @@ def __new__( loader_instance = super(MultiModalDataLoader, cls).__new__(cls) loader_instance.__init__(df, batch_size, n_past, n_future, num_workers) # Return train and test loader attributes - return loader_instance.datalaoders, loader_instance.scalers + return loader_instance.dataloaders, loader_instance.scalers From 6cec7dabcbfeef3a07b133e4b35ffa3186c6950b Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Wed, 30 Dec 2020 23:34:15 +0100 Subject: [PATCH 172/211] inference, do scaling on orginal and generated data --- traja/models/generative_models/vae.py | 2 - traja/models/inference.py | 147 +++++++++++++++----------- traja/models/predictive_models/ae.py | 8 -- 3 files changed, 85 insertions(+), 72 deletions(-) diff --git a/traja/models/generative_models/vae.py b/traja/models/generative_models/vae.py index 17de371b..e23eba36 100644 --- a/traja/models/generative_models/vae.py +++ b/traja/models/generative_models/vae.py @@ -363,8 +363,6 @@ def forward(self, data, training=True, classify=False): decoder_out,latent_out or classifier out """ if not classify: - - if not is_classification: # Set the classifier grad off for param in self.classifier.parameters(): param.requires_grad = False diff --git a/traja/models/inference.py b/traja/models/inference.py index bfa60014..39b15cc8 100644 --- a/traja/models/inference.py +++ b/traja/models/inference.py @@ -46,54 +46,69 @@ def __init__( assert model is not None self.model = model(**self.model_hyperparameters) - def generate_batch(self, batch_size, num_future, classify=True): + ( + self.generated_self.generated_categoryegory, + self.target_data, + self.target_data_, + self.generated_data, + self.generated_data_, + ) = (None, None, None, None, None) + + def generate(self, num_steps, classify=True, scaler=None): self.model.to(device) if self.model_type == "vae": # Random noise z = ( - torch.empty(batch_size, self.model_hyperparameters["latent_size"]) + torch.empty( + self.model_hyperparameters["batch_size"], + self.model_hyperparameters["latent_size"], + ) .normal_(mean=0, std=0.1) .to(device) ) # Generate trajectories from the noise - out = self.model.decoder(z, num_future).cpu().detach().numpy() - out = out.reshape(out.shape[0] * out.shape[1], out.shape[2]) + self.generated_data = ( + self.model.decoder(z, num_steps).cpu().detach().numpy() + ) + self.generated_data = self.generated_data.reshape( + self.generated_data.shape[0] * self.generated_data.shape[1], + self.generated_data.shape[2], + ) if classify: try: - cat = self.model.classifier(z) + self.generated_category = self.model.classifier(z) print( "IDs in this batch of synthetic data", - torch.max(cat, 1).indices.detach() + 1, + torch.max(self.generated_category, 1).indices.detach() + 1, ) except Exception as error: print("Classifier not found: " + repr(error)) - plt.figure(figsize=(12, 4)) - plt.plot(out[:, 0], label="Generated x: Longitude") - plt.plot(out[:, 1], label="Generated y: Latitude") - plt.legend() - fig, ax = plt.subplots(nrows=2, ncols=5, figsize=(16, 5), sharey=True) fig.set_size_inches(20, 5) + # Scale original data and generated data + for i in range(2): for j in range(5): if classify: try: label = "Animal ID {}".format( - (torch.max(cat, 1).indices + 1).detach()[i + j] + ( + torch.max(self.generated_category, 1).indices + 1 + ).detach()[i + j] ) except Exception as error: print("Classifier not found:" + repr(error)) else: label = "" ax[i, j].plot( - out[:, 0][ - (i + j) * num_future : (i + j) * num_future + num_future + self.generated_data[:, 0][ + (i + j) * num_steps : (i + j) * num_steps + num_steps ], - out[:, 1][ - (i + j) * num_future : (i + j) * num_future + num_future + self.generated_data[:, 1][ + (i + j) * num_steps : (i + j) * num_steps + num_steps ], label=label, color="g", @@ -101,13 +116,13 @@ def generate_batch(self, batch_size, num_future, classify=True): ax[i, j].legend() plt.show() - return out + return self.generated_data elif self.model_type == "vaegan" or "custom": return NotImplementedError # TODO: State space models - def generate_timeseries(num_steps): + def generate_timeseries(self, num_steps): """Recurrently generate time series for infinite time steps. Args: @@ -140,10 +155,6 @@ def __init__( self.model_path = model_path self.model_hyperparameters = model_hyperparameters - # Batch size and time step size - self.batch_size = self.model_hyperparameters["batch_size"] - num_future = self.model_hyperparameters["num_future"] - if self.model_type == "ae": self.model = MultiModelAE(**self.model_hyperparameters) @@ -153,19 +164,24 @@ def __init__( if self.model_type == "irl": self.model = MultiModelIRL(**self.model_hyperparameters) - if self.model_type == "vaegan": - self.model = MultiModelVAEGAN(**self.model_hyperparameters) - if self.model_type == "custom": assert model is not None self.model = model(**self.model_hyperparameters) - def predict_batch(self, data_loader, num_future, scaler, classify=True): + ( + self.predicted_self.generated_categoryegory, + self.target_data, + self.target_data_, + self.predicted_data, + self.predicted_data_, + ) = (None, None, None, None, None) + + def predict(self, data_loader, num_steps, scaler, classify=True): """[summary] Args: data_loader ([type]): [description] - num_future ([type]): [description] + num_steps ([type]): [description] scaler (dict): Scalers of the target data. This scale the model predictions to the scale of the target (future steps). : This scaler will be returned by the traja data preprocessing and loading helper function. classify (bool, optional): [description]. Defaults to True. @@ -176,59 +192,66 @@ def predict_batch(self, data_loader, num_future, scaler, classify=True): self.model.to(device) if self.model_type == "ae": - for data, target, category in data_loader: - predicted_data, predicted_category = self.model( - data.float().to(device), training=False, classify=classify - ) + for data, target, self.generated_categoryegory in data_loader: + data, target = data.to(device), target.to(device) + enc_self.generated_data = self.model.encoder(data) + latent_self.generated_data = self.model.latent(enc_self.generated_data) + self.predicted_data = self.model.decoder(latent_self.generated_data) + if classify: + self.predicted_self.generated_categoryegory = self.model.classifier( + latent_self.generated_data + ) + target = target.cpu().detach().numpy() target = target.reshape( target.shape[0] * target.shape[1], target.shape[2] ) - predicted_data = predicted_data.cpu().detach().numpy() - predicted_data = predicted_data.reshape( - predicted_data.shape[0] * predicted_data.shape[1], - predicted_data.shape[2], + self.predicted_data = self.predicted_data.cpu().detach().numpy() + self.predicted_data = self.predicted_data.reshape( + self.predicted_data.shape[0] * self.predicted_data.shape[1], + self.predicted_data.shape[2], ) # Rescaling predicted data - for i in range(predicted_data.shape[1]): + for i in range(self.predicted_data.shape[1]): s_s = scaler[f"scaler_{i}"].inverse_transform( - predicted_data[:, i].reshape(-1, 1) + self.predicted_data[:, i].reshape(-1, 1) ) s_s = np.reshape(s_s, len(s_s)) - predicted_data[:, i] = s_s + self.predicted_data[:, i] = s_s - # TODO:Deprecated;Slicing the data into batches + # TODO:Depreself.generated_categoryed;Slicing the data into batches predicted_data = np.array( [ - predicted_data[i : i + num_future] - for i in range(0, len(predicted_data), num_future) + self.predicted_data[i : i + num_steps] + for i in range(0, len(self.predicted_data), num_steps) ] ) # Rescaling target data - target_data = target.copy() - for i in range(target_data.shape[1]): + self.target_data = target.copy() + for i in range(self.target_data.shape[1]): s_s = scaler["scaler_{}".format(i)].inverse_transform( - target_data[:, i].reshape(-1, 1) + self.target_data[:, i].reshape(-1, 1) ) s_s = np.reshape(s_s, len(s_s)) - target_data[:, i] = s_s - # TODO:Deprecated;Slicing the data into batches - target_data = np.array( + self.target_data[:, i] = s_s + # TODO:Depreself.generated_categoryed;Slicing the data into batches + self.target_data = np.array( [ - target_data[i : i + num_future] - for i in range(0, len(target_data), num_future) + self.target_data[i : i + num_steps] + for i in range(0, len(self.target_data), num_steps) ] ) - # Reshape [batch_size*num_future,input_dim] + # Reshape [batch_size*num_steps,input_dim] predicted_data_ = predicted_data.reshape( - predicted_data.shape[0] * predicted_data.shape[1], - predicted_data.shape[2], + self.predicted_data.shape[0] * self.predicted_data.shape[1], + self.predicted_data.shape[2], ) - target_data_ = target_data.reshape( - target_data.shape[0] * target_data.shape[1], target_data.shape[2] + self.target_data_ = self.target_data.reshape( + self.target_data.shape[0] * self.target_data.shape[1], + self.target_data.shape[2], ) fig, ax = plt.subplots(nrows=2, ncols=5, figsize=(16, 5), sharey=False) @@ -237,22 +260,22 @@ def predict_batch(self, data_loader, num_future, scaler, classify=True): for j in range(5): ax[i, j].plot( predicted_data_[:, 0][ - (i + j) * num_future : (i + j) * num_future + num_future + (i + j) * num_steps : (i + j) * num_steps + num_steps ], predicted_data_[:, 1][ - (i + j) * num_future : (i + j) * num_future + num_future + (i + j) * num_steps : (i + j) * num_steps + num_steps ], - label=f"Predicted ID {predicted_category[i+j]}", + label=f"Predicted ID {self.predicted_self.generated_categoryegory[i+j]}", ) ax[i, j].plot( - target_data_[:, 0][ - (i + j) * num_future : (i + j) * num_future + num_future + self.target_data_[:, 0][ + (i + j) * num_steps : (i + j) * num_steps + num_steps ], - target_data_[:, 1][ - (i + j) * num_future : (i + j) * num_future + num_future + self.target_data_[:, 1][ + (i + j) * num_steps : (i + j) * num_steps + num_steps ], - label=f"Target ID {category[i+j]}", + label=f"Target ID {self.generated_categoryegory[i+j]}", color="g", ) ax[i, j].legend() diff --git a/traja/models/predictive_models/ae.py b/traja/models/predictive_models/ae.py index a5170a69..3039d0b8 100644 --- a/traja/models/predictive_models/ae.py +++ b/traja/models/predictive_models/ae.py @@ -359,14 +359,6 @@ def forward(self, data, classify=False, training=True): ------- decoder_out,latent_out or classifier out """ - if not training: - # Encoder -->Latent --> Decoder - # |--> Classifier - enc_out = self.encoder(data) - latent_out = self.latent(enc_out) - decoder_out = self.decoder(latent_out) - classifier_out = self.classifier(latent_out) - return decoder_out, classifier_out if not classify: # Set the classifier grad off From 86552be41dd8cd7edee912f7f4b205f0e44673ea Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 1 Jan 2021 12:25:14 +0100 Subject: [PATCH 173/211] Update inference.py --- traja/models/inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/inference.py b/traja/models/inference.py index 39b15cc8..5f576d7c 100644 --- a/traja/models/inference.py +++ b/traja/models/inference.py @@ -47,7 +47,7 @@ def __init__( self.model = model(**self.model_hyperparameters) ( - self.generated_self.generated_categoryegory, + self.generated_category, self.target_data, self.target_data_, self.generated_data, From e854459b961814bd546ee3d8de83cb4f05efd3d2 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 1 Jan 2021 13:22:52 +0100 Subject: [PATCH 174/211] Update inference.py --- traja/models/inference.py | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/traja/models/inference.py b/traja/models/inference.py index 5f576d7c..a66a2a82 100644 --- a/traja/models/inference.py +++ b/traja/models/inference.py @@ -90,6 +90,47 @@ def generate(self, num_steps, classify=True, scaler=None): # Scale original data and generated data + # Rescaling predicted data + for i in range(self.generated_data.shape[1]): + s_s = scaler[f"scaler_{i}"].inverse_transform( + self.generated_data[:, i].reshape(-1, 1) + ) + s_s = np.reshape(s_s, len(s_s)) + self.generated_data[:, i] = s_s + + # TODO:Depreself.generated_categoryed;Slicing the data into batches + self.generated_data = np.array( + [ + self.generated_data[i : i + num_steps] + for i in range(0, len(self.generated_data), num_steps) + ] + ) + + # Rescaling target data + for i in range(self.target_data.shape[1]): + + s_s = scaler["scaler_{}".format(i)].inverse_transform( + self.target_data[:, i].reshape(-1, 1) + ) + s_s = np.reshape(s_s, len(s_s)) + self.target_data[:, i] = s_s + # TODO:Depreself.generated_categoryed;Slicing the data into batches + self.target_data = np.array( + [ + self.target_data[i : i + num_steps] + for i in range(0, len(self.target_data), num_steps) + ] + ) + + # Reshape [batch_size*num_steps,input_dim] + self.generated_data_ = self.generated_data.reshape( + self.generated_data.shape[0] * self.generated_data.shape[1], + self.generated_data.shape[2], + ) + self.target_data_ = self.target_data.reshape( + self.target_data.shape[0] * self.target_data.shape[1], + self.target_data.shape[2], + ) for i in range(2): for j in range(5): if classify: From d3507d9d69045814e0acd161e4a89c29d2023ed1 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 1 Jan 2021 13:31:43 +0100 Subject: [PATCH 175/211] Update inference.py --- traja/models/inference.py | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/traja/models/inference.py b/traja/models/inference.py index a66a2a82..9ba6cef3 100644 --- a/traja/models/inference.py +++ b/traja/models/inference.py @@ -46,13 +46,7 @@ def __init__( assert model is not None self.model = model(**self.model_hyperparameters) - ( - self.generated_category, - self.target_data, - self.target_data_, - self.generated_data, - self.generated_data_, - ) = (None, None, None, None, None) + (self.generated_category, self.generated_data,) = (None, None, None, None, None) def generate(self, num_steps, classify=True, scaler=None): @@ -106,31 +100,12 @@ def generate(self, num_steps, classify=True, scaler=None): ] ) - # Rescaling target data - for i in range(self.target_data.shape[1]): - - s_s = scaler["scaler_{}".format(i)].inverse_transform( - self.target_data[:, i].reshape(-1, 1) - ) - s_s = np.reshape(s_s, len(s_s)) - self.target_data[:, i] = s_s - # TODO:Depreself.generated_categoryed;Slicing the data into batches - self.target_data = np.array( - [ - self.target_data[i : i + num_steps] - for i in range(0, len(self.target_data), num_steps) - ] - ) - # Reshape [batch_size*num_steps,input_dim] - self.generated_data_ = self.generated_data.reshape( + self.generated_data = self.generated_data.reshape( self.generated_data.shape[0] * self.generated_data.shape[1], self.generated_data.shape[2], ) - self.target_data_ = self.target_data.reshape( - self.target_data.shape[0] * self.target_data.shape[1], - self.target_data.shape[2], - ) + for i in range(2): for j in range(5): if classify: From ffd386fbb8a3d74c12d36128f527a5e56ae41786 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 1 Jan 2021 13:35:54 +0100 Subject: [PATCH 176/211] Update inference.py --- traja/models/inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/inference.py b/traja/models/inference.py index 9ba6cef3..5e630488 100644 --- a/traja/models/inference.py +++ b/traja/models/inference.py @@ -46,7 +46,7 @@ def __init__( assert model is not None self.model = model(**self.model_hyperparameters) - (self.generated_category, self.generated_data,) = (None, None, None, None, None) + (self.generated_category, self.generated_data,) = (None, None) def generate(self, num_steps, classify=True, scaler=None): From c9ad562c76dc9dcefb34f57a719ad5340588c192 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Wed, 30 Dec 2020 11:31:43 +0000 Subject: [PATCH 177/211] Add simply HybridTrainer test --- traja/models/utils.py | 8 +++++--- traja/tests/test_nn.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/traja/models/utils.py b/traja/models/utils.py index 7bfaddaa..f9cc51b7 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -57,11 +57,13 @@ def save(model, hyperparameters, PATH=None): if PATH is None: PATH = os.getcwd() + "model.pt" torch.save(model.state_dict(), PATH) - _dir, _ = os.path.split(PATH) + hyperdir, _ = os.path.split(PATH) if hyperparameters is not None: - with open("./hypers.json", "w") as fp: + with open(os.path.join(hyperdir, "hypers.json"), "w") as fp: json.dump(hyperparameters, fp, sort_keys=False) - print(f"Model saved at {_dir}") + if hyperdir == "": + hyperdir = "." + print("Model and hyperparameters saved at {hyperdir}") def load(model, PATH=None): diff --git a/traja/tests/test_nn.py b/traja/tests/test_nn.py index 0f132a82..ba64b21d 100644 --- a/traja/tests/test_nn.py +++ b/traja/tests/test_nn.py @@ -7,7 +7,7 @@ data_url = "https://raw.githubusercontent.com/traja-team/traja-research/dataset_und_notebooks/dataset_analysis/jaguar5.csv" df = pd.read_csv(data_url, error_bad_lines=False) -model_save_path = './model.pt' +model_save_path = 'model' # Hyperparameters batch_size = 10 From c2a7c64cf7d0920a81bad62bcd27db5d14fb95a7 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Wed, 30 Dec 2020 12:03:30 +0000 Subject: [PATCH 178/211] Update gitignore --- .gitignore | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e2a46752..71c802a7 100644 --- a/.gitignore +++ b/.gitignore @@ -113,8 +113,13 @@ _build/ # BUILD FILES *.zip +hypers.json +model docs/source/gallery docs/source/savefig docs/source/reference -docs/source/autoexamples + +# Editor files +*.swp +*.swo From 9d8764df1ecf871674cf31cd596c24e4515c800d Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Wed, 30 Dec 2020 12:41:15 +0000 Subject: [PATCH 179/211] Update pip requirements --- docs/requirements.txt | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 932d4323..3de08e47 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -7,11 +7,6 @@ scipy sphinx sphinx-gallery fastdtw -pillow -tzlocal -sphinx_rtd_theme -pytest -pytest-cov -codecov -readline -rpy2 +plotly +networkx +seaborn \ No newline at end of file From 0b70e124577edb4752d9ab892c8c428b5dc22a6d Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Wed, 30 Dec 2020 12:41:36 +0000 Subject: [PATCH 180/211] Make epochs a dynamic argument in trainer --- traja/models/train.py | 19 ++++++------------- traja/tests/test_nn.py | 26 ++++++++++++++++---------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 892b5e3a..6bfb5434 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -32,7 +32,6 @@ class HybridTrainer(object): dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, with dropout probability equal to dropout num_classifier_layers: Number of layers in the classifier - epochs: Number of epochs to train the network batch_size: Number of samples in a batch num_future: Number of time steps to be predicted forward num_past: Number of past time steps otherwise, length of sequences in each batch of data. @@ -60,7 +59,6 @@ def __init__( reset_state: bool, latent_size: int, dropout: float, - epochs: int, batch_size: int, num_future: int, num_past: int, @@ -94,7 +92,6 @@ def __init__( self.latent_size = latent_size self.num_classifier_layers = num_classifier_layers self.num_future = num_future - self.epochs = epochs self.batch_size = batch_size self.num_past = num_past self.dropout = dropout @@ -145,7 +142,7 @@ def __init__( def __str__(self): return f"Training model type {self.model_type}" - def fit(self, dataloaders, model_save_path=None): + def fit(self, train_loader, test_loader, model_save_path=None, training_mode='forecasting', epochs=50): """ This method implements the batch- wise training and testing protocol for both time series forecasting and classification of the timeseriesis_classification @@ -156,21 +153,20 @@ def fit(self, dataloaders, model_save_path=None): train_loader: Dataloader object of train dataset with batch data [data,target,category] test_loader: Dataloader object of test dataset with [data,target,category] model_save_path: Directory path to save the model + training_mode: Type of training ('forecasting', 'classification') + epochs: Number of epochs to train """ assert model_save_path is not None, f"Model path {model_save_path} unknown" + assert training_mode in ['forecasting', 'classification'], f'Training mode {classification} unknown' self.model.to(device) encoder_optimizer, latent_optimizer, decoder_optimizer, classifier_optimizer = self.model_optimizers.values() encoder_scheduler, latent_scheduler, decoder_scheduler, classifier_scheduler = self.model_lrschedulers.values() - # Training mode: Switch from Generative to classifier training mode - training_mode = 'forecasting' - - train_loader, test_loader = dataloaders.values() # Training - for epoch in range(self.epochs): + for epoch in range(epochs): test_loss_forecasting = 0 test_loss_classification = 0 if epoch > 0: # Initial step is to test and set LR schduler @@ -200,7 +196,7 @@ def fit(self, dataloaders, model_save_path=None): decoder_optimizer.step() latent_optimizer.step() - elif self.classify and training_mode != "forecasting": + elif self.classify and training_mode == "classification": if self.model_type == "vae": classifier_out, latent_out, mu, logvar = self.model( data, training=True, classify=True @@ -219,9 +215,6 @@ def fit(self, dataloaders, model_save_path=None): print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss / (idx + 1))) - if (epoch + 1) * 2 == self.epochs and self.classify: - training_mode = "classification" - # Testing if epoch % 10 == 0: with torch.no_grad(): diff --git a/traja/tests/test_nn.py b/traja/tests/test_nn.py index ba64b21d..afb8a218 100644 --- a/traja/tests/test_nn.py +++ b/traja/tests/test_nn.py @@ -5,16 +5,18 @@ from traja.datasets import dataset from traja.models.train import LSTMTrainer, HybridTrainer, CustomTrainer -data_url = "https://raw.githubusercontent.com/traja-team/traja-research/dataset_und_notebooks/dataset_analysis/jaguar5.csv" -df = pd.read_csv(data_url, error_bad_lines=False) -model_save_path = 'model' -# Hyperparameters -batch_size = 10 -num_past = 10 -num_future = 5 -if __name__ == '__main__': +def test_from_df(): + data_url = "https://raw.githubusercontent.com/traja-team/traja-research/dataset_und_notebooks/dataset_analysis/jaguar5.csv" + df = pd.read_csv(data_url, error_bad_lines=False) + model_save_path = 'model' + + # Hyperparameters + batch_size = 10 + num_past = 10 + num_future = 5 + # Prepare the dataloader train_loader, test_loader = dataset.MultiModalDataLoader(df, batch_size=batch_size, @@ -34,7 +36,6 @@ num_classes=9, # Uncomment to create and train classifier network num_classifier_layers=4, classifier_hidden_size= 32, - epochs=10, batch_size=batch_size, num_future=num_future, num_past=num_past, @@ -44,4 +45,9 @@ # Train the model - trainer.fit(train_loader, test_loader, model_save_path) + trainer.fit(train_loader, test_loader, model_save_path, epochs=10, training_mode='forecasting') + trainer.fit(train_loader, test_loader, model_save_path, epochs=10, training_mode='classification') + + +if __name__ == '__main__': + test_from_df() From 8546e7bb82ea44d7d7059f75480e0aa47d3801d5 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Wed, 30 Dec 2020 20:24:27 +0000 Subject: [PATCH 181/211] Ignore model parameter files --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 71c802a7..531a5e17 100644 --- a/.gitignore +++ b/.gitignore @@ -123,3 +123,7 @@ docs/source/reference # Editor files *.swp *.swo + + +# Model parameter files +*.pt From f54b684bdfabc800dd6980fa3e81f6e9775e2fea Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Wed, 30 Dec 2020 20:24:52 +0000 Subject: [PATCH 182/211] Make model an argument in VAE trainer --- docs/requirements.txt | 10 ++-- traja/models/generative_models/vae.py | 3 ++ traja/models/predictive_models/ae.py | 3 ++ traja/models/predictive_models/lstm.py | 3 ++ traja/models/train.py | 75 +++++++------------------- traja/models/visualizer.py | 14 ----- traja/tests/test_models.py | 50 +++++++++-------- traja/tests/test_nn.py | 5 +- 8 files changed, 63 insertions(+), 100 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 3de08e47..1166c8a5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,12 +1,14 @@ pandas -ipython -numpy +numpy==1.18.5 matplotlib shapely scipy -sphinx +sklearn sphinx-gallery fastdtw plotly networkx -seaborn \ No newline at end of file +seaborn +torch +pytest +h5py \ No newline at end of file diff --git a/traja/models/generative_models/vae.py b/traja/models/generative_models/vae.py index e23eba36..680d150d 100644 --- a/traja/models/generative_models/vae.py +++ b/traja/models/generative_models/vae.py @@ -310,6 +310,9 @@ def __init__( self.reset_state = reset_state self.bidirectional = bidirectional + # Let the trainer know what kind of model this is + self.model_type = 'vae' + self.encoder = LSTMEncoder( input_size=self.input_size, num_past=self.num_past, diff --git a/traja/models/predictive_models/ae.py b/traja/models/predictive_models/ae.py index 3039d0b8..fcfa583b 100644 --- a/traja/models/predictive_models/ae.py +++ b/traja/models/predictive_models/ae.py @@ -286,6 +286,9 @@ def __init__( self.reset_state = reset_state self.bidirectional = bidirectional + # Let the trainer know what kind of model this is + self.model_type = 'ae' + self.encoder = LSTMEncoder( input_size=self.input_size, num_past=self.num_past, diff --git a/traja/models/predictive_models/lstm.py b/traja/models/predictive_models/lstm.py index 519904e8..15e8f3c9 100644 --- a/traja/models/predictive_models/lstm.py +++ b/traja/models/predictive_models/lstm.py @@ -50,6 +50,9 @@ def __init__( self.reset_state = reset_state self.bidirectional = bidirectional + # Let the trainer know what kind of model this is + self.model_type = 'lstm' + # RNN decoder self.lstm = torch.nn.LSTM( input_size=self.input_size, diff --git a/traja/models/train.py b/traja/models/train.py index 6bfb5434..f1fccc8e 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -50,23 +50,8 @@ class HybridTrainer(object): def __init__( self, - model_type: str, + model: torch.nn.Module, optimizer_type: str, - input_size: int, - output_size: int, - lstm_hidden_size: int, - num_lstm_layers: int, - reset_state: bool, - latent_size: int, - dropout: float, - batch_size: int, - num_future: int, - num_past: int, - num_classes: int = None, - classifier_hidden_size: int = None, - num_classifier_layers: int = None, - bidirectional: bool = False, - batch_first: bool = True, loss_type: str = "huber", lr: float = 0.001, lr_factor: float = 0.1, @@ -74,28 +59,12 @@ def __init__( ): assert ( - model_type in HybridTrainer.valid_models + model.model_type in HybridTrainer.valid_models ), "Model type is {model_type}, valid models are {}".format( HybridTrainer.valid_models ) - self.model_type = model_type - self.input_size = input_size - self.lstm_hidden_size = lstm_hidden_size - self.num_lstm_layers = num_lstm_layers - self.classifier_hidden_size = classifier_hidden_size - self.num_classifier_layers = num_classifier_layers - self.batch_first = batch_first - self.reset_state = reset_state - self.output_size = output_size - self.num_classes = num_classes - self.latent_size = latent_size - self.num_classifier_layers = num_classifier_layers - self.num_future = num_future - self.batch_size = batch_size - self.num_past = num_past - self.dropout = dropout - self.bidirectional = bidirectional + self.model_type = model.model_type self.loss_type = loss_type self.optimizer_type = optimizer_type self.lr = lr @@ -103,31 +72,27 @@ def __init__( self.scheduler_patience = scheduler_patience self.model_hyperparameters = { - "input_size": self.input_size, - "num_past": self.num_past, - "batch_size": self.batch_size, - "lstm_hidden_size": self.lstm_hidden_size, - "num_lstm_layers": self.num_lstm_layers, - "classifier_hidden_size": self.classifier_hidden_size, - "num_classifier_layers": self.num_classifier_layers, - "num_future": self.num_future, - "latent_size": self.latent_size, - "output_size": self.output_size, - "num_classes": self.num_classes, - "batch_first": self.batch_first, - "reset_state": self.reset_state, - "bidirectional": self.bidirectional, - "dropout": self.dropout, + "input_size": model.input_size, + "num_past": model.num_past, + "batch_size": model.batch_size, + "lstm_hidden_size": model.lstm_hidden_size, + "num_lstm_layers": model.num_lstm_layers, + "classifier_hidden_size": model.classifier_hidden_size, + "num_classifier_layers": model.num_classifier_layers, + "num_future": model.num_future, + "latent_size": model.latent_size, + "output_size": model.output_size, + "num_classes": model.num_classes, + "batch_first": model.batch_first, + "reset_state": model.reset_state, + "bidirectional": model.bidirectional, + "dropout": model.dropout, } - # Instantiate model instance based on model_type - if self.model_type == 'ae': - self.model = MultiModelAE(**self.model_hyperparameters) - elif self.model_type == "vae": - self.model = MultiModelVAE(**self.model_hyperparameters) + self.model = model # Classification task check - self.classify = True if self.classifier_hidden_size is not None else False + self.classify = True if model.classifier_hidden_size is not None else False # Model optimizer and the learning rate scheduler optimizer = Optimizer( diff --git a/traja/models/visualizer.py b/traja/models/visualizer.py index bfc78d92..684a2a31 100644 --- a/traja/models/visualizer.py +++ b/traja/models/visualizer.py @@ -1,22 +1,10 @@ -import os, sys -from matplotlib.pyplot import figimage import networkx as nx -import pandas as pd import numpy as np import scipy -import sklearn from sklearn import neighbors from scipy.sparse import csgraph from sklearn.neighbors import radius_neighbors_graph -from sklearn.neighbors import kneighbors_graph from mpl_toolkits.mplot3d import Axes3D -import seaborn as sns -import argparse, copy, h5py, os, sys, time, socket -import tensorflow as tf -import torch, torchvision, torch.nn as nn -import torch.optim as optim -import torchvision.transforms as transforms -from matplotlib import ticker, colors import plotly.express as px @@ -24,8 +12,6 @@ # matplotlib.use("TKAgg") import matplotlib.pyplot as plt -from matplotlib.axes import Axes -from matplotlib import cm from matplotlib import style # plt.switch_backend("TkAgg") diff --git a/traja/tests/test_models.py b/traja/tests/test_models.py index ed9fb3fe..01a39e72 100644 --- a/traja/tests/test_models.py +++ b/traja/tests/test_models.py @@ -1,6 +1,9 @@ import pandas as pd from traja.datasets import dataset -from traja.models.train import LSTMTrainer, LatentModelTrainer +from traja.models.train import LSTMTrainer, HybridTrainer +from traja.models.generative_models.vae import MultiModelVAE +from traja.models.predictive_models.ae import MultiModelAE + # Sample data data_url = "https://raw.githubusercontent.com/traja-team/traja-research/dataset_und_notebooks/dataset_analysis/jaguar5.csv" @@ -24,31 +27,32 @@ def test_aevae(): num_workers=1) model_save_path = './model.pt' + + model = MultiModelVAE(input_size=2, + output_size=2, + lstm_hidden_size=32, + num_lstm_layers=2, + num_classes=9, + latent_size=10, + dropout=0.1, + num_classifier_layers=4, + classifier_hidden_size=32, + batch_size=batch_size, + num_future=num_future, + num_past=num_past, + bidirectional=False, + batch_first=True, + reset_state=True) + # Model Trainer # Model types; "ae" or "vae" - trainer = LatentModelTrainer(model_type='ae', - optimizer_type='Adam', - device='cpu', - input_size=2, - output_size=2, - lstm_hidden_size=32, - num_lstm_layers=2, - reset_state=True, - num_classes=9, - latent_size=10, - dropout=0.1, - num_classifier_layers=4, - classifier_hidden_size=32, - epochs=10, - batch_size=batch_size, - num_future=num_future, - num_past=num_past, - bidirectional=False, - batch_first=True, - loss_type='huber') + trainer = HybridTrainer(model=model, + optimizer_type='Adam', + loss_type='huber') # Train the model - trainer.train(train_loader, test_loader, model_save_path) + trainer.fit(train_loader, test_loader, model_save_path, epochs=10, training_mode='forecasting') + trainer.fit(train_loader, test_loader, model_save_path, epochs=10, training_mode='classification') def test_lstm(): @@ -82,7 +86,7 @@ def test_lstm(): num_future=num_future, num_layers=2, output_size=2, - lr_factor=0.1, + scheduler_patience=3, batch_first=True, dropout=0.1, diff --git a/traja/tests/test_nn.py b/traja/tests/test_nn.py index afb8a218..a846edfb 100644 --- a/traja/tests/test_nn.py +++ b/traja/tests/test_nn.py @@ -1,7 +1,4 @@ import pandas as pd -import traja -from traja import models -from traja import datasets from traja.datasets import dataset from traja.models.train import LSTMTrainer, HybridTrainer, CustomTrainer @@ -23,7 +20,7 @@ def test_from_df(): n_past=num_past, n_future=num_future, num_workers=2) - + trainer = HybridTrainer(model_type='vae', # "ae" or "vae" optimizer_type='Adam', # ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop','LBFGS', 'ASGD', 'Adamax'] input_size=2, From 4b7101e1009dee9bfdabc07f0a24ac65d1111d0d Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 12:03:25 +0000 Subject: [PATCH 183/211] Make HybridTrainer train LSTMs --- traja/models/generative_models/vae.py | 7 +- traja/models/optimizers.py | 70 +++++++-------- traja/models/predictive_models/ae.py | 7 +- traja/models/predictive_models/lstm.py | 4 +- traja/models/train.py | 117 +++++++++++++++---------- traja/tests/test_models.py | 87 +++++++++++++----- 6 files changed, 186 insertions(+), 106 deletions(-) diff --git a/traja/models/generative_models/vae.py b/traja/models/generative_models/vae.py index 680d150d..9c45dd59 100644 --- a/traja/models/generative_models/vae.py +++ b/traja/models/generative_models/vae.py @@ -354,7 +354,7 @@ def __init__( dropout=self.dropout, ) - def forward(self, data, training=True, classify=False): + def forward(self, data, training=True, classify=False, latent=True): """ Parameters: ----------- @@ -382,7 +382,10 @@ def forward(self, data, training=True, classify=False): latent_out, mu, logvar = self.latent(enc_out, training=training) # Decoder decoder_out = self.decoder(latent_out) - return decoder_out, latent_out, mu, logvar + if latent: + return decoder_out, latent_out, mu, logvar + else: + return decoder_out else: # training_mode = 'classification' # Unfreeze classifier parameters and freeze all other diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index a25a2f41..53f995c4 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -38,7 +38,11 @@ def __init__(self, model_type, model, optimizer_type, classify=False): self.model = model self.optimizer_type = optimizer_type self.optimizers = {} - self.schedulers = {} + self.forecasting_schedulers = {} + self.classification_schedulers = {} + + self.forecasting_keys = ['encoder', 'decoder', 'latent'] + self.classification_keys = ['classifier'] def get_optimizers(self, lr=0.0001): """Optimizers for each network in the model @@ -52,24 +56,18 @@ def get_optimizers(self, lr=0.0001): """ if self.model_type in ["lstm", "custom"]: - self.optimizers = getattr(torch.optim, f"{self.optimizer_type}")( + self.optimizers['encoder'] = getattr(torch.optim, f"{self.optimizer_type}")( self.model.parameters(), lr=lr ) elif self.model_type in ["ae", "vae"]: keys = ["encoder", "decoder", "latent", "classifier"] - for network in keys: - if network != "classifier": - self.optimizers[network] = getattr( - torch.optim, f"{self.optimizer_type}" - )(getattr(self.model, f"{network}").parameters(), lr=lr) - - if self.classify: - self.optimizers["classifier"] = getattr( + for key in keys: + network = getattr( torch.optim, f"{self.optimizer_type}" - )(getattr(self.model, "classifier").parameters(), lr=lr) - else: - self.optimizers["classifier"] = None + )(getattr(self.model, f"{key}").parameters(), lr=lr) + if network is not None: + self.optimizers[key] = network elif self.model_type == "vaegan": return NotImplementedError @@ -77,7 +75,9 @@ def get_optimizers(self, lr=0.0001): else: # self.model_type == "irl": return NotImplementedError - return self.optimizers + forecasting_optimizers = [self.optimizers[key] for key in self.forecasting_keys if key in self.optimizers] + classification_optimizers = [self.optimizers[key] for key in self.classification_keys if key in self.optimizers] + return forecasting_optimizers, classification_optimizers def get_lrschedulers(self, factor: float, patience: int): @@ -91,35 +91,31 @@ def get_lrschedulers(self, factor: float, patience: int): Returns: [dict]: [description] """ - if self.model_type in ["lstm", "custom"]: - assert not isinstance(self.optimizers, dict) - self.schedulers = ReduceLROnPlateau( - self.optimizers, + + if self.model_type == "irl" or self.model_type == 'vaegan': + return NotImplementedError + + forecasting_keys = [key for key in self.forecasting_keys if key in self.optimizers] + classification_keys = [key for key in self.classification_keys if key in self.optimizers] + + for network in forecasting_keys: + self.forecasting_schedulers[network] = ReduceLROnPlateau( + self.optimizers[network], + mode="max", + factor=factor, + patience=patience, + verbose=True, + ) + for network in classification_keys: + self.classification_schedulers[network] = ReduceLROnPlateau( + self.optimizers[network], mode="max", factor=factor, patience=patience, verbose=True, ) - elif self.model_type in ["ae", "vae"]: - for network in self.optimizers.keys(): - if self.optimizers[network] is not None: - self.schedulers[network] = ReduceLROnPlateau( - self.optimizers[network], - mode="max", - factor=factor, - patience=patience, - verbose=True, - ) - if not self.classify: - self.schedulers["classifier"] = None - - elif self.model_type == "irl": - return NotImplementedError - - else: # self.model_type == 'vaegan': - return NotImplementedError - return self.schedulers + return self.forecasting_schedulers, self.classification_schedulers if __name__ == "__main__": diff --git a/traja/models/predictive_models/ae.py b/traja/models/predictive_models/ae.py index fcfa583b..218d4f3a 100644 --- a/traja/models/predictive_models/ae.py +++ b/traja/models/predictive_models/ae.py @@ -351,7 +351,7 @@ def get_classifier_parameters(self): assert self.classifier_hidden_size is not None, "Classifier not found" return [self.classifier.parameters()] - def forward(self, data, classify=False, training=True): + def forward(self, data, classify=False, training=True, latent=True): """ Parameters: ----------- @@ -380,7 +380,10 @@ def forward(self, data, classify=False, training=True): enc_out = self.encoder(data) latent_out = self.latent(enc_out) decoder_out = self.decoder(latent_out) - return decoder_out, latent_out + if latent: + return decoder_out, latent_out + else: + return decoder_out else: # Classify # Unfreeze classifier and freeze the rest diff --git a/traja/models/predictive_models/lstm.py b/traja/models/predictive_models/lstm.py index 15e8f3c9..e149b9aa 100644 --- a/traja/models/predictive_models/lstm.py +++ b/traja/models/predictive_models/lstm.py @@ -76,7 +76,9 @@ def _init_hidden(self): .to(device), ) - def forward(self, x): + def forward(self, x, training=True, classify=False, latent=False): + assert not classify, 'LSTM forecaster cannot classify!' + assert not latent, 'LSTM forecaster does not have a latent space!' # To feed the latent states into lstm decoder, repeat the tensor n_future times at second dim (h0, c0) = self._init_hidden() diff --git a/traja/models/train.py b/traja/models/train.py index f1fccc8e..21e82725 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -46,7 +46,7 @@ class HybridTrainer(object): """ - valid_models = ["ae", "vae"] + valid_models = ['ae', 'vae', 'lstm'] def __init__( self, @@ -71,36 +71,50 @@ def __init__( self.lr_factor = lr_factor self.scheduler_patience = scheduler_patience - self.model_hyperparameters = { - "input_size": model.input_size, - "num_past": model.num_past, - "batch_size": model.batch_size, - "lstm_hidden_size": model.lstm_hidden_size, - "num_lstm_layers": model.num_lstm_layers, - "classifier_hidden_size": model.classifier_hidden_size, - "num_classifier_layers": model.num_classifier_layers, - "num_future": model.num_future, - "latent_size": model.latent_size, - "output_size": model.output_size, - "num_classes": model.num_classes, - "batch_first": model.batch_first, - "reset_state": model.reset_state, - "bidirectional": model.bidirectional, - "dropout": model.dropout, - } + if model.model_type == 'lstm': + self.model_hyperparameters = { + "input_size": model.input_size, + "batch_size": model.batch_size, + "hidden_size": model.hidden_size, + "num_future": model.num_future, + "num_layers": model.num_layers, + "output_size": model.output_size, + "batch_first": model.batch_first, + "reset_state": model.reset_state, + "bidirectional": model.bidirectional, + "dropout": model.dropout, + } + else: + self.model_hyperparameters = { + "input_size": model.input_size, + "num_past": model.num_past, + "batch_size": model.batch_size, + "lstm_hidden_size": model.lstm_hidden_size, + "num_lstm_layers": model.num_lstm_layers, + "classifier_hidden_size": model.classifier_hidden_size, + "num_classifier_layers": model.num_classifier_layers, + "num_future": model.num_future, + "latent_size": model.latent_size, + "output_size": model.output_size, + "num_classes": model.num_classes, + "batch_first": model.batch_first, + "reset_state": model.reset_state, + "bidirectional": model.bidirectional, + "dropout": model.dropout, + } self.model = model # Classification task check - self.classify = True if model.classifier_hidden_size is not None else False + self.classify = True if model.model_type != 'lstm' and model.classifier_hidden_size is not None else False # Model optimizer and the learning rate scheduler optimizer = Optimizer( self.model_type, self.model, self.optimizer_type, classify=self.classify ) - self.model_optimizers = optimizer.get_optimizers(lr=self.lr) - self.model_lrschedulers = optimizer.get_lrschedulers( + self.forecasting_optimizers, self.classification_optimizers = optimizer.get_optimizers(lr=self.lr) + self.forecasting_schedulers, self.classification_schedulers = optimizer.get_lrschedulers( factor=self.lr_factor, patience=self.scheduler_patience ) @@ -127,8 +141,10 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo self.model.to(device) - encoder_optimizer, latent_optimizer, decoder_optimizer, classifier_optimizer = self.model_optimizers.values() - encoder_scheduler, latent_scheduler, decoder_scheduler, classifier_scheduler = self.model_lrschedulers.values() + + #forecasting_optimizers, classification_optimizers = self.model_optimizers.values() + + #forecasting_schedulers, classification_schedulers = self.model_lrschedulers.values() # Training for epoch in range(epochs): @@ -140,15 +156,22 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo total_loss = 0 for idx, (data, target, category) in enumerate(train_loader): # Reset optimizer states - encoder_optimizer.zero_grad() - latent_optimizer.zero_grad() - decoder_optimizer.zero_grad() - classifier_optimizer.zero_grad() + for optimizer in self.forecasting_optimizers: + optimizer.zero_grad() + if self.classify: + for optimizer in self.classification_optimizers: + optimizer.zero_grad() + + data, target, category = ( + data.float().to(device), + target.float().to(device), + category.to(device), + ) if training_mode == "forecasting": - if self.model_type == "ae": - decoder_out, latent_out = self.model( - data, training=True, classify=False + if self.model_type == "ae" or self.model_type == 'lstm': + decoder_out = self.model( + data, training=True, classify=False, latent=False ) loss = Criterion().ae_criterion(decoder_out, target) else: # vae @@ -157,16 +180,15 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo loss = Criterion().vae_criterion(decoder_out, target, mu, logvar) loss.backward() - encoder_optimizer.step() - decoder_optimizer.step() - latent_optimizer.step() + for optimizer in self.forecasting_optimizers: + optimizer.step() elif self.classify and training_mode == "classification": if self.model_type == "vae": classifier_out, latent_out, mu, logvar = self.model( data, training=True, classify=True ) - else: # "ae" + else: # 'ae', 'lstm' classifier_out = self.model( data, training=True, classify=True ) @@ -175,7 +197,8 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo ) loss.backward() - classifier_optimizer.step() + for optimizer in self.classification_optimizers: + optimizer.step() total_loss += loss print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss / (idx + 1))) @@ -190,9 +213,14 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo for idx, (data, target, category) in enumerate(list(test_loader)): data, target, category = data.float().to(device), target.float().to(device), category.to(device) # Time series forecasting test - if self.model_type == 'ae': - out, latent = self.model(data, training=False, is_classification=False) - test_loss_forecasting += Criterion().ae_criterion(out, target).item() + if self.model_type == 'ae' or self.model_type == 'lstm': + out = self.model( + data, training=False, classify=False, latent=False + ) + test_loss_forecasting += ( + Criterion().ae_criterion(out, target).item() + ) + else: decoder_out, latent_out, mu, logvar = self.model(data, training=False, is_classification=False) @@ -200,7 +228,7 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo # Classification test if self.classify: category = category.long() - if self.model_type == "ae": + if self.model_type == 'ae' or self.model_type == 'lstm': classifier_out = self.model( data, training=False, classify=True ) @@ -230,12 +258,13 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo ) # Scheduler metric is test set loss - if training_mode == 'forecasting': - encoder_scheduler.step(test_loss_forecasting) - decoder_scheduler.step(test_loss_forecasting) - latent_scheduler.step(test_loss_forecasting) + if training_mode == "forecasting": + for scheduler in self.forecasting_schedulers.values(): + scheduler.step(test_loss_forecasting) else: - classifier_scheduler.step(test_loss_classification) + if self.classify: + for scheduler in self.classification_schedulers.values(): + scheduler.step() # Save the model at target path utils.save(self.model, self.model_hyperparameters, PATH=model_save_path) diff --git a/traja/tests/test_models.py b/traja/tests/test_models.py index 01a39e72..5ab1d58b 100644 --- a/traja/tests/test_models.py +++ b/traja/tests/test_models.py @@ -3,6 +3,7 @@ from traja.models.train import LSTMTrainer, HybridTrainer from traja.models.generative_models.vae import MultiModelVAE from traja.models.predictive_models.ae import MultiModelAE +from traja.models.predictive_models.lstm import LSTM # Sample data @@ -55,6 +56,52 @@ def test_aevae(): trainer.fit(train_loader, test_loader, model_save_path, epochs=10, training_mode='classification') +def test_ae(): + """ + Test Autoencoder and variational auto encoder models for training/testing/generative network and + classification networks + + """ + # Hyperparameters + batch_size = 10 + num_past = 10 + num_future = 5 + # Prepare the dataloader + data_loaders, scalers = dataset.MultiModalDataLoader(df, + batch_size=batch_size, + n_past=num_past, + n_future=num_future, + num_workers=1) + + model_save_path = './model.pt' + + model = MultiModelAE(input_size=2, + output_size=2, + lstm_hidden_size=32, + num_lstm_layers=2, + num_classes=9, + latent_size=10, + dropout=0.1, + num_classifier_layers=4, + classifier_hidden_size=32, + batch_size=batch_size, + num_future=num_future, + num_past=num_past, + bidirectional=False, + batch_first=True, + reset_state=True) + + # Model Trainer + # Model types; "ae" or "vae" + trainer = HybridTrainer(model=model, + optimizer_type='Adam', + loss_type='huber') + + # Train the model + trainer.fit(data_loaders, model_save_path, epochs=10, training_mode='forecasting') + trainer.fit(data_loaders, model_save_path, epochs=10, training_mode='classification') + + def test_lstm(): """ Testing method for lstm model used for forecasting. @@ -68,29 +115,29 @@ def test_lstm(): assert num_past == num_future # Prepare the dataloader - train_loader, test_loader = dataset.MultiModalDataLoader(df, - batch_size=batch_size, - n_past=num_past, - n_future=num_future, - num_workers=1) + data_loaders, scalers = dataset.MultiModalDataLoader(df, + batch_size=batch_size, + n_past=num_past, + n_future=num_future, + num_workers=1) model_save_path = './model.pt' - # Model Trainer - trainer = LSTMTrainer(model_type='lstm', - optimizer_type='Adam', - device='cuda', - epochs=10, - input_size=2, + + # Model init + model = LSTM(input_size=2, + hidden_size=32, + num_layers=2, + output_size=2, + dropout=0.1, batch_size=batch_size, - hidden_size=32, num_future=num_future, - num_layers=2, - output_size=2, - - scheduler_patience=3, + bidirectional=False, batch_first=True, - dropout=0.1, - reset_state=True, - bidirectional=False) + reset_state=True) + + # Model Trainer + trainer = HybridTrainer(model=model, + optimizer_type='Adam', + loss_type='huber') # Train the model - trainer.train(train_loader, test_loader, model_save_path) \ No newline at end of file + trainer.fit(data_loaders, model_save_path, epochs=10, training_mode='forecasting') From 9fc90a28471534f6e915a3b9dfdd7f956e3fb5fd Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 12:45:53 +0000 Subject: [PATCH 184/211] Move optimizer tests to dedicated file --- traja/models/optimizers.py | 29 ----------------------------- traja/tests/test_optimizers.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 29 deletions(-) create mode 100644 traja/tests/test_optimizers.py diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 53f995c4..f00edc04 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -116,32 +116,3 @@ def get_lrschedulers(self, factor: float, patience: int): ) return self.forecasting_schedulers, self.classification_schedulers - - -if __name__ == "__main__": - # Test - model_type = "custom" - model = MultiModelAE( - input_size=2, - num_past=10, - batch_size=5, - num_future=5, - lstm_hidden_size=32, - num_lstm_layers=2, - classifier_hidden_size=32, - num_classifier_layers=4, - output_size=2, - num_classes=10, - latent_size=10, - batch_first=True, - dropout=0.2, - reset_state=True, - bidirectional=True, - ) - - # Get the optimizers - opt = Optimizer(model_type, model, optimizer_type="RMSprop") - model_optimizers = opt.get_optimizers(lr=0.1) - model_schedulers = opt.get_lrschedulers(factor=0.1, patience=10) - - print(model_optimizers, model_schedulers) diff --git a/traja/tests/test_optimizers.py b/traja/tests/test_optimizers.py new file mode 100644 index 00000000..a4703b8b --- /dev/null +++ b/traja/tests/test_optimizers.py @@ -0,0 +1,31 @@ +from traja.models.optimizers import Optimizer +from traja.models.predictive_models.ae import MultiModelAE + + +def test_get_optimizers(): + # Test + model_type = "custom" + model = MultiModelAE( + input_size=2, + num_past=10, + batch_size=5, + num_future=5, + lstm_hidden_size=32, + num_lstm_layers=2, + classifier_hidden_size=32, + num_classifier_layers=4, + output_size=2, + num_classes=10, + latent_size=10, + batch_first=True, + dropout=0.2, + reset_state=True, + bidirectional=True, + ) + + # Get the optimizers + opt = Optimizer(model_type, model, optimizer_type="RMSprop") + model_optimizers = opt.get_optimizers(lr=0.1) + model_schedulers = opt.get_lrschedulers(factor=0.1, patience=10) + + print(model_optimizers, model_schedulers) \ No newline at end of file From 7e7895f06bc9f8f54103e4362377ad084f2cf60d Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 12:46:52 +0000 Subject: [PATCH 185/211] Remove redundant test --- traja/tests/test_nn.py | 50 ------------------------------------------ 1 file changed, 50 deletions(-) delete mode 100644 traja/tests/test_nn.py diff --git a/traja/tests/test_nn.py b/traja/tests/test_nn.py deleted file mode 100644 index a846edfb..00000000 --- a/traja/tests/test_nn.py +++ /dev/null @@ -1,50 +0,0 @@ -import pandas as pd -from traja.datasets import dataset -from traja.models.train import LSTMTrainer, HybridTrainer, CustomTrainer - - - -def test_from_df(): - data_url = "https://raw.githubusercontent.com/traja-team/traja-research/dataset_und_notebooks/dataset_analysis/jaguar5.csv" - df = pd.read_csv(data_url, error_bad_lines=False) - model_save_path = 'model' - - # Hyperparameters - batch_size = 10 - num_past = 10 - num_future = 5 - - # Prepare the dataloader - train_loader, test_loader = dataset.MultiModalDataLoader(df, - batch_size=batch_size, - n_past=num_past, - n_future=num_future, - num_workers=2) - - trainer = HybridTrainer(model_type='vae', # "ae" or "vae" - optimizer_type='Adam', # ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop','LBFGS', 'ASGD', 'Adamax'] - input_size=2, - output_size=2, - lstm_hidden_size=32, - num_lstm_layers=2, - reset_state=True, - latent_size=10, - dropout=0.1, - num_classes=9, # Uncomment to create and train classifier network - num_classifier_layers=4, - classifier_hidden_size= 32, - batch_size=batch_size, - num_future=num_future, - num_past=num_past, - bidirectional=False, - batch_first=True, - loss_type='huber') # 'rmse' or 'huber' - - - # Train the model - trainer.fit(train_loader, test_loader, model_save_path, epochs=10, training_mode='forecasting') - trainer.fit(train_loader, test_loader, model_save_path, epochs=10, training_mode='classification') - - -if __name__ == '__main__': - test_from_df() From 5b02d1e86d6e25c07b308d80ed85016e5703d010 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 12:47:53 +0000 Subject: [PATCH 186/211] Remove redundant LSTM trainer --- traja/models/train.py | 146 ------------------------------------- traja/tests/test_models.py | 2 +- 2 files changed, 1 insertion(+), 147 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index 21e82725..61c12029 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -270,152 +270,6 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo utils.save(self.model, self.model_hyperparameters, PATH=model_save_path) -class LSTMTrainer: - """ - Wrapper for training and testing the LSTM model - Parameters: - ----------- - model_type: {'lstm'} - Type of model should be "LSTM" - optimizer_type: {'Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', 'LBFGS', 'ASGD', 'Adamax'} - Type of optimizer to use for training. - device: {'cuda', 'cpu'} - Target device to use for training the model - epochs: int, default=100 - Number of epochs to train the network - input_size: - The number of expected features in the input x - batch_size: - Number of samples in a batch - hidden_size: - The number of features in the hidden state h - num_future: - Number of time steps to be predicted forward - num_layers: - Number of layers in the LSTM model - output_size: - Output feature dimension - lr: - Optimizer learning rate - lr_factor: - Factor by which the learning rate will be reduced - scheduler_patience: - Number of epochs with no improvement after which learning rate will be reduced. - For example, if patience = 2, then we will ignore the first 2 epochs with no - improvement, and will only decrease the LR after the 3rd epoch if the loss still - hasn’t improved then. - batch_first: - If True, then the input and output tensors are provided as (batch, seq, feature) - dropout: If non-zero, introduces a Dropout layer on the outputs of each LSTM layer except the last layer, - with dropout probability equal to dropout - reset_state: - If True, will reset the hidden and cell state for each batch of data - bidirectional: - If True, becomes a bidirectional LSTM - """ - - def __init__( - self, - model_type: str, - optimizer_type: str, - epochs: int, - input_size: int, - batch_size: int, - hidden_size: int, - num_future: int, - num_layers: int, - output_size: int, - lr: float, - lr_factor: float, - scheduler_patience: int, - batch_first: True, - dropout: float, - reset_state: bool, - bidirectional: bool, - ): - self.model_type = model_type - self.optimizer_type = optimizer_type - self.epochs = epochs - self.input_size = input_size - self.batch_size = batch_size - self.hidden_size = hidden_size - self.num_future = num_future - self.num_layers = num_layers - self.output_size = output_size - self.batch_first = batch_first - self.lr = lr - self.lr_factor = lr_factor - self.scheduler_patience = scheduler_patience - self.dropout = dropout - self.reset_state = reset_state - self.bidirectional = bidirectional - - self.model_hyperparameters = {'input_size': self.input_size, - 'batch_size': self.batch_size, - 'hidden_size': self.hidden_size, - 'num_future': self.num_future, - 'num_layers': self.num_layers, - 'output_size': self.output_size, - 'batch_first': self.batch_first, - 'reset_state': self.reset_state, - 'bidirectional': self.bidirectional, - 'dropout': self.dropout - } - - self.model = LSTM(**self.model_hyperparameters) - optimizer = Optimizer(self.model_type, self.model, self.optimizer_type) - self.optimizer = optimizer.get_optimizers(lr=0.001) - self.scheduler = optimizer.get_lrschedulers(factor=self.lr_factor, patience=self.scheduler_patience) - - def fit(self, dataloaders, model_save_path): - - """ Implements the batch wise training and testing for time series forecasting. - Args: - dataloaders: Dictionary containing train and test dataloaders - train_loader: Dataloader object of train dataset with batch data [data,target,category] - test_loader: Dataloader object of test dataset with [data,target,category] - model_save_path: Directory path to save the model - Return: None""" - - assert self.model_type == 'lstm' - self.model.to(device) - train_loader, test_loader = dataloaders.values() - - for epoch in range(self.epochs): - if epoch > 0: - self.model.train() - total_loss = 0 - for idx, (data, target, _) in enumerate(train_loader): - self.optimizer.zero_grad() - data, target = data.float().to(device), target.float().to(device) - output = self.model(data) - loss = Criterion().lstm_criterion(output, target) - loss.backward() - self.optimizer.step() - total_loss += loss - - print('Epoch {} | loss {}'.format(epoch, total_loss / (idx + 1))) - - # Testing - if epoch % 10 == 0: - with torch.no_grad(): - self.model.eval() - test_loss_forecasting = 0 - for idx, (data, target, _) in enumerate(list(test_loader)): - data, target = data.float().to(device), target.float().to(device) - out = self.model(data) - test_loss_forecasting += Criterion().lstm_criterion(out, target).item() - - test_loss_forecasting /= len(test_loader.dataset) - print(f'====> Test set generator loss: {test_loss_forecasting:.4f}') - - # Scheduler metric is test set loss - self.scheduler.step(test_loss_forecasting) - - # Save the model at target path - utils.save(self.model, self.model_hyperparameters, PATH=model_save_path) - - class CustomTrainer: """ Wrapper for training and testing user defined models diff --git a/traja/tests/test_models.py b/traja/tests/test_models.py index 5ab1d58b..db5252c6 100644 --- a/traja/tests/test_models.py +++ b/traja/tests/test_models.py @@ -1,6 +1,6 @@ import pandas as pd from traja.datasets import dataset -from traja.models.train import LSTMTrainer, HybridTrainer +from traja.models.train import HybridTrainer from traja.models.generative_models.vae import MultiModelVAE from traja.models.predictive_models.ae import MultiModelAE from traja.models.predictive_models.lstm import LSTM From 1c99976e02580974d82ba24abbb26e32e4aa0322 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 12:53:38 +0000 Subject: [PATCH 187/211] Fix ae classification bug --- traja/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index 61c12029..c412f609 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -264,7 +264,7 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo else: if self.classify: for scheduler in self.classification_schedulers.values(): - scheduler.step() + scheduler.step(test_loss_classification) # Save the model at target path utils.save(self.model, self.model_hyperparameters, PATH=model_save_path) From 35b52e6e0fa73b1d9e769ba711dd81f298ea6982 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 12:55:01 +0000 Subject: [PATCH 188/211] Remove dead code --- traja/models/train.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index c412f609..9d9258d7 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -141,11 +141,7 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo self.model.to(device) - - #forecasting_optimizers, classification_optimizers = self.model_optimizers.values() - - #forecasting_schedulers, classification_schedulers = self.model_lrschedulers.values() - + train_loader, test_loader = dataloaders.values() # Training for epoch in range(epochs): test_loss_forecasting = 0 From af693f285812d54a21dc633c03259967f7f90ae2 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 13:12:57 +0000 Subject: [PATCH 189/211] Temporarily remove failing tests --- docs/requirements.txt | 3 ++- .../{test_rutils.py => broken_test_rutils.py} | 1 - traja/tests/test_accessor.py | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) rename traja/tests/{test_rutils.py => broken_test_rutils.py} (97%) diff --git a/docs/requirements.txt b/docs/requirements.txt index 1166c8a5..c4d19f01 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -11,4 +11,5 @@ networkx seaborn torch pytest -h5py \ No newline at end of file +h5py +rpy2 \ No newline at end of file diff --git a/traja/tests/test_rutils.py b/traja/tests/broken_test_rutils.py similarity index 97% rename from traja/tests/test_rutils.py rename to traja/tests/broken_test_rutils.py index 55918a0b..a0ef70bd 100644 --- a/traja/tests/test_rutils.py +++ b/traja/tests/broken_test_rutils.py @@ -2,7 +2,6 @@ import numpy as np import numpy.testing as npt -from rpy2 import rinterface from rpy2.rinterface import RRuntimeWarning import traja diff --git a/traja/tests/test_accessor.py b/traja/tests/test_accessor.py index c81c424e..ac97b44b 100644 --- a/traja/tests/test_accessor.py +++ b/traja/tests/test_accessor.py @@ -30,17 +30,17 @@ def test_xy(): assert xy.shape == (20, 2) -def test_calc_derivatives(): - df.traja.calc_derivatives() +#def test_calc_derivatives(): +# df.traja.calc_derivatives() -def test_get_derivatives(): - df.traja.get_derivatives() +#def test_get_derivatives(): +# df.traja.get_derivatives() -def test_speed_intervals(): - si = df.traja.speed_intervals(faster_than=100) - assert isinstance(si, traja.TrajaDataFrame) +#def test_speed_intervals(): +# si = df.traja.speed_intervals(faster_than=100) +# assert isinstance(si, traja.TrajaDataFrame) def test_to_shapely(): From 74484079a33e0adb047468dd318d5162645ea7fa Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 15:04:40 +0000 Subject: [PATCH 190/211] Add regressor training to HybridTrainer --- traja/datasets/dataset.py | 10 ++-- traja/models/generative_models/vae.py | 75 ++++++++++++++++++++++---- traja/models/inference.py | 3 +- traja/models/losses.py | 11 ++++ traja/models/optimizers.py | 27 +++++++--- traja/models/predictive_models/ae.py | 54 +++++++++++++++++-- traja/models/predictive_models/lstm.py | 3 +- traja/models/train.py | 57 +++++++++++++++----- traja/tests/test_models.py | 19 ++----- traja/tests/test_optimizers.py | 21 ++------ 10 files changed, 211 insertions(+), 69 deletions(-) diff --git a/traja/datasets/dataset.py b/traja/datasets/dataset.py index aa8019bd..bb65a42b 100644 --- a/traja/datasets/dataset.py +++ b/traja/datasets/dataset.py @@ -231,22 +231,26 @@ class TimeSeriesDataset(Dataset): Dataset (torch.utils.data.Dataset): Pyptorch dataset object """ - def __init__(self, data, target, category): + def __init__(self, data, target, category=None, parameters=None): r""" Args: data (array): Data target (array): Target category (array): Category + parameters (array): Parameters """ + self.data = data self.target = target self.category = category + self.parameters = parameters def __getitem__(self, index): x = self.data[index] y = self.target[index] - z = self.category[index] - return x, y, z + z = self.category[index] if self.category else torch.zeros(1) + w = self.parameters[index] if self.parameters else torch.zeros(1) + return x, y, z, w def __len__(self): return len(self.data) diff --git a/traja/models/generative_models/vae.py b/traja/models/generative_models/vae.py index 9c45dd59..76ec36f2 100644 --- a/traja/models/generative_models/vae.py +++ b/traja/models/generative_models/vae.py @@ -291,6 +291,9 @@ def __init__( num_classifier_layers: int = None, classifier_hidden_size: int = None, num_classes: int = None, + num_regressor_layers: int = None, + regressor_hidden_size: int = None, + num_regressor_parameters: int = None, ): super(MultiModelVAE, self).__init__() @@ -309,6 +312,9 @@ def __init__( self.dropout = dropout self.reset_state = reset_state self.bidirectional = bidirectional + self.num_regressor_layers = num_regressor_layers + self.regressor_hidden_size = regressor_hidden_size + self.num_regressor_parameters = num_regressor_parameters # Let the trainer know what kind of model this is self.model_type = 'vae' @@ -354,7 +360,17 @@ def __init__( dropout=self.dropout, ) - def forward(self, data, training=True, classify=False, latent=True): + if self.num_regressor_parameters is not None: + self.regressor = MLPClassifier( + input_size=self.latent_size, + hidden_size=self.regressor_hidden_size, + num_classes=self.num_regressor_parameters, + latent_size=self.latent_size, + num_classifier_layers=self.num_regressor_layers, + dropout=self.dropout, + ) + + def forward(self, data, training=True, classify=False, regress=False, latent=True): """ Parameters: ----------- @@ -365,10 +381,18 @@ def forward(self, data, training=True, classify=False, latent=True): ------- decoder_out,latent_out or classifier out """ - if not classify: - # Set the classifier grad off - for param in self.classifier.parameters(): - param.requires_grad = False + + assert not (classify and regress), 'Model cannot both classify and regress!' + + if not classify or regress: + # Set the classifier and regressor grads off + if self.num_classes is not None: + for param in self.classifier.parameters(): + param.requires_grad = False + if self.num_regressor_parameters is not None: + for param in self.regressor.parameters(): + param.requires_grad = False + for param in self.encoder.parameters(): param.requires_grad = True for param in self.decoder.parameters(): @@ -387,11 +411,15 @@ def forward(self, data, training=True, classify=False, latent=True): else: return decoder_out - else: # training_mode = 'classification' - # Unfreeze classifier parameters and freeze all other - # network parameters + elif classify: + # Unfreeze classifier and freeze the rest + assert self.num_classes is not None, "Classifier not found" + for param in self.classifier.parameters(): param.requires_grad = True + if self.num_regressor_parameters is not None: + for param in self.regressor.parameters(): + param.requires_grad = False for param in self.encoder.parameters(): param.requires_grad = False for param in self.decoder.parameters(): @@ -405,4 +433,33 @@ def forward(self, data, training=True, classify=False, latent=True): latent_out, mu, logvar = self.latent(enc_out, training=training) # Classifier classifier_out = self.classifier(mu) # Deterministic - return classifier_out, latent_out, mu, logvar + if latent: + return classifier_out, latent_out, mu, logvar + else: + return classifier_out + + elif regress: + # Unfreeze classifier and freeze the rest + assert self.num_regressor_parameters is not None, "Regressor not found" + + if self.num_classes is not None: + for param in self.classifier.parameters(): + param.requires_grad = False + for param in self.regressor.parameters(): + param.requires_grad = True + for param in self.encoder.parameters(): + param.requires_grad = False + for param in self.decoder.parameters(): + param.requires_grad = False + for param in self.latent.parameters(): + param.requires_grad = False + + # Encoder -->Latent --> Regressor + enc_out = self.encoder(data) + latent_out, mu, logvar = self.latent(enc_out, training=training) + + regressor_out = self.regressor(mu) # Deterministic + if latent: + return regressor_out, latent_out, mu, logvar + else: + return regressor_out \ No newline at end of file diff --git a/traja/models/inference.py b/traja/models/inference.py index 5e630488..20127c5b 100644 --- a/traja/models/inference.py +++ b/traja/models/inference.py @@ -172,7 +172,8 @@ def __init__( self.model_hyperparameters = model_hyperparameters if self.model_type == "ae": - self.model = MultiModelAE(**self.model_hyperparameters) + self.model = MultiModelAE(num_regressor_layers=2, regressor_hidden_size=32, num_regressor_parameters=3, + **self.model_hyperparameters) if self.model_type == "lstm": self.model = LSTM(**self.model_hyperparameters) diff --git a/traja/models/losses.py b/traja/models/losses.py index 93bcc1b0..0d05fd2d 100644 --- a/traja/models/losses.py +++ b/traja/models/losses.py @@ -72,6 +72,17 @@ def classifier_criterion(self, predicted, target): loss = self.crossentropy_loss(predicted, target) return loss + def regressor_criterion(self, predicted, target): + """ + Regressor loss function + :param predicted: Predicted parameter value + :param target: Target parameter value + :return: MSE loss + """ + + loss = self.mse_loss(predicted, target) + return loss + def lstm_criterion(self, predicted, target): loss = self.huber_loss(predicted, target) diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index f00edc04..0e12c77c 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -40,9 +40,11 @@ def __init__(self, model_type, model, optimizer_type, classify=False): self.optimizers = {} self.forecasting_schedulers = {} self.classification_schedulers = {} + self.regression_schedulers = {} self.forecasting_keys = ['encoder', 'decoder', 'latent'] self.classification_keys = ['classifier'] + self.regression_keys = ['regressor'] def get_optimizers(self, lr=0.0001): """Optimizers for each network in the model @@ -61,13 +63,13 @@ def get_optimizers(self, lr=0.0001): ) elif self.model_type in ["ae", "vae"]: - keys = ["encoder", "decoder", "latent", "classifier"] + keys = ['encoder', 'decoder', 'latent', 'classifier', 'regressor'] for key in keys: - network = getattr( - torch.optim, f"{self.optimizer_type}" - )(getattr(self.model, f"{key}").parameters(), lr=lr) + network = getattr(self.model, f"{key}", None) if network is not None: - self.optimizers[key] = network + self.optimizers[key] = getattr( + torch.optim, f"{self.optimizer_type}" + )(network.parameters(), lr=lr) elif self.model_type == "vaegan": return NotImplementedError @@ -77,7 +79,8 @@ def get_optimizers(self, lr=0.0001): forecasting_optimizers = [self.optimizers[key] for key in self.forecasting_keys if key in self.optimizers] classification_optimizers = [self.optimizers[key] for key in self.classification_keys if key in self.optimizers] - return forecasting_optimizers, classification_optimizers + regression_optimizers = [self.optimizers[key] for key in self.regression_keys if key in self.optimizers] + return forecasting_optimizers, classification_optimizers, regression_optimizers def get_lrschedulers(self, factor: float, patience: int): @@ -97,6 +100,7 @@ def get_lrschedulers(self, factor: float, patience: int): forecasting_keys = [key for key in self.forecasting_keys if key in self.optimizers] classification_keys = [key for key in self.classification_keys if key in self.optimizers] + regression_keys = [key for key in self.regression_keys if key in self.optimizers] for network in forecasting_keys: self.forecasting_schedulers[network] = ReduceLROnPlateau( @@ -115,4 +119,13 @@ def get_lrschedulers(self, factor: float, patience: int): verbose=True, ) - return self.forecasting_schedulers, self.classification_schedulers + for network in regression_keys: + self.regression_schedulers[network] = ReduceLROnPlateau( + self.optimizers[network], + mode="max", + factor=factor, + patience=patience, + verbose=True, + ) + + return self.forecasting_schedulers, self.classification_schedulers, self.regression_schedulers diff --git a/traja/models/predictive_models/ae.py b/traja/models/predictive_models/ae.py index 218d4f3a..04af63a3 100644 --- a/traja/models/predictive_models/ae.py +++ b/traja/models/predictive_models/ae.py @@ -267,6 +267,9 @@ def __init__( num_classifier_layers: int = None, classifier_hidden_size: int = None, num_classes: int = None, + num_regressor_layers: int = None, + regressor_hidden_size: int = None, + num_regressor_parameters: int = None, ): super(MultiModelAE, self).__init__() @@ -285,6 +288,9 @@ def __init__( self.dropout = dropout self.reset_state = reset_state self.bidirectional = bidirectional + self.num_regressor_layers = num_regressor_layers + self.regressor_hidden_size = regressor_hidden_size + self.num_regressor_parameters = num_regressor_parameters # Let the trainer know what kind of model this is self.model_type = 'ae' @@ -330,6 +336,16 @@ def __init__( dropout=self.dropout, ) + if self.num_regressor_parameters is not None: + self.regressor = MLPClassifier( + input_size=self.latent_size, + hidden_size=self.regressor_hidden_size, + num_classes=self.num_regressor_parameters, + latent_size=self.latent_size, + num_classifier_layers=self.num_regressor_layers, + dropout=self.dropout, + ) + def get_ae_parameters(self): """ Return: @@ -351,7 +367,7 @@ def get_classifier_parameters(self): assert self.classifier_hidden_size is not None, "Classifier not found" return [self.classifier.parameters()] - def forward(self, data, classify=False, training=True, latent=True): + def forward(self, data, classify=False, regress=False, training=True, latent=True): """ Parameters: ----------- @@ -362,12 +378,16 @@ def forward(self, data, classify=False, training=True, latent=True): ------- decoder_out,latent_out or classifier out """ + assert not (classify and regress), 'Model cannot both classify and regress!' - if not classify: - # Set the classifier grad off + if not classify or regress: + # Set the classifier and regressor grads off if self.num_classes is not None: for param in self.classifier.parameters(): param.requires_grad = False + if self.num_regressor_parameters is not None: + for param in self.regressor.parameters(): + param.requires_grad = False for param in self.encoder.parameters(): param.requires_grad = True @@ -385,12 +405,15 @@ def forward(self, data, classify=False, training=True, latent=True): else: return decoder_out - else: # Classify + elif classify: # Classify # Unfreeze classifier and freeze the rest assert self.num_classifier_layers is not None, "Classifier not found" for param in self.classifier.parameters(): param.requires_grad = True + if self.num_regressor_parameters is not None: + for param in self.regressor.parameters(): + param.requires_grad = False for param in self.encoder.parameters(): param.requires_grad = False for param in self.decoder.parameters(): @@ -404,3 +427,26 @@ def forward(self, data, classify=False, training=True, latent=True): classifier_out = self.classifier(latent_out) # Deterministic return classifier_out + + elif regress: + # Unfreeze regressor and freeze the rest + assert self.num_regressor_layers is not None, "Regressor not found" + + if self.num_classes is not None: + for param in self.classifier.parameters(): + param.requires_grad = False + for param in self.regressor.parameters(): + param.requires_grad = True + for param in self.encoder.parameters(): + param.requires_grad = False + for param in self.decoder.parameters(): + param.requires_grad = False + for param in self.latent.parameters(): + param.requires_grad = False + + # Encoder-->Latent-->Regressor + enc_out = self.encoder(data) + latent_out = self.latent(enc_out) + + regressor_out = self.regressor(latent_out) # Deterministic + return regressor_out \ No newline at end of file diff --git a/traja/models/predictive_models/lstm.py b/traja/models/predictive_models/lstm.py index e149b9aa..004fbdc9 100644 --- a/traja/models/predictive_models/lstm.py +++ b/traja/models/predictive_models/lstm.py @@ -76,8 +76,9 @@ def _init_hidden(self): .to(device), ) - def forward(self, x, training=True, classify=False, latent=False): + def forward(self, x, training=True, classify=False, regress=False, latent=False): assert not classify, 'LSTM forecaster cannot classify!' + assert not regress, 'LSTM forecaster cannot regress!' assert not latent, 'LSTM forecaster does not have a latent space!' # To feed the latent states into lstm decoder, repeat the tensor n_future times at second dim (h0, c0) = self._init_hidden() diff --git a/traja/models/train.py b/traja/models/train.py index 9d9258d7..261c3b42 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -105,16 +105,17 @@ def __init__( self.model = model - # Classification task check + # Classification, regression task checks self.classify = True if model.model_type != 'lstm' and model.classifier_hidden_size is not None else False + self.regress = True if model.model_type != 'lstm' and model.regressor_hidden_size is not None else False # Model optimizer and the learning rate scheduler optimizer = Optimizer( self.model_type, self.model, self.optimizer_type, classify=self.classify ) - self.forecasting_optimizers, self.classification_optimizers = optimizer.get_optimizers(lr=self.lr) - self.forecasting_schedulers, self.classification_schedulers = optimizer.get_lrschedulers( + self.forecasting_optimizers, self.classification_optimizers, self.regression_optimizers = optimizer.get_optimizers(lr=self.lr) + self.forecasting_schedulers, self.classification_schedulers, self.regression_schedulers = optimizer.get_lrschedulers( factor=self.lr_factor, patience=self.scheduler_patience ) @@ -137,7 +138,7 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo """ assert model_save_path is not None, f"Model path {model_save_path} unknown" - assert training_mode in ['forecasting', 'classification'], f'Training mode {classification} unknown' + assert training_mode in ['forecasting', 'classification', 'regression'], f'Training mode {training_mode} unknown' self.model.to(device) @@ -146,22 +147,27 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo for epoch in range(epochs): test_loss_forecasting = 0 test_loss_classification = 0 + test_loss_regression = 0 if epoch > 0: # Initial step is to test and set LR schduler # Training self.model.train() total_loss = 0 - for idx, (data, target, category) in enumerate(train_loader): + for idx, (data, target, category, parameters) in enumerate(train_loader): # Reset optimizer states for optimizer in self.forecasting_optimizers: optimizer.zero_grad() if self.classify: for optimizer in self.classification_optimizers: optimizer.zero_grad() + if self.regress: + for optimizer in self.regression_optimizers: + optimizer.zero_grad() - data, target, category = ( + data, target, category, parameters = ( data.float().to(device), target.float().to(device), category.to(device), + parameters.to(device) ) if training_mode == "forecasting": @@ -195,6 +201,17 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo loss.backward() for optimizer in self.classification_optimizers: optimizer.step() + + elif self.regress and training_mode == 'regression': + regressor_out = self.model(data, training=True, regress=True, latent=False) + loss = Criterion().regressor_criterion( + regressor_out, parameters + ) + + loss.backward() + for optimizer in self.regression_optimizers: + optimizer.step() + total_loss += loss print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss / (idx + 1))) @@ -206,8 +223,13 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo total = 0.0 correct = 0.0 self.model.eval() - for idx, (data, target, category) in enumerate(list(test_loader)): - data, target, category = data.float().to(device), target.float().to(device), category.to(device) + for idx, (data, target, category, parameters) in enumerate(list(test_loader)): + data, target, category, parameters = ( + data.float().to(device), + target.float().to(device), + category.to(device), + parameters.to(device) + ) # Time series forecasting test if self.model_type == 'ae' or self.model_type == 'lstm': out = self.model( @@ -241,6 +263,12 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo _, predicted = torch.max(classifier_out.data, 1) correct += (predicted == (category - 1)).sum().item() + if self.regress: + regressor_out = self.model(data, training=True, regress=True, latent=False) + test_loss_regression += Criterion().regressor_criterion( + regressor_out, parameters + ) + test_loss_forecasting /= len(test_loader.dataset) print( f"====> Mean test set generator loss: {test_loss_forecasting:.4f}" @@ -253,14 +281,19 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo f"====> Mean test set classifier loss: {test_loss_classification:.4f}; accuracy: {accuracy:.2f}" ) + if self.regress: + print(f'====> Mean test set regressor loss: {test_loss_regression:.4f}') + # Scheduler metric is test set loss if training_mode == "forecasting": for scheduler in self.forecasting_schedulers.values(): scheduler.step(test_loss_forecasting) - else: - if self.classify: - for scheduler in self.classification_schedulers.values(): - scheduler.step(test_loss_classification) + elif training_mode == 'classification': + for scheduler in self.classification_schedulers.values(): + scheduler.step(test_loss_classification) + elif training_mode == 'regression': + for scheduler in self.regression_schedulers.values(): + scheduler.step(test_loss_regression) # Save the model at target path utils.save(self.model, self.model_hyperparameters, PATH=model_save_path) diff --git a/traja/tests/test_models.py b/traja/tests/test_models.py index db5252c6..ae2e23e1 100644 --- a/traja/tests/test_models.py +++ b/traja/tests/test_models.py @@ -75,21 +75,10 @@ def test_ae(): model_save_path = './model.pt' - model = MultiModelAE(input_size=2, - output_size=2, - lstm_hidden_size=32, - num_lstm_layers=2, - num_classes=9, - latent_size=10, - dropout=0.1, - num_classifier_layers=4, - classifier_hidden_size=32, - batch_size=batch_size, - num_future=num_future, - num_past=num_past, - bidirectional=False, - batch_first=True, - reset_state=True) + model = MultiModelAE(input_size=2, num_past=num_past, batch_size=batch_size, num_future=num_future, + lstm_hidden_size=32, num_lstm_layers=2, output_size=2, latent_size=10, batch_first=True, + dropout=0.1, reset_state=True, bidirectional=False, num_classifier_layers=4, + classifier_hidden_size=32, num_classes=9) # Model Trainer # Model types; "ae" or "vae" diff --git a/traja/tests/test_optimizers.py b/traja/tests/test_optimizers.py index a4703b8b..e343bc20 100644 --- a/traja/tests/test_optimizers.py +++ b/traja/tests/test_optimizers.py @@ -5,23 +5,10 @@ def test_get_optimizers(): # Test model_type = "custom" - model = MultiModelAE( - input_size=2, - num_past=10, - batch_size=5, - num_future=5, - lstm_hidden_size=32, - num_lstm_layers=2, - classifier_hidden_size=32, - num_classifier_layers=4, - output_size=2, - num_classes=10, - latent_size=10, - batch_first=True, - dropout=0.2, - reset_state=True, - bidirectional=True, - ) + model = MultiModelAE(input_size=2, num_past=10, batch_size=5, num_future=5, lstm_hidden_size=32, num_lstm_layers=2, + output_size=2, latent_size=10, batch_first=True, dropout=0.2, reset_state=True, + bidirectional=True, num_classifier_layers=4, classifier_hidden_size=32, num_classes=10, + num_regressor_layers=2, regressor_hidden_size=32, num_regressor_parameters=3) # Get the optimizers opt = Optimizer(model_type, model, optimizer_type="RMSprop") From ac672db94d1e6b3c27b0a0c2c3b8de6f81e252a0 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 16:28:45 +0000 Subject: [PATCH 191/211] Make test_loader not generate a list --- traja/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index 261c3b42..8150e1b2 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -223,7 +223,7 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo total = 0.0 correct = 0.0 self.model.eval() - for idx, (data, target, category, parameters) in enumerate(list(test_loader)): + for idx, (data, target, category, parameters) in enumerate(test_loader): data, target, category, parameters = ( data.float().to(device), target.float().to(device), From 44e29e730f0d15601a17ea7e65839242ca8a4b6c Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 17:02:26 +0000 Subject: [PATCH 192/211] Assert that train data ratio is positive --- traja/datasets/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/traja/datasets/utils.py b/traja/datasets/utils.py index b95ce362..3d69eabf 100644 --- a/traja/datasets/utils.py +++ b/traja/datasets/utils.py @@ -77,7 +77,8 @@ def shuffle_split(train_data:np.array, target_data:np.array,target_category:np.a # Shuffle the IDs and the corresponding sequence , preserving the order train_data, target_data, target_category = shuffle(train_data, target_data, target_category) - assert train_ratio<=1.0,"Train data ratio should be less than or equal to 1 " + assert train_ratio > 0, "Train data ratio should be greater than zero" + assert train_ratio <= 1.0, "Train data ratio should be less than or equal to 1 " # Train test split split = int(train_ratio*len(train_data)) From 47b3f15ba366d969eb766038eecf23f8886ee2e5 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 17:04:08 +0000 Subject: [PATCH 193/211] Format utils file --- traja/datasets/utils.py | 72 ++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/traja/datasets/utils.py b/traja/datasets/utils.py index 3d69eabf..0beba662 100644 --- a/traja/datasets/utils.py +++ b/traja/datasets/utils.py @@ -14,7 +14,8 @@ logger = logging.getLogger(__name__) -def get_class_distribution(targets): + +def get_class_distribution(targets): """Compute class distribution, returns number of classes and their count in the targets Args: @@ -24,8 +25,9 @@ def get_class_distribution(targets): [type]: [description] """ targets_ = np.unique(targets, return_counts=True) - return targets_[0],targets_[1] - + return targets_[0], targets_[1] + + def generate_dataset(df, n_past, n_future): """ df : Dataframe @@ -34,16 +36,16 @@ def generate_dataset(df, n_past, n_future): Returns: X: Past steps Y: Future steps (Sequence target) - Z: Sequence category""" - + Z: Sequence category""" + # Split the dataframe with respect to IDs - series_ids = dict(tuple(df.groupby('ID'))) # Dict of ids as keys and x,y,id as values + series_ids = dict(tuple(df.groupby('ID'))) # Dict of ids as keys and x,y,id as values train_data, target_data, target_category = list(), list(), list() - + for id in series_ids.keys(): - X, Y, Z= list(), list(), list() + X, Y, Z = list(), list(), list() # Drop the column ids and convert the pandas into arrays - series = series_ids[id].drop(columns = ['ID']).to_numpy() + series = series_ids[id].drop(columns=['ID']).to_numpy() for window_start in range(len(series)): past_end = window_start + n_past future_end = past_end + n_future @@ -54,14 +56,15 @@ def generate_dataset(df, n_past, n_future): Y.append(future) # For each sequence length set target category Z.append(int(id)) - + train_data.extend(np.array(X)) target_data.extend(np.array(Y)) target_category.extend(np.array(Z)) return train_data, target_data, target_category -def shuffle_split(train_data:np.array, target_data:np.array,target_category:np.array, train_ratio:float): + +def shuffle_split(train_data: np.array, target_data: np.array, target_category: np.array, train_ratio: float): """[summary] Args: @@ -73,25 +76,26 @@ def shuffle_split(train_data:np.array, target_data:np.array,target_category:np.a Returns: [type]: [description] """ - + # Shuffle the IDs and the corresponding sequence , preserving the order train_data, target_data, target_category = shuffle(train_data, target_data, target_category) assert train_ratio > 0, "Train data ratio should be greater than zero" assert train_ratio <= 1.0, "Train data ratio should be less than or equal to 1 " - + # Train test split - split = int(train_ratio*len(train_data)) + split = int(train_ratio * len(train_data)) train_x = train_data[:split] train_y = target_data[:split] train_z = target_category[:split] test_x = train_data[split:] - test_y = target_data[split :] + test_y = target_data[split:] test_z = target_category[split:] - - return [train_x,train_y,train_z],[test_x,test_y,test_z] + + return [train_x, train_y, train_z], [test_x, test_y, test_z] + def scale_data(data, sequence_length): """[summary] @@ -103,21 +107,22 @@ def scale_data(data, sequence_length): Returns: [type]: [description] """ - assert len(data[0].shape)==2 - scalers={} + assert len(data[0].shape) == 2 + scalers = {} data = np.vstack(data) - + for i in range(data.shape[1]): - scaler = MinMaxScaler(feature_range=(-1,1)) - s_s = scaler.fit_transform(data[:,i].reshape(-1,1)) - s_s=np.reshape(s_s,len(s_s)) - scalers['scaler_'+ str(i)] = scaler - data[:,i]=s_s + scaler = MinMaxScaler(feature_range=(-1, 1)) + s_s = scaler.fit_transform(data[:, i].reshape(-1, 1)) + s_s = np.reshape(s_s, len(s_s)) + scalers['scaler_' + str(i)] = scaler + data[:, i] = s_s # Slice the data into batches data = [data[i:i + sequence_length] for i in range(0, len(data), sequence_length)] return data, scalers -def weighted_random_samplers(train_z,test_z): + +def weighted_random_samplers(train_z, test_z): """[summary] Args: @@ -131,18 +136,19 @@ def weighted_random_samplers(train_z,test_z): # Prepare weighted random sampler: train_target_list = torch.tensor(train_z).type(torch.LongTensor) test_target_list = torch.tensor(test_z).type(torch.LongTensor) - + # Number of classes and their frequencies train_targets_, train_class_count = get_class_distribution(train_target_list) test_targets_, test_class_count = get_class_distribution(test_target_list) # Compute class weights - train_class_weights = 1./torch.tensor(train_class_count, dtype=torch.float) - test_class_weights = 1./torch.tensor(test_class_count, dtype=torch.float) - + train_class_weights = 1. / torch.tensor(train_class_count, dtype=torch.float) + test_class_weights = 1. / torch.tensor(test_class_count, dtype=torch.float) + # Assign weights to original target list - train_class_weights_all = train_class_weights[train_target_list-1] # Note the targets start from 1, to python idx to apply,-1 - test_class_weights_all = test_class_weights[test_target_list-1] + train_class_weights_all = train_class_weights[train_target_list - 1] # Note the targets start from 1, to python idx + # to apply,-1 + test_class_weights_all = test_class_weights[test_target_list - 1] # Weighted samplers train_weighted_sampler = WeightedRandomSampler( @@ -155,4 +161,4 @@ def weighted_random_samplers(train_z,test_z): num_samples=len(test_class_weights_all), replacement=True ) - return train_weighted_sampler, test_weighted_sampler \ No newline at end of file + return train_weighted_sampler, test_weighted_sampler From 40d8525221bdc44b79822417198eb6741ab229fc Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 17:04:58 +0000 Subject: [PATCH 194/211] Format code --- traja/models/__init__.py | 6 +- traja/models/experiment.py | 5 - traja/models/generative_models/vae.py | 118 ++++++++++++------------ traja/models/inference.py | 57 ++++++------ traja/models/manifold.py | 4 - traja/models/nn.py | 6 +- traja/models/optimizers.py | 7 +- traja/models/predictive_models/ae.py | 122 ++++++++++++------------- traja/models/predictive_models/lstm.py | 31 ++++--- traja/models/train.py | 57 ++++++------ traja/models/utils.py | 9 +- traja/models/visualizer.py | 18 ++-- 12 files changed, 201 insertions(+), 239 deletions(-) diff --git a/traja/models/__init__.py b/traja/models/__init__.py index f6629e51..6f89c5c8 100644 --- a/traja/models/__init__.py +++ b/traja/models/__init__.py @@ -1,8 +1,8 @@ # from .nn import LSTM from traja.models.generative_models.vae import MultiModelVAE +from traja.models.generative_models.vaegan import MultiModelVAEGAN from traja.models.predictive_models.ae import MultiModelAE -from traja.models.predictive_models.lstm import LSTM from traja.models.predictive_models.irl import MultiModelIRL -from traja.models.generative_models.vaegan import MultiModelVAEGAN -from .utils import TimeDistributed, read_hyperparameters, save, load +from traja.models.predictive_models.lstm import LSTM from .inference import * +from .utils import TimeDistributed, read_hyperparameters, save, load diff --git a/traja/models/experiment.py b/traja/models/experiment.py index 5e805571..05f596a1 100644 --- a/traja/models/experiment.py +++ b/traja/models/experiment.py @@ -5,7 +5,6 @@ import matplotlib.pyplot as plt import numpy as np -from torch.utils.data import sampler try: import torch @@ -18,11 +17,7 @@ import os import pandas as pd from time import time -from sklearn.preprocessing import MinMaxScaler from datetime import datetime -from sklearn.model_selection import train_test_split -from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler -import torchvision.transforms as transforms device = torch.device("cuda" if torch.cuda.is_available() else "cpu") diff --git a/traja/models/generative_models/vae.py b/traja/models/generative_models/vae.py index 76ec36f2..dcc6066f 100644 --- a/traja/models/generative_models/vae.py +++ b/traja/models/generative_models/vae.py @@ -24,9 +24,10 @@ trainer.train_latent_model(train_dataloader, test_dataloader, model_save_path=PATH)""" import torch -from traja.models.utils import TimeDistributed from torch import nn +from traja.models.utils import TimeDistributed + device = "cuda" if torch.cuda.is_available() else "cpu" @@ -46,16 +47,16 @@ class LSTMEncoder(torch.nn.Module): """ def __init__( - self, - input_size: int, - num_past: int, - batch_size: int, - hidden_size: int, - num_lstm_layers: int, - batch_first: bool, - dropout: float, - reset_state: bool, - bidirectional: bool, + self, + input_size: int, + num_past: int, + batch_size: int, + hidden_size: int, + num_lstm_layers: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool, ): super(LSTMEncoder, self).__init__() @@ -81,11 +82,11 @@ def __init__( def _init_hidden(self): return ( torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad_() - .to(device), + .requires_grad_() + .to(device), torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad_() - .to(device), + .requires_grad_() + .to(device), ) @@ -116,7 +117,6 @@ def reparameterize(self, mu, logvar, training= True): return mu def forward(self, x, training=True): - z_variables = self.latent(x) # [batch_size, latent_size*2] mu, logvar = torch.chunk(z_variables, 2, dim=1) # [batch_size,latent_size] # Reparameterize @@ -127,7 +127,6 @@ def forward(self, x, training=True): class LSTMDecoder(torch.nn.Module): - """ Implementation of Decoder network using LSTM layers input_size: The number of expected features in the input x num_future: Number of time steps to be predicted given the num_past steps @@ -143,17 +142,17 @@ class LSTMDecoder(torch.nn.Module): """ def __init__( - self, - batch_size: int, - num_future: int, - hidden_size: int, - num_lstm_layers: int, - output_size: int, - latent_size: int, - batch_first: bool, - dropout: float, - reset_state: bool, - bidirectional: bool, + self, + batch_size: int, + num_future: int, + hidden_size: int, + num_lstm_layers: int, + output_size: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool, ): super(LSTMDecoder, self).__init__() self.batch_size = batch_size @@ -183,11 +182,11 @@ def __init__( def _init_hidden(self): return ( torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad_() - .to(device), + .requires_grad_() + .to(device), torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad_() - .to(device), + .requires_grad_() + .to(device), ) def forward(self, x, num_future=None): @@ -222,13 +221,13 @@ class MLPClassifier(torch.nn.Module): """ def __init__( - self, - input_size: int, - hidden_size: int, - num_classes: int, - latent_size: int, - num_classifier_layers: int, - dropout: float, + self, + input_size: int, + hidden_size: int, + num_classes: int, + latent_size: int, + num_classifier_layers: int, + dropout: float, ): super(MLPClassifier, self).__init__() self.latent_size = latent_size @@ -257,7 +256,6 @@ def forward(self, x): class MultiModelVAE(torch.nn.Module): - """Implementation of Multimodel Variational autoencoders; This Module wraps the Variational Autoencoder models [Encoder,Latent[Sampler],Decoder]. If classify=True, then the wrapper also include classification layers @@ -275,25 +273,25 @@ class MultiModelVAE(torch.nn.Module): """ def __init__( - self, - input_size: int, - num_past: int, - batch_size: int, - num_future: int, - lstm_hidden_size: int, - num_lstm_layers: int, - output_size: int, - latent_size: int, - batch_first: bool, - dropout: float, - reset_state: bool, - bidirectional: bool = False, - num_classifier_layers: int = None, - classifier_hidden_size: int = None, - num_classes: int = None, - num_regressor_layers: int = None, - regressor_hidden_size: int = None, - num_regressor_parameters: int = None, + self, + input_size: int, + num_past: int, + batch_size: int, + num_future: int, + lstm_hidden_size: int, + num_lstm_layers: int, + output_size: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool = False, + num_classifier_layers: int = None, + classifier_hidden_size: int = None, + num_classes: int = None, + num_regressor_layers: int = None, + regressor_hidden_size: int = None, + num_regressor_parameters: int = None, ): super(MultiModelVAE, self).__init__() @@ -462,4 +460,4 @@ def forward(self, data, training=True, classify=False, regress=False, latent=Tru if latent: return regressor_out, latent_out, mu, logvar else: - return regressor_out \ No newline at end of file + return regressor_out diff --git a/traja/models/inference.py b/traja/models/inference.py index 20127c5b..82529b5e 100644 --- a/traja/models/inference.py +++ b/traja/models/inference.py @@ -1,27 +1,25 @@ """Generate time series from model""" -import plotly.express as px +import matplotlib.pyplot as plt +import numpy as np import torch -from traja.models.predictive_models.ae import MultiModelAE + from traja.models.generative_models.vae import MultiModelVAE from traja.models.generative_models.vaegan import MultiModelVAEGAN +from traja.models.predictive_models.ae import MultiModelAE from traja.models.predictive_models.irl import MultiModelIRL from traja.models.predictive_models.lstm import LSTM -from traja.models.utils import load -import matplotlib.pyplot as plt -import os -import numpy as np device = "cuda" if torch.cuda.is_available() else "cpu" class Generator: def __init__( - self, - model_type: str = None, - model_path: str = None, - model_hyperparameters: dict = None, - model: torch.nn.Module = None, + self, + model_type: str = None, + model_path: str = None, + model_hyperparameters: dict = None, + model: torch.nn.Module = None, ): """Generate a batch of future steps from a random latent state of Multi variate multi label models @@ -58,8 +56,8 @@ def generate(self, num_steps, classify=True, scaler=None): self.model_hyperparameters["batch_size"], self.model_hyperparameters["latent_size"], ) - .normal_(mean=0, std=0.1) - .to(device) + .normal_(mean=0, std=0.1) + .to(device) ) # Generate trajectories from the noise self.generated_data = ( @@ -112,7 +110,7 @@ def generate(self, num_steps, classify=True, scaler=None): try: label = "Animal ID {}".format( ( - torch.max(self.generated_category, 1).indices + 1 + torch.max(self.generated_category, 1).indices + 1 ).detach()[i + j] ) except Exception as error: @@ -121,10 +119,10 @@ def generate(self, num_steps, classify=True, scaler=None): label = "" ax[i, j].plot( self.generated_data[:, 0][ - (i + j) * num_steps : (i + j) * num_steps + num_steps + (i + j) * num_steps: (i + j) * num_steps + num_steps ], self.generated_data[:, 1][ - (i + j) * num_steps : (i + j) * num_steps + num_steps + (i + j) * num_steps: (i + j) * num_steps + num_steps ], label=label, color="g", @@ -152,11 +150,11 @@ def generate_timeseries(self, num_steps): class Predictor: def __init__( - self, - model_type: str = None, - model_path: str = None, - model_hyperparameters: dict = None, - model: torch.nn.Module = None, + self, + model_type: str = None, + model_path: str = None, + model_hyperparameters: dict = None, + model: torch.nn.Module = None, ): """Generate a batch of future steps from a random latent state of Multi variate multi label models @@ -240,14 +238,13 @@ def predict(self, data_loader, num_steps, scaler, classify=True): # TODO:Depreself.generated_categoryed;Slicing the data into batches predicted_data = np.array( [ - self.predicted_data[i : i + num_steps] + self.predicted_data[i: i + num_steps] for i in range(0, len(self.predicted_data), num_steps) ] ) # Rescaling target data self.target_data = target.copy() for i in range(self.target_data.shape[1]): - s_s = scaler["scaler_{}".format(i)].inverse_transform( self.target_data[:, i].reshape(-1, 1) ) @@ -256,7 +253,7 @@ def predict(self, data_loader, num_steps, scaler, classify=True): # TODO:Depreself.generated_categoryed;Slicing the data into batches self.target_data = np.array( [ - self.target_data[i : i + num_steps] + self.target_data[i: i + num_steps] for i in range(0, len(self.target_data), num_steps) ] ) @@ -277,22 +274,22 @@ def predict(self, data_loader, num_steps, scaler, classify=True): for j in range(5): ax[i, j].plot( predicted_data_[:, 0][ - (i + j) * num_steps : (i + j) * num_steps + num_steps + (i + j) * num_steps: (i + j) * num_steps + num_steps ], predicted_data_[:, 1][ - (i + j) * num_steps : (i + j) * num_steps + num_steps + (i + j) * num_steps: (i + j) * num_steps + num_steps ], - label=f"Predicted ID {self.predicted_self.generated_categoryegory[i+j]}", + label=f"Predicted ID {self.predicted_self.generated_categoryegory[i + j]}", ) ax[i, j].plot( self.target_data_[:, 0][ - (i + j) * num_steps : (i + j) * num_steps + num_steps + (i + j) * num_steps: (i + j) * num_steps + num_steps ], self.target_data_[:, 1][ - (i + j) * num_steps : (i + j) * num_steps + num_steps + (i + j) * num_steps: (i + j) * num_steps + num_steps ], - label=f"Target ID {self.generated_categoryegory[i+j]}", + label=f"Target ID {self.generated_categoryegory[i + j]}", color="g", ) ax[i, j].legend() diff --git a/traja/models/manifold.py b/traja/models/manifold.py index 90244697..5e99db34 100644 --- a/traja/models/manifold.py +++ b/traja/models/manifold.py @@ -1,13 +1,9 @@ -from sklearn.manifold import * - - class Manifold: """Wrap all the manifold functionalities provided by scikit learn. Provide interface to apply non-linear dimensionality reduction techniques, visualize and infer the strucure of the data using neural networks """ def __init__(self, manifold_type): - pass def __new__(cls): diff --git a/traja/models/nn.py b/traja/models/nn.py index d2f26e36..6740d398 100644 --- a/traja/models/nn.py +++ b/traja/models/nn.py @@ -2,9 +2,9 @@ """Pytorch visualization code modified from Chad Jensen's implementation (https://discuss.pytorch.org/t/lstm-for-sequence-prediction/22021/3).""" import logging + import matplotlib.pyplot as plt import numpy as np -from torch.utils.data import sampler try: import torch @@ -17,11 +17,7 @@ import os import pandas as pd from time import time -from sklearn.preprocessing import MinMaxScaler from datetime import datetime -from sklearn.model_selection import train_test_split -from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler -import torchvision.transforms as transforms nb_steps = 10 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index 0e12c77c..f0ac88d7 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -1,10 +1,5 @@ import torch from torch.optim.lr_scheduler import ReduceLROnPlateau -from traja.models.generative_models.vae import MultiModelVAE -from traja.models.predictive_models.ae import MultiModelAE -from traja.models.predictive_models.lstm import LSTM -from traja.models.predictive_models.irl import MultiModelIRL -from traja.models.generative_models.vaegan import MultiModelVAEGAN class Optimizer: @@ -74,7 +69,7 @@ def get_optimizers(self, lr=0.0001): elif self.model_type == "vaegan": return NotImplementedError - else: # self.model_type == "irl": + else: # self.model_type == "irl": return NotImplementedError forecasting_optimizers = [self.optimizers[key] for key in self.forecasting_keys if key in self.optimizers] diff --git a/traja/models/predictive_models/ae.py b/traja/models/predictive_models/ae.py index 04af63a3..54bc24a7 100644 --- a/traja/models/predictive_models/ae.py +++ b/traja/models/predictive_models/ae.py @@ -1,12 +1,12 @@ import torch -from traja.models.utils import TimeDistributed from torch import nn +from traja.models.utils import TimeDistributed + device = "cuda" if torch.cuda.is_available() else "cpu" class LSTMEncoder(torch.nn.Module): - """ Implementation of Encoder network using LSTM layers Parameters: ----------- @@ -24,18 +24,17 @@ class LSTMEncoder(torch.nn.Module): """ def __init__( - self, - input_size: int, - num_past: int, - batch_size: int, - hidden_size: int, - num_lstm_layers: int, - batch_first: bool, - dropout: float, - reset_state: bool, - bidirectional: bool, + self, + input_size: int, + num_past: int, + batch_size: int, + hidden_size: int, + num_lstm_layers: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool, ): - super(LSTMEncoder, self).__init__() self.input_size = input_size @@ -60,11 +59,11 @@ def __init__( def _init_hidden(self): return ( torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad_() - .to(device), + .requires_grad_() + .to(device), torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad_() - .to(device), + .requires_grad_() + .to(device), ) def forward(self, x): @@ -92,7 +91,6 @@ def forward(self, x): class LSTMDecoder(torch.nn.Module): - """ Implementation of Decoder network using LSTM layers Parameters: ------------ @@ -111,17 +109,17 @@ class LSTMDecoder(torch.nn.Module): """ def __init__( - self, - batch_size: int, - num_future: int, - hidden_size: int, - num_lstm_layers: int, - output_size: int, - latent_size: int, - batch_first: bool, - dropout: float, - reset_state: bool, - bidirectional: bool, + self, + batch_size: int, + num_future: int, + hidden_size: int, + num_lstm_layers: int, + output_size: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool, ): super(LSTMDecoder, self).__init__() self.batch_size = batch_size @@ -151,11 +149,11 @@ def __init__( def _init_hidden(self): return ( torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad_() - .to(device), + .requires_grad_() + .to(device), torch.zeros(self.num_lstm_layers, self.batch_size, self.hidden_size) - .requires_grad_() - .to(device), + .requires_grad_() + .to(device), ) def forward(self, x, num_future=None): @@ -180,9 +178,6 @@ def forward(self, x, num_future=None): class MLPClassifier(torch.nn.Module): - """ MLP classifier - """ - """ MLP classifier: Classify the input data using the latent embeddings Parameters: ----------- @@ -195,13 +190,13 @@ class MLPClassifier(torch.nn.Module): """ def __init__( - self, - input_size: int, - hidden_size: int, - num_classes: int, - latent_size: int, - num_classifier_layers: int, - dropout: float, + self, + input_size: int, + hidden_size: int, + num_classes: int, + latent_size: int, + num_classifier_layers: int, + dropout: float, ): super(MLPClassifier, self).__init__() self.latent_size = latent_size @@ -230,7 +225,6 @@ def forward(self, x): class MultiModelAE(torch.nn.Module): - """Implementation of Multimodel autoencoders; This Module wraps the Autoencoder models [Encoder,Latent,Decoder]. If classify=True, then the wrapper also include classification layers @@ -251,25 +245,25 @@ class MultiModelAE(torch.nn.Module): """ def __init__( - self, - input_size: int, - num_past: int, - batch_size: int, - num_future: int, - lstm_hidden_size: int, - num_lstm_layers: int, - output_size: int, - latent_size: int, - batch_first: bool, - dropout: float, - reset_state: bool, - bidirectional: bool = False, - num_classifier_layers: int = None, - classifier_hidden_size: int = None, - num_classes: int = None, - num_regressor_layers: int = None, - regressor_hidden_size: int = None, - num_regressor_parameters: int = None, + self, + input_size: int, + num_past: int, + batch_size: int, + num_future: int, + lstm_hidden_size: int, + num_lstm_layers: int, + output_size: int, + latent_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool = False, + num_classifier_layers: int = None, + classifier_hidden_size: int = None, + num_classes: int = None, + num_regressor_layers: int = None, + regressor_hidden_size: int = None, + num_regressor_parameters: int = None, ): super(MultiModelAE, self).__init__() @@ -449,4 +443,4 @@ def forward(self, data, classify=False, regress=False, training=True, latent=Tru latent_out = self.latent(enc_out) regressor_out = self.regressor(latent_out) # Deterministic - return regressor_out \ No newline at end of file + return regressor_out diff --git a/traja/models/predictive_models/lstm.py b/traja/models/predictive_models/lstm.py index 004fbdc9..3c245343 100644 --- a/traja/models/predictive_models/lstm.py +++ b/traja/models/predictive_models/lstm.py @@ -1,5 +1,6 @@ """Implementation of Multimodel LSTM""" import torch + from traja.models.utils import TimeDistributed device = "cuda" if torch.cuda.is_available() else "cpu" @@ -25,17 +26,17 @@ class LSTM(torch.nn.Module): """ def __init__( - self, - batch_size: int, - num_future: int, - hidden_size: int, - num_layers: int, - output_size: int, - input_size: int, - batch_first: bool, - dropout: float, - reset_state: bool, - bidirectional: bool, + self, + batch_size: int, + num_future: int, + hidden_size: int, + num_layers: int, + output_size: int, + input_size: int, + batch_first: bool, + dropout: float, + reset_state: bool, + bidirectional: bool, ): super(LSTM, self).__init__() @@ -69,11 +70,11 @@ def __init__( def _init_hidden(self): return ( torch.zeros(self.num_layers, self.batch_size, self.hidden_size) - .requires_grad_() - .to(device), + .requires_grad_() + .to(device), torch.zeros(self.num_layers, self.batch_size, self.hidden_size) - .requires_grad_() - .to(device), + .requires_grad_() + .to(device), ) def forward(self, x, training=True, classify=False, regress=False, latent=False): diff --git a/traja/models/train.py b/traja/models/train.py index 8150e1b2..8f744b61 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -1,14 +1,10 @@ -from traja.models.generative_models.vae import MultiModelVAE -from traja.models.predictive_models.ae import MultiModelAE -from traja.models.predictive_models.lstm import LSTM -from traja.models.predictive_models.irl import MultiModelIRL -from traja.models.generative_models.vaegan import MultiModelVAEGAN +import matplotlib.pyplot as plt +import torch + from . import utils from . import visualizer from .losses import Criterion from .optimizers import Optimizer -import torch -import matplotlib.pyplot as plt device = 'cuda' if torch.cuda.is_available() else 'cpu' @@ -49,17 +45,17 @@ class HybridTrainer(object): valid_models = ['ae', 'vae', 'lstm'] def __init__( - self, - model: torch.nn.Module, - optimizer_type: str, - loss_type: str = "huber", - lr: float = 0.001, - lr_factor: float = 0.1, - scheduler_patience: int = 10, + self, + model: torch.nn.Module, + optimizer_type: str, + loss_type: str = "huber", + lr: float = 0.001, + lr_factor: float = 0.1, + scheduler_patience: int = 10, ): assert ( - model.model_type in HybridTrainer.valid_models + model.model_type in HybridTrainer.valid_models ), "Model type is {model_type}, valid models are {}".format( HybridTrainer.valid_models ) @@ -114,7 +110,8 @@ def __init__( self.model_type, self.model, self.optimizer_type, classify=self.classify ) - self.forecasting_optimizers, self.classification_optimizers, self.regression_optimizers = optimizer.get_optimizers(lr=self.lr) + self.forecasting_optimizers, self.classification_optimizers, self.regression_optimizers = optimizer.get_optimizers( + lr=self.lr) self.forecasting_schedulers, self.classification_schedulers, self.regression_schedulers = optimizer.get_lrschedulers( factor=self.lr_factor, patience=self.scheduler_patience ) @@ -138,7 +135,8 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo """ assert model_save_path is not None, f"Model path {model_save_path} unknown" - assert training_mode in ['forecasting', 'classification', 'regression'], f'Training mode {training_mode} unknown' + assert training_mode in ['forecasting', 'classification', + 'regression'], f'Training mode {training_mode} unknown' self.model.to(device) @@ -255,8 +253,11 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo data, training=False, classify=True ) - test_loss_classification += Criterion().classifier_criterion(classifier_out, - category - 1).item() + test_loss_classification += ( + Criterion() + .classifier_criterion(classifier_out, category - 1) + .item() + ) # Compute number of correct samples total += category.size(0) @@ -318,14 +319,14 @@ class CustomTrainer: """ def __init__( - self, - model: torch.nn.Module, - optimizer_type: None, - criterion: None, - epochs: int, - lr: float = 0.001, - lr_factor: float = 0.001, - scheduler_patience: int = 10, + self, + model: torch.nn.Module, + optimizer_type: None, + criterion: None, + epochs: int, + lr: float = 0.001, + lr_factor: float = 0.001, + scheduler_patience: int = 10, ): self.model = model self.optimizer_type = optimizer_type @@ -481,7 +482,6 @@ def train(self): # TODO class Trainer: - """Wraps all above Trainers. Instantiate and return the Trainer of model type """ def __init__(self, *model_hyperparameters, **kwargs): @@ -516,4 +516,3 @@ def __new__(cls): # Return the instance of the trainer return NotImplementedError - diff --git a/traja/models/utils.py b/traja/models/utils.py index f9cc51b7..74c0347f 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -1,9 +1,7 @@ +import json +import os + import torch -import matplotlib.pyplot as plt -import numpy as np -import collections -from numpy import math -import os, json class TimeDistributed(torch.nn.Module): @@ -99,4 +97,3 @@ def read_hyperparameters(hyperparameter_json): """ with open(hyperparameter_json) as f_in: return json.load(f_in) - diff --git a/traja/models/visualizer.py b/traja/models/visualizer.py index 684a2a31..832cd43f 100644 --- a/traja/models/visualizer.py +++ b/traja/models/visualizer.py @@ -1,18 +1,14 @@ +# matplotlib.use("TKAgg") +import matplotlib.pyplot as plt import networkx as nx import numpy as np +import plotly.express as px import scipy -from sklearn import neighbors +from matplotlib import style +from mpl_toolkits.mplot3d import Axes3D from scipy.sparse import csgraph +from sklearn import neighbors from sklearn.neighbors import radius_neighbors_graph -from mpl_toolkits.mplot3d import Axes3D -import plotly.express as px - - -import matplotlib - -# matplotlib.use("TKAgg") -import matplotlib.pyplot as plt -from matplotlib import style # plt.switch_backend("TkAgg") @@ -292,7 +288,6 @@ def show(self, X, spec_embed, fig3): class Network: def __init__(self, activity, weights): - pass def show(self): @@ -307,4 +302,3 @@ def __init__(self, inputs, manifold): def show(self): fig = None return fig - From 9dcf8042e3bc29f11d460ba98175a87db48160cb Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 18:00:03 +0000 Subject: [PATCH 195/211] Work around torch collate behaviour --- traja/models/train.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/traja/models/train.py b/traja/models/train.py index 8f744b61..3ca43a3f 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -161,6 +161,8 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo for optimizer in self.regression_optimizers: optimizer.zero_grad() + if type(category) == list: + category = category[0] data, target, category, parameters = ( data.float().to(device), target.float().to(device), @@ -222,6 +224,9 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo correct = 0.0 self.model.eval() for idx, (data, target, category, parameters) in enumerate(test_loader): + if type(category) == list: + category = category[0] + print(category) data, target, category, parameters = ( data.float().to(device), target.float().to(device), From 67262a8704a1fdfa5c3e38df0865f1880500a5e9 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 18:27:45 +0000 Subject: [PATCH 196/211] Remove debug --- traja/models/train.py | 1 - 1 file changed, 1 deletion(-) diff --git a/traja/models/train.py b/traja/models/train.py index 3ca43a3f..5ab9a763 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -226,7 +226,6 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo for idx, (data, target, category, parameters) in enumerate(test_loader): if type(category) == list: category = category[0] - print(category) data, target, category, parameters = ( data.float().to(device), target.float().to(device), From f5be6e48ba3f0e5fe916330dea4c05903e585ebb Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 19:11:07 +0000 Subject: [PATCH 197/211] Add missing parenthesis --- traja/models/generative_models/vae.py | 2 +- traja/models/predictive_models/ae.py | 2 +- traja/models/train.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/traja/models/generative_models/vae.py b/traja/models/generative_models/vae.py index dcc6066f..d6e9a50b 100644 --- a/traja/models/generative_models/vae.py +++ b/traja/models/generative_models/vae.py @@ -382,7 +382,7 @@ def forward(self, data, training=True, classify=False, regress=False, latent=Tru assert not (classify and regress), 'Model cannot both classify and regress!' - if not classify or regress: + if not (classify or regress): # Set the classifier and regressor grads off if self.num_classes is not None: for param in self.classifier.parameters(): diff --git a/traja/models/predictive_models/ae.py b/traja/models/predictive_models/ae.py index 54bc24a7..c790f90b 100644 --- a/traja/models/predictive_models/ae.py +++ b/traja/models/predictive_models/ae.py @@ -374,7 +374,7 @@ def forward(self, data, classify=False, regress=False, training=True, latent=Tru """ assert not (classify and regress), 'Model cannot both classify and regress!' - if not classify or regress: + if not (classify or regress): # Set the classifier and regressor grads off if self.num_classes is not None: for param in self.classifier.parameters(): diff --git a/traja/models/train.py b/traja/models/train.py index 5ab9a763..c3795711 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -185,7 +185,7 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo for optimizer in self.forecasting_optimizers: optimizer.step() - elif self.classify and training_mode == "classification": + elif training_mode == "classification": if self.model_type == "vae": classifier_out, latent_out, mu, logvar = self.model( data, training=True, classify=True @@ -202,7 +202,7 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo for optimizer in self.classification_optimizers: optimizer.step() - elif self.regress and training_mode == 'regression': + elif training_mode == 'regression': regressor_out = self.model(data, training=True, regress=True, latent=False) loss = Criterion().regressor_criterion( regressor_out, parameters From c1f0058401267186e4c783da42f99f6c8092b73f Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Thu, 31 Dec 2020 19:20:55 +0000 Subject: [PATCH 198/211] Make parameters floating --- traja/models/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index c3795711..a0f679bc 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -167,7 +167,7 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo data.float().to(device), target.float().to(device), category.to(device), - parameters.to(device) + parameters.float().to(device) ) if training_mode == "forecasting": @@ -230,7 +230,7 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo data.float().to(device), target.float().to(device), category.to(device), - parameters.to(device) + parameters.float().to(device) ) # Time series forecasting test if self.model_type == 'ae' or self.model_type == 'lstm': From 8adbd5d76965c28739a5c6713fed2aa6db089b17 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 1 Jan 2021 14:35:12 +0100 Subject: [PATCH 199/211] Update inference.py --- traja/models/inference.py | 68 ++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/traja/models/inference.py b/traja/models/inference.py index 82529b5e..6a39fabd 100644 --- a/traja/models/inference.py +++ b/traja/models/inference.py @@ -15,11 +15,11 @@ class Generator: def __init__( - self, - model_type: str = None, - model_path: str = None, - model_hyperparameters: dict = None, - model: torch.nn.Module = None, + self, + model_type: str = None, + model_path: str = None, + model_hyperparameters: dict = None, + model: torch.nn.Module = None, ): """Generate a batch of future steps from a random latent state of Multi variate multi label models @@ -56,8 +56,8 @@ def generate(self, num_steps, classify=True, scaler=None): self.model_hyperparameters["batch_size"], self.model_hyperparameters["latent_size"], ) - .normal_(mean=0, std=0.1) - .to(device) + .normal_(mean=0, std=0.1) + .to(device) ) # Generate trajectories from the noise self.generated_data = ( @@ -110,7 +110,7 @@ def generate(self, num_steps, classify=True, scaler=None): try: label = "Animal ID {}".format( ( - torch.max(self.generated_category, 1).indices + 1 + torch.max(self.generated_category, 1).indices + 1 ).detach()[i + j] ) except Exception as error: @@ -119,10 +119,10 @@ def generate(self, num_steps, classify=True, scaler=None): label = "" ax[i, j].plot( self.generated_data[:, 0][ - (i + j) * num_steps: (i + j) * num_steps + num_steps + (i + j) * num_steps : (i + j) * num_steps + num_steps ], self.generated_data[:, 1][ - (i + j) * num_steps: (i + j) * num_steps + num_steps + (i + j) * num_steps : (i + j) * num_steps + num_steps ], label=label, color="g", @@ -150,11 +150,11 @@ def generate_timeseries(self, num_steps): class Predictor: def __init__( - self, - model_type: str = None, - model_path: str = None, - model_hyperparameters: dict = None, - model: torch.nn.Module = None, + self, + model_type: str = None, + model_path: str = None, + model_hyperparameters: dict = None, + model: torch.nn.Module = None, ): """Generate a batch of future steps from a random latent state of Multi variate multi label models @@ -170,8 +170,12 @@ def __init__( self.model_hyperparameters = model_hyperparameters if self.model_type == "ae": - self.model = MultiModelAE(num_regressor_layers=2, regressor_hidden_size=32, num_regressor_parameters=3, - **self.model_hyperparameters) + self.model = MultiModelAE( + num_regressor_layers=2, + regressor_hidden_size=32, + num_regressor_parameters=3, + **self.model_hyperparameters, + ) if self.model_type == "lstm": self.model = LSTM(**self.model_hyperparameters) @@ -184,7 +188,7 @@ def __init__( self.model = model(**self.model_hyperparameters) ( - self.predicted_self.generated_categoryegory, + self.predicted_category, self.target_data, self.target_data_, self.predicted_data, @@ -207,15 +211,13 @@ def predict(self, data_loader, num_steps, scaler, classify=True): self.model.to(device) if self.model_type == "ae": - for data, target, self.generated_categoryegory in data_loader: + for data, target, self.generated_category in data_loader: data, target = data.to(device), target.to(device) - enc_self.generated_data = self.model.encoder(data) - latent_self.generated_data = self.model.latent(enc_self.generated_data) - self.predicted_data = self.model.decoder(latent_self.generated_data) + self.predicted_data = self.model.encoder(data) + self.predicted_data = self.model.latent(self.generated_data) + self.predicted_data = self.model.decoder(self.generated_data) if classify: - self.predicted_self.generated_categoryegory = self.model.classifier( - latent_self.generated_data - ) + self.generated_category = self.model.classifier(self.predicted_data) target = target.cpu().detach().numpy() target = target.reshape( @@ -238,7 +240,7 @@ def predict(self, data_loader, num_steps, scaler, classify=True): # TODO:Depreself.generated_categoryed;Slicing the data into batches predicted_data = np.array( [ - self.predicted_data[i: i + num_steps] + self.predicted_data[i : i + num_steps] for i in range(0, len(self.predicted_data), num_steps) ] ) @@ -253,7 +255,7 @@ def predict(self, data_loader, num_steps, scaler, classify=True): # TODO:Depreself.generated_categoryed;Slicing the data into batches self.target_data = np.array( [ - self.target_data[i: i + num_steps] + self.target_data[i : i + num_steps] for i in range(0, len(self.target_data), num_steps) ] ) @@ -274,22 +276,22 @@ def predict(self, data_loader, num_steps, scaler, classify=True): for j in range(5): ax[i, j].plot( predicted_data_[:, 0][ - (i + j) * num_steps: (i + j) * num_steps + num_steps + (i + j) * num_steps : (i + j) * num_steps + num_steps ], predicted_data_[:, 1][ - (i + j) * num_steps: (i + j) * num_steps + num_steps + (i + j) * num_steps : (i + j) * num_steps + num_steps ], - label=f"Predicted ID {self.predicted_self.generated_categoryegory[i + j]}", + label=f"Predicted ID {self.generated_categoryegory[i+j]}", ) ax[i, j].plot( self.target_data_[:, 0][ - (i + j) * num_steps: (i + j) * num_steps + num_steps + (i + j) * num_steps : (i + j) * num_steps + num_steps ], self.target_data_[:, 1][ - (i + j) * num_steps: (i + j) * num_steps + num_steps + (i + j) * num_steps : (i + j) * num_steps + num_steps ], - label=f"Target ID {self.generated_categoryegory[i + j]}", + label=f"Target ID {self.generated_category[i + j]}", color="g", ) ax[i, j].legend() From 31e4f6f00bb5d07747f6bc53284beb30f0bebea9 Mon Sep 17 00:00:00 2001 From: Saran-nns Date: Fri, 1 Jan 2021 14:36:05 +0100 Subject: [PATCH 200/211] Update utils.py --- traja/models/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traja/models/utils.py b/traja/models/utils.py index 74c0347f..b399ca86 100644 --- a/traja/models/utils.py +++ b/traja/models/utils.py @@ -61,7 +61,7 @@ def save(model, hyperparameters, PATH=None): json.dump(hyperparameters, fp, sort_keys=False) if hyperdir == "": hyperdir = "." - print("Model and hyperparameters saved at {hyperdir}") + print(f"Model and hyperparameters saved at {hyperdir}") def load(model, PATH=None): From 52894c3e3aa46f421d1ef1b02d47f22f9b38f271 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Fri, 1 Jan 2021 14:02:26 +0000 Subject: [PATCH 201/211] Simplify classifier step in trainer --- traja/models/train.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/traja/models/train.py b/traja/models/train.py index a0f679bc..1a1f661f 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -186,14 +186,9 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo optimizer.step() elif training_mode == "classification": - if self.model_type == "vae": - classifier_out, latent_out, mu, logvar = self.model( - data, training=True, classify=True - ) - else: # 'ae', 'lstm' - classifier_out = self.model( - data, training=True, classify=True - ) + classifier_out = self.model( + data, training=True, classify=True, latent=False + ) loss = Criterion().classifier_criterion( classifier_out, (category - 1).long() ) From 02707300f6ef86259a830e14702b61a8c5e53149 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Fri, 1 Jan 2021 14:49:00 +0000 Subject: [PATCH 202/211] Add torch and other libraries to requirements.txt --- requirements.txt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 816d0e5a..29082480 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,13 @@ pandas -numpy +numpy==1.18.5 matplotlib shapely scipy -fastdtw \ No newline at end of file +sklearn +fastdtw +plotly +networkx +seaborn +torch +pytest +h5py \ No newline at end of file From 0a33ad812897be264fea58a9329a56f83fc21f0a Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Fri, 1 Jan 2021 14:55:55 +0000 Subject: [PATCH 203/211] Move pytest to requirements-dev.txt --- .travis.yml | 26 ++++++++++++-------------- requirements-dev.txt | 21 +++++++++++++-------- requirements.txt | 1 - 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index b87c5109..2277f67c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,20 +23,18 @@ before_install: - sudo chmod 2777 /usr/local/lib/R/site-library install: - - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$PATH" - - hash -r - - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - - # Update dependencies - - conda env create -n test-environment -f environment.yml python=$TRAVIS_PYTHON_VERSION - - source activate test-environment - # Doc/Test requirements - - pip install -r docs/requirements.txt - - pip install . - +- wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh + -O miniconda.sh +- bash miniconda.sh -b -p $HOME/miniconda +- export PATH="$HOME/miniconda/bin:$PATH" +- hash -r +- conda config --set always_yes yes --set changeps1 no +- conda update -q conda +- conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION +- source activate test-environment +- pip install -r requirements-dev.txt +- pip install --upgrade pytest flake8 sphinx +- pip install . script: - cd docs && make doctest && cd .. - py.test . --cov-report term --cov=traja --ignore tests/test_rutils.py diff --git a/requirements-dev.txt b/requirements-dev.txt index 9821e8b0..b82a4ec7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,14 @@ +pandas +numpy==1.18.5 +matplotlib +shapely +psutil +scipy +sklearn +fastdtw +plotly +networkx +seaborn +torch pytest -Sphinx -sphinx_gallery -sphinx_rtd_theme -pillow -black -pre-commit -pyinstaller -pyqt5 +h5py \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 29082480..d27f79da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,5 +9,4 @@ plotly networkx seaborn torch -pytest h5py \ No newline at end of file From 3489c5e55021b1679faac0dee5398edb226bfd79 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Fri, 1 Jan 2021 15:38:54 +0000 Subject: [PATCH 204/211] Cleaning up rebase --- traja/models/generative_models/vae.py | 19 ++++++++---------- traja/models/optimizers.py | 9 +++++++-- traja/models/predictive_models/ae.py | 2 +- traja/models/train.py | 29 ++++++++++++++++++--------- traja/tests/test_models.py | 16 +++++++-------- 5 files changed, 44 insertions(+), 31 deletions(-) diff --git a/traja/models/generative_models/vae.py b/traja/models/generative_models/vae.py index d6e9a50b..70027b00 100644 --- a/traja/models/generative_models/vae.py +++ b/traja/models/generative_models/vae.py @@ -89,7 +89,6 @@ def _init_hidden(self): .to(device), ) - def forward(self, x): (h0, c0) = self._init_hidden() enc_output, _ = self.lstm_encoder(x, (h0.detach(), c0.detach())) @@ -108,8 +107,9 @@ def __init__(self, hidden_size: int, latent_size: int, dropout: float): self.hidden_size = hidden_size self.dropout = dropout self.latent = torch.nn.Linear(self.hidden_size, self.latent_size * 2) - - def reparameterize(self, mu, logvar, training= True): + + @staticmethod + def reparameterize(mu, logvar, training=True): if training: std = logvar.mul(0.5).exp_() eps = std.data.new(std.size()).normal_() @@ -230,7 +230,7 @@ def __init__( dropout: float, ): super(MLPClassifier, self).__init__() - self.latent_size = latent_size + self.input_size = input_size self.hidden_size = hidden_size self.num_classes = num_classes @@ -398,11 +398,9 @@ def forward(self, data, training=True, classify=False, regress=False, latent=Tru for param in self.latent.parameters(): param.requires_grad = True - # Encoder + # Encoder -->Latent --> Decoder enc_out = self.encoder(data) - # Latent - latent_out, mu, logvar = self.latent(enc_out, training=training) - # Decoder + latent_out, mu, logvar = self.latent(enc_out) decoder_out = self.decoder(latent_out) if latent: return decoder_out, latent_out, mu, logvar @@ -425,11 +423,10 @@ def forward(self, data, training=True, classify=False, regress=False, latent=Tru for param in self.latent.parameters(): param.requires_grad = False - # Encoder + # Encoder -->Latent --> Classifier enc_out = self.encoder(data) - # Latent latent_out, mu, logvar = self.latent(enc_out, training=training) - # Classifier + classifier_out = self.classifier(mu) # Deterministic if latent: return classifier_out, latent_out, mu, logvar diff --git a/traja/models/optimizers.py b/traja/models/optimizers.py index f0ac88d7..72f1e4f3 100644 --- a/traja/models/optimizers.py +++ b/traja/models/optimizers.py @@ -11,6 +11,8 @@ def __init__(self, model_type, model, optimizer_type, classify=False): latent and classifier. In case of 'lstm', var optimizers is an optimizer for lstm and TimeDistributed(linear layer) :param model_type: Type of model 'ae', 'vae' or 'lstm' :param model: Model instance + :param classify: If True, will return the Optimizer and scheduler for classifier + :param optimizer_type: Optimizer to be used; Should be one in ['Adam', 'Adadelta', 'Adagrad', 'AdamW', 'SparseAdam', 'RMSprop', 'Rprop', 'LBFGS', 'ASGD', 'Adamax'] """ @@ -32,6 +34,7 @@ def __init__(self, model_type, model, optimizer_type, classify=False): self.model_type = model_type self.model = model self.optimizer_type = optimizer_type + self.classify = classify self.optimizers = {} self.forecasting_schedulers = {} self.classification_schedulers = {} @@ -49,7 +52,8 @@ def get_optimizers(self, lr=0.0001): lr (float, optional): Optimizer learning rate. Defaults to 0.0001. Returns: - [type]: [description] + dict: Optimizers + """ if self.model_type in ["lstm", "custom"]: @@ -87,7 +91,8 @@ def get_lrschedulers(self, factor: float, patience: int): patience (int, optional): [description]. Defaults to 10. Returns: - [dict]: [description] + [dict]: Learning rate schedulers + """ if self.model_type == "irl" or self.model_type == 'vaegan': diff --git a/traja/models/predictive_models/ae.py b/traja/models/predictive_models/ae.py index c790f90b..f0dddbfa 100644 --- a/traja/models/predictive_models/ae.py +++ b/traja/models/predictive_models/ae.py @@ -199,7 +199,7 @@ def __init__( dropout: float, ): super(MLPClassifier, self).__init__() - self.latent_size = latent_size + self.input_size = input_size self.hidden_size = hidden_size self.num_classes = num_classes diff --git a/traja/models/train.py b/traja/models/train.py index 1a1f661f..e1ab9ead 100644 --- a/traja/models/train.py +++ b/traja/models/train.py @@ -6,7 +6,7 @@ from .losses import Criterion from .optimizers import Optimizer -device = 'cuda' if torch.cuda.is_available() else 'cpu' +device = "cuda" if torch.cuda.is_available() else "cpu" class HybridTrainer(object): @@ -119,7 +119,7 @@ def __init__( def __str__(self): return f"Training model type {self.model_type}" - def fit(self, train_loader, test_loader, model_save_path=None, training_mode='forecasting', epochs=50): + def fit(self, dataloaders, model_save_path=None, training_mode='forecasting', epochs=50): """ This method implements the batch- wise training and testing protocol for both time series forecasting and classification of the timeseriesis_classification @@ -177,9 +177,12 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo ) loss = Criterion().ae_criterion(decoder_out, target) else: # vae - decoder_out, latent_out, mu, logvar = self.model(data, training=True, - is_classification=False) - loss = Criterion().vae_criterion(decoder_out, target, mu, logvar) + decoder_out, latent_out, mu, logvar = self.model( + data, training=True, classify=False + ) + loss = Criterion().vae_criterion( + decoder_out, target, mu, logvar + ) loss.backward() for optimizer in self.forecasting_optimizers: @@ -209,7 +212,11 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo total_loss += loss - print('Epoch {} | {} loss {}'.format(epoch, training_mode, total_loss / (idx + 1))) + print( + "Epoch {} | {} loss {}".format( + epoch, training_mode, total_loss / (idx + 1) + ) + ) # Testing if epoch % 10 == 0: @@ -237,9 +244,13 @@ def fit(self, train_loader, test_loader, model_save_path=None, training_mode='fo ) else: - decoder_out, latent_out, mu, logvar = self.model(data, training=False, - is_classification=False) - test_loss_forecasting += Criterion().vae_criterion(decoder_out, target, mu, logvar) + decoder_out, latent_out, mu, logvar = self.model( + data, training=False, classify=False + ) + test_loss_forecasting += Criterion().vae_criterion( + decoder_out, target, mu, logvar + ) + # Classification test if self.classify: category = category.long() diff --git a/traja/tests/test_models.py b/traja/tests/test_models.py index ae2e23e1..a0221cb5 100644 --- a/traja/tests/test_models.py +++ b/traja/tests/test_models.py @@ -5,11 +5,11 @@ from traja.models.predictive_models.ae import MultiModelAE from traja.models.predictive_models.lstm import LSTM - # Sample data data_url = "https://raw.githubusercontent.com/traja-team/traja-research/dataset_und_notebooks/dataset_analysis/jaguar5.csv" df = pd.read_csv(data_url, error_bad_lines=False) + def test_aevae(): """ Test Autoencoder and variational auto encoder models for training/testing/generative network and @@ -21,11 +21,11 @@ def test_aevae(): num_past = 10 num_future = 5 # Prepare the dataloader - train_loader, test_loader = dataset.MultiModalDataLoader(df, - batch_size=batch_size, - n_past=num_past, - n_future=num_future, - num_workers=1) + data_loaders, scalers = dataset.MultiModalDataLoader(df, + batch_size=batch_size, + n_past=num_past, + n_future=num_future, + num_workers=1) model_save_path = './model.pt' @@ -52,8 +52,8 @@ def test_aevae(): loss_type='huber') # Train the model - trainer.fit(train_loader, test_loader, model_save_path, epochs=10, training_mode='forecasting') - trainer.fit(train_loader, test_loader, model_save_path, epochs=10, training_mode='classification') + trainer.fit(data_loaders, model_save_path, epochs=10, training_mode='forecasting') + trainer.fit(data_loaders, model_save_path, epochs=10, training_mode='classification') def test_ae(): From 45eae94e2ccc96dee67c75e5eb647a30a731c7dd Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Fri, 1 Jan 2021 16:08:24 +0000 Subject: [PATCH 205/211] Update docs requirements.txt --- docs/requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index c4d19f01..13a440b0 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -4,7 +4,9 @@ matplotlib shapely scipy sklearn +sphinx sphinx-gallery +sphinx_rtd_theme fastdtw plotly networkx @@ -12,4 +14,5 @@ seaborn torch pytest h5py +ipython rpy2 \ No newline at end of file From 7ffb2969d8e5bc3f04b271f2df9ad52bda4fa4f3 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Fri, 1 Jan 2021 16:09:22 +0000 Subject: [PATCH 206/211] Update docs example files --- .../reference/traja.TrajaDataFrame.examples | 75 +++++++++++++++++-- docs/source/reference/traja.generate.examples | 55 ++++++-------- 2 files changed, 88 insertions(+), 42 deletions(-) diff --git a/docs/source/reference/traja.TrajaDataFrame.examples b/docs/source/reference/traja.TrajaDataFrame.examples index 5b97ec7e..7ff8e5a8 100644 --- a/docs/source/reference/traja.TrajaDataFrame.examples +++ b/docs/source/reference/traja.TrajaDataFrame.examples @@ -1,7 +1,7 @@ Examples using ``traja.TrajaDataFrame`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. raw:: html @@ -9,9 +9,10 @@ Examples using ``traja.TrajaDataFrame`` .. only:: html - .. figure:: /gallery/images/thumb/sphx_glr_plot_3d_thumb.png + .. figure:: /gallery/images/thumb/sphx_glr_plot_3d_thumb.png + :alt: 3D Plotting with traja - :ref:`sphx_glr_gallery_plot_3d.py` + :ref:`sphx_glr_gallery_plot_3d.py` .. raw:: html @@ -19,17 +20,18 @@ Examples using ``traja.TrajaDataFrame`` .. only:: not html - * :ref:`sphx_glr_gallery_plot_3d.py` + * :ref:`sphx_glr_gallery_plot_3d.py` .. raw:: html -
+
.. only:: html - .. figure:: /gallery/images/thumb/sphx_glr_plot_with_traja_thumb.png + .. figure:: /gallery/images/thumb/sphx_glr_plot_average_direction_thumb.png + :alt: Average direction for each grid cell - :ref:`sphx_glr_gallery_plot_with_traja.py` + :ref:`sphx_glr_gallery_plot_average_direction.py` .. raw:: html @@ -37,4 +39,61 @@ Examples using ``traja.TrajaDataFrame`` .. only:: not html - * :ref:`sphx_glr_gallery_plot_with_traja.py` + * :ref:`sphx_glr_gallery_plot_average_direction.py` + +.. raw:: html + +
+ +.. only:: html + + .. figure:: /gallery/images/thumb/sphx_glr_plot_collection_thumb.png + :alt: Plotting Multiple Trajectories + + :ref:`sphx_glr_gallery_plot_collection.py` + +.. raw:: html + +
+ +.. only:: not html + + * :ref:`sphx_glr_gallery_plot_collection.py` + +.. raw:: html + +
+ +.. only:: html + + .. figure:: /gallery/images/thumb/sphx_glr_plot_comparing_thumb.png + :alt: Comparing + + :ref:`sphx_glr_gallery_plot_comparing.py` + +.. raw:: html + +
+ +.. only:: not html + + * :ref:`sphx_glr_gallery_plot_comparing.py` + +.. raw:: html + +
+ +.. only:: html + + .. figure:: /gallery/images/thumb/sphx_glr_plot_grid_thumb.png + :alt: Plotting trajectories on a grid + + :ref:`sphx_glr_gallery_plot_grid.py` + +.. raw:: html + +
+ +.. only:: not html + + * :ref:`sphx_glr_gallery_plot_grid.py` diff --git a/docs/source/reference/traja.generate.examples b/docs/source/reference/traja.generate.examples index dc9caec3..eefc59cd 100644 --- a/docs/source/reference/traja.generate.examples +++ b/docs/source/reference/traja.generate.examples @@ -1,7 +1,7 @@ Examples using ``traja.generate`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. raw:: html @@ -9,9 +9,10 @@ Examples using ``traja.generate`` .. only:: html - .. figure:: /gallery/images/thumb/sphx_glr_plot_3d_thumb.png + .. figure:: /gallery/images/thumb/sphx_glr_plot_3d_thumb.png + :alt: 3D Plotting with traja - :ref:`sphx_glr_gallery_plot_3d.py` + :ref:`sphx_glr_gallery_plot_3d.py` .. raw:: html @@ -19,7 +20,7 @@ Examples using ``traja.generate`` .. only:: not html - * :ref:`sphx_glr_gallery_plot_3d.py` + * :ref:`sphx_glr_gallery_plot_3d.py` .. raw:: html @@ -27,9 +28,10 @@ Examples using ``traja.generate`` .. only:: html - .. figure:: /gallery/images/thumb/sphx_glr_plot_average_direction_thumb.png + .. figure:: /gallery/images/thumb/sphx_glr_plot_average_direction_thumb.png + :alt: Average direction for each grid cell - :ref:`sphx_glr_gallery_plot_average_direction.py` + :ref:`sphx_glr_gallery_plot_average_direction.py` .. raw:: html @@ -37,7 +39,7 @@ Examples using ``traja.generate`` .. only:: not html - * :ref:`sphx_glr_gallery_plot_average_direction.py` + * :ref:`sphx_glr_gallery_plot_average_direction.py` .. raw:: html @@ -45,9 +47,10 @@ Examples using ``traja.generate`` .. only:: html - .. figure:: /gallery/images/thumb/sphx_glr_plot_collection_thumb.png + .. figure:: /gallery/images/thumb/sphx_glr_plot_collection_thumb.png + :alt: Plotting Multiple Trajectories - :ref:`sphx_glr_gallery_plot_collection.py` + :ref:`sphx_glr_gallery_plot_collection.py` .. raw:: html @@ -55,7 +58,7 @@ Examples using ``traja.generate`` .. only:: not html - * :ref:`sphx_glr_gallery_plot_collection.py` + * :ref:`sphx_glr_gallery_plot_collection.py` .. raw:: html @@ -63,9 +66,10 @@ Examples using ``traja.generate`` .. only:: html - .. figure:: /gallery/images/thumb/sphx_glr_plot_comparing_thumb.png + .. figure:: /gallery/images/thumb/sphx_glr_plot_comparing_thumb.png + :alt: Comparing - :ref:`sphx_glr_gallery_plot_comparing.py` + :ref:`sphx_glr_gallery_plot_comparing.py` .. raw:: html @@ -73,7 +77,7 @@ Examples using ``traja.generate`` .. only:: not html - * :ref:`sphx_glr_gallery_plot_comparing.py` + * :ref:`sphx_glr_gallery_plot_comparing.py` .. raw:: html @@ -81,9 +85,10 @@ Examples using ``traja.generate`` .. only:: html - .. figure:: /gallery/images/thumb/sphx_glr_plot_grid_thumb.png + .. figure:: /gallery/images/thumb/sphx_glr_plot_grid_thumb.png + :alt: Plotting trajectories on a grid - :ref:`sphx_glr_gallery_plot_grid.py` + :ref:`sphx_glr_gallery_plot_grid.py` .. raw:: html @@ -91,22 +96,4 @@ Examples using ``traja.generate`` .. only:: not html - * :ref:`sphx_glr_gallery_plot_grid.py` - -.. raw:: html - -
- -.. only:: html - - .. figure:: /gallery/images/thumb/sphx_glr_plot_with_traja_thumb.png - - :ref:`sphx_glr_gallery_plot_with_traja.py` - -.. raw:: html - -
- -.. only:: not html - - * :ref:`sphx_glr_gallery_plot_with_traja.py` + * :ref:`sphx_glr_gallery_plot_grid.py` From d3577ef71ce43fac50809bcb2c7718c6c19ec05e Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Fri, 1 Jan 2021 16:29:00 +0000 Subject: [PATCH 207/211] Testing Travis dependencies workaround --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2277f67c..a7a01da4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,7 @@ install: - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION - source activate test-environment - pip install -r requirements-dev.txt +- pip install -r docs/requirements.txt - pip install --upgrade pytest flake8 sphinx - pip install . script: From 8b45766ef18f12ee70c24260f400ef429080c9fe Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Fri, 1 Jan 2021 17:18:08 +0000 Subject: [PATCH 208/211] Remove docs rpy2 requirement --- docs/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 13a440b0..6ea846eb 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -14,5 +14,4 @@ seaborn torch pytest h5py -ipython -rpy2 \ No newline at end of file +ipython \ No newline at end of file From 3635e133c223f25d846deac8424ee6aa1bbdeb59 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Fri, 1 Jan 2021 18:27:17 +0000 Subject: [PATCH 209/211] Remove obsolete, broken config --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a7a01da4..870bdc9f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ install: - pip install . script: - cd docs && make doctest && cd .. - - py.test . --cov-report term --cov=traja --ignore tests/test_rutils.py + - py.test . after_success: - codecov From 4b08011aaf95c809589493df0b1402e560e048d4 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sat, 2 Jan 2021 13:36:05 +0000 Subject: [PATCH 210/211] Fix Python 3.8 changed-behaviour bug --- .travis.yml | 2 +- traja/trajectory.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 870bdc9f..a29afda5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,8 @@ dist: xenial language: python python: - - '3.6' - '3.7' + - '3.8' git: depth: false diff --git a/traja/trajectory.py b/traja/trajectory.py index ffedc1e2..285d7f68 100644 --- a/traja/trajectory.py +++ b/traja/trajectory.py @@ -730,9 +730,9 @@ def resample_time(trj: TrajaDataFrame, step_time: str, new_fps: Optional[bool] = x y time 1970-01-01 00:00:00.000 0.000000 0.000000 - 1970-01-01 00:00:00.050 0.999571 4.293384 + 1970-01-01 00:00:00.050 0.919113 4.022971 1970-01-01 00:00:00.100 -1.298510 5.423373 - 1970-01-01 00:00:00.150 -6.056916 4.874502 + 1970-01-01 00:00:00.150 -6.057524 4.708803 1970-01-01 00:00:00.200 -10.347759 2.108385 """ From e814c3e241f8c971d8b3496123eaae3d3e4d1272 Mon Sep 17 00:00:00 2001 From: Wolf Byttner Date: Sat, 2 Jan 2021 14:20:28 +0000 Subject: [PATCH 211/211] Set Pandas version >= 1.2.0 --- docs/requirements.txt | 2 +- requirements-dev.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 6ea846eb..31882dd5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -pandas +pandas>=1.2.0 numpy==1.18.5 matplotlib shapely diff --git a/requirements-dev.txt b/requirements-dev.txt index b82a4ec7..52d6cc2f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -pandas +pandas>=1.2.0 numpy==1.18.5 matplotlib shapely diff --git a/requirements.txt b/requirements.txt index d27f79da..1bd588ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pandas +pandas>=1.2.0 numpy==1.18.5 matplotlib shapely