Merge pull request #1 from RicherMans/github2

Updated to use ignite framework, simplifies a lot of stats

README.md

@@ -8,24 +8,27 @@ Source code for the paper [Text-based Depression Detection: What Triggers An Ale
 The required python packages can be found in `requirements.txt`.
 
 ```
-tqdm==4.24.0
-allennlp==0.8.2
-tableprint==0.8.0
-tabulate==0.8.2
-fire==0.1.3
-nltk==3.3
+torch==1.2.0
 kaldi_io==0.9.1
-scipy==1.2.1
-torchnet==0.0.4
-pandas==0.24.1
-numpy==1.16.2
-bert_serving_client==1.8.3
-imbalanced_learn==0.4.3
-torch==0.4.1.post2
-gensim==3.7.1
+bert_serving_server==1.9.6
+pytorch_ignite==0.2.0
+numpy==1.16.4
+librosa==0.7.0
+tabulate==0.8.3
+mistletoe==0.7.2
+scipy==1.3.0
+tqdm==4.32.2
+pandas==0.24.2
+fire==0.1.3
+imbalanced_learn==0.5.0
+allennlp==0.8.5
+gensim==3.8.0
+ignite==1.1.0
 imblearn==0.0
-scikit_learn==0.20.3
-PyYAML==5.1
+nltk==3.4.5
+plotnine==0.6.0
+scikit_learn==0.21.3
+PyYAML==5.1.2
 ```
 
 
@@ -70,14 +73,9 @@ The main script of this repo is `run.py`.
 The code is centered around the config files placed at `config/`. Each parameter in these files can be modified for each run using google-fire e.g., if one opts to run a different model, just pass `--model GRU`. 
 
 `run.py` the following options ( ran as `python run.py OPTION`):
-* `train`: Trains a model given a config file (default is `config/text_lstm_deep.yaml`)
-* `stats`: Prints the evaluation results on the development set
-* `trainstats`: Convenience function to run train and evaluate in one.
-* `search`: Parameter search for learning rate, momentum and nesterov (SGD)
-* `searchadam`: Learning rate search for adam optimizer
-* `ex`: Extracts features from a given network (not finished),
-* `fwd`: Debugging function to forward some features through a network
-* `fuse`: Fusion of two models or more. Fusion is done by averaging each output.
+* `train`: Trains a model given a config file (default is `config/text_lstm_deep.yaml`).
+* `evaluate`: Evaluates a given trained model directory. Just pass the result of `train` to it.
+* `evaluates`: Same as evaluate but runs multiple passed directories e.g., passed as glob (`experiment/*/*/*`), and returns an outputfile as well as a table report of the results. Useful for multiple runs with different seeds.
 
 ## Notes
 

dataset.py

@@ -18,7 +18,6 @@ class ListDataset(torch.utils.data.Dataset):
     Arguments:
         *lists (List): List that have the same size of the first dimension.
     """
-
     def __init__(self, *lists):
         assert all(len(lists[0]) == len(a_list) for a_list in lists)
         self.lists = lists
@@ -30,38 +29,32 @@ def __len__(self):
         return len(self.lists[0])
 
 
-def seq_collate_fn(data_batches):
-    """seq_collate_fn
-
-    Helper function for torch.utils.data.Dataloader
-
-    :param data_batches: iterateable
-    """
-    data_batches.sort(key=lambda x: len(x[0]), reverse=True)
-
-    def merge_seq(dataseq, dim=0):
-        lengths = [seq.shape for seq in dataseq]
-        # Assuming duration is given in the first dimension of each sequence
-        maxlengths = tuple(np.max(lengths, axis=dim))
-
-        # For the case that the lenthts are 2dimensional
-        lengths = np.array(lengths)[:, dim]
-        # batch_mean = np.mean(np.concatenate(dataseq),axis=0, keepdims=True)
-        # padded = np.tile(batch_mean, (len(dataseq), maxlengths[0], 1))
-        padded = np.zeros((len(dataseq),) + maxlengths)
-        for i, seq in enumerate(dataseq):
-            end = lengths[i]
-            padded[i, :end] = seq[:end]
-        return padded, lengths
-    features, targets = zip(*data_batches)
-    features_seq, feature_lengths = merge_seq(features)
-    return torch.from_numpy(features_seq), torch.tensor(targets)
-
-
-def create_dataloader(
-        kaldi_string, label_dict, transform=None,
-        batch_size: int = 16, num_workers: int = 1, shuffle: bool = True
-):
+def pad(tensorlist, batch_first=True, padding_value=0.):
+    # In case we have 3d tensor in each element, squeeze the first dim (usually 1)
+    if len(tensorlist[0].shape) == 3:
+        tensorlist = [ten.squeeze() for ten in tensorlist]
+    # In case of len == 1 padding will throw an error
+    if len(tensorlist) == 1:
+        return torch.as_tensor(tensorlist)
+    tensorlist = [torch.as_tensor(item) for item in tensorlist]
+    return torch.nn.utils.rnn.pad_sequence(tensorlist,
+                                           batch_first=batch_first,
+                                           padding_value=padding_value)
+
+
+def sequential_collate(batches):
+    # sort length wise
+    batches.sort(key=lambda x: len(x), reverse=True)
+    features, targets = zip(*batches)
+    return pad(features), torch.as_tensor(targets)
+
+
+def create_dataloader(kaldi_string,
+                      label_dict,
+                      transform=None,
+                      batch_size: int = 16,
+                      num_workers: int = 1,
+                      shuffle: bool = True):
     """create_dataloader
 
     :param kaldi_string: copy-feats input
@@ -83,25 +76,30 @@ def valid_feat(item):
     features = []
     labels = []
     # Directly filter out all utterances without labels
-    for idx, (k, feat) in enumerate(filter(valid_feat, kaldi_io.read_mat_ark(kaldi_string))):
+    for idx, (k, feat) in enumerate(
+            filter(valid_feat, kaldi_io.read_mat_ark(kaldi_string))):
         if transform:
             feat = transform(feat)
         features.append(feat)
         labels.append(label_dict[k])
     assert len(features) > 0, "No features were found, are the labels correct?"
     # Shuffling means that this is training dataset, so oversample
     if shuffle:
-        random_oversampler = RandomOverSampler(random_state=0)
+        sampler = RandomOverSampler()
         # Assume that label is Score, Binary, we take the binary to oversample
         sample_index = 1 if len(labels[0]) == 2 else 0
         # Dummy X data, y is the binary label
-        _, _ = random_oversampler.fit_resample(
-            torch.ones(len(features), 1), [l[sample_index] for l in labels])
+        _, _ = sampler.fit_resample(torch.ones(len(features), 1),
+                                    [l[sample_index] for l in labels])
         # Get the indices for the sampled data
-        indicies = random_oversampler.sample_indices_
+        indicies = sampler.sample_indices_
         # reindex, new data is oversampled in the minority class
-        features, labels = [features[id]
-                            for id in indicies], [labels[id] for id in indicies]
+        features, labels = [features[id] for id in indicies
+                            ], [labels[id] for id in indicies]
     dataset = ListDataset(features, labels)
     # Configure weights to reduce number of unseen utterances
-    return data.DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, collate_fn=seq_collate_fn, shuffle=shuffle)
+    return data.DataLoader(dataset,
+                           batch_size=batch_size,
+                           num_workers=num_workers,
+                           collate_fn=sequential_collate,
+                           shuffle=shuffle)

losses.py

@@ -4,9 +4,7 @@
 
 
 class MAELoss(torch.nn.Module):
-
     """Docstring for MAELoss. """
-
     def __init__(self):
         """TODO: to be defined1. """
         torch.nn.Module.__init__(self)
@@ -19,9 +17,7 @@ def forward(self, input, target):
 
 
 class RMSELoss(torch.nn.Module):
-
     """Docstring for RMSELoss. """
-
     def __init__(self):
         """ """
         torch.nn.Module.__init__(self)
@@ -33,10 +29,35 @@ def forward(self, input, target):
         return torch.sqrt(self.loss(input, target))
 
 
-class MSELoss(torch.nn.Module):
+class CosineLoss(torch.nn.Module):
+    """description"""
+    def __init__(self):
+        torch.nn.Module.__init__(self)
+        self.norm = torch.nn.functional.normalize
 
-    """Docstring for MSELoss. """
+    @staticmethod
+    def label_to_onehot(tar, nlabels=2):
+        if tar.ndimension() == 1:
+            tar = tar.unsqueeze(-1)  # add singleton [B, 1]
+        tar_onehot = tar.new_zeros((len(tar), nlabels)).detach()
+        tar_onehot.scatter_(1, tar.long(), 1)
+        return tar_onehot.float()
 
+    def forward(self, input, target):
+        target = CosineLoss.label_to_onehot(target)
+        if input.ndimension() == 2:
+            input = input.unsqueeze(-1)  # add singleton dimension
+        if target.ndimension() == 2:
+            target = target.unsqueeze(1)  # Add singleton dimension
+        norm_input = self.norm(input, p=2, dim=1)
+        #Input shape: [Bx1xC]
+        #Target shape: [BxCx1]
+        cos_loss = 1 - torch.bmm(target, norm_input)
+        return cos_loss.mean()
+
+
+class MSELoss(torch.nn.Module):
+    """Docstring for MSELoss. """
     def __init__(self):
         """ """
         torch.nn.Module.__init__(self)
@@ -49,24 +70,18 @@ def forward(self, input, target):
 
 
 class HuberLoss(torch.nn.Module):
-
     """Docstring for HuberLoss. """
-
     def __init__(self):
         """ """
         torch.nn.Module.__init__(self)
-
         self.loss = torch.nn.SmoothL1Loss()
 
     def forward(self, input, target):
-        target = target.float()
-        return self.loss(input, target)
+        return self.loss(input, target.float())
 
 
 class DepressionLoss(torch.nn.Module):
-
     """Docstring for DepressionLoss. """
-
     def __init__(self):
         """ """
         torch.nn.Module.__init__(self)
@@ -75,45 +90,57 @@ def __init__(self):
         self.bin_loss = BCEWithLogitsLoss()
 
     def forward(self, input, target):
-        return self.score_loss(input[:, 0], target[:, 0]) + self.bin_loss(input[:, 1], target[:, 1])
-
+        return self.score_loss(input[:, 0], target[:, 0]) + self.bin_loss(
+            input[:, 1], target[:, 1])
 
-class DepressionLossMSE(torch.nn.Module):
 
+class DepressionLossSmoothCos(torch.nn.Module):
     """Docstring for DepressionLoss. """
-
     def __init__(self):
         """ """
         torch.nn.Module.__init__(self)
 
-        self.score_loss = MSELoss()
-        self.bin_loss = BCEWithLogitsLoss()
+        self.score_loss = HuberLoss()
+        self.cos_loss = CosineLoss()
 
     def forward(self, input, target):
-        score_loss = self.score_loss(input[:, 0], target[:, 0])
-        binary_loss = self.bin_loss(input[:, 1], target[:, 1])
-        return score_loss + binary_loss
+        target = target.long()
+        phq8_pred, phq8_tar = input[:, 0], target[:, 0]
+        binary_pred, binary_tar = input[:, 1:3], target[:, 1]
+        return self.score_loss(phq8_pred, phq8_tar) + self.cos_loss(
+            binary_pred, binary_tar)
 
 
 class DepressionLossSmooth(torch.nn.Module):
-
     """Docstring for DepressionLoss. """
-
-    def __init__(self):
+    def __init__(self, reduction='sum'):
         """ """
         torch.nn.Module.__init__(self)
 
         self.score_loss = HuberLoss()
-        self.bin_loss = BCEWithLogitsLoss()
+        self.bce = BCEWithLogitsLoss()
+        self.weight = torch.nn.Parameter(torch.tensor(0.))
+        self.reduction = reduction
+        self.eps = 0.01
 
     def forward(self, input, target):
-        return self.score_loss(input[:, 0], target[:, 0]) + self.bin_loss(input[:, 1], target[:, 1])
+        phq8_pred, phq8_tar = input[:, 0], target[:, 0]
+        binary_pred, binary_tar = input[:, 1], target[:, 1]
+        score_loss, bin_loss = self.score_loss(phq8_pred, phq8_tar), self.bce(
+            binary_pred, binary_tar)
+        weight = torch.clamp(torch.sigmoid(self.weight),
+                             min=self.eps,
+                             max=1 - self.eps)
+        stacked_loss = (weight * score_loss) + ((1 - weight) * bin_loss)
+        if self.reduction == 'mean':
+            stacked_loss = stacked_loss.mean()
+        elif self.reduction == 'sum':
+            stacked_loss = stacked_loss.sum()
+        return stacked_loss
 
 
 class BCEWithLogitsLoss(torch.nn.Module):
-
     """Docstring for BCEWithLogitsLoss. """
-
     def __init__(self):
         """TODO: to be defined1. """
         torch.nn.Module.__init__(self)