From 5c3870fb1a49fc0a9c181683a839c3100890bce2 Mon Sep 17 00:00:00 2001 From: Daksh Date: Sat, 19 Dec 2020 16:21:33 +0100 Subject: [PATCH 1/2] first version --- rasa/utils/tensorflow/layers.py | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/rasa/utils/tensorflow/layers.py b/rasa/utils/tensorflow/layers.py index a9017094e945..aba25b46f86f 100644 --- a/rasa/utils/tensorflow/layers.py +++ b/rasa/utils/tensorflow/layers.py @@ -853,16 +853,33 @@ def _loss_softmax( ) -> tf.Tensor: """Define softmax loss.""" - logits = tf.concat( + softmax_logits = tf.concat( + [sim_pos, sim_neg_il, sim_neg_ll, sim_neg_ii, sim_neg_li], axis=-1 + ) + + sigmoid_logits = tf.concat( [sim_pos, sim_neg_il, sim_neg_ll, sim_neg_ii, sim_neg_li], axis=-1 ) # create label_ids for softmax - label_ids = tf.zeros_like(logits[..., 0], tf.int32) + softmax_label_ids = tf.zeros_like(softmax_logits[..., 0], tf.int32) + + sigmoid_label_ids = tf.concat( + [ + tf.expand_dims(tf.ones_like(sigmoid_logits[..., 0], tf.float32), -1), + tf.zeros_like(sigmoid_logits[..., 1:], tf.float32), + ], + axis=-1, + ) - loss = tf.nn.sparse_softmax_cross_entropy_with_logits( - labels=label_ids, logits=logits + softmax_loss = tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=softmax_label_ids, logits=softmax_logits ) + sigmoid_loss = tf.nn.sigmoid_cross_entropy_with_logits( + labels=sigmoid_label_ids, logits=sigmoid_logits + ) + + loss = softmax_loss + tf.reduce_mean(sigmoid_loss, axis=-1) if self.scale_loss: # in case of cross entropy log_likelihood = -loss @@ -878,6 +895,14 @@ def _loss_softmax( else: loss = tf.reduce_mean(loss, axis=-1) + tf.print( + tf.reduce_mean(sim_pos), + tf.reduce_mean(sim_neg_ii), + tf.reduce_mean(sim_neg_il), + tf.reduce_mean(sim_neg_ll), + tf.reduce_mean(sim_neg_li), + ) + # average the loss over the batch return tf.reduce_mean(loss) From 8cff4ec91f953f465c42e5f634d07eb3247c5ea6 Mon Sep 17 00:00:00 2001 From: Daksh Date: Mon, 21 Dec 2020 16:23:41 +0100 Subject: [PATCH 2/2] remove extra terms from softmax --- rasa/utils/tensorflow/layers.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/rasa/utils/tensorflow/layers.py b/rasa/utils/tensorflow/layers.py index aba25b46f86f..07f16851edde 100644 --- a/rasa/utils/tensorflow/layers.py +++ b/rasa/utils/tensorflow/layers.py @@ -853,9 +853,7 @@ def _loss_softmax( ) -> tf.Tensor: """Define softmax loss.""" - softmax_logits = tf.concat( - [sim_pos, sim_neg_il, sim_neg_ll, sim_neg_ii, sim_neg_li], axis=-1 - ) + softmax_logits = tf.concat([sim_pos, sim_neg_il, sim_neg_li], axis=-1) sigmoid_logits = tf.concat( [sim_pos, sim_neg_il, sim_neg_ll, sim_neg_ii, sim_neg_li], axis=-1 @@ -864,7 +862,7 @@ def _loss_softmax( # create label_ids for softmax softmax_label_ids = tf.zeros_like(softmax_logits[..., 0], tf.int32) - sigmoid_label_ids = tf.concat( + sigmoid_labels = tf.concat( [ tf.expand_dims(tf.ones_like(sigmoid_logits[..., 0], tf.float32), -1), tf.zeros_like(sigmoid_logits[..., 1:], tf.float32), @@ -876,7 +874,7 @@ def _loss_softmax( labels=softmax_label_ids, logits=softmax_logits ) sigmoid_loss = tf.nn.sigmoid_cross_entropy_with_logits( - labels=sigmoid_label_ids, logits=sigmoid_logits + labels=sigmoid_labels, logits=sigmoid_logits ) loss = softmax_loss + tf.reduce_mean(sigmoid_loss, axis=-1)