Add epsilon to prevent NaNs with tiny attention weights

jkhlot · Oct 18, 2017 · 1aadc4f · 1aadc4f
1 parent ac27e25
commit 1aadc4f
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/deepmoji/attlayer.py b/deepmoji/attlayer.py
@@ -44,7 +44,7 @@ def call(self, x, mask=None):
         if mask is not None:
             mask = K.cast(mask, K.floatx())
             ai = ai * mask
-        att_weights = ai / K.sum(ai, axis=1, keepdims=True)
+        att_weights = ai / (K.sum(ai, axis=1, keepdims=True) + K.epsilon())
         weighted_input = x * K.expand_dims(att_weights)
         result = K.sum(weighted_input, axis=1)
         if self.return_attention: