Merge pull request graykode#4 from likejazz/patch-1

Add `W` parameter as written in paper.

Author: graykode

1-1.NNLM/NNLM-Torch.py

@@ -39,14 +39,15 @@ def __init__(self):
         super(NNLM, self).__init__()
 
         self.H = nn.Parameter(torch.randn(n_step * n_class, n_hidden).type(dtype))
+        self.W = nn.Parameter(torch.randn(n_step * n_class, n_class).type(dtype))
         self.d = nn.Parameter(torch.randn(n_hidden).type(dtype))
         self.U = nn.Parameter(torch.randn(n_hidden, n_class).type(dtype))
         self.b = nn.Parameter(torch.randn(n_class).type(dtype))
 
     def forward(self, X):
         input = X.view(-1, n_step * n_class) # [batch_size, n_step * n_class]
         tanh = nn.functional.tanh(self.d + torch.mm(input, self.H)) # [batch_size, n_hidden]
-        output = torch.mm(tanh, self.U) + self.b # [batch_size, n_class]
+        output = self.b + torch.mm(input, self.W) + torch.mm(tanh, self.U) # [batch_size, n_class]
         return output
 
 model = NNLM()
@@ -76,4 +77,4 @@ def forward(self, X):
 predict = model(input_batch).data.max(1, keepdim=True)[1]
 
 # Test
-print([sen.split()[:2] for sen in sentences], '->', [number_dict[n.item()] for n in predict.squeeze()])
+print([sen.split()[:2] for sen in sentences], '->', [number_dict[n.item()] for n in predict.squeeze()])