add graykode#10 ipynb file

Author: graykode

1-1.NNLM/NNLM_Torch.ipynb

@@ -19,10 +19,10 @@
       "metadata": {
         "id": "mvlw9p3tPJjr",
         "colab_type": "code",
-        "outputId": "a9d7624b-4a3b-4078-9a89-11c2d6d177d5",
+        "outputId": "2a4a8f52-315e-42b3-9d49-e9c7c3358979",
         "colab": {
           "base_uri": "https://localhost:8080/",
-          "height": 161
+          "height": 129
         }
       },
       "cell_type": "code",
@@ -45,8 +45,9 @@
         "n_class = len(word_dict) # number of Vocabulary\n",
         "\n",
         "# NNLM Parameter\n",
-        "n_step = 2 # n-1\n",
-        "n_hidden = 2 # h\n",
+        "n_step = 2 # n-1 in paper\n",
+        "n_hidden = 2 # h in paper\n",
+        "m = 2 # m in paper\n",
         "\n",
         "def make_batch(sentences):\n",
         "    input_batch = []\n",
@@ -57,7 +58,7 @@
         "        input = [word_dict[n] for n in word[:-1]]\n",
         "        target = word_dict[word[-1]]\n",
         "\n",
-        "        input_batch.append(np.eye(n_class)[input])\n",
+        "        input_batch.append(input)\n",
         "        target_batch.append(target)\n",
         "\n",
         "    return input_batch, target_batch\n",
@@ -66,17 +67,18 @@
         "class NNLM(nn.Module):\n",
         "    def __init__(self):\n",
         "        super(NNLM, self).__init__()\n",
-        "\n",
-        "        self.H = nn.Parameter(torch.randn(n_step * n_class, n_hidden).type(dtype))\n",
-        "        self.W = nn.Parameter(torch.randn(n_step * n_class, n_class).type(dtype))\n",
+        "        self.C = nn.Embedding(n_class, m)\n",
+        "        self.H = nn.Parameter(torch.randn(n_step * m, n_hidden).type(dtype))\n",
+        "        self.W = nn.Parameter(torch.randn(n_step * m, n_class).type(dtype))\n",
         "        self.d = nn.Parameter(torch.randn(n_hidden).type(dtype))\n",
         "        self.U = nn.Parameter(torch.randn(n_hidden, n_class).type(dtype))\n",
         "        self.b = nn.Parameter(torch.randn(n_class).type(dtype))\n",
         "\n",
         "    def forward(self, X):\n",
-        "        input = X.view(-1, n_step * n_class) # [batch_size, n_step * n_class]\n",
-        "        tanh = nn.functional.tanh(self.d + torch.mm(input, self.H)) # [batch_size, n_hidden]\n",
-        "        output = self.b + torch.mm(input, self.W) + torch.mm(tanh, self.U) # [batch_size, n_class]\n",
+        "        X = self.C(X)\n",
+        "        X = X.view(-1, n_step * m) # [batch_size, n_step * n_class]\n",
+        "        tanh = torch.tanh(self.d + torch.mm(X, self.H)) # [batch_size, n_hidden]\n",
+        "        output = self.b + torch.mm(X, self.W) + torch.mm(tanh, self.U) # [batch_size, n_class]\n",
         "        return output\n",
         "\n",
         "model = NNLM()\n",
@@ -85,7 +87,7 @@
         "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
         "\n",
         "input_batch, target_batch = make_batch(sentences)\n",
-        "input_batch = Variable(torch.Tensor(input_batch))\n",
+        "input_batch = Variable(torch.LongTensor(input_batch))\n",
         "target_batch = Variable(torch.LongTensor(target_batch))\n",
         "\n",
         "# Training\n",
@@ -108,24 +110,16 @@
         "# Test\n",
         "print([sen.split()[:2] for sen in sentences], '->', [number_dict[n.item()] for n in predict.squeeze()])"
       ],
-      "execution_count": 0,
+      "execution_count": 1,
       "outputs": [
         {
           "output_type": "stream",
           "text": [
-            "/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py:1320: UserWarning: nn.functional.tanh is deprecated. Use torch.tanh instead.\n",
-            "  warnings.warn(\"nn.functional.tanh is deprecated. Use torch.tanh instead.\")\n"
-          ],
-          "name": "stderr"
-        },
-        {
-          "output_type": "stream",
-          "text": [
-            "Epoch: 1000 cost = 0.283353\n",
-            "Epoch: 2000 cost = 0.058013\n",
-            "Epoch: 3000 cost = 0.023128\n",
-            "Epoch: 4000 cost = 0.011383\n",
-            "Epoch: 5000 cost = 0.006090\n",
+            "Epoch: 1000 cost = 0.147408\n",
+            "Epoch: 2000 cost = 0.026562\n",
+            "Epoch: 3000 cost = 0.010481\n",
+            "Epoch: 4000 cost = 0.005095\n",
+            "Epoch: 5000 cost = 0.002696\n",
             "[['i', 'like'], ['i', 'love'], ['i', 'hate']] -> ['dog', 'coffee', 'milk']\n"
           ],
           "name": "stdout"