fix graykode#30 vocab_size to seq_len

typanda · Jul 8, 2019 · 6e171b9 · 6e171b9
1 parent 3b3a80d
commit 6e171b9
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 12 deletions.
diff --git a/5-1.Transformer/Transformer(Greedy_decoder)-Torch.py b/5-1.Transformer/Transformer(Greedy_decoder)-Torch.py
@@ -138,7 +138,7 @@ class Encoder(nn.Module):
     def __init__(self):
         super(Encoder, self).__init__()
         self.src_emb = nn.Embedding(src_vocab_size, d_model)
-        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(src_vocab_size, d_model),freeze=True)
+        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(src_len+1, d_model),freeze=True)
         self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])
 
     def forward(self, enc_inputs): # enc_inputs : [batch_size x source_len]
@@ -154,7 +154,7 @@ class Decoder(nn.Module):
     def __init__(self):
         super(Decoder, self).__init__()
         self.tgt_emb = nn.Embedding(tgt_vocab_size, d_model)
-        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(tgt_vocab_size, d_model),freeze=True)
+        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(tgt_len+1, d_model),freeze=True)
         self.layers = nn.ModuleList([DecoderLayer() for _ in range(n_layers)])
 
     def forward(self, dec_inputs, enc_inputs, enc_outputs): # dec_inputs : [batch_size x target_len]

diff --git a/5-1.Transformer/Transformer(Greedy_decoder)_Torch.ipynb b/5-1.Transformer/Transformer(Greedy_decoder)_Torch.ipynb
@@ -93,6 +93,11 @@
     }
    ],
    "source": [
+    "'''\n",
+    "  code by Tae Hwan Jung(Jeff Jung) @graykode, Derek Miller @dmmiller612\n",
+    "  Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch\n",
+    "              https://github.com/JayParks/transformer\n",
+    "'''\n",
     "'''\n",
     "  code by Tae Hwan Jung(Jeff Jung) @graykode, Derek Miller @dmmiller612\n",
     "  Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch\n",
@@ -233,7 +238,7 @@
     "    def __init__(self):\n",
     "        super(Encoder, self).__init__()\n",
     "        self.src_emb = nn.Embedding(src_vocab_size, d_model)\n",
-    "        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(src_vocab_size, d_model),freeze=True)\n",
+    "        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(src_len+1, d_model),freeze=True)\n",
     "        self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])\n",
     "\n",
     "    def forward(self, enc_inputs): # enc_inputs : [batch_size x source_len]\n",
@@ -249,7 +254,7 @@
     "    def __init__(self):\n",
     "        super(Decoder, self).__init__()\n",
     "        self.tgt_emb = nn.Embedding(tgt_vocab_size, d_model)\n",
-    "        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(tgt_vocab_size, d_model),freeze=True)\n",
+    "        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(tgt_len+1, d_model),freeze=True)\n",
     "        self.layers = nn.ModuleList([DecoderLayer() for _ in range(n_layers)])\n",
     "\n",
     "    def forward(self, dec_inputs, enc_inputs, enc_outputs): # dec_inputs : [batch_size x target_len]\n",
@@ -351,10 +356,20 @@
   },
   "kernelspec": {
    "name": "python3",
+   "language": "python",
    "display_name": "Python 3"
   },
-  "accelerator": "GPU"
+  "accelerator": "GPU",
+  "pycharm": {
+   "stem_cell": {
+    "cell_type": "raw",
+    "source": [],
+    "metadata": {
+     "collapsed": false
+    }
+   }
+  }
  },
  "nbformat": 4,
  "nbformat_minor": 0
-}
+}
diff --git a/5-1.Transformer/Transformer-Torch.py b/5-1.Transformer/Transformer-Torch.py
@@ -137,7 +137,7 @@ class Encoder(nn.Module):
     def __init__(self):
         super(Encoder, self).__init__()
         self.src_emb = nn.Embedding(src_vocab_size, d_model)
-        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(src_vocab_size, d_model),freeze=True)
+        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(src_len+1, d_model),freeze=True)
         self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])
 
     def forward(self, enc_inputs): # enc_inputs : [batch_size x source_len]
@@ -153,7 +153,7 @@ class Decoder(nn.Module):
     def __init__(self):
         super(Decoder, self).__init__()
         self.tgt_emb = nn.Embedding(tgt_vocab_size, d_model)
-        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(tgt_vocab_size, d_model),freeze=True)
+        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(tgt_len+1, d_model),freeze=True)
         self.layers = nn.ModuleList([DecoderLayer() for _ in range(n_layers)])
 
     def forward(self, dec_inputs, enc_inputs, enc_outputs): # dec_inputs : [batch_size x target_len]

diff --git a/5-1.Transformer/Transformer_Torch.ipynb b/5-1.Transformer/Transformer_Torch.ipynb
@@ -250,7 +250,7 @@
     "    def __init__(self):\n",
     "        super(Encoder, self).__init__()\n",
     "        self.src_emb = nn.Embedding(src_vocab_size, d_model)\n",
-    "        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(src_vocab_size, d_model),freeze=True)\n",
+    "        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(src_len+1, d_model),freeze=True)\n",
     "        self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])\n",
     "\n",
     "    def forward(self, enc_inputs): # enc_inputs : [batch_size x source_len]\n",
@@ -266,7 +266,7 @@
     "    def __init__(self):\n",
     "        super(Decoder, self).__init__()\n",
     "        self.tgt_emb = nn.Embedding(tgt_vocab_size, d_model)\n",
-    "        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(tgt_vocab_size, d_model),freeze=True)\n",
+    "        self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(tgt_len+1, d_model),freeze=True)\n",
     "        self.layers = nn.ModuleList([DecoderLayer() for _ in range(n_layers)])\n",
     "\n",
     "    def forward(self, dec_inputs, enc_inputs, enc_outputs): # dec_inputs : [batch_size x target_len]\n",
@@ -345,10 +345,20 @@
   },
   "kernelspec": {
    "name": "python3",
+   "language": "python",
    "display_name": "Python 3"
   },
-  "accelerator": "GPU"
+  "accelerator": "GPU",
+  "pycharm": {
+   "stem_cell": {
+    "cell_type": "raw",
+    "source": [],
+    "metadata": {
+     "collapsed": false
+    }
+   }
+  }
  },
  "nbformat": 4,
  "nbformat_minor": 0
-}
+}