From 2cc31240e5d90656dc7fad4310fd11938c812205 Mon Sep 17 00:00:00 2001 From: graykode Date: Fri, 14 Aug 2020 16:13:01 +0900 Subject: [PATCH 1/3] #57 (refactor) tensorflow v1 codes are archived in archive folder, torch is main code, remove all jupyter notebook files --- 1-1.NNLM/{NNLM-Torch.py => NNLM.py} | 0 1-1.NNLM/NNLM_Tensor.ipynb | 115 ------ 1-1.NNLM/NNLM_Torch.ipynb | 130 ------ ...tmax).py => Word2Vec-Skipgram(Softmax).py} | 0 .../Word2Vec_Skipgram_Tensor(NCE_loss).ipynb | 138 ------- .../Word2Vec_Skipgram_Tensor(Softmax).ipynb | 136 ------ .../Word2Vec_Skipgram_Torch(Softmax).ipynb | 152 ------- 2-1.TextCNN/{TextCNN-Torch.py => TextCNN.py} | 0 2-1.TextCNN/TextCNN_Tensor.ipynb | 142 ------- 2-1.TextCNN/TextCNN_Torch.ipynb | 165 -------- 3-1.TextRNN/{TextRNN-Torch.py => TextRNN.py} | 0 3-1.TextRNN/TextRNN_Tensor.ipynb | 121 ------ 3-1.TextRNN/TextRNN_Torch.ipynb | 136 ------ .../{TextLSTM-Torch.py => TextLSTM.py} | 0 3-2.TextLSTM/TextLSTM_Tensor.ipynb | 122 ------ 3-2.TextLSTM/TextLSTM_Torch.ipynb | 130 ------ 3-3.Bi-LSTM/{Bi-LSTM-Torch.py => Bi-LSTM.py} | 0 3-3.Bi-LSTM/Bi_LSTM_Tensor.ipynb | 127 ------ 3-3.Bi-LSTM/Bi_LSTM_Torch.ipynb | 150 ------- 4-1.Seq2Seq/{Seq2Seq-Torch.py => Seq2Seq.py} | 0 4-1.Seq2Seq/Seq2Seq_Tensor.ipynb | 149 ------- 4-1.Seq2Seq/Seq2Seq_Torch.ipynb | 172 -------- ...ention)-Torch.py => Seq2Seq(Attention).py} | 0 .../Seq2Seq(Attention)_Tensor.ipynb | 171 -------- .../Seq2Seq(Attention)_Torch.ipynb | 194 --------- ...ention)-Torch.py => Bi-LSTM(Attention).py} | 0 .../Bi_LSTM(Attention)_Tensor.ipynb | 152 ------- .../Bi_LSTM(Attention)_Torch.ipynb | 164 -------- ...orch.py => Transformer(Greedy_decoder).py} | 0 .../Transformer(Greedy_decoder)_Torch.ipynb | 391 ------------------ .../{Transformer-Torch.py => Transformer.py} | 0 5-1.Transformer/Transformer_Torch.ipynb | 380 ----------------- 5-2.BERT/{BERT-Torch.py => BERT.py} | 0 5-2.BERT/BERT_Torch.ipynb | 305 -------------- .../tensorflow/v1/1-1.NNLM/NNLM.py | 0 .../Word2Vec-Skipgram(NCE_loss).py | 0 .../Word2Vec-Skipgram(Softmax).py | 0 .../tensorflow/v1/2-1.TextCNN/TextCNN.py | 0 .../tensorflow/v1/3-1.TextRNN/TextRNN.py | 0 .../tensorflow/v1/3-2.TextLSTM/TextLSTM.py | 0 .../tensorflow/v1/3-3.Bi-LSTM/Bi-LSTM.py | 0 .../tensorflow/v1/4-1.Seq2Seq/Seq2Seq.py | 0 .../Seq2Seq(Attention).py | 0 .../Bi-LSTM(Attention).py | 0 44 files changed, 3842 deletions(-) rename 1-1.NNLM/{NNLM-Torch.py => NNLM.py} (100%) delete mode 100644 1-1.NNLM/NNLM_Tensor.ipynb delete mode 100644 1-1.NNLM/NNLM_Torch.ipynb rename 1-2.Word2Vec/{Word2Vec-Skipgram-Torch(Softmax).py => Word2Vec-Skipgram(Softmax).py} (100%) delete mode 100644 1-2.Word2Vec/Word2Vec_Skipgram_Tensor(NCE_loss).ipynb delete mode 100644 1-2.Word2Vec/Word2Vec_Skipgram_Tensor(Softmax).ipynb delete mode 100644 1-2.Word2Vec/Word2Vec_Skipgram_Torch(Softmax).ipynb rename 2-1.TextCNN/{TextCNN-Torch.py => TextCNN.py} (100%) delete mode 100644 2-1.TextCNN/TextCNN_Tensor.ipynb delete mode 100644 2-1.TextCNN/TextCNN_Torch.ipynb rename 3-1.TextRNN/{TextRNN-Torch.py => TextRNN.py} (100%) delete mode 100644 3-1.TextRNN/TextRNN_Tensor.ipynb delete mode 100644 3-1.TextRNN/TextRNN_Torch.ipynb rename 3-2.TextLSTM/{TextLSTM-Torch.py => TextLSTM.py} (100%) delete mode 100644 3-2.TextLSTM/TextLSTM_Tensor.ipynb delete mode 100644 3-2.TextLSTM/TextLSTM_Torch.ipynb rename 3-3.Bi-LSTM/{Bi-LSTM-Torch.py => Bi-LSTM.py} (100%) delete mode 100644 3-3.Bi-LSTM/Bi_LSTM_Tensor.ipynb delete mode 100644 3-3.Bi-LSTM/Bi_LSTM_Torch.ipynb rename 4-1.Seq2Seq/{Seq2Seq-Torch.py => Seq2Seq.py} (100%) delete mode 100644 4-1.Seq2Seq/Seq2Seq_Tensor.ipynb delete mode 100644 4-1.Seq2Seq/Seq2Seq_Torch.ipynb rename 4-2.Seq2Seq(Attention)/{Seq2Seq(Attention)-Torch.py => Seq2Seq(Attention).py} (100%) delete mode 100644 4-2.Seq2Seq(Attention)/Seq2Seq(Attention)_Tensor.ipynb delete mode 100644 4-2.Seq2Seq(Attention)/Seq2Seq(Attention)_Torch.ipynb rename 4-3.Bi-LSTM(Attention)/{Bi-LSTM(Attention)-Torch.py => Bi-LSTM(Attention).py} (100%) delete mode 100644 4-3.Bi-LSTM(Attention)/Bi_LSTM(Attention)_Tensor.ipynb delete mode 100644 4-3.Bi-LSTM(Attention)/Bi_LSTM(Attention)_Torch.ipynb rename 5-1.Transformer/{Transformer(Greedy_decoder)-Torch.py => Transformer(Greedy_decoder).py} (100%) delete mode 100644 5-1.Transformer/Transformer(Greedy_decoder)_Torch.ipynb rename 5-1.Transformer/{Transformer-Torch.py => Transformer.py} (100%) delete mode 100644 5-1.Transformer/Transformer_Torch.ipynb rename 5-2.BERT/{BERT-Torch.py => BERT.py} (100%) delete mode 100644 5-2.BERT/BERT_Torch.ipynb rename 1-1.NNLM/NNLM-Tensor.py => archive/tensorflow/v1/1-1.NNLM/NNLM.py (100%) rename 1-2.Word2Vec/Word2Vec-Skipgram-Tensor(NCE_loss).py => archive/tensorflow/v1/1-2.Word2Vec/Word2Vec-Skipgram(NCE_loss).py (100%) rename 1-2.Word2Vec/Word2Vec-Skipgram-Tensor(Softmax).py => archive/tensorflow/v1/1-2.Word2Vec/Word2Vec-Skipgram(Softmax).py (100%) rename 2-1.TextCNN/TextCNN-Tensor.py => archive/tensorflow/v1/2-1.TextCNN/TextCNN.py (100%) rename 3-1.TextRNN/TextRNN-Tensor.py => archive/tensorflow/v1/3-1.TextRNN/TextRNN.py (100%) rename 3-2.TextLSTM/TextLSTM-Tensor.py => archive/tensorflow/v1/3-2.TextLSTM/TextLSTM.py (100%) rename 3-3.Bi-LSTM/Bi-LSTM-Tensor.py => archive/tensorflow/v1/3-3.Bi-LSTM/Bi-LSTM.py (100%) rename 4-1.Seq2Seq/Seq2Seq-Tensor.py => archive/tensorflow/v1/4-1.Seq2Seq/Seq2Seq.py (100%) rename 4-2.Seq2Seq(Attention)/Seq2Seq(Attention)-Tensor.py => archive/tensorflow/v1/4-2.Seq2Seq(Attention)/Seq2Seq(Attention).py (100%) rename 4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention)-Tensor.py => archive/tensorflow/v1/4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention).py (100%) diff --git a/1-1.NNLM/NNLM-Torch.py b/1-1.NNLM/NNLM.py similarity index 100% rename from 1-1.NNLM/NNLM-Torch.py rename to 1-1.NNLM/NNLM.py diff --git a/1-1.NNLM/NNLM_Tensor.ipynb b/1-1.NNLM/NNLM_Tensor.ipynb deleted file mode 100644 index 5ff2c6d..0000000 --- a/1-1.NNLM/NNLM_Tensor.ipynb +++ /dev/null @@ -1,115 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "4dTc8F-IObVT", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 125.0 - }, - "outputId": "20bfe275-33af-4655-b2ee-cecdea69cce7" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1000 cost = 0.468996\n", - "Epoch: 2000 cost = 0.125292\n", - "Epoch: 3000 cost = 0.045533\n", - "Epoch: 4000 cost = 0.020888\n", - "Epoch: 5000 cost = 0.010938\n", - "[['i', 'like'], ['i', 'love'], ['i', 'hate']] -> ['dog', 'coffee', 'milk']\n" - ] - } - ], - "source": [ - "# code by Tae Hwan Jung @graykode\n", - "import tensorflow as tf\n", - "import numpy as np\n", - "\n", - "tf.reset_default_graph()\n", - "\n", - "sentences = [ \"i like dog\", \"i love coffee\", \"i hate milk\"]\n", - "\n", - "word_list = \" \".join(sentences).split()\n", - "word_list = list(set(word_list))\n", - "word_dict = {w: i for i, w in enumerate(word_list)}\n", - "number_dict = {i: w for i, w in enumerate(word_list)}\n", - "n_class = len(word_dict) # number of Vocabulary\n", - "\n", - "# NNLM Parameter\n", - "n_step = 2 # number of steps ['i like', 'i love', 'i hate']\n", - "n_hidden = 2 # number of hidden units\n", - "\n", - "def make_batch(sentences):\n", - " input_batch = []\n", - " target_batch = []\n", - "\n", - " for sen in sentences:\n", - " word = sen.split()\n", - " input = [word_dict[n] for n in word[:-1]]\n", - " target = word_dict[word[-1]]\n", - "\n", - " input_batch.append(np.eye(n_class)[input])\n", - " target_batch.append(np.eye(n_class)[target])\n", - "\n", - " return input_batch, target_batch\n", - "\n", - "# Model\n", - "X = tf.placeholder(tf.float32, [None, n_step, n_class]) # [batch_size, number of steps, number of Vocabulary]\n", - "Y = tf.placeholder(tf.float32, [None, n_class])\n", - "\n", - "input = tf.reshape(X, shape=[-1, n_step * n_class]) # [batch_size, n_step * n_class]\n", - "H = tf.Variable(tf.random_normal([n_step * n_class, n_hidden]))\n", - "d = tf.Variable(tf.random_normal([n_hidden]))\n", - "U = tf.Variable(tf.random_normal([n_hidden, n_class]))\n", - "b = tf.Variable(tf.random_normal([n_class]))\n", - "\n", - "tanh = tf.nn.tanh(d + tf.matmul(input, H)) # [batch_size, n_hidden]\n", - "model = tf.matmul(tanh, U) + b # [batch_size, n_class]\n", - "\n", - "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=model, labels=Y))\n", - "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n", - "prediction =tf.argmax(model, 1)\n", - "\n", - "# Training\n", - "init = tf.global_variables_initializer()\n", - "sess = tf.Session()\n", - "sess.run(init)\n", - "\n", - "input_batch, target_batch = make_batch(sentences)\n", - "\n", - "for epoch in range(5000):\n", - " _, loss = sess.run([optimizer, cost], feed_dict={X: input_batch, Y: target_batch})\n", - " if (epoch + 1)%1000 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - "# Predict\n", - "predict = sess.run([prediction], feed_dict={X: input_batch})\n", - "\n", - "# Test\n", - "input = [sen.split()[:2] for sen in sentences]\n", - "print([sen.split()[:2] for sen in sentences], '->', [number_dict[n] for n in predict[0]])" - ] - } - ], - "metadata": { - "colab": { - "name": "NNLM-Tensor.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/1-1.NNLM/NNLM_Torch.ipynb b/1-1.NNLM/NNLM_Torch.ipynb deleted file mode 100644 index 12e7030..0000000 --- a/1-1.NNLM/NNLM_Torch.ipynb +++ /dev/null @@ -1,130 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "mvlw9p3tPJjr", - "colab_type": "code", - "outputId": "2a4a8f52-315e-42b3-9d49-e9c7c3358979", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 129.0 - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1000 cost = 0.147408\n", - "Epoch: 2000 cost = 0.026562\n", - "Epoch: 3000 cost = 0.010481\n", - "Epoch: 4000 cost = 0.005095\n", - "Epoch: 5000 cost = 0.002696\n", - "[['i', 'like'], ['i', 'love'], ['i', 'hate']] -> ['dog', 'coffee', 'milk']\n" - ] - } - ], - "source": [ - "# code by Tae Hwan Jung @graykode\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from torch.autograd import Variable\n", - "\n", - "dtype = torch.FloatTensor\n", - "\n", - "sentences = [ \"i like dog\", \"i love coffee\", \"i hate milk\"]\n", - "\n", - "word_list = \" \".join(sentences).split()\n", - "word_list = list(set(word_list))\n", - "word_dict = {w: i for i, w in enumerate(word_list)}\n", - "number_dict = {i: w for i, w in enumerate(word_list)}\n", - "n_class = len(word_dict) # number of Vocabulary\n", - "\n", - "# NNLM Parameter\n", - "n_step = 2 # n-1 in paper\n", - "n_hidden = 2 # h in paper\n", - "m = 2 # m in paper\n", - "\n", - "def make_batch(sentences):\n", - " input_batch = []\n", - " target_batch = []\n", - "\n", - " for sen in sentences:\n", - " word = sen.split()\n", - " input = [word_dict[n] for n in word[:-1]]\n", - " target = word_dict[word[-1]]\n", - "\n", - " input_batch.append(input)\n", - " target_batch.append(target)\n", - "\n", - " return input_batch, target_batch\n", - "\n", - "# Model\n", - "class NNLM(nn.Module):\n", - " def __init__(self):\n", - " super(NNLM, self).__init__()\n", - " self.C = nn.Embedding(n_class, m)\n", - " self.H = nn.Parameter(torch.randn(n_step * m, n_hidden).type(dtype))\n", - " self.W = nn.Parameter(torch.randn(n_step * m, n_class).type(dtype))\n", - " self.d = nn.Parameter(torch.randn(n_hidden).type(dtype))\n", - " self.U = nn.Parameter(torch.randn(n_hidden, n_class).type(dtype))\n", - " self.b = nn.Parameter(torch.randn(n_class).type(dtype))\n", - "\n", - " def forward(self, X):\n", - " X = self.C(X)\n", - " X = X.view(-1, n_step * m) # [batch_size, n_step * n_class]\n", - " tanh = torch.tanh(self.d + torch.mm(X, self.H)) # [batch_size, n_hidden]\n", - " output = self.b + torch.mm(X, self.W) + torch.mm(tanh, self.U) # [batch_size, n_class]\n", - " return output\n", - "\n", - "model = NNLM()\n", - "\n", - "criterion = nn.CrossEntropyLoss()\n", - "optimizer = optim.Adam(model.parameters(), lr=0.001)\n", - "\n", - "input_batch, target_batch = make_batch(sentences)\n", - "input_batch = Variable(torch.LongTensor(input_batch))\n", - "target_batch = Variable(torch.LongTensor(target_batch))\n", - "\n", - "# Training\n", - "for epoch in range(5000):\n", - "\n", - " optimizer.zero_grad()\n", - " output = model(input_batch)\n", - "\n", - " # output : [batch_size, n_class], target_batch : [batch_size] (LongTensor, not one-hot)\n", - " loss = criterion(output, target_batch)\n", - " if (epoch + 1)%1000 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - "# Predict\n", - "predict = model(input_batch).data.max(1, keepdim=True)[1]\n", - "\n", - "# Test\n", - "print([sen.split()[:2] for sen in sentences], '->', [number_dict[n.item()] for n in predict.squeeze()])\n" - ] - } - ], - "metadata": { - "colab": { - "name": "NNLM-Torch.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/1-2.Word2Vec/Word2Vec-Skipgram-Torch(Softmax).py b/1-2.Word2Vec/Word2Vec-Skipgram(Softmax).py similarity index 100% rename from 1-2.Word2Vec/Word2Vec-Skipgram-Torch(Softmax).py rename to 1-2.Word2Vec/Word2Vec-Skipgram(Softmax).py diff --git a/1-2.Word2Vec/Word2Vec_Skipgram_Tensor(NCE_loss).ipynb b/1-2.Word2Vec/Word2Vec_Skipgram_Tensor(NCE_loss).ipynb deleted file mode 100644 index 5978cde..0000000 --- a/1-2.Word2Vec/Word2Vec_Skipgram_Tensor(NCE_loss).ipynb +++ /dev/null @@ -1,138 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "0PIghxafQ6R8", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 437.0 - }, - "outputId": "92e0fb28-7307-4a92-d601-980997471481" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1000 cost = 4.152723\n", - "Epoch: 2000 cost = 3.183771\n", - "Epoch: 3000 cost = 2.936138\n", - "Epoch: 4000 cost = 2.803021\n", - "Epoch: 5000 cost = 3.062087\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAecAAAFKCAYAAAAnj5dkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl0U3X+//Fnkqal0DVtKgrqyCaI\ngOULKgKDVFDcEIXa4jLqMG4I6ogim19QARdUdARlEQFRAQuoiAuyK/uPb5FNx8KggoDS0jZtaWna\nJL8/GCKVUmhpm0vu63HOnOnN5+be99t4feUuudfi8/l8iIiIiGFYA12AiIiIlKVwFhERMRiFs4iI\niMEonEVERAxG4SwiImIwCmcRERGDCQl0AcdkZuYHuoQTxMbWJSenMNBl1Aqz9GqWPkG9BiOz9Anm\n6NXpjDzpmPacKxASYgt0CbXGLL2apU9Qr8HILH2CuXotj8JZRETEYIIunCdNmsAnn8wLdBkiIiJV\nZphzztXloYcGBLoEERGRMxKwcP71119JTU3lnnvuYd68eZSWehkx4llmznyHnTszuPzyKxk2bCTL\nly9l+vQpeDwe4uOdPP30CNxuN/37/4PPPvuakJCjLQwdOogrrujAjh3badCgIffe+w9++mk3r776\nIllZWYSG2hk2bCTNm18SqJZFREROS0APa+fk5OB0Olm8eDFNmjRh5MihDB/+LDNnzmbp0sVs3vx/\nvPzyaF544VU+/HA+HTp04uWXx3LRRY2Ii4tj69bvADhy5Aj/93+b6NLlGv+yvV4vQ4c+SY8eNzBn\nzgKefHIoQ4YMorS0NFDtioiInJaAhnNpaSk9evQAoFGjJrRocQkxMTFER8cQFxfP5s3/R2JiOxo2\nPB+Am2/uxebNmygtLeXqq69h9epVAGzYsJYWLVoSGxvrX/Yvv/xMbm42N954CwCtW19GTEws27dv\nreUuRUREKieg55xtNht16tQBwGq1Eh5e1z9mtVqx20OJjPzjd2ARERH4fD5crlyuvvoahg17kkcf\nHcQ336zkmmu6l1l2QUE+R44c4c47+/hfO3z4MC6Xq4a7EhEROTOGviDM4XCwY8cfe7p5eXlYrVb/\nnrXVamPnzgw2blzPo48+Uea98fFO6tWrx4cfzq/tskVERM6IoX9KFRISwnffbWbfvl8B+PTT+bRv\nf4X/IrCuXa/h3Xen0LRpM6KjY8q8t379c3E6z2HFiqUA5ObmMnLkMIqKimq3CRERkUoy9J6z05nA\nkCEjGDr06IVc557bgMGDh/nHr776Gvr1u4shQ0ac8F6LxcKzz45l3LixTJ36NlarlZSUOwkPD6/N\nFkRERCrN4vP5fIEuAmrm3trFJR5cBcVER4QRZq/8reCczkhD3vO7JpilV7P0Ceo1GJmlTzBHrxXd\nW9vQe85V5fF6mbt8F5szMsnOK8YRFUZiMycpSU2wWQ19JF9ERCQ4w3nu8l0s3fSrf/pQXrF/+o5u\nzQJVloiIyGkJut3I4hIPmzMyyx3bnJFFcYmnlisSERGpnKALZ1dBMdl5xeWO5eQfwVVQ/piIiIhR\nVDmcx44dS0pKCqmpqWzdWvauW2vXrqVPnz6kpKQwceLEMy6yMqIjwnBEhZU7FhtZh+iI8sdERESM\nokrhvHHjRn755Rfmzp3LmDFjGDNmTJnx0aNH8+abbzJ79mzWrFnDrl27qqXY0xFmt5HYzFnuWGKz\n+CpdtS0iIlKbqhTO69ato1u3bgA0btwYl8tFQUEBAHv37iU6Oppzzz0Xq9VKly5dWLduXfVVfBpS\nkprQrV1D4qLqYLVAXFQdurVrSEpSk1qtQ0REpCqqdLV2VlYWLVu29E87HA4yMzOJiIggMzMTh8NR\nZmzv3r1nXmkl2KxW7ujWjN5dGp/R75xFREQCoVp+SlUd9zGJja1LSEj1B2jDM3x/RT8SDzZm6dUs\nfYJ6DUZm6RPM1eufVSmcExISyMrK8k8fPHgQp9NZ7tjvv/9OQkLCKZeZk1NYlVJqlBnuUHOMWXo1\nS5+gXoORWfoEc/Ra0ZePKp1z7tixI4sXLwZgx44dJCQkEBERAUDDhg0pKCjg119/pbS0lBUrVtCx\nY8eqrEZERMSUqrTn3LZtW1q2bElqaioWi4WRI0eyYMECIiMj6d69O6NGjWLQoEEA3HDDDVx00UXV\nWrSIiEgwC+oHX5wpMxxWOcYsvZqlT1CvwcgsfYI5eq32w9oiYjzp6ZtISelVLcv64ovPeOyx/tWy\nLBGpPIWziIiIwSicRYLMhAmvk5p6Gz169GDbti0UFxczbtxY+va9jTvv7MObb47H4zn6AJhdu3by\n8MN/5447enPvvXewYcOJNww6fLiAu+++nRUrltZ2KyKmFZSPjBQxq99+O0Dz5i0YMOBxVqz4ktde\ne4mkpGs5ePB3Zs36CI+nlAEDHmDp0sV0796DUaOGcc89/ejevQf//vf3/POfA5g//zP/8rxeL88+\nO4Jrr72erl27BbAzEXPRnrNIEAkNDSUpqTsA119/PTt3ZrBq1XJ69ryVkJAQwsLq0L379WzcuJ4D\nB/Zz6NAhunW7DoDmzS+hfv36/PDD9/7lTZo0gZiYWO6++76A9CNiVgpnkSASFRWN1Xp0sz5274HD\nhwuIjIzyzxMZGUlOTg45OTlERERisViOG4siJycbgB9//J758+fidJ76JkIiUr0UziJBJD//j5+e\n5OXlAUcD2+VyHfe6C4fDgcPhID/fVeb2uy6XC4cjDoD4eCfvvTeXhQs/5ocfdtRSByICCmeRoFJc\nfIRVq1YAsHjxYlq0uIQuXbry+eef4vF4KCoqYvHiL+jQoRPnnnseTmcCy5Z9DcC2bVvIzj5EixZH\nH2oTF+ekQYOGDBz4T8aMGUVxcXHA+hIxG10QJhJELrjgQnbs2MrkyROw20MYMuR/adSoCfv37+Pu\nu2/HYrHQtWs3kpK6YbFYePbZsYwb9wLTp0+lTp1wnn/+RcLDw8ss89prr2flyuVMmfIWAwf+M0Cd\niZiL7hBWATPcoeYYs/Rqlj7hzHr1ekvwlORjs0ditdqrubLqZ5bP1Sx9gjl6regOYdpzFhE/n89L\nzr6vKcr9EU+JC5s9mvCYi4ltcC0Wi86CidQWhbOI+OXs+5qCzI3+aU+Jyz/taNgjUGWJmI6+CosI\ncPRQdlHuj+WOFeVm4PWW1HJFIualcBYxiUmTJvDJJ/NOOu4pycdT4jrJmAtPSXCf/xMxEh3WFjGJ\nhx4aUOG4zR6JzR5dbkDb7NHY7Ce/eEVEqpf2nEUM5sCB/dxyy3V88MFM+va9jdTU29i+fRtPPfUY\nvXpdz9ixz57weMjjp3fv3sWDD97HjTfeSGrqrcyfPxeAMWNGMWPGOwD8+98/8Pe/30Vq6m0MGPAA\n+/fvw2q1Ex5zcbk1hcc0Oyuu2hYJFgpnEQPKzc3F4Yhj9uwFNGnShJEjhzJ8+LPMnDmbpUsXs3//\nryd977vvTqVXr958/vnnTJo0nU2bNuJ2u8vMM2rUMO6//2HmzFnAX/96NePHvwxAbINriXBejs0e\nA1iw2WOIcF5ObINra7JdEfkTHdYWMSCPx0NS0tGnQDVq1ASAmJgYAOLi4snKyjrpe2NjHaxcuYz/\n+Z/WOJ3n88ILr5YZ37PnF3Jzc+nQoSMAt912O7fc0hsAi8WKo2EPvOddc1b9zlkk2GjPWcSAbDYb\nYWF1ALBarYSH1/WPWa1WWre+7KTvffjhgTRq1ITHH3+c2267kQUL0sqMu1y5/odiAP99WlVYmXms\nVjv2MIeCWSRAFM4iZ6GtW7/D6/X6p49/4EXdunV58MFHWLJkCWPHjuOddyaxZ88v/vHo6Bjy8vL8\n7y8tLeXAgf21V7yInJLCWeQsFBcXz6FDWeTkZOPxeFiy5Ev/2ODB/2T37v8A0KhRY+rViyjzWMjz\nz7+AhIQEVq1aDsCiRZ/w8stjarcBEamQzjmLnIUaNGjIDTf05L777uScc+rTo8eN7NyZAUCfPik8\n++wIfD4PHo+XW2/tw/nnX+B/r8Vi4bnnXuT5559h8uSJxMXFM2zYyEC1IiLl0IMvKmCGG68fY5Ze\ng73P5557hu++S+fgwd9ZsGABCQkXlDvfgQP7SU29lVWrNtRyhTUj2D/XY8zSJ5ijVz34QsQkli5d\nzOzZC2jQoGGF/3Er8ZTg8/lwe9yE2kJruUoRORWFs0iQGDDgAbxeL088MYDDhw8zceIEGjRozCuv\nvMCWLZvxer00atSElqlX8N2eLXjx0f/NgWSu34u1GPr3f5Tu3fVwCxEj0AVhIkFiwoQpALz55mTq\n1Dn6M6yNG9dz4MB+PvxwPnPmfExxtIfF67/C5XaBz0d+cQENH7iU9sl/ZerUtwNZvogcR+EsEsRi\nYmL4+efdfPPNCvIO51GvcwJRTeOODvrA0aY+ANmR+WRmHgxgpSJyPIWzSBC75JJLefzxp5g3by69\nb72Jze+vxlP030c/Wi1YQ20AuNz5eDyeAFYqIsdTOIsEua5du/Hmm5OZ89HHhHisHFyz54R5YsKi\n4LjfQotIYCmcRYLY558v9D+JKj42ngsvvKjc+Zo7mqFoFjEOhbNIEOvcuQs//vgDqam3cuedfQhz\n2bi1z+3EhEUDEFcnlqsbduK6C7sGuFIROZ5uQlIBM/wI/hiz9GqWPqHiXt0eNzn5h6h3xEvd2His\nf3rwxdnGLJ+rWfoEc/Sqm5CIiJ/P4yE3LY3Dm9NxZWcT4nAQkdgWZ3IqFpst0OWJCApnEdPJTJtD\n7tIl/unSQ4f80wmpdwaqLBE5js45i5iIt7iYgs3p5Y4VbN6Mt7i4lisSkfIonEVMpNTlojQ7u/yx\nnGxKXa5arkhEyqNwFjGRkOhoQhyO8sdiHYRER9dyRSJSHoWziIlYw8KISGxb7lhEYuJZf9W2SLDQ\nBWEiJuNMTgWOnmMuzckmJNZBRGKi/3URCTyFs4jJWGw2ElLvJP7WPpS6XIRER2uPWcRgdFhbxKSs\nYWGEJiSYNpi//347TzwxAIAxY0b5b3PaqVM7Dh78PZCliSicRcScLrnkUl57bUKgyxApl8JZRILO\ngQP7ueWW6/jgg5n07Xsbqam3sX37Np566jF69bqesWOfJT19EykpvSpczuTJExk6dBBer7eWKhc5\nSuecRSQo5ebm4nDEMXv2AkaMGMzIkUOZNu19LBa49dYbaN26TYXvX7bsazZsWMfEiVOxWrUfI7VL\n4SwiQcnj8ZCU1A2ARo2aABATEwNAXFw8WVlZJ31vRsa/mTp1Em++OYnw8PCaL1bkT/R1UESCks1m\nIyysDgBWq5Xw8Lr+MavVSuvWl530vePGvYDFAlFRUTVep0h5FM4iIn8ycuRomjVrzttv64IxCQyF\ns4jInzRseD5PPDGYFSuWkp6+KdDliAkpnEVEyhEdHcNTTw3lhReeo7DwcKDLEZOx+Hw+X2XfVFJS\nwpAhQ9i/fz82m40XXniB888/v8w8LVu2pG3bP+7hO2PGDGwVPMg9MzO/smXUOKcz0pB11QSz9GqW\nPkG9BiOz9Anm6NXpjDzpWJWu1l60aBFRUVG8+uqrrF69mldffZXXX3+9zDwRERHMmjWrKosXEQkY\nr7cET0k+NnskVqs90OWISVUpnNetW0evXkd/vH/VVVcxbNiwai1KRKS2+XxecvZ9TVHuj3hKXNjs\n0YTHXExsg2uxWHQGUGpXlf6Ny8rKwvHfZ8JarVYsFgtut7vMPG63m0GDBpGamsr06dPPvFIRkRqU\ns+9rCjI34ilxAeApcVGQuZGcfV8HuDIxo1PuOaelpZGWllbmtS1btpSZLu+09eDBg+nZsycWi4W7\n7rqLdu3a0apVq5OuJza2LiEhJz8nHSgVnRMINmbp1Sx9gno9XV6Pm99+2FnumDt/J3GOW7DaQqu8\n/Oqkz9QcThnOycnJJCcnl3ltyJAhZGZm0rx5c0pKSvD5fISGlv0Xt2/fvv6/r7zySjIyMioM55yc\nwsrWXuPMcEHCMWbp1Sx9gnqtjJLibNxHcsodcx/J5bcDB7CHOaq8/OqizzS4VPTlo0qHtTt27MhX\nX30FwIoVK7jiiivKjO/evZtBgwbh8/koLS0lPT2dpk2bVmVVIiI1zmaPxGaPPslYNDa7effgJDCq\ndEHYDTfcwNq1a+nbty+hoaG8+OKLAEyZMoX27duTmJhI/fr16dOnD1arlaSkJFq3bl2thYuIVBer\n1U54zMUUZG48YSw8ppmu2pZaV6XfOdcEIx6+MMNhlWPM0qtZ+gT1Wll/XK2dcdzV2s0MdbW2PtPg\nUu2/cxYRCTYWixVHwx54z7tGv3OWgFM4i4gcx2q1YzXAxV9ibsY4ViMiIiJ+CmcRERGDUTiLiIgY\njMJZRETEYBTOIiIiBqNwFjkLpadvIiWlF5MmTeCTT+YB0KlTOw4e/D3AlYlIddBPqUTOYg89NCDQ\nJYhIDdCes8hZbMyYUcyY8c4Jr0+ePJH+/fvj9Xr56afdDBjwAKmpt/G3v6Xw739/H4BKRaQyFM4i\nQWbZsq/ZsGEdr7zyCgBDhz5Jjx43MGfOAp58cihDhgyitLQ0wFWKSEUUziJBJCPj30ydOomXXnqN\nunXr8ssvP5Obm82NN94CQOvWlxETE8v27VsDXKmIVETnnEWCyLhxL1C3bl2ioqIAKCjI58iRI9x5\nZx//PIcPH8blcgWqRBE5DQpnkSAycuRoPv10AW+/PYExY54lPt5JvXr1+PDD+YEuTUQqQYe1RYJI\nw4bn88QTg1mxYikbNmygfv1zcTrPYcWKpQDk5uYycuQwioqKAlypiFRE4SwSZKKjY3jqqaEMHTqU\noqJCnn12LPPnf8Qdd/RmwID7adfucsLDwwNdpohUwOLz+XyBLgIw5EO1zfCw72PM0qtZ+gT1GozM\n0ieYo1enM/KkY9pzFglyJSUeXDlFlJR4Al2KiJwmXRAmEqS8Hi+rl+7kp4wsCvKKiYgK46Jm8VyV\n1BirNXDfy91uN8uWfc31198UsBpEjE57ziJB6uvPvmfbpn0U5BUDUJBXzLZN+1i7/D8BrSsj40e+\n+uqLgNYgYnTacxYJQiUlHn7c/lu5Yz9nZHFFl0bY7bZqW9+XXy5i5sx3AWjZsiVPP/0Mixd/wZw5\n7+PxeIiLi+eZZ54jNDSU4cOf5PDhw/Tv/w/eeuvEW4+KiPacRYJSYYEbV275P5cqyC+msMBdbes6\ncGA/Eye+wYQJk5k9ez5FRUf46KMPGT/+ZcaPn8icOR/ToEFDZsx4B4cjjgcfHEDLlq0VzCIV0J6z\nSBCqGxFKdEw4rpwTAzoiMoy6EaHVtq6NG9fTqlVr4uOdwNEbodhsNlJS7sRutwPQpk0iixfrULbI\n6VI4iwQhu93GxZfWZ+O3P50w9pdm8dV6SNvlyiUi4o+fhISFheHxeJg69W3WrPkGj8dDYWEh559/\nQbWtUyTYKZxFgtS1N19CUZGbnzOyKMgvJiIyjL/892rt6hQdHVPmQRqHDxfwzTcrWbPmGyZMmEpM\nTAwLF37M119/Wa3rFQlmCmeRIGW1WenUrSlXdGlEYYGbuhGh1brHfEyHDh15++03OXBgP/Xrn8u4\ncS/QqFFj6tc/l5iYGFyuXJYvX+K/ZWhISAiFhQX4fD4sFku11yMSDHRBmEiQs9ttRMeG10gwAyQk\nnMPgwcN59NGH6dv3NiwWC9deez0ul4uUlF6MGjWc++/vz8GDv/Pmm+Np3foysrKy6NWrBx6Pbowi\nUh7dvrMCZrh93DFm6dUsfYJ6DUZm6RPM0atu3ykiAVdc4uFgTiHFuo2oyCnpnLOI1CiP18vc5bvY\nnJFJdl4xjqgwEps5SUlqgi2AtxEVMTKFs4jUqLnLd7F006/+6UN5xf7pO7o1C1RZIoamr60iUmOK\nSzxszsgsd2xzRpYOcYuchMJZRGqMq6CY7P8+eOPPcvKP4Coof0zE7BTOIlJjoiPCcESFlTsWG1mH\n6Ijyx0TMTuEsIjUmzG4jsZmz3LHEZvGE1dBvr0XOdrogTERqVEpSE+DoOeac/CPERtYhsVm8/3UR\nOZHCWURqlM1q5Y5uzejdpTGugmKiI8K0xyxyCgpnEakVYXYbCbF1A12GyFlB55xFREQMRuEsIiJi\nMApnERERg1E4i4iIGIzCWURExGAUziIiIgajcBYRETEYhbOIiIjBKJxFREQMRuEsIiJiMApnERER\ng6lyOG/cuJEOHTqwYsWKcscXLlxI7969SU5OJi0trcoFioiImE2VHnyxZ88epk+fTtu2bcsdLyws\nZOLEicybNw+73U6fPn3o3r07MTExZ1SsiIiIGVRpz9npdDJhwgQiIyPLHd+yZQutWrUiMjKSOnXq\n0LZtW9LT08+oUBEREbOo0p5zeHh4heNZWVk4HA7/tMPhIDMzsyqrEhERMZ1ThnNaWtoJ54wHDhxI\n586dT3slPp/vlPPExtYlJMR4D2B3Oss/OhCMzNKrWfoE9RqMzNInmKvXPztlOCcnJ5OcnFyphSYk\nJJCVleWfPnjwIJdddlmF78nJKazUOmqD0xlJZmZ+oMuoFWbp1Sx9gnoNRmbpE8zRa0VfPmrkp1Rt\n2rRh27Zt5OXlcfjwYdLT02nXrl1NrEpERCToVOmc88qVK5k2bRq7d+9mx44dzJo1i3fffZcpU6bQ\nvn17EhMTGTRoEP369cNisfDII4+c9OIxERERKcviO50TwrXAiIcvzHBY5Riz9GqWPkG9BiOz9Anm\n6LXWD2uLiIhI1SmcRUREDEbhLCIiYjAKZxEREYNROIuIiBiMwllERMRgFM4iIiIGo3AWERExGIWz\niIiIwSicRUREDEbhLCIiYjAKZxEREYNROIuIiBiMwllERMRgFM4iIiIGo3AWERExGIWziIiIwSic\nRUREDEbhLCIiYjAKZxEREYNROIuIiBiMwllERMRgFM4iIiIGo3AWERExGIWziIiIwSicRUREDEbh\nLCIiYjAKZxEREYNROIuIiBiMwllERMRgFM4iIiIGo3AWERExGIWziIiIwSicRUREDEbhLCIiZ2TL\nlu/o0+fmQJcRVBTOIiJnqeef/19Wr/6mWpb1xRef8dhj/atlWXLmQgJdgIiIVM0zzzwXsHXPmPEO\nCxd+THR0NJ06dQGguLiYf/3rVdLTN2G1Wrnyyo707/8oNpuNDRvW8dJLowkPD+f22+9g4sQ3mDlz\nNueee17AejAyhbOIiEF89tknzJnzPh6Ph7i4eJ555jnS0zexdu1q6tWrx44dWwELzz33Io0aNWbA\ngAe4+eZeXHfdDXTq1I7Bg4czb94cCgoKGD58FAsXfsz27Vv5y18a8dJLrxESEsLq1auYMuUtSkpK\nCA+vy9Chz9C06cWVqvOnn3Yzd+6HfPBBGtHRMYwY8TQAH300m4MHf2fWrI/weEoZMOABli5dTLdu\n1zFmzCiGDv1fOnToyMSJb3DkSFEN/BMMHjqsLSJiADk52Ywf/zLjx09kzpyPadCgITNmvAPA+vVr\nuPXWZBYvXkxiYjvS0maXuwyXK5f33ptLUlJ3hg8fTL9+DzJ79gJ2797Fd9+lU1payujRoxg8eASz\nZy+gc+cuTJjwRqVr3bIlncsua4vDEYfNZuO6664HYN261fTseSshISGEhdWhe/fr2bhxPXv37sHt\ndtOhQ0cAevdOwev1Vumfk1konEVEDCA21sHixatISDgHgDZtEtm/fx8Af/lLI5o3bwHAxRdfzO+/\n/1buMjp3vhqAxo2b0KBBQy644EJCQ0Np2PB8srIyCQkJYdGiJVx6aasT1lEZeXl5RERE+KcjI6MA\nyM3N8f999PVIcnJyyM/PK/N6fHx8pddpNjqsLSJiAB6Ph3femcSaNd/g8XgoLCzk/PMvAKBevT+C\n0Gq14fGUv9dZt27d/85jJTw8vMx7ju2ppqXN4auvFuF2u3G73VgslkrXGhkZRUFBgX86NzcHAIcj\nDpfL5X89L8+Fw+GgXr16FBUV+l/Pzj5U6XWajfacRUQMYNmyJaxZ8w0TJkxl9uwF9Ov3YLWvY9u2\nLXzwwUxefPE1Zs9ewJAhI6q0nEsvbcW2bd+Rk5ODx+Nh8eIvAbjqqk58/vmneDweioqKWLz4Czp0\n6ETDhhdQWlpKevomAD75ZH6VvhSYifacRUQMIDc3m/r1zyUmJgaXK5fly5dQVFS9F03l5OQQGxvL\nOefU58iRI3z55eccOVKEz+er1HKaNr2YW27pTb9+dxEVFU23bteye/cuevdOYf/+fdx99+1YLBa6\ndu1GUlI3LBYLTz45hLFjnyUiIpKUlDuwWq0K6AoonEVEDKBbt+tYsmQxKSm9OO+8Btx/f3+GDHmC\nCRNer/TV1CdzxRUd+PjjNFJSehEf7+SxxwaxY8c2RowYTMeOf63Usu6//2Huv/9h//Rdd90LwJNP\nDi13/m7drqNbt+sAKCoqYsyYUURERFatEROw+Cr7lamGZGbmB7qEEzidkYasqyaYpVez9AnqNRid\nzX3+4x9/o0/yHbS9/K+s/WYx89I+5P330046/9nc6+lyOk/+5UR7ziIiJuUtLqbU5SIkOhprWFiN\nrcfj9dKiQx9ee/MtSkpeJzQsnB69H8Dj9WKz6tKn8iicRURMxufxkJk2h4LN6ZRmZxPicBCR2BZn\ncioWm63a1zd3+S52ZEbQsNPj/te2HTj6+h3dmlX7+oKBvrKIiJhMZtoccpcuofTQIfD5KD10iNyl\nS8hMm1Pt6you8bA5I7Pcsc0ZWRSXeKp9ncFA4SwiYiLe4mIKNqeXO1aweTPe4uJqXZ+roJjsvPKX\nmZN/BFdB9a4vWFQ5nDdu3EiHDh1YsWJFueMtW7bk7rvv9v/P49G3IxGRQCt1uSjNzi5/LCeb0uNu\nIlIdoiPCcESVfz47NrIO0RE1d677bFalc8579uxh+vTptG3b9qTzREREMGvWrCoXJiIi1S8kOpoQ\nh+PoIe0/j8U6CImOrtb1hdltJDZzsnTTryeMJTaLJ8xe/ee4g0GV9pydTicTJkwgMlK/URMROZtY\nw8KISCx/xyoiMbFGrtpOSWpCt3YNiYuqg9UCcVF16NauISlJTap9XcGiSnvOx9+z9WTcbjeDBg1i\n3759XHfdddx3331VWZWIiFSuL74qAAAU4UlEQVQzZ3IqcPQcc2lONiGxDiISE/2vVzeb1cod3ZrR\nu0tjXAXFREeEaY/5FE4ZzmlpaaSllf2h+MCBA+ncuXOF7xs8eDA9e/bEYrFw11130a5dO1q1anXS\n+WNj6xISYrwPq6IfiQcbs/Rqlj5BvQaj6uozYeBDeIqLcWfnEOqIxVaDv3M+XsNKzGuWz7Q8pwzn\n5ORkkpOTK73gvn37+v++8sorycjIqDCcc3IKTzoWKGa4Q80xZunVLH2Ceg1GNdJnSD3IcwPu6l3u\nGTLDZ1rRl48a+SnV7t27GTRoED6f779PIkmnadOmNbEqERGRoFOlc84rV65k2rRp7N69mx07djBr\n1izeffddpkyZQvv27UlMTKR+/fr06dMHq9VKUlISrVu3ru7aRUREgpIefFEBMxxWOcYsvZqlT1Cv\nwcgsfYI5eq31w9oiIiJSdQpnERERg1E4i4iIGIzCWURExGAUziIiIgajcBYRETEYhbOIiIjBKJxF\nREQMRuEsIiJiMApnERERg1E4i4iIGIzCWURExGAUziIiIgajcBYRETEYhbOIiIjBKJxFRCRopKdv\nIiWlV42v5/vvt/PEEwNqbPkKZxERkUq65JJLee21CTW2/JAaW7KIiMhp+uyzT5gz5308Hg9xcfGM\nH/8qs2bN5rffDuBy5fKf/+wiISGBsWNfITbWQZ8+N3PrrX1YtmwJBw/+xi239Ob++x8us0y3281b\nb73B+vXrKC0toWfPW/nb3/5eLfWmp2/ipZdGM3fuJ9WyvD/TnrOIiARUTk4248e/zPjxE5kz52Ma\nNGjIW2+9BcCqVSt4/PGnmD9/Eeed14BZs6b737d9+zamTp3JrFkfsWBBGjt3ZpRZ7ocfvsdPP/3E\ne+/NYdasj1i5chlr1nxbq71VlcJZREQCKjbWweLFq0hIOAeANm0S2bt3LwBt2/4P553XAIC//jWJ\nbdu2+t/Xo8eN2Gw2YmMdtGlzGdu3by2z3DVrvuG22/oQGhpKeHg4PXrcyKpVy2upqzOjw9oiIhJQ\nHo+Hd96ZxJo13+DxeCgsLKRx40YAREVF++eLjIwkPz/fPx0VFXXcWBT5+XlllpufX8C//vUakydP\nBKCkpIQWLVrWZCvVRuEsIiIBtWzZEtas+YYJE6YSExPDwoUfs3LlEgByc3P98+Xn55UJZJfrj7G8\nPFeZMYD4+Hj69r2bjh0713AH1U+HtUVEJKByc7OpX/9cYmJicLlyWb58CYcPHwZg69bv+P333wBY\nsWIZbdok+t+3bNkSvF4v2dmH2Lp1C61bJ5ZZbufOXVi06BM8Hg8+n48ZM95h/fq1tdfYGdCes4iI\nBFS3btexZMliUlJ6cd55Dbj//v4MGzaIwsJC2re/gtdee4mdOzM455z6PP74U/73XXRRI+6//x5+\n+20/ffqk0KhRY9LTN/nHb7vtdg4cOMDdd9+Oz+ejefNLuP32OwLRYqVZfD6fL9BFAGRm5p96plrm\ndEYasq6aYJZezdInqNdgZJY+4WivL774CpmZBxky5JkTxvv0uZlnnnmeNm0uC0B11cPpjDzpmA5r\ni4hIUCop8eDKKaKkxHPGy3J73GQWHsLtcVdDZaemw9oiIhJUvF4va5f/h58ysijIKyYiKoyLmsVz\nVVJjrNbK7ZN6vB4W7PqcrZk7yCnOJTYshtbOltzW5EZsVlsNdaBwFhERg+rX78GTjs2b99lJx9Yu\n/w/bNu3zTxfkFfunO3VrWqkaFuz6nJW/rvZPZxfn+KeTm/Ws1LIqQ4e1RUQkaJSUePgpI6vcsZ8z\nsip1iNvtcbM1c0e5Y9uydtToIW6Fs4iIBI3CAjcFecXljhXkF1NYcPqB6irOJ6c4t9yx7CO5uIpr\n7uI8hbOIiASNuhGhRESFlTsWERlG3YjQ015WdFgksWEx5Y456sQQHXbyq63PlMJZRESCht1u46Jm\n8eWO/aVZPHb76V/EFWoLpbWz/Nt9topvSajt9IO+snRBmIiI1Jpvv13J1KlvU1R0hIYNG/K3v/2d\nYcOeYuHCxdjtdgBGjBhMhw5X0L37zSd95OP8+XNZsCANn89HvXr1GDp0JI0aNQbgqqSj//9zRhYF\n+cVERIbxl/9erV1ZtzW5ETh6jjn7SC6OOjG0im/pf72mKJxFRKRW7Nv3K88/P5JJk6bRqFETZs2a\nzvz5c0lISGDDhrV06tSF4uJiNm7cwHPPjeK99/545KPH4+GRR/5B48ZNSUxsy9Spk1iwYBF169Zj\n+fKlrFu32h/OVquVTt2ackWXRhQWuKkbEVqpPebj2aw2kpv15JbGPXAV5xMdFlmje8zH6LC2iIjU\nig0b1pGY2JZGjZoAcMstvVm9+huSkrqzZMliAP7f/1tPs2YXc84555z0kY+hoWFYLBYWLfqU7OxD\nJCV148477zlhfXa7jejY8CoH8/FCbaE468bVSjCD9pxFRKSWFBTks2XLZu64o7f/tYiICNq1u4L3\n359BUVER33yzkqSk7sDJH/kYEhLCG2+8xXvvTWfatMk0btyUQYOG0Lhxk4D0VRMUziIiUivi4520\na3c5o0e/fMJYo0ZN+Pbblaxdu5qHHx743/lP/sjHZs2aM3r0S5SUlPDBBzN55ZWxvP32uzXeQ23R\nYW0REakVl1/egS1bvmPfvl8B+P777bz++isAdO9+HVOmvEXjxk2JjXUAJ3/k43/+s4sRI56mpKQE\nu91O8+aXAJZAtVUjtOcsIiK1Ij4+nqefHs6wYU9RWlpC3bp1efTRQQAkJXXnzTfH+6/GhpM/8jE8\nPJzzzjuPu+++nZAQO3Xr1uWJJ54OVFs1Qo+MrIDZHs9mhl7N0ieo12AUzH263W6Sk29m1qyPiIqK\nDupej9EjI0VExNDmzv2QDh06ERUVfVrzuz1eDh1x4/Z4a7iywNBhbRERCag77uhNbKyDMWPGnXJe\nj8/Hl3uy+CG3gFx3KTGhIbSIieD6C+KxWYLnvLPCWUREAurDD+ef9rxf7sli7cE/HkaR4y71T990\nobPaawsUHdYWEZGzgtvj5YfcgnLHfsgtCKpD3ApnERE5K+SXlJLrLi13LNddSn5J+WNnI4WziIic\nFSLtIcSEln82NiY0hEh78JypVTiLiMhZIdRmpUVMRLljLWIiCLUFT6QFz9cMEREJetdfcPRZzeVd\nrR1MFM4iInLWsFks3HShk2sbxpFfUkqkPSSo9piPqVI4l5aWMnz4cPbs2YPH42Hw4MG0a9euzDwL\nFy5k5syZWK1Wbr/9dpKTk6ulYBERkVCblbhaenxjIFQpnD/99FPCw8OZPXs2O3fuZOjQocybN88/\nXlhYyMSJE5k3bx52u50+ffrQvXt3YmJiqq1wERGRYFWlYwE9e/Zk6NChADgcDnJzc8uMb9myhVat\nWhEZGUmdOnVo27Yt6enpZ16tiIiICVRpz9lut/v/njlzJjfddFOZ8aysLBwOh3/a4XCQmZlZxRJF\nRETM5ZThnJaWRlpaWpnXBg4cSOfOnfnggw/YsWMHkyZNqnAZp/Pgq9jYuoSE2E45X22r6KkhwcYs\nvZqlT1CvwcgsfYK5ev2zU4ZzcnJyuRdzpaWlsXz5ct56660ye9IACQkJZGVl+acPHjzIZZddVuF6\ncnIKT7fmWmOGR5YdY5ZezdInqNdgZJY+wRy9VvsjI/fu3cucOXOYMGECYWFhJ4y3adOGbdu2kZeX\nx+HDh0lPTz/ham4REREpX5XOOaelpZGbm8sDDzzgf23atGnMmDGD9u3bk5iYyKBBg+jXrx8Wi4VH\nHnmEyEjzHp4QERGpDIvvdE4I1wIjHr4ww2GVY8zSq1n6BPUajMzSJ5ij12o/rC0iIiI1R+EsIiJi\nMApnERERg1E4i4iIGIzCWURExGAUziIiIgajcBYRETEYhbOIiIjBKJxFREQMRuEsIiJiMApnERER\ng1E4i4hIjTpwYD9dulxRqffs3r2b775Lr6GKjE/hLCIihrN06VJTh3OVHhkpIiJSWYsWfUpa2mzy\n8/N5+OGBXHPNtYwfP45NmzZQWlpK69ZtGDp0JOvXr2Xy5MmEhISQn5/PwIH/5NNPFzB37ge43W5a\ntmzFsGH/S1hYnUC3VGMUziIiUuO8Xi+lpSXMnDmHFSuW8vbbb2K329m6dTOzZn2Ex+OhX7+7WLbs\na6677ga6d+9OXNw53HvvP9iyZTPvvDOJ6dM/ID7eybhxY5k6dRIDBjwe6LZqjA5ri4hIjfP5fPTo\ncRMAzZo1JzPzIFdffQ3vvDOLkJAQwsLCaN78Evbv33fCe9es+YZrrulOfLwTgF69erNq1Yparb+2\nac9ZRERqnM1mo06do4ehrVYrXq+XnJwcXn/9ZX788UesVgvZ2YdITu57wnvz8wv49tsVbNy4HgCv\n10dpaUmt1l/bFM4iIhIQU6a8RUhICO+9N4fQ0FCefXZEufPFx8fTo8dNQX0Y+890WFtERAIiNzeb\nRo2aEBoays6dGWzbtoWioiIAQkJCKCgoAKBTpy6sWrWCnJwcAL79diXvvz8jUGXXCu05i4hIQKSm\n3sXo0aP44ovPaN06kQEDHufFF5/nkksupWvXrgwaNIjfftvP6NEv87e/3cfAgQ/i83mJjXXw1FPD\nAl1+jbL4fD5foIsAyMzMD3QJJ3A6Iw1ZV00wS69m6RPUazAyS59gjl6dzsiTjumwtoiIGJ7b4+XQ\nETdujzfQpdQKHdYWERHD8vh8fLknix9yC8h1lxITGkKLmAiuvyAem8US6PJqjMJZREQM68s9Waw9\nmOufznGX+qdvutAZqLJqnA5ri4iIIbk9Xn7ILSh37IfcgqA+xK1wFhERQ8ovKSXXXVruWK67lPyS\n8seCgcJZREQMKdIeQkxo+WdfY0JDiLQH75lZhbOIiBhSqM1Ki5iIcsdaxEQQagveCAverx0iInLW\nu/6CeIByr9YOZgpnERExLJvFwk0XOrm2YRz5JaVE2kOCeo/5GIWziIgYXqjNSpwtNNBl1Jrg//oh\nIiJyllE4i4iIGIzCWURExGAUziIiIgajcBYRETEYhbOIiIjBKJxFREQMRuEsIiJiMApnERERg1E4\ni4iIGIzF5/P5Al2EiIiI/EF7ziIiIgajcBYRETEYhbOIiIjBKJxFREQMRuEsIiJiMApnERERgwkJ\ndAGBVlpayvDhw9mzZw8ej4fBgwfTrl27MvN88cUXvPvuu1itVjp06MA///lPFixYwBtvvMEFF1wA\nwFVXXcXDDz8ciBZOy+n0uXDhQmbOnInVauX2228nOTmZkpIShgwZwv79+7HZbLzwwgucf/75Aeri\n9G3cuJHHHnuMsWPH0rVr1zJj27dv56WXXvJP79q1i4kTJ7JmzRo+++wzzjnnHAB69uxJcnJyrdZd\nFRX1CtCyZUvatm3rn54xYwZer/es+1xP1WcwbKfHnKrXYNlWT1VzsG2rleIzuXnz5vlGjhzp8/l8\nvoyMDF/v3r3LjBcWFvq6du3qy8/P93m9Xl+fPn18O3fu9M2fP9/34osvBqDiqjlVn4cPH/Zde+21\nvry8PF9RUZHvxhtv9OXk5PgWLFjgGzVqlM/n8/m+/fZb32OPPVbbpVfaL7/84nvooYd8/fv39y1f\nvrzCeV0ul+/OO+/0eTwe37/+9S/frFmzaqnK6nE6vV5++eUnvHa2fa6n6jNYtlOf79S9BtO2Wpma\nz/ZttbJMf1i7Z8+eDB06FACHw0Fubm6Z8fDwcBYuXEhERAQWi4WYmJgT5jkbnKrPLVu20KpVKyIj\nI6lTpw5t27YlPT2ddevW0b17d+DoXkd6enqt115ZTqeTCRMmEBkZecp5p02bxj333IPVenZuCpXp\n9Xhn2+d6qj6DZTuFU/caTNtqZWo+27fVyjJHlxWw2+2EhYUBMHPmTG666aYT5omIiADgxx9/ZN++\nfbRp0wY4euipX79+3HPPPXz//fe1V3QVnKrPrKwsHA6Hf9rhcJCZmVnmdavVisViwe12117hVRAe\nHo7NZjvlfEeOHGH16tVcc801/te++uor7rvvPh588EH27t1bk2VWi9Pp1e12M2jQIFJTU5k+fTrA\nWfe5nk6fwbCdwql7DaZt9XRrDoZttbJMdc45LS2NtLS0Mq8NHDiQzp0788EHH7Bjxw4mTZpU7nt/\n/vlnnnzySV599VXsdjtt2rTB4XBw9dVXs3nzZp5++mk+++yz2mjjlM6kz2N8J7mr68leD5SKej2V\npUuXcvXVV/u/iXfp0oUrr7yS9u3b8/nnnzN69GgmT55cI3VXRVV7HTx4MD179sRisXDXXXedcK0B\nGOtzPZPP9GzaTuHMej3mbN5Wt2zZUmb6ZDWfbdtqdTBVOCcnJ5d70UBaWhrLly/nrbfewm63nzD+\n22+/8cgjj/Dyyy/TokULABo3bkzjxo0BSExMJDs7G4/Hc1p7bDWtKn0mJCSQlZXlnz548CCXXXYZ\nCQkJZGZm0rx5c0pKSvD5fISGhtZ4D6frZL2ejhUrVtC3b1//dOvWrf1/JyUl8corr5xxfdWpqr0e\n3+OVV15JRkaGoT/XqvZ5tm2nULVeg2lbHTJkyGnVfLZtq9XB9Ie19+7dy5w5c5gwYYL/sO+fDR8+\nnFGjRtGyZUv/a1OnTmXRokUAZGRk4HA4DLPBl+dUfbZp04Zt27aRl5fH4cOHSU9Pp127dnTs2JGv\nvvoKOLqBXHHFFbVdeo3Zvn07zZs390+PHj2aTZs2AUcPhTZt2jRQpVWb3bt3M2jQIHw+H6WlpaSn\np9O0adOg/FyDYTs9HcG0rZ5uzWbYVv/M9E+leu211/j8888577zz/K9NmzaNGTNm0L59e2JiYujV\nq1eZb2r33nsvLVu25KmnnvL/R2/YsGFl5jGaU/WZmJjIV199xbRp0/yHP3v27InH42HEiBH8/PPP\nhIaG8uKLL3LuuecGsJNTW7lyJdOmTWP37t04HA6cTifvvvsuU6ZM8fcK0KFDB9atW+d/348//sjI\nkSMJCQnBYrEwevRoLrzwwkC1cVpOp9dx48axfv16rFYrSUlJPPzww2fd53qqPoNlO4XT+0yDZVs9\nWc3BuK1WlunDWURExGhMf1hbRETEaBTOIiIiBqNwFhERMRiFs4iIiMEonEVERAxG4SwiImIwCmcR\nERGDUTiLiIgYzP8HGchs4S7MCOEAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - " reference : https://github.com/golbin/TensorFlow-Tutorials/blob/master/04%20-%20Neural%20Network%20Basic/03%20-%20Word2Vec.py\n", - "'''\n", - "import tensorflow as tf\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "tf.reset_default_graph()\n", - "\n", - "# 3 Words Sentence\n", - "sentences = [ \"i like dog\", \"i like cat\", \"i like animal\",\n", - " \"dog cat animal\", \"apple cat dog like\", \"dog fish milk like\",\n", - " \"dog cat eyes like\", \"i like apple\", \"apple i hate\",\n", - " \"apple i movie book music like\", \"cat dog hate\", \"cat dog like\"]\n", - "\n", - "word_sequence = \" \".join(sentences).split()\n", - "word_list = \" \".join(sentences).split()\n", - "word_list = list(set(word_list))\n", - "word_dict = {w: i for i, w in enumerate(word_list)}\n", - "\n", - "# Word2Vec Parameter\n", - "batch_size = 20\n", - "embedding_size = 2 # To show 2 dim embedding graph\n", - "num_sampled = 10 # for negative sampling, less than batch_size\n", - "voc_size = len(word_list)\n", - "\n", - "def random_batch(data, size):\n", - " random_inputs = []\n", - " random_labels = []\n", - " random_index = np.random.choice(range(len(data)), size, replace=False)\n", - "\n", - " for i in random_index:\n", - " random_inputs.append(data[i][0]) # target\n", - " random_labels.append([data[i][1]]) # context word\n", - "\n", - " return random_inputs, random_labels\n", - "\n", - "# Make skip gram of one size window\n", - "skip_grams = []\n", - "for i in range(1, len(word_sequence) - 1):\n", - " target = word_dict[word_sequence[i]]\n", - " context = [word_dict[word_sequence[i - 1]], word_dict[word_sequence[i + 1]]]\n", - "\n", - " for w in context:\n", - " skip_grams.append([target, w])\n", - "\n", - "# Model\n", - "inputs = tf.placeholder(tf.int32, shape=[batch_size])\n", - "labels = tf.placeholder(tf.int32, shape=[batch_size, 1]) # To use tf.nn.nce_loss, [batch_size, 1]\n", - "\n", - "embeddings = tf.Variable(tf.random_uniform([voc_size, embedding_size], -1.0, 1.0))\n", - "selected_embed = tf.nn.embedding_lookup(embeddings, inputs)\n", - "\n", - "nce_weights = tf.Variable(tf.random_uniform([voc_size, embedding_size], -1.0, 1.0))\n", - "nce_biases = tf.Variable(tf.zeros([voc_size]))\n", - "\n", - "# Loss and optimizer\n", - "cost = tf.reduce_mean(tf.nn.nce_loss(nce_weights, nce_biases, labels, selected_embed, num_sampled, voc_size))\n", - "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n", - "\n", - "# Training\n", - "with tf.Session() as sess:\n", - " init = tf.global_variables_initializer()\n", - " sess.run(init)\n", - "\n", - " for epoch in range(5000):\n", - " batch_inputs, batch_labels = random_batch(skip_grams, batch_size)\n", - " _, loss = sess.run([optimizer, cost], feed_dict={inputs: batch_inputs, labels: batch_labels})\n", - "\n", - " if (epoch + 1) % 1000 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - " trained_embeddings = embeddings.eval()\n", - "\n", - "for i, label in enumerate(word_list):\n", - " x, y = trained_embeddings[i]\n", - " plt.scatter(x, y)\n", - " plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')\n", - "plt.show()" - ] - } - ], - "metadata": { - "colab": { - "name": "Word2Vec-Tensor(NCE_loss).ipynb", - "version": "0.3.2", - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/1-2.Word2Vec/Word2Vec_Skipgram_Tensor(Softmax).ipynb b/1-2.Word2Vec/Word2Vec_Skipgram_Tensor(Softmax).ipynb deleted file mode 100644 index 675c840..0000000 --- a/1-2.Word2Vec/Word2Vec_Skipgram_Tensor(Softmax).ipynb +++ /dev/null @@ -1,136 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "aSeI4R7URNY7", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 437.0 - }, - "outputId": "5d7620db-36f1-48af-cd82-6dbc7abb04c5" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1000 cost = 2.294210\n", - "Epoch: 2000 cost = 2.031863\n", - "Epoch: 3000 cost = 1.944698\n", - "Epoch: 4000 cost = 1.989209\n", - "Epoch: 5000 cost = 1.808382\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd8AAAFKCAYAAABcq1WoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4E3X+B/B37rb0TJvKUS/ALoeg\nZVFEQKAUrIuLXKUc4q66KCgogqIcbkEuAcVdKQi2CAWE1gJyquUo9yGyRS6PgoggILQ0SZMeSXP8\n/ujPSCVQWtJJZvJ+PY/Pw8w3yXw+zCPvzHcmMzKn0+kEERERCUbu7QKIiIj8DcOXiIhIYAxfIiIi\ngTF8iYiIBMbwJSIiEhjDl4iISGBKoTZUUGASalMeExERBL2+1NtleIyU+mEvvktK/UipF0Ba/Yih\nF50u5IZjPPK9CaVS4e0SPEpK/bAX3yWlfqTUCyCtfsTeC8OXyIcsXJiKdetWe7sMIqpjgk07E1H1\nhg8f6e0SiEgADF+iWrh06SKGD38WAwYMxqZN6+F0ApMmTUFGRjpOncrHww8/ggkTUpCbuw1LlnwM\nu92OqCgd3nxzEqxWK1566V/YuHELlMrK/wXHjx+Ldu3a4+TJE2jUKAb//Oe/8PPPZ/D++++isLAQ\narUKEyakoFmzFl7unIg8gdPORLVkMBig1UZi1aq1aNq0KVJSxmPixCnIyFiFbdtycOTI/zB79jTM\nnPk+Vq5cg/btO2L27Bm4997GiIyMxLFj3wIAysrK8L//HUbnzt1cn+1wODB+/OtITPwbMjPX4vXX\nx+Ott8bCZrN5q10i8iCGL1Et2e12xMcnAAAaN26K5s1bIDw8HGFh4YiMjMKRI/9DXFxbxMTcCQD4\n+99748iRw7DZbOjSpRv27t0FANizZw+aN2+JiIgI12f/8stZGAxF6NnzKQBA69YPIjw8AidOHBO4\nSyKqC5x2JqolhUIBjSYAACCXyxEYGOQak8vlUKnUCAn546cGwcHBcDqdMBoN6NKlGyZMeB2vvDIW\n27ZtQ7du3at8ttlsQnl5OYYM6e9aV1JSAqPRWMddEZEQGL5EdUSr1eLkyT+OVIuLiyGXy11HxnK5\nAqdO5WPv3r144YVRVd4bFaVDvXr1sHLlGqHLJiIBcNqZqI4olUp8++0RXLjwKwBg/fo1eOihdq6L\nrLp27YZPPvkYzZs3R1hYeJX31q/fADrdHdixYxuAyvPLKSkTUFZWJmwTRFQneORLVEd0umi89dYk\njB9feaFUgwaNMG7cBNd4ly7d8PzzT2PatGnXvVcmk2HKlBmYM2cG0tI+glwuR3LyEAQGBgrZAhHV\nEZnT6XQKsSEx3l5SpwsRZd03IqV+2IvvklI/UuoFkFY/YuiFt5ckEiGHowIVliI4HBXeLoWIPIzT\nzkQ+xul0QH9hC8oMP8JeYYRCFYbA8L8golEPyGT8vkwkBbcVvrNnz8b//vc/2Gw2vPjii+jRo4en\n6iLyW/oLW2AuOORatlcYXcvamERvlUVEHlTr8D148CBOnTqFrKws6PV69OnTh+FLdJscjgqUGX50\nO1ZmyIejYTfI5SqBqyIiT6t1+D700ENo3bo1ACA0NBRlZWWw2+1QKMT9mCcib7JXmGCvcH8jDXuF\nEfYKE+QarcBVEZGn1foEkkKhQFBQ5R19Vq9ejccee4zBS3SbFKoQKFRhNxgLg0J146sniUg8bvun\nRtu2bcOiRYvwySefVLmV3p/ZbHbRP/yYSAjnf1iPK+f2Xrc++q6OuLPZU16oiIg87bYuuNqzZw8W\nLlyI9PT0mwYvAOj1pbezKa8Qw+/IakJK/Ui5F422C4LLrCgz5F9ztXMsNNououhZyvtG7KTUjxh6\nudnvfGsdviaTCbNnz8bSpUsRHh5e/RuI6JbIZHJoYxLhaNgN9goTFKoQXmRFJDG1Dt8vvvgCer0e\no0ePdq2bNWsWGjZs6JHCiPydXK7ixVVEElXr8E1OTkZycrInayEiIvILvF0OERGRwBi+REREAmP4\nEhERCYzhS0REJDCGLxERkcAYvkREJAnvvPM2+vbtiY4d2+LHH3+44esuXbqIzp3bCVjZ9fg8XyIi\nkoRt23KwatVaNGoU4+1SqsXwJSIi0Rs58gU4HA6MGTMSJSUlmD59Dlq2vB/vvTcTR48egcPhQJMm\n92HixBTXezZtWo/s7FUwmUwYMWIUuncX7nnZnHYmIiLRS039GAAwb94iBAQEAAAOHTqIS5cuYuXK\nNcjM/Bz33tsYJ04cBwA4HA7YbBXIyMjEqFGvIS3tI0HrZfgSEZEkhYeH4+zZM9i9ewfKy8sxbNgI\ntGvXHgDgdDqRmPgkACA2thkKCq4IWhvDl4iIJKlFi/sxevQbWL06C716PY7JkyfCZKp8EpJCoXAd\nIcvlcjgcDkFrY/gSEZFkde2agHnzFmHNmo2wWMqxcuUyb5cEgOFLREQStXnzBixdmg4ACA0Nw113\n3QOZTOblqioxfImISJI6deqMH3/8HgMH9sGQIf1x9uwZDBw4xNtlAQBkTqfTKcSGCgpMQmzGo3S6\nEFHWfSNS6oe9+C4p9SOlXgBp9SOGXnS6kBuO8ciXiIj8kqXCjiv6Ulgq7IJvmzfZICIiv2J3OJCV\nexpH8gtQVGyBNlSDuFgdkuObQiEX5piU4UtERH4lK/c0th3+1bV8tdjiWh6cECtIDZx2JiIiv2Gp\nsONIfoHbsSP5hYJNQTN8iYjIbxjNFhQVW9yO6U3lMJrdj3kaw5eIiPxGWLAG2lCN27GIkACEBbsf\n8zSGLxER+Q2NSoG4WJ3bsbjYKGhUCkHq4AVXRETkV5LjmwKoPMerN5UjIiQAcbFRrvVCYPgSEZFf\nUcjlGJwQi36dm8BotiAsWCPYEe/vGL5EROSXNCoFoiOCvLJtnvMlIiISGMOXiIhIYAxfIiIigTF8\niYiIBMbwJSIiEhjDl4iISGAMXyIiIoExfImIiATG8CUiIhIYw5eIiEhgDF8iIiKBMXyJiIgExvAl\nIiISGMOXiIhIYAxfIiIigTF8iYiIBMbwJSIiEhjDl4iISGAMXyIiIoExfImIiATG8CUiIhIYw5eI\niEhgDF8iIiKBMXyJiIgExvAlIiISGMOXiIhIYAxfIiIigTF8iYiIBMbwJSIiEhjDl4iISGAMXyIi\nIoHdVvjm5+cjISEBK1as8FQ9REREklfr8C0tLcXUqVPRvn17T9ZDREQkebUOX7VajbS0NERHR3uy\nHiIiIslT1vqNSiWUylt/e0REEJRKRW035zU6XYi3S/AoKfXDXnyXlPqRUi+AtPoRcy+1Dt+a0utL\nhdqUx+h0ISgoMHm7DI+RUj/sxXdJqR8p9QJIqx8x9HKzLwe82pmIiEhgDF8iIiKB1Xra+cSJE5g1\naxYuXLgApVKJnJwczJs3D+Hh4Z6sj4iISHJqHb73338/li9f7slaiIiI/AKnnYmIiATG8CUSUF7e\nYSQn9/bIZ33xxUa8+upLHvksIhIWw5eIiEhggv3Ol4j+kJr6H+zduxtyuQyzZr2L6Oi78OGH7yMv\n7zDkcjkeeaQDXnrpFSgUCpw+fQrvvz8TRqMRarUGI0aMQrt2VW/rWlJixvDhz+G5515A164JXuqK\niG4Vw5dIYL/9dgnNmjXHyJGjsWHD53jnnXfw2GPdcOXKZSxf/hnsdhtGjnwB27bloHv3REyePAH/\n+Mfz6N49ET/88B1ee20k1qzZ6Po8h8OBKVMmoUePJxi8RCLBaWciganVasTHdwcAxMd3x/fff49d\nu3LRq1cfKJVKaDQB6N79CRw6dBCXLl3E1atXkZDwOACgWbMWqF+/Pr7//jvX5y1cmIrw8AgMHfqs\nV/ohoppj+BIJLDQ0DHJ55f969erVA1A5bRwSEup6TUhICPR6PfR6PYKDQyCTya4ZC4VeXwQA+PHH\n77BmTRZ0Oj7ghEhMGL5EAjOZTNf9OTQ0DEaj0bW+uNgIrVYLrVYLk8kIp9PpGjMajdBqIwEAUVE6\nLFuWhQ0bPsf3358UqAMiul0MXyKBWSzl2LVrBwBg587taNWqFTp37orNm9fDbrejrKwMOTlfoH37\njmjQoCF0umhs374FAHD8+FEUFV1F8+YtAQCRkTo0ahSDUaNew/Tpk2GxWLzWFxHdOl5wRSSwu+66\nGydPHsOiRamQy+WYM2c2tNqGuHjxAoYOHQCZTIauXRMQH58AmUyGKVNmYM6cmViyJA0BAYGYOvVd\nBAYGVvnMHj2ewM6dufj44wUYNeo1L3VGRLdK5rx2PqsO+fqjn9wRwyOrakJK/bAX3yWlfqTUCyCt\nfsTQCx8pSCRBlgo7ruhLYamwe7sUIqohTjsTiYzd4UBW7mkcyS9AUbEF2lAN4mJ1SI5vCoWc36eJ\nxIDhSyQyWbmnse3wr67lq8UW1/LghFhvlUVENcCvyUQiYqmw40h+gduxI/mFnIImEgmGL5GIGM0W\nFBW7/zmR3lQOo5k/NSISA4YvkYiEBWugDdW4HYsICUBYsPsxIvItDF8iEdGoFIiL1bkdi4uNgkal\nELgi8nW/P0N64cJUrFq1CgDQsWNbXLly2cuV+TdecEUkMsnxTQFUnuPVm8oRERKAuNgo13oid4YP\nHymK38b6C4Yvkcgo5HIMTohFv85NYDRbEBas4REvVWv69MmIjW2CpKShVdYvWjQfZ8+ewfTpc/DL\nL2fx/vvvorCwEGq1ChMmpKBZsxZeqljaGL5EIqVRKRAdEeTtMkjEtm/fgq+/PoD589MAAOPHv46n\nn34GTz7ZG8eOfYu33hqL1as3QqlkVHgaz/kSEfmh/PwfkJa2ELNmzUVgYCB++eUsDIYi9Oz5FACg\ndesHER4egRMnjnm5Umni1xkiIj80Z85MBAUFITS08jnSZrMJ5eXlGDKkv+s1JSUlVR51SZ7D8CUi\n8kMpKdOwfv1afPRRKkaPfh1RUTrUq1cPK1eu8XZpfoHTzkREfigm5k6MGTMOO3ZsQ17eYdSv3wA6\n3R3YsWMbAMBgMCAlZQLKysq8XKk0MXyJiPxUWFg43nhjPGbOfAdlZaWYMmUG1qz5DIMH98PIkcPQ\ntu3D1z07mjyDz/O9Can9Jk5K/bAX3yWlfqTUCyCtfsTQC5/nS0RE1bLarSgovQqr3ertUiSPF1wR\nEfk5u8OOtac341jBSegtBkRowtFa1xJ9m/aEQs4buNQFhi8RkZ9be3ozdv6617VcZNG7lpNie3mr\nLEnjtDMRkR+z2q04VnDS7djxwpOcgq4jDF8iIj9mtJigtxjcjhWVG2C0+PZFTWLF8CUi8mNhmhBE\naMLdjmkDwhGmufEVu1R7DF8iIj+mVqjRWtfS7VirqJZQK9QCV+QfeMEVEZGf69u0J4DKc7xF5QZo\nA8LRKqqlaz15HsOXiMjPKeQKJMX2wlNNEmG0mBCmCeERbx1j+BIREYDKKWhdUKS3y/ALPOdLREQk\nMIYvERGRwBi+REREAmP4EhERCYzhS0REJDCGLxERkcAYvkRERAJj+BIREQmM4UtERCQwhi8REZHA\nGL5EREQCY/gSEREJjOFLREQkMIYvERGRwBi+REREAmP4EhERCYzhS0REJDCGLxERkcCUtX3jjBkz\ncPToUchkMkyYMAGtW7f2ZF1ERESSVavwPXToEH755RdkZWXhp59+woQJE5CVleXp2oiIiCSpVtPO\nBw4cQEJCAgCgSZMmMBqNMJvNHi2MiIhIqmoVvoWFhYiIiHAta7VaFBQUeKwoIiIiKav1Od9rOZ3O\nal8TEREEpVLhic0JSqcL8XYJHiWlftiL75JSP1LqBZBWP2LupVbhGx0djcLCQtfylStXoNPpbvoe\nvb60NpvyKp0uBAUFJm+X4TFS6oe9+C4p9SOlXgBp9SOGXm725aBW084dOnRATk4OAODkyZOIjo5G\ncHBw7aojIiLyM7U68m3Tpg1atmyJgQMHQiaTISUlxdN1ERERSVatz/m+/vrrnqyDiIjIb/AOV0RE\nRAJj+BIREQmM4UtERCQwhi8REZHAGL5EREQCY/gSEREJjOFLREQkMIYvERGRwBi+REREAmP4EhER\nCYzhS0REJDCGLxERkcAYvkRERAJj+BIRScClSxfRuXO7Gr3n3Lmz+PbbvDqqiG6G4UtE5Kd27drJ\n8PWSWj/Pl4iIfM+mTeuRnb0KJpMJI0aMQrduPfDBB3Nw+PDXcDjsuP/+1hg/PgUHD+7HihVLoFKp\nYDKZMGrUa1i/fi2ysj6F1WpFy5atMGHCv6HRBHi7JUli+BIRSYTD4YDNVoGMjEzs2LENH300DyqV\nCseOHcHy5Z9Bqw1Cr15PYfv2LXj88b/hsce6olGjGPzzn//C0aNHkJ6+EEuWfIqoKB3mzJmBtLSF\nGDlytLfbkiROOxMRSYTT6URi4pMAgNjYZigouIIuXbohPX05lEolNBoNmjVrgYsXL1z33n37dqNb\nt+6IitIBAHr37oddu3YIWr8/4ZEvEZFEKBQKBARUThPL5XI4HA7o9Xr85z+z8eOPP0KlUqCgoABJ\nSYOue6/JZMaePTtw6NBBAIDD4YTNViFo/f6E4UtEJGEff7wASqUSy5ZlolGjSIwc+arb10VFRSEx\n8UlOMwuE085ERBJmMBShceOmUKvV+OGHH3D8+FGUlZUBAJRKJcxmMwCgY8fO2LVrB/R6PQBgz56d\nWLFiqbfKljwe+RIRSdjAgU9j2rTJ+OKLjWjX7mGMHDka7747FS1a3I8OHTphypRJ+O23i5g2bTae\neeZZjBr1IpxOByIitHjjjQneLl+yZE6n0ynEhgoKTEJsxqN0uhBR1n0jUuqHvfguKfUjpV4AafUj\nhl50upAbjnHamYiIAAAORwUqLEVwOHihVV1j+BIReVBe3mEkJ/f2dhk14nQ6UPTrV7j03QJc+i4V\nl75bgKJfv4LT6fB2aZLFc75ERH5Of2ELzAWHXMv2CqNrWRuT6K2yJI3hS0QEYOPGdcjMXAG73Y7I\nyCi8/fY7yMxcip9+Oguj0YCffjqN6OhozJjxHiIitOjf/+/o06c/tm/fiitXfsNTT/XDsGEjqnym\n1WrFggX/xcGDB2CzVaBXrz545pnnvNShew5HBcoMP7odKzPkw9GwG+RylcBVSR+nnYnI7+n1Rfjg\ng9n44IP5yMz8HI0axWDp0nQAwK5dOzB69BtYs2YTGjZshOXLl7jed+LEcaSlZWD58s+wdm02Tp3K\nr/K5K1cuw88//4xlyzKxfPln2LlzO/bt2yNob9WxV5hgrzDeYMwIe4VvX9QkVn4TvgsXpmLdutXe\nLoOIfFBEhBY5ObsQHX0HAOCBB+Jct2Bs0+avaNiwEQDgscficfz4Mdf7EhN7QqFQICJCiwceeBAn\nThyr8rn79u1G3779oVarERgYiMTEnti1K1egrm6NQhUChSrsBmNhUKhufMUu1Z7fTDsPHz7S2yUQ\nkY+y2+1IT1+Ifft2w263o7S0FHfeeRcAIDT0j2AKCQmByfTHkWBoaOg1Y6EwmYqrfK7JZMaHH87F\nokXzAQAVFRVo3rxlXbZSY3K5CoHhf6lyzvd3geGxnHKuIz4XvpcuXcTw4c9iwIDB2LRpPZxOYNKk\nKcjISMepU/l4+OFHkJjYE7NmTUNW1joAlVcX/r585sxpzJo1HSUlJbDZKpCUNBD9+iVj+vTJrqd3\n/PDD95g9ezpKS0sRFRWFCRNSXN9sicj/bN++Ffv27UZqahrCw8OxYcPn2LLlSwCAwWBwvc5kKq4S\nuEbjH2PFxcYqY0DlLRsHDRqKDh061XEHtyeiUQ8Aled47RVGKFRhCAyPda0nz/PJaWeDwQCtNhKr\nVq1F06ZNkZIyHhMnTkFGxips25aDixd/veF7P/kkDb1798OKFZ9h4cIlOHz4EKxWa5XXTJ48AcOG\njUBm5lo89lgXfPDB7LpuiYh8mMFQhPr1GyA8PBxGowG5uVtdt2A8duxbXL78GwBgx47teOCBONf7\ntm/fCofDgaKiqzh27Chat46r8rmdOnXGpk3rYLfb4XQ6sXRpOg4e3C9cY7dIJpNDG5OIBi1GoEGL\nl9GgxQhoYxIhk/lkREiCzx35ApVTQPHxCQCAxo2bAgDCw8MBAJGRUSgsLLzheyMitNi5czsaN26K\n2Ni/YObM96uMnzv3CwwGA9q37wAA6Nt3AJ56ql9dtEFEIpGQ8Di2bs1BcnJvNGzYCMOGvYS33hqD\nkpISPPRQO8ydOwunTuXjjjvqY/ToN1zvu/fexhg27B/47beL6N8/GY0bN0Fe3mHXeN++A3Dp0iUM\nHToATqcTzZq1wIABg73R4i2Ry1WQa7TeLsMv+GT4KhQKaDR/PBYrMDDINSaXy9G69YP48stNbt87\nYsQoLF++BP/+91uwWq0YOvRZ9O2b5Bo3Gg0IDg52LSuVSiiVPvnXQEQC0WojkZaWUWXdxo1bkJm5\nFPXq1cPUqe+6fd9f//ownnvuhSrr2rRp6zolplKpMHr063VTNImaKOcUjh37Fg7HH3deufYCiKCg\nILz44svIylqHGTPmID19Ic6d+8U1HhYWjuLiYtf7bTYbLl26KFzxROQ3LBV2XNGXwlJh93Yp5GNE\nGb6RkVG4erUQen0R7HY7tm790jU2btxrOHPmJwBA48ZNUK9eMGQymWv8zjvvQnR0tOty/02b1mH2\n7OnCNkBEkmZ3OLByWz4mpR3E+EUHMSntIFZuy4fdwds11pbVar3hjKcYiXK+tVGjGPztb73w7LND\ncMcd9ZGY2NP14/b+/ZMxZcok2GyVNwbv06e/6ycDACCTyfDOO+9i6tS3sWjRfERGVl7tTET0Z6NG\njbrhk3NWr954w/dl5Z7GtsN/XBh6tdjiWh6cEOvZIv1Efv6P+OqrL/DEE096uxSP4CMFb0IMj6yq\nCSn1w158l5T6qU0vlgo7JqUdxNViy3VjkaEBmDasHTQqhadKrBFf2zdffrkJGRmfAABatmyJN998\nGzk5X1x3m0+1Wo1nnx2MkpISxMY2w4IF6T7Xizt8pOCfWO1WFJRehdVurf7FREQ1YDRbUOQmeAFA\nbyqH0ex+zN9cunQR8+f/F6mpi7Bq1RqUlZXjs89Wur3Np1YbiRdfHImWLVtjwYJ0b5fuEaKcdq4t\nu8OOtac341jBSegtBkRowtFa1xJ9m/aEQu6db6JEJC1hwRpoQzVuj3wjQgIQFqzxQlW+59Chg2jV\nqjWionQAgJSUaVAoFEhOHgKVqvKuWg88EIecnC+8WWad8avwXXt6M3b+ute1XGTRu5aTYnt5qywi\nkhCNSoG4WF2Vc76/i4uN8tqUs6+p/NnnH9OyGo0GdrsdaWkfub3Np9T4zbSz1W7FsYKTbseOF57k\nFDQReUxyfFMktI1BZGgA5LLKc70JbWOQHN/U26X5jLCw8Cq35ywpMWPLli9dt/lctWotnn/+RS9W\nWLf85sjXaDFBbzG4HSsqN8BoMUEXFClwVUQkRQq5HIMTYtGvcxMYzRaEBWt4xPsn7dt3wEcfzcOl\nSxdRv34DzJkzE40bN7nhbT6VSiVKS81wOp1Vfj4qVn5z5BumCUGEJtztmDYgHGEaPjaLiDxLo1Ig\nOiKIwetGdPQdGDduIl55ZQQGDeoLmUyGHj2egNFoRHJyb0yePBHDhr2EK1cuY968D9C69YMoLCxE\n796JsNvFf9MSvznyVSvUaK1rWeWc7+9aRbWEWqH2QlVERP4rPj7BdR//3/35Np/rV2+EzWiEMiwM\nn38unYuv/CZ8AaBv054AKs/xFpUboA0IR6uolq71RETkG5x2OwqyM2E+kgdbURGUWi2C49pAlzQQ\nMoX4ZxL8KnwVcgWSYnvhqSaJMFpMCNOE8IiXiMgHFWRnwrBtq2vZdvWqazl64BBvleUxfnPO91pq\nhRq6oEgGLxGRD3JYLDAfyXM7Zj5yBA6L+G9U4pfhS0REvstmNMJWVOR+TF8Em9EocEWex/AlIiKf\nogwLg1KrdT8WoYUyLEzgijyP4UtERD5FrtEgOK6N27HguDjINeK/RadfXXBFRETioEsaCKDyHK9N\nXwRlhBbBcXGu9WLH8CUiIp8jUygQPXAIovr0d/3OVwpHvL9j+BIRkc+SazRQR0d7uwyP4zlfIiIi\ngTF8iYiIBFbr8D106BDat2+PHTt2eLIeIiIiyatV+J47dw5LlixBmzbuLwUnIiKiG6tV+Op0OqSm\npiIkpO4fw/fddycwZsxIAMD06ZOxdGk6AKBjx7a4cuVynW+fiIjI02p1tXNgYGCN3xMREQSlsuZP\noujcuT06d24PAAgIUKFePQ10usrQj4wMdv25rtT15wtNSv2wF98lpX6k1AsgrX7E3Eu14ZudnY3s\n7Owq60aNGoVOnTrVaEN6fel16y5duojhw5/FgAGDsWnTejidwKRJU5CRkY5Tp/Lx8MOPIDGxJ2bN\nmoasrHUoL69ASYkFBQUmAMDVq2YoFCYsWjQfZ8+ewfTpcyCXe+4aMp0uxLUtKZBSP+zFd0mpHyn1\nAkirHzH0crMvB9WGb1JSEpKSkjxa0LUMBgO02kisWrUWkyaNQ0rKeCxevAIyGdCnz9/QuvUDN33/\n9u1b8PXXBzB/fppHg5eIiKiueP0mG3a7HfHxCQCAxo2bAgDCw8MBAJGRUSgsLLzhe/Pzf0Ba2kLM\nm7ewVlPhRERE3lCrQ8WdO3di6NCh2LNnD+bOnYvnnnuu1gUoFApoNAGVxcjlCAwM+qM4uRytWz94\nw/fOmTMTMhkQGhpa6+0TEREJrVZHvl26dEGXLl08XErNpaRMw/r1a/HRR6kYPfp1b5dDRER0S0R9\nkjQm5k6MGTMOO3ZsQ17eYW+XQ0REdEtEHb4AEBYWjjfeGI+ZM99BaWmJt8shkTh69Fv07/93b5dB\nRH5K5nQ6nUJsyNOXhFdU2FFqtiIoWA2Vqua/H74VYriUvSak1M/t9nL06LeYOvVtrF690YNV1Y6U\n9gsgrX6k1AsgrX7E0Mtt/dTI1zgcDuzP/Qk/5xfCXGxBcKgG98ZG4dH4JvypEd3U0qXp2LDhc4SF\nhaFjx84AAIvFgg8/fB95eYchl8vxyCMd8NJLr0ChUODrrw9g1qxpCAwMxIABgzF//n+RkbEKDRo0\n9HInRCR2okur/bk/4fjhCzBS0eRSAAASBklEQVQXWwAA5mILjh++gP25P3m5MvJlP/98BllZK5Ge\nvgzp6ctx+vQpAMBnn63ClSuXsXz5Z/jkkxU4duwItm3Lgd1ux/Tpk/HGGxPw6aer8euv51FeXubl\nLohIKkQVvhUVdvyc7/53v2fzC1FRYRe4IhKLo0fz8OCDbaDVRkKhUODxx58AABw4sBe9evWBUqmE\nRhOA7t2fwKFDB3H+/DlYrVa0b98BANCvXzIcDoc3WyAiCRFV+Jaara4j3j8zmywoNVsFrojEori4\nGMHBwa7lkJDK34YbDHrXnyvXh0Cv18NkKq6yPioqSrhiiUjyRBW+QcFqBIdq3I4Fh2gQFKwWuCIS\ni5CQUJjNZteywaAHAGi1kTAaja71xcVGaLVa1KtXD2Vlf9yPvKjoqnDFEpHkiSp8VSoF7o11fwRy\nT2xUnV31TOJ3//2tcPz4t9Dr9bDb7cjJ+RIA8OijHbF583rY7XaUlZUhJ+cLtG/fETExd8Fms7l+\nP75u3RrIZDJvtkBEEiK6q50fjW8CoPIcr9lkQXCIBvf8/9XORDdy331/wVNP9cPzzz+N0NAwJCT0\nwJkzp9GvXzIuXryAoUMHQCaToWvXBMTHJ0Amk+H119/CjBlTEBwcguTkwZDL5QxgIvII/s73JsTw\nO7KakFI/QvdSVlaG7t074auvdlY5d+wJUtovgLT6kVIvgLT6EUMvN/udr6imna+lUikQFhHIqWaq\nM//61zPI2ZqDq+VWbNmag3vuudfjwUtE/kl0085EQrA7nfjrgGfx3/RUVCyYB3VgEHr/61XYnU4o\nOPVMRLeJ4UvkxpfnCnE+IgYPvv6ua93P/7/+ybt13iuMiCRBtNPORHXFanfge4PZ7dj3BjOsdt5s\ng4huD8OX6E9MFTYYrDa3YwarDaYK92NERLeK4Uv0JyEqJcLV7s/IhKuVCFHxbA2J39Sp/8bevbs9\n8llffLERr776kkc+y1/wXxGiP1Er5GgeHoz9VwzXjTUPD4Zawe+sJH5vv/2Ot0vwawxfIjeeuKvy\nTmrfG8wwWG0IVyvRPDzYtZ7I12zcuA6ZmStgt9sRGRmFt99+B3l5h7F//17Uq1cPR49+C41GhZSU\nGWjcuAlGjnwBf/97bzz++N/QsWNbjBs3EatXZ8JsNmPixMnYsOFznDhxDPfc0xizZs2FUqnE3r27\n8PHHC1BRUYHAwCCMH/827rvvL95uXZT4FZ7IDYVMhifv1uHV++/GmFZ349X778aTd+v4MyPySXp9\nET74YDY++GA+MjM/R6NGMVi6NB0AcPDgPvTpk4TMzLVo164dsrNXuf0Mo9GAZcuyEB/fHRMnjsPz\nz7+IVavW4syZ0/j22zzYbDZMmzYZ48ZNwqpVa9GpU2ekpv5XuCYlhuFLdBNqhRyRAWpONZNPi4jQ\nIidnF6Kj7wAAPPBAHC5evAAAuOeexmjWrDkAoEWLFrh8+Te3n9GpUxcAQJMmTdGoUQzuuutuqNVq\nxMTcicLCAiiVSmzatBX339/qum1QzXHamYhI5Ox2O9LTF2Lfvt2w2+0oLS3FnXfeBQCoV++Pu7Ip\nFArYb/BTuaCgIACAXC5HYGCga71crnA9yzo7OxNffbUJVqsVVquV9zq/DQxfIiKR2759K/bt243U\n1DSEh4djw4bPsWXLlx7dxvHjR/HppxlIS8tAgwYN8c03BzFr1nSPbsOfcC6NiEjkDIYi1K/fAOHh\n4TAaDcjN3YqysjKPbkOv1yMiIgJ33FEf5eXl+PLLzSgvL4NAz+aRHIYvEZHIJSQ8DqPRiOTk3pg8\neSKGDXsJV65cRmrqfzy2jXbt2iMqSofk5N547bWXMWDAINSrF4xJk8Z5bBv+RLSPFBSCGB5ZVRNS\n6oe9+C4p9SOlXgBp9SOGXiT5SEEiIqr03XcnMGbMSEG3abU7cLXcynud1xIvuCIiErkWLe7H3Lmp\ngmzL7nTiy3OFbm9Aw9/B3zoe+RIRiVxe3mEkJ/cWZFtfnivE/isG6K02OAHorTbsv2LAl+cKBdm+\nVDB8iYjolvBxm57D8CUiolvCx216DsOXiIhuCR+36TkMXyIiuiW/P27THT5us2b4NYWIiG4ZH7fp\nGQxfIiK6Zb8/brNHTCRMFTaEqJQ84q0Fhi8Rkci1adMWWVnrBN2mWiFHpEIt6DalhF9XiIhExmGx\nwHrlChwWi7dLoVrikS8RkUg47XYUZGfCfCQPtqIiKLVaBMe1gS5pIGQKhbfLoxpg+BIRiURBdiYM\n27a6lm1Xr7qWowcO8VZZVAucdiYiEgGHxQLzkTy3Y+YjRzgFLTIMXyIiEbAZjbAVFbkf0xfBZjQK\nXBHdDoYvEZEIKMPCoNRq3Y9FaKEMCxO4IrodDF8iIhGQazQIjmvjdiw4Lg5yjUbgiuh28IIrIiKR\n0CUNBFB5jtemL4IyQovguDjXehIPhi8RkUjIFApEDxyCqD79YTMaoQwL4xGvSDF8iYhERq7RQB0d\n7e0y6DbwnC8REZHAGL5EREQCY/gSEREJjOFLREQkMIYvERGRwBi+REREAmP4EhERCYzhS0REJDCG\nLxERkcBqdYcrm82GiRMn4ty5c7Db7Rg3bhzatm3r6drIg/bs2YklSz6GyVSCmJgYPPPMc5gw4Q1s\n2JADlUoFAJg0aRxat34QvXv3x4IF/8XBgwdgs1WgV68+eOaZ5wAAa9ZkYe3abDidTtSrVw/jx6eg\nceMm3myNiEh0ahW+69evR2BgIFatWoVTp05h/PjxWL16tadrIw+5cOFXTJ2agqysTERENMDy5Uuw\nZk0WoqOj8fXX+9GxY2dYLBYcOvQ1Ro9+AytXLsPPP/+MZcsyYbfb8fLL/0KTJvchLq4N0tIWYu3a\nTQgKqofc3G04cGAvw5eIqIZqNe3cq1cvjB8/HgCg1WphMBg8WhR51tdfH0BcXBvExsYCAJ56qh/2\n7t2N+Pju2Lo1BwDwzTcHERv7F0RF6bBv32707dsfarUagYGBSEzsiV27cqFWayCTybBp03oUFV1F\nfHwChgz5hzdbIyISpVod+f4+TQkAGRkZePLJJz1WEHme2WzC0aNHkJiYCLvdAQAIDg5G27btsGLF\nUpSVlWH37p2Ij+8OADCZzPjww7lYtGg+AKCiogLNm7eEUqnEf/+7AMuWLcHixYvQpMl9GDv2LTRp\n0tRrvRERiZHM6XQ6b/aC7OxsZGdnV1k3atQodOrUCZ9++ilyc3OxcOHCKoHsjs1mh1KpuP2KqcbW\nrl2LnTt34sMPP7xuLDk5GUOGDMHMmTOxadMmREZGYujQoXjuuefQtWvXG36m1WpFeno6du/ejczM\nzLosn4hIcqo98k1KSkJSUtJ167Ozs5Gbm4sFCxZUG7wAoNeX1q5CL9LpQlBQYPJ2GbetefM4zJ49\nB+fPn0dAQDi+++4Etmz5CqNHv44uXRLw/vtzce+9TeFwqFFQYEK7dh3w6aer0Lx5HORyOTIyFqNZ\nsxbQ6aKxZEkaUlKmQaVS4c47m8Bm2+mVvyOp7BtAWr0A0upHSr0A0upHDL3odCE3HKvVtPP58+eR\nmZmJFStWQMMHOfu8qKgovPnmRLz88ssoL7cgKCgIr7wyFgAQH98d8+Z94LqaGQD69h2AS5cuYejQ\nAXA6nWjWrAUGDBiMwMBANGzYEEOHDoBSqUJQUBDGjHnTW20REYlWtdPO7sydOxebN29Gw4YNXesW\nL14MtVp9w/f4+jcUd8Twzaom3PVjtVqRlPR3LF/+GUJDw6r9jIoKO0rNVgQFq6FSee80gpT2jZR6\nAaTVj5R6AaTVjxh68fiR75gxYzBmzJhaF0S+IytrJdq371ht8DocDuzP/Qk/5xfCXGxBcKgG98ZG\n4dH4JpDLea8WIqKaqFX4kjQMHtwPERFaTJ8+p9rX7s/9CccPX3Atm4struWOCffVWY1ERFLE8PVj\nK1euuaXXVVTY8XN+oduxs/mFaNe5sVenoImIxIbzhVStUrMV5mKL2zGzyYJSs1XgioiIxI3hS9UK\nClYjONT9Ve3BIRoEBd/4QjsiIroew5eqpVIpcG9slNuxe2KjOOVMRFRDPOdLt+TR+MqHJ5zNL4TZ\nZEFwiAb3/P/VzkREVDMMX7olcrkcHRPuQ7vOjX3id75ERGLG8KUaUakUCIsI9HYZRESixnO+RERE\nAmP4EhERCYzhS0REJDCGLxERkcAYvkRERAJj+BIREQmM4UtERCQwhi8REZHAGL5EREQCY/gSEREJ\nTOZ0Op3eLoKIiMif8MiXiIhIYAxfIiIigTF8iYiIBMbwJSIiEhjDl4iISGAMXyIiIoExfK9hs9nw\n5ptvYtCgQRgwYAAOHz583Ws2bNiAfv36ISkpCdnZ2V6o8tYdOnQI7du3x44dO9yOt2zZEkOHDnX9\nZ7fbBa6wZqrrRyz7pqKiAmPHjsWgQYPw9NNP4/z589e9Rgz7ZsaMGUhOTsbAgQNx7NixKmP79+9H\n//79kZycjPnz53upwpq5WT/x8fEYPHiwa39cvnzZS1Xemvz8fCQkJGDFihXXjYlx39ysH7HtGxcn\nuaxevdqZkpLidDqdzvz8fGe/fv2qjJeUlDh79OjhLC4udpaVlTl79uzp1Ov1Xqi0er/88otz+PDh\nzpdeesmZm5vr9jUPP/ywwFXVXnX9iGnfrF271jl58mSn0+l07tmzx/nqq69e9xpf3zdff/2184UX\nXnA6nU7n6dOnnQMGDKgy/sQTTzgvXrzotNvtzkGDBjlPnTrljTJvWXX9dO3a1Wk2m71RWo2VlJQ4\nn376aeekSZOcy5cvv25cbPumun7EtG+uxSPfa/Tq1Qvjx48HAGi1WhgMhirjR48eRatWrRASEoKA\ngAC0adMGeXl53ii1WjqdDqmpqQgJCfF2KR5RXT9i2jcHDhxA9+7dAQCPPvqoz9Z5MwcOHEBCQgIA\noEmTJjAajTCbzQCA8+fPIywsDA0aNIBcLkfnzp1x4MABb5ZbrZv1IzZqtRppaWmIjo6+bkyM++Zm\n/YgZw/caKpUKGo0GAJCRkYEnn3yyynhhYSG0Wq1rWavVoqCgQNAab1VgYCAUCsVNX2O1WjF27FgM\nHDgQS5YsEaiy2qmuHzHtm2trlcvlkMlksFqtVV7j6/umsLAQERERruVr/74LCgpEsy9+d7N+fpeS\nkoJBgwbhvffeg9OHbwyoVCoREBDgdkyM++Zm/fxOLPvmWkpvF+At2dnZ150XHDVqFDp16oRPP/0U\nJ0+exMKFC2/6Gb6yk2/Wy82MGzcOvXr1gkwmw9NPP422bduiVatWdVnqLaltP9fy5X1z9OjRKsvu\navXVfXMjvvL37Sl/7ueVV15Bp06dEBYWhpdffhk5OTlITEz0UnV0LbHuG78N36SkJCQlJV23Pjs7\nG7m5uViwYAFUKlWVsejoaBQWFrqWr1y5ggcffLDOa63OjXqpzqBBg1x/fuSRR5Cfn+8T/8DXph8x\n7Zu33noLBQUFaNasGSoqKuB0OqFWq6u8xlf3ze/c/X3rdDq3Y5cvX/b5KcOb9QMAvXv3dv35scce\nQ35+vij+gf8zMe6b6oh133Da+Rrnz59HZmYmUlNTXdPP13rggQdw/PhxFBcXo6SkBHl5eWjbtq0X\nKr19Z86cwdixY+F0OmGz2ZCXl4f77rvP22XVmpj2TYcOHfDVV18BAHbs2IF27dpVGRfDvunQoQNy\ncnIAACdPnkR0dDSCg4MBADExMTCbzfj1119hs9mwY8cOdOjQwZvlVutm/ZhMJjz//POuUwPffPON\nz+2PWyXGfXMzYt43fKrRNebOnYvNmzejYcOGrnWLFy/G0qVL8dBDDyEuLg5fffUVFi9e7JoO7NWr\nlxcrvrGdO3di8eLFOHPmDLRaLXQ6HT755BN8/PHHrl7mzJmDgwcPQi6XIz4+HiNGjPB22Td0K/2I\nZd/Y7XZMmjQJZ8+ehVqtxrvvvosGDRqIbt+89957OHz4MGQyGVJSUvDdd98hJCQE3bt3xzfffIP3\n3nsPANCjRw88//zzXq62ejfrJyMjA+vWrYNGo0GLFi3w9ttvQyaTebtkt06cOIFZs2bhwoULUCqV\nuOOOOxAfH4+YmBhR7pvq+hHTvrkWw5eIiEhgnHYmIiISGMOXiIhIYAxfIiIigTF8iYiIBMbwJSIi\nEhjDl4iISGAMXyIiIoExfImIiAT2f66afYe2UguxAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - "'''\n", - "import tensorflow as tf\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "tf.reset_default_graph()\n", - "\n", - "# 3 Words Sentence\n", - "sentences = [ \"i like dog\", \"i like cat\", \"i like animal\",\n", - " \"dog cat animal\", \"apple cat dog like\", \"dog fish milk like\",\n", - " \"dog cat eyes like\", \"i like apple\", \"apple i hate\",\n", - " \"apple i movie book music like\", \"cat dog hate\", \"cat dog like\"]\n", - "\n", - "word_sequence = \" \".join(sentences).split()\n", - "word_list = \" \".join(sentences).split()\n", - "word_list = list(set(word_list))\n", - "word_dict = {w: i for i, w in enumerate(word_list)}\n", - "\n", - "# Word2Vec Parameter\n", - "batch_size = 20\n", - "embedding_size = 2 # To show 2 dim embedding graph\n", - "voc_size = len(word_list)\n", - "\n", - "def random_batch(data, size):\n", - " random_inputs = []\n", - " random_labels = []\n", - " random_index = np.random.choice(range(len(data)), size, replace=False)\n", - "\n", - " for i in random_index:\n", - " random_inputs.append(np.eye(voc_size)[data[i][0]]) # target\n", - " random_labels.append(np.eye(voc_size)[data[i][1]]) # context word\n", - "\n", - " return random_inputs, random_labels\n", - "\n", - "# Make skip gram of one size window\n", - "skip_grams = []\n", - "for i in range(1, len(word_sequence) - 1):\n", - " target = word_dict[word_sequence[i]]\n", - " context = [word_dict[word_sequence[i - 1]], word_dict[word_sequence[i + 1]]]\n", - "\n", - " for w in context:\n", - " skip_grams.append([target, w])\n", - "\n", - "# Model\n", - "inputs = tf.placeholder(tf.float32, shape=[None, voc_size])\n", - "labels = tf.placeholder(tf.float32, shape=[None, voc_size])\n", - "\n", - "# W and WT is not Traspose relationship\n", - "W = tf.Variable(tf.random_uniform([voc_size, embedding_size], -1.0, 1.0))\n", - "WT = tf.Variable(tf.random_uniform([embedding_size, voc_size], -1.0, 1.0))\n", - "\n", - "hidden_layer = tf.matmul(inputs, W) # [batch_size, embedding_size]\n", - "output_layer = tf.matmul(hidden_layer, WT) # [batch_size, voc_size]\n", - "\n", - "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=output_layer, labels=labels))\n", - "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n", - "\n", - "with tf.Session() as sess:\n", - " init = tf.global_variables_initializer()\n", - " sess.run(init)\n", - "\n", - " for epoch in range(5000):\n", - " batch_inputs, batch_labels = random_batch(skip_grams, batch_size)\n", - " _, loss = sess.run([optimizer, cost], feed_dict={inputs: batch_inputs, labels: batch_labels})\n", - "\n", - " if (epoch + 1)%1000 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - " trained_embeddings = W.eval()\n", - "\n", - "for i, label in enumerate(word_list):\n", - " x, y = trained_embeddings[i]\n", - " plt.scatter(x, y)\n", - " plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')\n", - "plt.show()" - ] - } - ], - "metadata": { - "colab": { - "name": "Word2Vec-Tensor(Softmax).ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/1-2.Word2Vec/Word2Vec_Skipgram_Torch(Softmax).ipynb b/1-2.Word2Vec/Word2Vec_Skipgram_Torch(Softmax).ipynb deleted file mode 100644 index 5a76713..0000000 --- a/1-2.Word2Vec/Word2Vec_Skipgram_Torch(Softmax).ipynb +++ /dev/null @@ -1,152 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "3PgIns-jRj_8", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 437.0 - }, - "outputId": "3956131d-9474-4ca2-bb14-19ec689ad61c" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1000 cost = 2.263604\n", - "Epoch: 2000 cost = 2.092835\n", - "Epoch: 3000 cost = 1.889603\n", - "Epoch: 4000 cost = 2.024243\n", - "Epoch: 5000 cost = 1.588786\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd8AAAFKCAYAAABcq1WoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8U1X+//FXlja0dA1NVUBUwAKi\nIPxQBwXRWhA3ZC+i+HVEhtEBh1FRNgdQQcERNxSwqCAoVBYVRYZ9V2QYEAQdi7sC2pYmaWvbtFl+\nfzATh7GA1vamSd7Px8OH3Jzk3s85lLx7zz25MQUCgQAiIiJiGHOoCxAREYk2Cl8RERGDKXxFREQM\npvAVERExmMJXRETEYApfERERg1mNOlBBQYlRhzJcamo8TmdZqMuIGhpv42nMjaXxNlZdjbfDkXjC\nNp351gKr1RLqEqKKxtt4GnNjabyNFYrxVviKiIgYTOErIvXOkSOH6dbtkl/1mm+++YoPP9xdRxWJ\n1C6Fr4hEhM2bNyl8JWwYtuBKROTXeuedt1iyZBElJSXceedIrrqqB08++Ti7dn2A1+ulXbv2jB07\nkR073mPhwpeJiYmhpKSEkSP/wltvLSc391UqKytp2/YCxo37KzZbg1B3SQRQ+IpIPeX3+/F6q5g/\nfzEbN65j1qxniYmJYd++PSxY8Do+n4+hQ29h/fo1XH31tVx++ZU0adKU2267g7179zB37mxefvlV\n0tIcPP74VHJyZjNixKhQd0sE0LSziNRTgUCAnj2vByAjozUFBflcccVVzJ27AKvVis1mo3Xr8zh8\n+NDPXrt9+xauuqo7aWkOAHr37sfmzRsNrV/kZHTmKyL1ksVioUGDY9PEZrMZv9+P0+nkqaem8+mn\nn2I2mygqOsqAATf97LUlJaVs3bqRnTt3AOD3B/B6qwytX+RkFL4iEjZeeOF5rFYrr7yymNjYWCZP\nnlDt89LS0ujZ83pNM0u9pWlnEQkbLlcRzZu3JDY2loMH8/joo72Ul5cDYLVaKS0tBaBLl25s3rwR\np9MJwNatm1i4cF6oyhb5GZ35ikjYGDToFh55ZBLvvvs27dp1YMSIUTz22MOcd975XHZZVyZPnsD3\n3x/mkUemc+utv2fkyOEEAn5SU+2MHj0u1OWLBJkCgUDAiANF8r2dHY7EiO5ffaPxNp7G3Fgab2PV\n1Xjr3s4iEnX8/iqqPEX4/VpoJfWPpp1FJKIEAn6ch9ZQ7voUX5UbS0wycSmtSG3SA5NJ5xtSPyh8\nRSSiOA+tobRgZ3DbV+UObtub9gxVWSLH0a+BIhIx/P4qyl2fVttW7srTFLTUGwpfEYkYvqoSfFXu\nE7S58VVpEZPUDwpfEYkYlphELDHJJ2hLxhJz4tWnIkZS+IpIxDCbY4hLaVVtW1xKBmZzjMEViVRP\nC65EJKKkNukBHLvG+9Nq54zg4yL1gcJXRCKKyWTG3rQn/sZX4asqwRKTqDNeqXc07SwSIR5++K9s\n27alVvb17rtv8+c/31Ur+woVszmGGJtdwSv1ks58RSLEgw8+FOoSROQXUviK1FNvv/0mixcvxOfz\n0ahRGg8++BC7d+/ivfe20bBhQ/bu/RCr1cJDDz1G8+YtGDHiD9xwQ2+uvvpaunTpxP33j2fp0sWU\nlpYyfvwkVqx4g/3793H22c2ZNm0GVquVbds288ILz1NVVUVcXDxjxz7IuedWv2BJRGqPpp1F6iGn\ns4gnn5zOk08+x+LFb9CkSVPmzZsLwI4d2+nTZwCLFy+nQ4dOLFmyqNp9uN0uXnkll8zM7owffz9D\nhw5n0aLlfPHFZ3z44W68Xi+PPDKJ+++fwKJFy+natRszZz5tXCdFopjCV6QeSk21s3r1ZtLTTwOg\nffsOHD58CICzz25O69ZtAGjVqhU//PB9tfvo2vUKAFq0aEmTJk1p1uwsYmNjadr0TAoLC7Barbzz\nzlrOP/+Cnx1DROqWpp1F6iGfz8fcubPZvn0LPp+PsrIyzjyzGQANGyYEn2c2W/D5/NXuIz4+/t/P\nMRMXF3fca/z+Y69ZsmQxf//7O1RWVlJZWYnJZKqrLonIf1H4itRD69evZfv2LcycmUNKSgorVrzB\nmjWravUYH320l1dfnU9OznzOOKMx//jHDqZNm1KrxxCR6mnaWaQecrmKOP30M0hJScHtdrFhw1rK\ny8tr9RhOp5PU1FROO+10KioqWLVqJRUV5QQCgVo9joj8nMJXpB7Kyroat9tNdnZvJk0az7Bhd5Gf\n/wMzZz5Va8e45JLOpKU5yM7uzV/+8icGDryJhg0TmDDh/lo7hohUzxQw6NfcgoLI/TYRhyMxovtX\n32i860ZVlY+y0kriE2KJibEc16YxN5bG21h1Nd4Ox4m/yEPXfEWinN/v570Nn/NlXiGlxR4Skmyc\nk5HGpZktMJs1OSZSFxS+IlHuvQ2f89Gunz5iVFrsCW53yTo3VGWJRDT9WisSxaqqfHyZV1ht21d5\nhVRV+QyuSCQ6KHxFolhZaSWlxZ5q20pLPJSVVhpckUh0+E3TztOnT+ef//wnXq+X4cOH06OHvi9T\nJJzEJ8SSkGSrNoATEm3EJ8SGoCqRyFfj8N2xYwcHDx4kNzcXp9NJnz59FL4iYSYmxsI5GWnHXfP9\nj7Mz0n626llEakeNw/eiiy6iXbt2ACQlJVFeXo7P58Ni0T9WkXByaWYL4Ng13tISDwmJNs7+92pn\nEakbNQ5fi8USvHfs0qVLufzyyxW8p7B16yZycmZRXl5B06ZNufXW2xk3bjQrVqwmJubYF35PmHA/\n7dpdSO/e/Xn++afZseN9vN4qevXqw6233g7AsmW5LF++hEAgQMOGDRk7diLNm+uNUmrGbDbTJetc\nLunW/ISf8xWR2vWbb7Kxbt065syZw0svvURi4ok/UOz1+rBao/cf9LfffsuNN97I4sWLycjIYM6c\nORw4cIAvv/ySUaNGcdVVV+HxeOjcuTOrVq1i6dKl/OMf/+CFF17A6/Vy8803c/fdd3PRRRdx5ZVX\nsnHjRhISEli1ahXfffcdw4YNC3UXRUTkF/pNC662bt3K7NmzmTt37kmDF8DpLPsth6rXfsndUd59\ndy0XXtiR1NQzKCgoISvrep555hl+//thLFv2Ju3aXcy2bZs599xWmM3xrF27jltuuQ23+9hCmKys\nnqxYsZLWrS8kEID5818lK+tqOnXqQqdOkX0Hsf+lu/8YT2NuLI23scLqDlclJSVMnz6defPmkZKS\nUtPdRI3S0hL27t3D4MH9go8lJCTQqdMlLFw4j/LycrZs2URmZncASkpKeeaZGcyZ8xwAVVVVtGnT\nFqvVytNPP88rr7zMiy/OoUWLc7n33jG0aNEyJP0SEZFfr8bh++677+J0Ohk1alTwsWnTptG4ceNa\nKSzSpKU56NTpYh55ZPrP2po3b8nWrZt4771t3HnnyH8/P42bbhrCZZd1/dnzMzJa88gj06iqquLV\nV+fzt79NZdasl+q8DyIiUjtqfJON7Oxstm3bxoIFC4L/KXhP7OKLO7N374ccOvQdAB9/vJ+nnvob\nAN27X80LLzxPixbnkppqB6Br1268886b+Hw+AoEA8+bNZceO9/j888+YMOEBqqqqiImJoXXr8wB9\nAbqISDjRvZ0NkpaWxgMPjGfcuNF4vVXEx8dz9933ApCZ2Z1nn30yuJoZoG/fgRw5coQhQwYSCARo\n3fo8Bg4cTFxcHI0bN2bIkIFYrTHEx8dzzz0PhKpbIiJSA/pKwVrwWy/WV1ZWMmDADSxY8DpJSckn\nf67PT0mVl8QYK7GW6Lw7qBajGE9jbiyNt7HCasGV1J7c3Nfo3LnLSYPXFwiw6ptCPnGV4qr0khJr\npU1KAtc0S8Ni0rSziEg4UfiG2ODB/UhNtTNlyuMnfd6qbwp5L98V3HZWeoPb15/lqNMaRUSkdil8\nQ+y115ad8jmVPj+fuEqrbfvEVUqPpo2idgpaRCQc6R07DJRUeXFVeqttc1V6Kamqvk1EROonhW8Y\nSIyxkhJb/SRFSqyVxBhNYIiIhBOFbxiItZhpk5JQbVublARNOYuIhBmdMoWJa5qlAVS72llERMKL\nwjdMWEwmrj/LQY+mjaL+c74iIuFO4RtmYi1mGlliQ12GiIj8Bjp1EhERMZjCV0RExGAKXxEREYMp\nfEVERAym8BWJYrt37yI7u3eoyxCJOgpfERERg+mjRiJh6O2332Tx4oX4fD4aNUrjwQcfYuXKFXz/\n/RHcbheff/4Z6enpTJ36N1JT7fTvfwN9+vRn/fq15Od/z4039mPYsDuP22dlZSXPP/80O3a8j9db\nRa9efbj11ttD1EORyKYzX5Ew43QW8eST03nyyedYvPgNmjRpyrx5cwHYvHkjo0aNZtmyd2jcuAkL\nFrwcfN3+/R+RkzOfBQteZ/nyJRw8mHfcfl977RW+/PJLXnllMQsWvM6mTevZvn2roX0TiRYKX5Ew\nk5pqZ/XqzaSnnwZA+/YdOHz4EAAdO/4/GjduAsDll2fy0Uf7gq/r2fM6LBYLqal22re/kP379x23\n3+3bt9C3b39iY2OJi4ujZ8/r2Lx5g0G9EokumnYWCTM+n4+5c2ezffsWfD4fZWVlnHlmMwCSkpKD\nz0tMTKSkpCS4nZSU9F9tSZSUFB+335KSUp55ZgZz5jwHQFVVFW3atK3LrohELYWvSJhZv34t27dv\nYebMHFJSUlix4g3WrFkFgMvlCj6vpKT4uMB1u39qKy52H9cGkJaWxk03DeGyy7rWcQ9ERNPOImHG\n5Sri9NPPICUlBbfbxYYNaykvLwdg374P+eGH7wHYuHE97dt3CL5u/fq1+P1+ioqOsm/fXtq163Dc\nfrt27cY777yJz+cjEAgwb95cdux4z7iOiUQRnfmKhJmsrKtZu3Y12dm9ady4CcOG3cWYMfdQVlbG\nRRddwowZ0zh4MI/TTjudUaNGB193zjnNGTbs//j++8P0759N8+Yt2L17V7C9b9+BHDlyhCFDBhII\nBGjd+jwGDhwcii6KRDxTIBAIGHGggoKSUz8pTDkciRHdv/pG4129F1+cQ0FBPmPGPPiztv79b+DB\nBx+mffsLa7RvjbmxNN7GqqvxdjgST9imaWcRCaqq8uF2llNV5Qt1KSIRTdPOIoLf7+e9DZ/zZV4h\npcUeEpJsnJORxqWZLTCb9Tu6SG1T+IpEiKFDh5+wbenSt0/62vc2fM5Huw4Ft0uLPcHtLlnn1k6B\nIhKkX2lFolxVlY8v8wqrbfsqr1BT0CJ1QOErEuXKSispLfZU21Za4qGstNLgikQin8JXJMrFJ8SS\nkGSrti0h0UZ8QqzBFYlEPoWvSJSLibFwTkZatW1nZ6QRE2MxuCKRyKcFVyLCpZktgGPXeEtLPCQk\n2jj736udRaT2KXxFBLPZTJesc7mkW3PKSiuJT4jVGa9IHVL4ikhQTIyF5NS4UJchEvF0zVdERMRg\nCl8RERGDKXxFREQMpvAVERExmMJXRETEYApfERERgyl8RUREDKbwFRERMZjCV0RExGAKXxEREYMp\nfEVERAym8BURETGYwldERMRgCl8RERGDKXwlZPbu/ZD+/W8IdRkiIoZT+IpI1KusrGTVqndCXYZE\nEWuoC5DoMm/eXFaseIPk5GS6dOkGgMfj4ZlnnmD37l2YzWZ+97vLuOuuu7FYLHzwwftMm/YIcXFx\nDBw4mOeee5q3316BzZYc4p5IJMnL+5S///1drrnm+lCXIlFCZ75imC+//ILc3NeYO/cV5s5dwGef\nHQTg9dcXkZ//AwsWvM5LLy1k3749rFu3Gp/Px5Qpkxg9ehyvvrqU7777loqK8hD3QsLFqlXvMGhQ\nXwYN6svDDz9IZWUlb7/9Jjff3J9Bg/rwpz8N4/vvj1BUdJTx4+/jwIF93HXXHaEuW6KEwlcMs3fv\nbi68sCN2eyMsFgtXX30NAO+/v41evfpgtVqx2RrQvfs17Ny5g2+//YbKyko6d74MgH79svH7/aHs\ngoSJI0cO89xzTzNz5hwWLVpGeXkFr7/+Gk8+OZ0nn3yOxYvfoEmTpsybNxe7vRHDh4+gbdt2PP/8\n3FCXLlFC085imOLiYhISEoLbiYlJALhczuCfjz2eiNPppKSk+LjH09LSjCtWwtrOnTu44IJ2pKU5\nAJg48REsFgvZ2TcTExMDQPv2HVi9+t1QlilRTOErhklMTKK0tDS47XI5AbDbG+F2u4OPFxe7sdvt\nNGzYkPLysuDjRUVHjStWwprb7SIhITG4bbPZ8Pl85OTMYvv2Lfh8PsrKyjjzzGYhrFKimaadxTDn\nn38BH330IU6nE5/Px+rVqwC49NIurFz5Fj6fj/LyclavfpfOnbvQtGkzvF4vu3fvAuDNN5dhMplC\n2QU+/ng/99wzIqQ1yKklJ6fgdruC2z/+WMqaNavYvn0LM2fmsGjRcoYOHR7CCiXa/abwzcvLIysr\ni4ULF9ZWPRLBzj23FTfe2I+hQ29h6NAhtGvXHjh2LTc9/TSGDBnIHXcM4dJLu5KZmUVsbCz33TeG\nqVMnc9ttgznzzGaYzeaQBvB5553PjBkzQ3Z8+WU6d76Mffv2cuTIYQKBAI8//igFBfmcfvoZpKQc\nC+YNG9ZSXn5sAZ/VaqWsrJRAIBDiyiVamAI1/GkrKytj+PDhnH322bRq1YpbbrnlpM8vKCipUYHh\nwOFIjOj+1Rfl5eV0796VXbt2UVERmhp2797FtGmPkJv7ZmgKCJFw/BnfsGEds2Y9i8Vipk2btgwf\n/icefHAMxcVuGjduwh133MmYMffQo8c1DBgwiDvvHIrf72P58nexWCwhrT0cxzuc1dV4OxyJJ2yr\n8TXf2NhYcnJyyMnJqekuRE5p6NAh3ND/Jq7t0ZP169dw9tnnkJiYSEWF3pjk5DIzs8jMzDrusZyc\n+cdtv/32muCf33hDi6/EODUOX6vVitWq9VpSN3yBAKu+KcRx7U3Menkus3JmkdgwgUnjJoa6NIkQ\nlb5K3J4Skm2JxFpiQ12ORBnD0jM1NR6rNbRTOXXpZNML8ust/vhb3st3YW52Lv9v9LTg4z84jn10\nJFTjnZISj8Vijsq/70jps8/vY8HeZfzju30UlhWRFm/noqbtGNK+HxZz/XmPipTxDhdGj7dh4et0\nlp36SWFK12dqV6XPz+7Dzmrbdh920qdVE4qLfjS4qmNcrjJ8Pn/U/X1H0s/4krwVbPpuW3C7oOwo\n7+ZtpKysigEZvUJY2U8iabzDQSiu+eqjRlLvlFR5cVV6q21zVXpxV1QZXJFEikpfJfsKDlTb9lHh\nASp9lQZXJNGqxuG7f/9+hgwZwhtvvMErr7zCkCFDcLlcp36hyCkkxlhJia1+UiYl1kpygxiDK5JI\n4faU4PRU/z5VVOHC7dHZphijxtPO559/PgsWLKjNWkQAiLWYaZOSwHv5P3+TbJOSgM0Sugmbjh07\nRd3HjCJJsi2RVFsKRZ6fX9awN0gh2abrrGIMTTtLvXRNszQuTU8hNdaKCUiNtXJpegrXNDP2/s6e\nKh/5zjI8VT5Djyt1I9YSSztHWwBKv3TyyVPvc2Tt5xT+4xAXpLUls9ul5Of/EOIqJRros0JSL1lM\nJq4/y0GPpo0oqfKSGGMl1sAzXp/fT+6Gz9iTV0BRsQd7ko0OGQ6yM1tiMet31nDWt+V1AGw5tAmA\n82/oxAVpbenb8jqe5qEQVibRROEr9VqsxUyjEHwGM3fDZ6zb9V1w+2ixJ7g9OCvD8Hqk9ljMFgZk\n9OKs4nSmNzhIzIZyfmyajyXj+I8ZzZnzHF999QVTpjzO119/xRNPPEZhYSGxsTGMGzeR1q3PC1EP\nJBLoV3iR/+Gp8rEnr6Datj15hZqCjhBWsxWLyYLZ9PO3wfXr1/DBB+/z178+AsDYsffRs+e1LF68\nnPvuG8uYMffi9Va/Il/kl1D4ivwPd6mHomJPtW3OkgrcpdW3SWTIy/sXOTmzmTZtBnFxcXz99Ve4\nXEVcd92NALRrdyEpKans378vxJVKONO0s8j/SE6wYU+ycbSaAE5NbEBygi0EVYlRHn/8UeLj40lK\nSgKgtLSEiooKbr65f/A5P/7443HfQS3yayl8Rf6HLcZChwzHcdd8/6NDRhq2mPpzC0KpfRMnPsJb\nby1n1qyZjBp1H2lpDho2bMhrry0LdWkSQTTtLFKN7MyWZHVqSqOkBphN0CipAVmdmpKd2TLUpUkd\na9r0TO655342blzH7t27OP30M3A4TmPjxnUAuFwuJk4cF/wuYJGa0JmvSDUsZjODszLo160F7lIP\nyQk2nfFGkeTkFEaPHsujjz7E/PmLmDx5Ko8/PpWcnFmYzWays28mLi4u1GVKGDMFAoGAEQeK5JuE\n6yboxtJ4G09jbiyNt7H0xQoiIvWQ3+OhMj8fv0cr3aV2aNpZROQEAj4fBUsWU7pnN96iIqx2Owkd\nOuIYMAiTRZchpOYUviIiJ1CwZDGudWuD296jR4Pb6YNuDlVZEgE07SwiUg2/x0Ppnt3VtpXu2aMp\naPlNFL4iItXwut14i4qqb3MW4dVNNuQ3UPiKiFTDmpyM1W6vvi3VjjU52eCKJJIofEVEqmG22Ujo\n0LHatoQOHTDbdJtRqTktuBIROQHHgEHAsWu8XmcR1lQ7CR06BB8XqSmFr4jICZgsFtIH3Uxan/54\n3W6syck645VaofAVETkFs81GbHp6qMuQCKJrviIiIgZT+IqIiBhM4SsiImIwha+IiIjBFL4iIiIG\nU/iKiIgYTOErIiJiMIWviIiIwRS+IiIiBlP4ioiIGEzhKyIiYjCFr4iIiMEUviIiIgZT+IqIiBhM\n4SsiImIwha+IiIjBFL4iIiIGU/iKiIgYTOErIiJiMIWviIiIwRS+IiIiBlP4ioiIGEzhKyIiYjCF\nr4iIiMEUviIiIgZT+IqIiBhM4SsiImIwha+IiIjBFL4iIiIGU/iKyEnNnj2TRYsWhboMkYhiDXUB\nIlK//fGPI3A4EikoKAl1KSIRQ+ErEiGOHDnMH//4ewYOHMw777xFIAATJkxm/vy5HDyYx8UX/45x\n4yayYcM6Xn75BXw+H2lpDh54YAKVlZXcddcdvP32GqzWY28LY8feyyWXdObAgf1kZLRgwIAhfPnl\nFzzxxGMUFhYSGxvDuHETad36vBD3XCT8aNpZJIK4XC7s9kYsWrScli1bMnHiWMaPn8z8+YtYt241\ne/b8k+nTH+HRR5/gtdeW0blzF6ZPn8o55zSnUaNG7Nv3IQAVFRX885+76NbtquC+/X4/Y8feR8+e\n17J48XLuu28sY8bci9frDVV3RcKWwlckgvh8PjIzswBo3rwlbdqcR0pKCsnJKTRqlMaePf+kQ4dO\nNG16JgA33NCbPXt24fV6ueKKq9i2bTMAH3zwHm3atCU1NTW476+//gqXq4jrrrsRgHbtLiQlJZX9\n+/cZ3EuR8KfwjRIff7yfe+4ZAcCUKZOYN28uAF26dCI//4dQlia1yGKxYLM1AMBsNhMXFx9sM5vN\nxMTEkpiYGHwsISGBQCCA2+36d/huAWDLlk1cdVX34/ZdWlpCRUUFN9/cn8GD+zF4cD+cziLcbrcB\nPROJLLrmGyXOO+98ZsyYGeoyJMTsdjsHDvx0plpcXIzZbA6eGZvNFg4ezGPnzh3cffc9x702Lc1B\nw4YNee21ZUaXLRJxdOYbAY4cOcyNN17Nq6/O56ab+jJoUF/27/+I0aP/TO/e1zB16mR2795Fdnbv\nk+5nzpznGDv2Xvx+v0GVi9GsVisffriHQ4e+A+Ctt5Zx0UWXBBdZXXnlVbz00guce24Gyckpx732\n9NPPwOE4jY0b1wHHri9PnDiO8vJyYzshEgF05hsh/nuhzYQJ9zNx4lhefHEhJhP06XMt7dq1P+nr\n169fwwcfvM9zz+VgNut3skjlcKQzZswExo49tlDqjDOacP/944LtV1xxFUOH3sKYMRN+9lqTycTk\nyVN5/PGp5OTMwmw2k519M3FxcUZ2QSQi1Dh8p06dyt69ezGZTIwbN4527drVZl3yK/3vQhuAlJRj\nZy6NGqVRWFh4wtfm5f2LnJzZPPvsbL2RhrEzzmjM5s0fBLdvu+2O49pzc98M/vmKK66iOq1atWbb\ntl3HPTZ+/KTg53zPOutsZs58oRarFolONTrF2blzJ19//TW5ublMmTKFKVOm1HZd8iudaqFNu3YX\nnvC1jz/+KCYTJCUl1XmdEt78Hg+V+fn4PZ5QlyJyQg899CB9+15Hly6d+PTTf53weUeOHKZbt0sM\nrOwnNTrzff/998nKOnaW1aJFC9xuN6WlpSQkJNRqcWKMiRMf4a23ljNr1kxGjbov1OVIPRTw+chf\n/Cqle3bjLSrCareT0KEjjgGDMFksoS5P5Djr1q1m0aLlNGnSNNSlnFCNwrewsJC2bdsGt+12OwUF\nBScN39TUeKzWyP1H6nAknvpJdcTjaXhcDQ0b2mjQICa4bbGYSUmJx2Ix43Ak0qBBDA0b2oLt7du3\n4eKLH+KGG26gV69rueSS0Pwm+GuEcryj0Rc5L+Fatza47T16FNe6tcQ1iKX5sNtDWFnk0s94zQwZ\nMgS/38/o0XdTWlrKs88+y4UXXsjEiRPZtWsXfr+fVq1a8dhjj2G3H3vvXLp0KfPnz6e4uJjRo0dz\n/fXX13mdtbLgKhAInPI5TmdZbRyqXgr1fW+Lin4ECNbw448eKiqqgts+nx+Xqwyfz09BQQkVFVX8\n+KMn2H70aCnp6Q25994xPPDAGObPX0R8fMPQdOYXCPV4Rxu/x0PRzp3VthW8/wENr+mF2WYzuKrI\npp/xmpsx43m6dOnEU0/N4q677sDlKmflyrV8+eXXLFiwBIC5c2ezadN7NGt2Fn6/n6qqKl566TU2\nblzHE0/M4JJLutVKLSf7BapG4Zuenn7cAp78/HwcDkdNdiW14JcutPnP/8ePnxRs++/FNV26dKNL\nl9r5oZPI4XW78RRUv2DP6yzC63YTm55ucFUiv1xKSgpfffUFW7Zs5OKLOzNs2J3AsWu+gUCA3r17\nU1rqJSOjNQUF+YbUVKMFV5dddhmrV68G4MCBA6Snp+t6bxjyVPnId5bhqfKFuhSpx6zJydgcadW3\npdqxJicbXJHIr3PeeeczatQuSq3kAAAO8UlEQVRoli7NpVevq5k0aTwlJcdmFiwWS/BTHmaz2bD7\nHNTozLdjx460bduWQYMGYTKZmDhxYm3XJXXI5/eTu+Ez9uQVUFTswZ5ko0OGg+zMllj0GV/5H2ab\nDfvFF3PknZU/a0vo0EFTzhIWrrwyiyuvzKK42M2jjz7Ea6+9Qq9efUJWT42v+d53n1bFhqvcDZ+x\nbtd3we2jxZ7g9uCsjFCVJfXYObf/H+UVlZTu2YPXWYQ11U5Chw44BgwKdWkip7Ry5QoKCvK57bY7\nSEpKplmzszGZTCGtSXe4ijKeKh978gqqbduTV0i/bi2wxUTuqnSpGZPFQvqgm0nr0x+v2401OVln\nvBI2unbtxqOPPsSgQX2wWCw0bXom48dP4scffwxZTabAL1mqXAsieeVeOK1MzHeWMXbODqr7Szeb\nYOoffkd6anw1rfVHOI13pNCYG0vjbay6Gu+TrXbWBb4ok5xgw55U/RlLamIDkhN0NiMi0cXvq6TK\nU4TfX2XYMTXtHGVsMRY6ZDiOu+b7Hx0y0jTlLCJRIxDw4zy0hu8/OUhlhRNLTDJxKa1IbdIDk6lu\nz00VvlEoO/PYFy/sySvEWVJBamIDOmSkBR8XEYkGzkNrKC346QYyvip3cNvetGedHlvhG4UsZjOD\nszLo160F7lIPyQk2nfGKSFTx+6sod31abVu5Kw9/46swm2Pq7Pi65hvFbDEW0lPjFbwiEnV8VSX4\nqtwnaHPjq6rbBW8KXxERiTqWmEQsMdXfnc0Sk4wlpm6/2ELhKyIiUcdsjiEupVW1bXEpGXU65Qy6\n5isiIlEqtUkPACpLDlJZ4fr3aueM4ON1SeErIiJRyWQyY2/ak0b2G/n+yBEsMYl1fsb7HwpfERGJ\namZLLDE2u7HHNPRoIiIiovAVERExmsJXRETEYGEbvrNnz+TNN5eGugwREZFfLWwXXP3xjyNCXYKI\niEiN1PmZ73fffUeXLl149dX53HRTXwYN6sv+/R8xevSf6d37GqZOnczu3bvIzu4dfM1/b3/xxWcM\nH/57brllIIMG9WHZslwApkyZxLx5cwH4178+4fbbb2HQoL6MGPEHDh8+VNfdEhERqTFDpp2dTid2\neyMWLVpOy5YtmThxLOPHT2b+/EWsW7eaw4d//vV2//HSSzn07t2PhQtfZ/bsl9m1ayeVlZXHPWfS\npHEMG3Ynixcv5/LLr+DJJ6fXdZdERERqzJBpZ6/XS2ZmFgDNmx/72rqUlBQAGjVKo7Cw8ISvTU21\ns2nTepo3b0lGRiseffSJ49q/+eZrXC4XnTtfBkDfvgO58cZ+ddENERGRWmHIma/FYsFma3DsgGYz\ncXHxPxVgNtOu3YUnfO2dd46kefOW/PWvY+jb9zqWL19yXLvb7SIhISG4bbVasdlstdwDERGR2lMv\nFlzt2/chfr8/uF1S8tNXOcXHxzN8+J8YPvxPfPLJAe699246dbo42J6cnEJxcTF+vx+z2YzX66Wg\nIJ8zzmhsaB9ERER+qXrxUaNGjdI4erQQp7MIn8/H2rWrgm333/8XvvjicwCaN29Bw4YJmEymYPuZ\nZzYjPT2dzZs3APDOO28yffoUYzsgIiLyK9SLM98mTZpy7bW9+P3vb+a0006nZ8/rOHgwD4D+/bOZ\nPHkCXm8VAH369OfMM5sFX2symXjoocd4+OEHmTPnORo1SmPcuIkh6YeIiMgvYQoEAgEjDlRQUHLq\nJ4UphyMxovtX32i8jacxN5bG21h1Nd4OR+IJ2+rFtPNv4anyke8sw1PlC3UpIiIiv0i9mHauCZ/f\nT+6Gz9iTV0BRsQd7ko0OGQ6yM1tiMYf97xQiIhLBwjZ8czd8xrpdP92c42ixJ7g9OCsjVGWJiIic\nUlieInqqfOzJK6i2bU9eoaagRUSkXgvL8HWXeigq9lTb5iypwF1afZuIiEh9EJbhm5xgw55U/V2s\nUhMbkJygO1yJiEj9FZbha4ux0CHDUW1bh4w0bDEWgysSERH55cJ2wVV25rEvaNiTV4izpILUxAZ0\nyEgLPi4iIlJfhW34WsxmBmdl0K9bC9ylHpITbDrjFRGRsBC24fsfthgL6anxp36iiIhIPRGW13xF\nRETCmcJXRETEYApfERERgyl8RUREDKbwFRERMZjCV0RExGAKXxEREYMpfEVERAym8BURETGYwldE\nRMRgCl8RERGDKXxFREQMpvAVERExmMJXRETEYApfERERgyl8RUREDKbwFRERMZjCV0RExGAKXxER\nEYMpfEVERAym8BURETGYwldERMRgNQ7fnTt30rlzZzZu3Fib9YiIiES8GoXvN998w8svv0zHjh1r\nux4REZGIV6PwdTgczJw5k8TExNquR0REJOJZa/KiuLi42q5DREQkapwyfJcsWcKSJUuOe2zkyJF0\n7dr1Vx0oNTUeq9Xy66oLIw6HZgGMpPE2nsbcWBpvYxk93qcM3wEDBjBgwIDffCCns+w376O+cjgS\nKSgoCXUZUUPjbTyNubE03saqq/E+WaDro0YiIiIGq1H4btq0iSFDhrB161ZmzJjB7bffXtt1iYiI\nRKwaLbi64ooruOKKK2q5FBERkeigaWcRERGDKXxFREQMpvAVERExmMJXRETEYApfERERgyl8RURE\nDKbwFRERMZjCV0RExGAKXxEREYMpfEVERAym8BURETGYwldERMRgCl8RERGDKXxFREQMpvAVEREx\nmMJXRETEYApfERERgyl8RUREDKbwFRERMZjCV0RExGAKXxEREYMpfEVERAym8BURETGYwldERMRg\nCl8RERGDKXxFREQMpvAVERExmMJXRETEYApfOandu3eRnd27Vvb17rtv8+c/31Ur+xIRCWcKXxER\nEYNZQ12AhIeZM59i27YtmM0mxo79KxkZrXnmmSfYvXsXZrOZ3/3uMu66624sFguffXaQJ554FLfb\nTWysjTvvHMkll3Q+bn8//ljKH/94O7ff/geuvDIrRL0SEQkNha+c0vffH6F16zaMGDGKFSveYMaM\naWRm9iA//wcWLHgdn8/LiBF/YN261XTv3pNJk8bxf/83lO7de/Kvf33MX/4ygmXL3g7uz+/3M3ny\nBHr0uEbBKyJRSdPOckqxsbFkZnYHIDOzOwcP5rF58wZ69eqD1WrFZmtA9+7XsHPnDo4cOczRo0fJ\nyroagNatz+P000/nk08+Du5v9uyZpKSkMmTI70PSHxGRUFP4yiklJSVjNh/7UWnYsCFwbNo4MTEp\n+JzExEScTidOp5OEhERMJtN/tSXhdBYB8OmnH7NsWS4OR7qBPRARqV8UvnJKJSUlP/tzUlIybrc7\n+HhxsRu73Y7dbqekxE0gEAi2ud1u7PZGAKSlOXjllVxWrHiDTz45YFAPRETqF4WvnJLHU8HmzRsB\n2LRpPW3anEe3bleycuVb+Hw+ysvLWb36XTp37sIZZzTG4Uhn/fo1AHz00V6Kio7Spk1bABo1ctCk\nSVNGjvwLU6ZMwuPxhKxfIiKhogVXckrNmp3FgQP7mDNnJmazmfHjJ9G8eUsOHz7EkCEDMZlMXHll\nFpmZWZhMJiZPnsrjjz/Kyy/n0KBBHA8//BhxcXHH7bNHj2vYtGkDL7zwPCNH/iVEPRMRCQ1T4L/n\nB+tQQUHJqZ8UphyOxIjuX22p9FXi9pSQbEsk1hJb4/1ovI2nMTeWxttYdTXeDkfiCdt05it1zuf3\nsfyzlewrOIDT4yLVlkI7R1v6trwOi9kS6vJERAyn8JU6t/yzlWz6bltwu8jjDG4PyOgVqrJEREJG\nC66kTlX6KtlXUP2q5o8KD1DpqzS4IhGR0FP4Sp1ye0pwelzVthVVuHB7dF1LRKKPwlfqVLItkVRb\nSrVt9gYpJNtOvCBBRCRSKXylTsVaYmnnaFtt2wVpbX/TqmcRkXClBVdS5/q2vA44do23qMKFvUEK\nF6S1DT4uIhJtFL5S5yxmCwMyenFji5618jlfEZFwp/AVw8RaYnHENwp1GSIiIadrviIiIgZT+IqI\niBhM4SsiImIwha+IiIjBFL4iIiIGU/iKiIgYTOErIiJiMIWviIiIwUyBQCAQ6iJERESiic58RURE\nDKbwFRERMZjCV0RExGAKXxEREYMpfEVERAym8BURETGYwrcWeL1eHnjgAW666SYGDhzIrl27Ql1S\nxNu5cyedO3dm48aNoS4lok2dOpXs7GwGDRrEvn37Ql1OVMjLyyMrK4uFCxeGupSoMH36dLKzs+nX\nrx9r1qwx7LhWw44Uwd566y3i4uJYtGgRBw8eZOzYsSxdujTUZUWsb775hpdffpmOHTuGupSItnPn\nTr7++mtyc3P5/PPPGTduHLm5uaEuK6KVlZXx8MMP07lz51CXEhV27NjBwYMHyc3Nxel00qdPH3r0\n6GHIsXXmWwt69erF2LFjAbDb7bhcrhBXFNkcDgczZ84kMTEx1KVEtPfff5+srCwAWrRogdvtprS0\nNMRVRbbY2FhycnJIT08PdSlR4aKLLuLpp58GICkpifLycnw+nyHHVvjWgpiYGGw2GwDz58/n+uuv\nD3FFkS0uLg6LxRLqMiJeYWEhqampwW273U5BQUEIK4p8VquVBg0ahLqMqGGxWIiPjwdg6dKlXH75\n5Ya9t2ja+VdasmQJS5YsOe6xkSNH0rVrV1599VUOHDjA7NmzQ1Rd5DnZeIuxdCdaiVTr1q1j6dKl\nvPTSS4YdU+H7Kw0YMIABAwb87PElS5awYcMGnn/+eWJiYkJQWWQ60XhL3UtPT6ewsDC4nZ+fj8Ph\nCGFFIrVv69atzJ49m7lz5xp6KUvTzrXg22+/ZfHixcycOTM4/SwS7i677DJWr14NwIEDB0hPTych\nISHEVYnUnpKSEqZPn86cOXNISUkx9Nj6VqNaMGPGDFauXEnjxo2Dj7344ovExsaGsKrItWnTJl58\n8UW++OIL7HY7DofD0OmiaPK3v/2NXbt2YTKZmDhxIq1btw51SRFt//79TJs2jUOHDmG1WjnttNN4\n9tlnDQ+GaJGbm8uzzz7LOeecE3xs2rRpx72X1xWFr4iIiME07SwiImIwha+IiIjBFL4iIiIGU/iK\niIgYTOErIiJiMIWviIiIwRS+IiIiBlP4ioiIGOz/A0vBwFbJIDthAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - "'''\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from torch.autograd import Variable\n", - "import matplotlib.pyplot as plt\n", - "\n", - "dtype = torch.FloatTensor\n", - "\n", - "# 3 Words Sentence\n", - "sentences = [ \"i like dog\", \"i like cat\", \"i like animal\",\n", - " \"dog cat animal\", \"apple cat dog like\", \"dog fish milk like\",\n", - " \"dog cat eyes like\", \"i like apple\", \"apple i hate\",\n", - " \"apple i movie book music like\", \"cat dog hate\", \"cat dog like\"]\n", - "\n", - "word_sequence = \" \".join(sentences).split()\n", - "word_list = \" \".join(sentences).split()\n", - "word_list = list(set(word_list))\n", - "word_dict = {w: i for i, w in enumerate(word_list)}\n", - "\n", - "# Word2Vec Parameter\n", - "batch_size = 20 # To show 2 dim embedding graph\n", - "embedding_size = 2 # To show 2 dim embedding graph\n", - "voc_size = len(word_list)\n", - "\n", - "def random_batch(data, size):\n", - " random_inputs = []\n", - " random_labels = []\n", - " random_index = np.random.choice(range(len(data)), size, replace=False)\n", - "\n", - " for i in random_index:\n", - " random_inputs.append(np.eye(voc_size)[data[i][0]]) # target\n", - " random_labels.append(data[i][1]) # context word\n", - "\n", - " return random_inputs, random_labels\n", - "\n", - "# Make skip gram of one size window\n", - "skip_grams = []\n", - "for i in range(1, len(word_sequence) - 1):\n", - " target = word_dict[word_sequence[i]]\n", - " context = [word_dict[word_sequence[i - 1]], word_dict[word_sequence[i + 1]]]\n", - "\n", - " for w in context:\n", - " skip_grams.append([target, w])\n", - "\n", - "# Model\n", - "class Word2Vec(nn.Module):\n", - " def __init__(self):\n", - " super(Word2Vec, self).__init__()\n", - "\n", - " # W and WT is not Traspose relationship\n", - " self.W = nn.Parameter(-2 * torch.rand(voc_size, embedding_size) + 1).type(dtype) # voc_size > embedding_size Weight\n", - " self.WT = nn.Parameter(-2 * torch.rand(embedding_size, voc_size) + 1).type(dtype) # embedding_size > voc_size Weight\n", - "\n", - " def forward(self, X):\n", - " # X : [batch_size, voc_size]\n", - " hidden_layer = torch.matmul(X, self.W) # hidden_layer : [batch_size, embedding_size]\n", - " output_layer = torch.matmul(hidden_layer, self.WT) # output_layer : [batch_size, voc_size]\n", - " return output_layer\n", - "\n", - "model = Word2Vec()\n", - "\n", - "criterion = nn.CrossEntropyLoss()\n", - "optimizer = optim.Adam(model.parameters(), lr=0.001)\n", - "\n", - "# Training\n", - "for epoch in range(5000):\n", - "\n", - " input_batch, target_batch = random_batch(skip_grams, batch_size)\n", - "\n", - " input_batch = Variable(torch.Tensor(input_batch))\n", - " target_batch = Variable(torch.LongTensor(target_batch))\n", - "\n", - " optimizer.zero_grad()\n", - " output = model(input_batch)\n", - "\n", - " # output : [batch_size, voc_size], target_batch : [batch_size] (LongTensor, not one-hot)\n", - " loss = criterion(output, target_batch)\n", - " if (epoch + 1)%1000 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - "for i, label in enumerate(word_list):\n", - " W, WT = model.parameters()\n", - " x,y = float(W[i][0]), float(W[i][1])\n", - " plt.scatter(x, y)\n", - " plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')\n", - "plt.show()\n" - ] - } - ], - "metadata": { - "colab": { - "name": "Word2Vec-Torch(Softmax).ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/2-1.TextCNN/TextCNN-Torch.py b/2-1.TextCNN/TextCNN.py similarity index 100% rename from 2-1.TextCNN/TextCNN-Torch.py rename to 2-1.TextCNN/TextCNN.py diff --git a/2-1.TextCNN/TextCNN_Tensor.ipynb b/2-1.TextCNN/TextCNN_Tensor.ipynb deleted file mode 100644 index a693a7a..0000000 --- a/2-1.TextCNN/TextCNN_Tensor.ipynb +++ /dev/null @@ -1,142 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "K4kObERVUvo1", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 125.0 - }, - "outputId": "203a5ac0-0006-4616-ff0b-439f9917b3bf" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 001000 cost = 0.001538\n", - "Epoch: 002000 cost = 0.000269\n", - "Epoch: 003000 cost = 0.000093\n", - "Epoch: 004000 cost = 0.000040\n", - "Epoch: 005000 cost = 0.000020\n", - "sorry hate you is Bad Mean...\n" - ] - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - " Reference : https://github.com/ioatr/textcnn\n", - "'''\n", - "import tensorflow as tf\n", - "import numpy as np\n", - "\n", - "tf.reset_default_graph()\n", - "\n", - "# Text-CNN Parameter\n", - "embedding_size = 2 # n-gram\n", - "sequence_length = 3\n", - "num_classes = 2 # 0 or 1\n", - "filter_sizes = [2,2,2] # n-gram window\n", - "num_filters = 3\n", - "\n", - "# 3 words sentences (=sequence_length is 3)\n", - "sentences = [\"i love you\",\"he loves me\", \"she likes baseball\", \"i hate you\",\"sorry for that\", \"this is awful\"]\n", - "labels = [1,1,1,0,0,0] # 1 is good, 0 is not good.\n", - "\n", - "word_list = \" \".join(sentences).split()\n", - "word_list = list(set(word_list))\n", - "word_dict = {w: i for i, w in enumerate(word_list)}\n", - "vocab_size = len(word_dict)\n", - "\n", - "inputs = []\n", - "for sen in sentences:\n", - " inputs.append(np.asarray([word_dict[n] for n in sen.split()]))\n", - "\n", - "outputs = []\n", - "for out in labels:\n", - " outputs.append(np.eye(num_classes)[out]) # ONE-HOT : To using Tensor Softmax Loss function\n", - "\n", - "# Model\n", - "X = tf.placeholder(tf.int32, [None, sequence_length])\n", - "Y = tf.placeholder(tf.int32, [None, num_classes])\n", - "\n", - "W = tf.Variable(tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0))\n", - "embedded_chars = tf.nn.embedding_lookup(W, X) # [batch_size, sequence_length, embedding_size]\n", - "embedded_chars = tf.expand_dims(embedded_chars, -1) # add channel(=1) [batch_size, sequence_length, embedding_size, 1]\n", - "\n", - "pooled_outputs = []\n", - "for i, filter_size in enumerate(filter_sizes):\n", - " filter_shape = [filter_size, embedding_size, 1, num_filters]\n", - " W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1))\n", - " b = tf.Variable(tf.constant(0.1, shape=[num_filters]))\n", - "\n", - " conv = tf.nn.conv2d(embedded_chars, # [batch_size, sequence_length, embedding_size, 1]\n", - " W, # [filter_size(n-gram window), embedding_size, 1, num_filters(=3)]\n", - " strides=[1, 1, 1, 1],\n", - " padding='VALID')\n", - " h = tf.nn.relu(tf.nn.bias_add(conv, b))\n", - " pooled = tf.nn.max_pool(h,\n", - " ksize=[1, sequence_length - filter_size + 1, 1, 1], # [batch_size, filter_height, filter_width, channel]\n", - " strides=[1, 1, 1, 1],\n", - " padding='VALID')\n", - " pooled_outputs.append(pooled) # dim of pooled : [batch_size(=6), output_height(=1), output_width(=1), channel(=1)]\n", - "\n", - "num_filters_total = num_filters * len(filter_sizes)\n", - "h_pool = tf.concat(pooled_outputs, num_filters) # h_pool : [batch_size(=6), output_height(=1), output_width(=1), channel(=1) * 3]\n", - "h_pool_flat = tf.reshape(h_pool, [-1, num_filters_total]) # [batch_size, ]\n", - "\n", - "# Model-Training\n", - "Weight = tf.get_variable('W', shape=[num_filters_total, num_classes], \n", - " initializer=tf.contrib.layers.xavier_initializer())\n", - "Bias = tf.Variable(tf.constant(0.1, shape=[num_classes]))\n", - "model = tf.nn.xw_plus_b(h_pool_flat, Weight, Bias) \n", - "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=model, labels=Y))\n", - "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n", - "\n", - "# Model-Predict\n", - "hypothesis = tf.nn.softmax(model)\n", - "predictions = tf.argmax(hypothesis, 1)\n", - "# Training\n", - "init = tf.global_variables_initializer()\n", - "sess = tf.Session()\n", - "sess.run(init)\n", - "\n", - "for epoch in range(5000):\n", - " _, loss = sess.run([optimizer, cost], feed_dict={X: inputs, Y: outputs})\n", - " if (epoch + 1)%1000 == 0:\n", - " print('Epoch:', '%06d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - "# Test\n", - "test_text = 'sorry hate you'\n", - "tests = []\n", - "tests.append(np.asarray([word_dict[n] for n in test_text.split()]))\n", - "\n", - "predict = sess.run([predictions], feed_dict={X: tests})\n", - "result = predict[0][0]\n", - "if result == 0:\n", - " print(test_text,\"is Bad Mean...\")\n", - "else:\n", - " print(test_text,\"is Good Mean!!\")" - ] - } - ], - "metadata": { - "colab": { - "name": "TextCNN-Tensor.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/2-1.TextCNN/TextCNN_Torch.ipynb b/2-1.TextCNN/TextCNN_Torch.ipynb deleted file mode 100644 index b6cb34d..0000000 --- a/2-1.TextCNN/TextCNN_Torch.ipynb +++ /dev/null @@ -1,165 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "id": "KgNWMAfrWz8r", - "colab_type": "code", - "colab": {} - }, - "outputs": [], - "source": [ - "# http://pytorch.org/\n", - "from os.path import exists\n", - "from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag\n", - "platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())\n", - "cuda_output = !ldconfig -p|grep cudart.so|sed -e 's/.*\\.\\([0-9]*\\)\\.\\([0-9]*\\)$/cu\\1\\2/'\n", - "accelerator = cuda_output[0] if exists('/dev/nvidia0') else 'cpu'" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "dzhILRnNVS-a", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 125.0 - }, - "outputId": "d29a4372-2558-4dad-8417-3469ccdd0ce0" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1000 cost = 0.709715\n", - "Epoch: 2000 cost = 0.593492\n", - "Epoch: 3000 cost = 0.156285\n", - "Epoch: 4000 cost = 0.092555\n", - "Epoch: 5000 cost = 0.076225\n", - "sorry hate you is Bad Mean...\n" - ] - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - "'''\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from torch.autograd import Variable\n", - "import torch.nn.functional as F\n", - "\n", - "dtype = torch.FloatTensor\n", - "\n", - "# Text-CNN Parameter\n", - "embedding_size = 2 # n-gram\n", - "sequence_length = 3\n", - "num_classes = 2 # 0 or 1\n", - "filter_sizes = [2, 2, 2] # n-gram window\n", - "num_filters = 3\n", - "\n", - "# 3 words sentences (=sequence_length is 3)\n", - "sentences = [\"i love you\", \"he loves me\", \"she likes baseball\", \"i hate you\", \"sorry for that\", \"this is awful\"]\n", - "labels = [1, 1, 1, 0, 0, 0] # 1 is good, 0 is not good.\n", - "\n", - "word_list = \" \".join(sentences).split()\n", - "word_list = list(set(word_list))\n", - "word_dict = {w: i for i, w in enumerate(word_list)}\n", - "vocab_size = len(word_dict)\n", - "\n", - "inputs = []\n", - "for sen in sentences:\n", - " inputs.append(np.asarray([word_dict[n] for n in sen.split()]))\n", - "\n", - "targets = []\n", - "for out in labels:\n", - " targets.append(out) # To using Torch Softmax Loss function\n", - "\n", - "input_batch = Variable(torch.LongTensor(inputs))\n", - "target_batch = Variable(torch.LongTensor(targets))\n", - "\n", - "\n", - "class TextCNN(nn.Module):\n", - " def __init__(self):\n", - " super(TextCNN, self).__init__()\n", - "\n", - " self.num_filters_total = num_filters * len(filter_sizes)\n", - " self.W = nn.Parameter(torch.empty(vocab_size, embedding_size).uniform_(-1, 1)).type(dtype)\n", - " self.Weight = nn.Parameter(torch.empty(self.num_filters_total, num_classes).uniform_(-1, 1)).type(dtype)\n", - " self.Bias = nn.Parameter(0.1 * torch.ones([num_classes])).type(dtype)\n", - "\n", - " def forward(self, X):\n", - " embedded_chars = self.W[X] # [batch_size, sequence_length, sequence_length]\n", - " embedded_chars = embedded_chars.unsqueeze(1) # add channel(=1) [batch, channel(=1), sequence_length, embedding_size]\n", - "\n", - " pooled_outputs = []\n", - " for filter_size in filter_sizes:\n", - " # conv : [input_channel(=1), output_channel(=3), (filter_height, filter_width), bias_option]\n", - " conv = nn.Conv2d(1, num_filters, (filter_size, embedding_size), bias=True)(embedded_chars)\n", - " h = F.relu(conv)\n", - " # mp : ((filter_height, filter_width))\n", - " mp = nn.MaxPool2d((sequence_length - filter_size + 1, 1))\n", - " # pooled : [batch_size(=6), output_height(=1), output_width(=1), output_channel(=3)]\n", - " pooled = mp(h).permute(0, 3, 2, 1)\n", - " pooled_outputs.append(pooled)\n", - "\n", - " h_pool = torch.cat(pooled_outputs, len(filter_sizes)) # [batch_size(=6), output_height(=1), output_width(=1), output_channel(=3) * 3]\n", - " h_pool_flat = torch.reshape(h_pool, [-1, self.num_filters_total]) # [batch_size(=6), output_height * output_width * (output_channel * 3)]\n", - "\n", - " model = torch.mm(h_pool_flat, self.Weight) + self.Bias # [batch_size, num_classes]\n", - " return model\n", - "\n", - "model = TextCNN()\n", - "\n", - "criterion = nn.CrossEntropyLoss()\n", - "optimizer = optim.Adam(model.parameters(), lr=0.001)\n", - "\n", - "# Training\n", - "for epoch in range(5000):\n", - " optimizer.zero_grad()\n", - " output = model(input_batch)\n", - "\n", - " # output : [batch_size, num_classes], target_batch : [batch_size] (LongTensor, not one-hot)\n", - " loss = criterion(output, target_batch)\n", - " if (epoch + 1) % 1000 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - "# Test\n", - "test_text = 'sorry hate you'\n", - "tests = [np.asarray([word_dict[n] for n in test_text.split()])]\n", - "test_batch = Variable(torch.LongTensor(tests))\n", - "\n", - "# Predict\n", - "predict = model(test_batch).data.max(1, keepdim=True)[1]\n", - "if predict[0][0] == 0:\n", - " print(test_text,\"is Bad Mean...\")\n", - "else:\n", - " print(test_text,\"is Good Mean!!\")" - ] - } - ], - "metadata": { - "colab": { - "name": "TextCNN-Torch.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/3-1.TextRNN/TextRNN-Torch.py b/3-1.TextRNN/TextRNN.py similarity index 100% rename from 3-1.TextRNN/TextRNN-Torch.py rename to 3-1.TextRNN/TextRNN.py diff --git a/3-1.TextRNN/TextRNN_Tensor.ipynb b/3-1.TextRNN/TextRNN_Tensor.ipynb deleted file mode 100644 index 10a4660..0000000 --- a/3-1.TextRNN/TextRNN_Tensor.ipynb +++ /dev/null @@ -1,121 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "IUgjtn9xY6G0", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 179.0 - }, - "outputId": "7e3ed467-8e97-4c68-bff3-7c56b8f307c0" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:From :42: BasicRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "This class is equivalent as tf.keras.layers.SimpleRNNCell, and will be replaced by that in Tensorflow 2.0.\n", - "Epoch: 1000 cost = 0.130863\n", - "Epoch: 2000 cost = 0.022833\n", - "Epoch: 3000 cost = 0.008204\n", - "Epoch: 4000 cost = 0.003774\n", - "Epoch: 5000 cost = 0.001942\n", - "[['i', 'like'], ['i', 'love'], ['i', 'hate']] -> ['dog', 'coffee', 'milk']\n" - ] - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - "'''\n", - "import tensorflow as tf\n", - "import numpy as np\n", - "\n", - "tf.reset_default_graph()\n", - "\n", - "sentences = [ \"i like dog\", \"i love coffee\", \"i hate milk\"]\n", - "\n", - "word_list = \" \".join(sentences).split()\n", - "word_list = list(set(word_list))\n", - "word_dict = {w: i for i, w in enumerate(word_list)}\n", - "number_dict = {i: w for i, w in enumerate(word_list)}\n", - "n_class = len(word_dict)\n", - "\n", - "# TextRNN Parameter\n", - "n_step = 2 # number of cells(= number of Step)\n", - "n_hidden = 5 # number of hidden units in one cell\n", - "\n", - "def make_batch(sentences):\n", - " input_batch = []\n", - " target_batch = []\n", - " \n", - " for sen in sentences:\n", - " word = sen.split()\n", - " input = [word_dict[n] for n in word[:-1]]\n", - " target = word_dict[word[-1]]\n", - "\n", - " input_batch.append(np.eye(n_class)[input])\n", - " target_batch.append(np.eye(n_class)[target])\n", - "\n", - " return input_batch, target_batch\n", - "\n", - "# Model\n", - "X = tf.placeholder(tf.float32, [None, n_step, n_class]) # [batch_size, n_step, n_class]\n", - "Y = tf.placeholder(tf.float32, [None, n_class]) # [batch_size, n_class]\n", - "\n", - "W = tf.Variable(tf.random_normal([n_hidden, n_class]))\n", - "b = tf.Variable(tf.random_normal([n_class]))\n", - "\n", - "cell = tf.nn.rnn_cell.BasicRNNCell(n_hidden)\n", - "outputs, states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)\n", - "\n", - "# outputs : [batch_size, n_step, n_hidden]\n", - "outputs = tf.transpose(outputs, [1, 0, 2]) # [n_step, batch_size, n_hidden]\n", - "outputs = outputs[-1] # [batch_size, n_hidden]\n", - "model = tf.matmul(outputs, W) + b # model : [batch_size, n_class]\n", - "\n", - "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=model, labels=Y))\n", - "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n", - "\n", - "prediction = tf.cast(tf.argmax(model, 1), tf.int32)\n", - "\n", - "# Training\n", - "init = tf.global_variables_initializer()\n", - "sess = tf.Session()\n", - "sess.run(init)\n", - "\n", - "input_batch, target_batch = make_batch(sentences)\n", - "\n", - "for epoch in range(5000):\n", - " _, loss = sess.run([optimizer, cost], feed_dict={X: input_batch, Y: target_batch})\n", - " if (epoch + 1)%1000 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - " \n", - "input = [sen.split()[:2] for sen in sentences]\n", - "\n", - "predict = sess.run([prediction], feed_dict={X: input_batch})\n", - "print([sen.split()[:2] for sen in sentences], '->', [number_dict[n] for n in predict[0]])" - ] - } - ], - "metadata": { - "colab": { - "name": "TextRNN-Tensor.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/3-1.TextRNN/TextRNN_Torch.ipynb b/3-1.TextRNN/TextRNN_Torch.ipynb deleted file mode 100644 index 74bae01..0000000 --- a/3-1.TextRNN/TextRNN_Torch.ipynb +++ /dev/null @@ -1,136 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "32Fac6LDZJmE", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 125.0 - }, - "outputId": "7c9fab8d-0e3f-4994-aecc-0b3d00a59ffc" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1000 cost = 0.064772\n", - "Epoch: 2000 cost = 0.014707\n", - "Epoch: 3000 cost = 0.005852\n", - "Epoch: 4000 cost = 0.002837\n", - "Epoch: 5000 cost = 0.001509\n", - "[['i', 'like'], ['i', 'love'], ['i', 'hate']] -> ['dog', 'coffee', 'milk']\n" - ] - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - "'''\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from torch.autograd import Variable\n", - "\n", - "dtype = torch.FloatTensor\n", - "\n", - "sentences = [ \"i like dog\", \"i love coffee\", \"i hate milk\"]\n", - "\n", - "word_list = \" \".join(sentences).split()\n", - "word_list = list(set(word_list))\n", - "word_dict = {w: i for i, w in enumerate(word_list)}\n", - "number_dict = {i: w for i, w in enumerate(word_list)}\n", - "n_class = len(word_dict)\n", - "\n", - "# TextRNN Parameter\n", - "batch_size = len(sentences)\n", - "n_step = 2 # number of cells(= number of Step)\n", - "n_hidden = 5 # number of hidden units in one cell\n", - "\n", - "def make_batch(sentences):\n", - " input_batch = []\n", - " target_batch = []\n", - "\n", - " for sen in sentences:\n", - " word = sen.split()\n", - " input = [word_dict[n] for n in word[:-1]]\n", - " target = word_dict[word[-1]]\n", - "\n", - " input_batch.append(np.eye(n_class)[input])\n", - " target_batch.append(target)\n", - "\n", - " return input_batch, target_batch\n", - "\n", - "# to Torch.Tensor\n", - "input_batch, target_batch = make_batch(sentences)\n", - "input_batch = Variable(torch.Tensor(input_batch))\n", - "target_batch = Variable(torch.LongTensor(target_batch))\n", - "\n", - "class TextRNN(nn.Module):\n", - " def __init__(self):\n", - " super(TextRNN, self).__init__()\n", - "\n", - " self.rnn = nn.RNN(input_size=n_class, hidden_size=n_hidden)\n", - " self.W = nn.Parameter(torch.randn([n_hidden, n_class]).type(dtype))\n", - " self.b = nn.Parameter(torch.randn([n_class]).type(dtype))\n", - "\n", - " def forward(self, hidden, X):\n", - " X = X.transpose(0, 1) # X : [n_step, batch_size, n_class]\n", - " outputs, hidden = self.rnn(X, hidden)\n", - " # outputs : [n_step, batch_size, num_directions(=1) * n_hidden]\n", - " # hidden : [num_layers(=1) * num_directions(=1), batch_size, n_hidden]\n", - " outputs = outputs[-1] # [batch_size, num_directions(=1) * n_hidden]\n", - " model = torch.mm(outputs, self.W) + self.b # model : [batch_size, n_class]\n", - " return model\n", - "\n", - "model = TextRNN()\n", - "\n", - "criterion = nn.CrossEntropyLoss()\n", - "optimizer = optim.Adam(model.parameters(), lr=0.001)\n", - "\n", - "# Training\n", - "for epoch in range(5000):\n", - " optimizer.zero_grad()\n", - "\n", - " # hidden : [num_layers * num_directions, batch, hidden_size]\n", - " hidden = Variable(torch.zeros(1, batch_size, n_hidden))\n", - " # input_batch : [batch_size, n_step, n_class]\n", - " output = model(hidden, input_batch)\n", - "\n", - " # output : [batch_size, n_class], target_batch : [batch_size] (LongTensor, not one-hot)\n", - " loss = criterion(output, target_batch)\n", - " if (epoch + 1) % 1000 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - "input = [sen.split()[:2] for sen in sentences]\n", - "\n", - "# Predict\n", - "hidden = Variable(torch.zeros(1, batch_size, n_hidden))\n", - "predict = model(hidden, input_batch).data.max(1, keepdim=True)[1]\n", - "print([sen.split()[:2] for sen in sentences], '->', [number_dict[n.item()] for n in predict.squeeze()])" - ] - } - ], - "metadata": { - "colab": { - "name": "TextRNN-Torch.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/3-2.TextLSTM/TextLSTM-Torch.py b/3-2.TextLSTM/TextLSTM.py similarity index 100% rename from 3-2.TextLSTM/TextLSTM-Torch.py rename to 3-2.TextLSTM/TextLSTM.py diff --git a/3-2.TextLSTM/TextLSTM_Tensor.ipynb b/3-2.TextLSTM/TextLSTM_Tensor.ipynb deleted file mode 100644 index a2a1830..0000000 --- a/3-2.TextLSTM/TextLSTM_Tensor.ipynb +++ /dev/null @@ -1,122 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "HggP9sCLZUrt", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 269.0 - }, - "outputId": "ee905d35-c761-4ce1-bd06-f0313843b4a9" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:From :38: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "This class is deprecated, please use tf.nn.rnn_cell.LSTMCell, which supports all the feature this cell currently has. Please replace the existing code with tf.nn.rnn_cell.LSTMCell(name='basic_lstm_cell').\n", - "Epoch: 0100 cost = 0.017741\n", - "Epoch: 0200 cost = 0.004202\n", - "Epoch: 0300 cost = 0.001867\n", - "Epoch: 0400 cost = 0.001056\n", - "Epoch: 0500 cost = 0.000681\n", - "Epoch: 0600 cost = 0.000476\n", - "Epoch: 0700 cost = 0.000352\n", - "Epoch: 0800 cost = 0.000271\n", - "Epoch: 0900 cost = 0.000215\n", - "Epoch: 1000 cost = 0.000174\n", - "['mak', 'nee', 'coa', 'wor', 'lov', 'hat', 'liv', 'hom', 'has', 'sta'] -> ['e', 'd', 'l', 'd', 'e', 'e', 'e', 'e', 'h', 'r']\n" - ] - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - "'''\n", - "import tensorflow as tf\n", - "import numpy as np\n", - "\n", - "tf.reset_default_graph()\n", - "\n", - "char_arr = [c for c in 'abcdefghijklmnopqrstuvwxyz']\n", - "word_dict = {n: i for i, n in enumerate(char_arr)}\n", - "number_dict = {i: w for i, w in enumerate(char_arr)}\n", - "n_class = len(word_dict) # number of class(=number of vocab)\n", - "\n", - "seq_data = ['make', 'need', 'coal', 'word', 'love', 'hate', 'live', 'home', 'hash', 'star']\n", - "\n", - "# TextLSTM Parameters\n", - "n_step = 3\n", - "n_hidden = 128\n", - "\n", - "def make_batch(seq_data):\n", - " input_batch, target_batch = [], []\n", - "\n", - " for seq in seq_data:\n", - " input = [word_dict[n] for n in seq[:-1]] # 'm', 'a' , 'k' is input\n", - " target = word_dict[seq[-1]] # 'e' is target\n", - " input_batch.append(np.eye(n_class)[input])\n", - " target_batch.append(np.eye(n_class)[target])\n", - "\n", - " return input_batch, target_batch\n", - "\n", - "# Model\n", - "X = tf.placeholder(tf.float32, [None, n_step, n_class]) # [batch_size, n_step, n_class]\n", - "Y = tf.placeholder(tf.float32, [None, n_class]) # [batch_size, n_class]\n", - "\n", - "W = tf.Variable(tf.random_normal([n_hidden, n_class]))\n", - "b = tf.Variable(tf.random_normal([n_class]))\n", - "\n", - "cell = tf.nn.rnn_cell.BasicLSTMCell(n_hidden)\n", - "outputs, states = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)\n", - "\n", - "# outputs : [batch_size, n_step, n_hidden]\n", - "outputs = tf.transpose(outputs, [1, 0, 2]) # [n_step, batch_size, n_hidden]\n", - "outputs = outputs[-1] # [batch_size, n_hidden]\n", - "model = tf.matmul(outputs, W) + b # model : [batch_size, n_class]\n", - "\n", - "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=model, labels=Y))\n", - "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n", - "\n", - "prediction = tf.cast(tf.argmax(model, 1), tf.int32)\n", - "\n", - "# Training\n", - "init = tf.global_variables_initializer()\n", - "sess = tf.Session()\n", - "sess.run(init)\n", - "\n", - "input_batch, target_batch = make_batch(seq_data)\n", - "\n", - "for epoch in range(1000):\n", - " _, loss = sess.run([optimizer, cost], feed_dict={X: input_batch, Y: target_batch})\n", - " if (epoch + 1)%100 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - "inputs = [sen[:3] for sen in seq_data]\n", - "\n", - "predict = sess.run([prediction], feed_dict={X: input_batch})\n", - "print(inputs, '->', [number_dict[n] for n in predict[0]])" - ] - } - ], - "metadata": { - "colab": { - "name": "TextLSTM-Tensor.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/3-2.TextLSTM/TextLSTM_Torch.ipynb b/3-2.TextLSTM/TextLSTM_Torch.ipynb deleted file mode 100644 index 2d779dc..0000000 --- a/3-2.TextLSTM/TextLSTM_Torch.ipynb +++ /dev/null @@ -1,130 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "sX4M-kbQZtmS", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 215.0 - }, - "outputId": "ce4ac3be-dc76-4ae3-8db3-5df57944a5a2" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 0100 cost = 0.017576\n", - "Epoch: 0200 cost = 0.003831\n", - "Epoch: 0300 cost = 0.001706\n", - "Epoch: 0400 cost = 0.000970\n", - "Epoch: 0500 cost = 0.000627\n", - "Epoch: 0600 cost = 0.000439\n", - "Epoch: 0700 cost = 0.000325\n", - "Epoch: 0800 cost = 0.000250\n", - "Epoch: 0900 cost = 0.000199\n", - "Epoch: 1000 cost = 0.000162\n", - "['mak', 'nee', 'coa', 'wor', 'lov', 'hat', 'liv', 'hom', 'has', 'sta'] -> ['e', 'd', 'l', 'd', 'e', 'e', 'e', 'e', 'h', 'r']\n" - ] - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - "'''\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from torch.autograd import Variable\n", - "\n", - "dtype = torch.FloatTensor\n", - "\n", - "char_arr = [c for c in 'abcdefghijklmnopqrstuvwxyz']\n", - "word_dict = {n: i for i, n in enumerate(char_arr)}\n", - "number_dict = {i: w for i, w in enumerate(char_arr)}\n", - "n_class = len(word_dict) # number of class(=number of vocab)\n", - "\n", - "seq_data = ['make', 'need', 'coal', 'word', 'love', 'hate', 'live', 'home', 'hash', 'star']\n", - "\n", - "# TextLSTM Parameters\n", - "n_step = 3\n", - "n_hidden = 128\n", - "\n", - "def make_batch(seq_data):\n", - " input_batch, target_batch = [], []\n", - "\n", - " for seq in seq_data:\n", - " input = [word_dict[n] for n in seq[:-1]] # 'm', 'a' , 'k' is input\n", - " target = word_dict[seq[-1]] # 'e' is target\n", - " input_batch.append(np.eye(n_class)[input])\n", - " target_batch.append(target)\n", - "\n", - " return Variable(torch.Tensor(input_batch)), Variable(torch.LongTensor(target_batch))\n", - "\n", - "class TextLSTM(nn.Module):\n", - " def __init__(self):\n", - " super(TextLSTM, self).__init__()\n", - "\n", - " self.lstm = nn.LSTM(input_size=n_class, hidden_size=n_hidden)\n", - " self.W = nn.Parameter(torch.randn([n_hidden, n_class]).type(dtype))\n", - " self.b = nn.Parameter(torch.randn([n_class]).type(dtype))\n", - "\n", - " def forward(self, X):\n", - " input = X.transpose(0, 1) # X : [n_step, batch_size, n_class]\n", - "\n", - " hidden_state = Variable(torch.zeros(1, len(X), n_hidden)) # [num_layers(=1) * num_directions(=1), batch_size, n_hidden]\n", - " cell_state = Variable(torch.zeros(1, len(X), n_hidden)) # [num_layers(=1) * num_directions(=1), batch_size, n_hidden]\n", - "\n", - " outputs, (_, _) = self.lstm(input, (hidden_state, cell_state))\n", - " outputs = outputs[-1] # [batch_size, n_hidden]\n", - " model = torch.mm(outputs, self.W) + self.b # model : [batch_size, n_class]\n", - " return model\n", - "\n", - "input_batch, target_batch = make_batch(seq_data)\n", - "\n", - "model = TextLSTM()\n", - "\n", - "criterion = nn.CrossEntropyLoss()\n", - "optimizer = optim.Adam(model.parameters(), lr=0.001)\n", - "\n", - "output = model(input_batch)\n", - "\n", - "# Training\n", - "for epoch in range(1000):\n", - " optimizer.zero_grad()\n", - "\n", - " output = model(input_batch)\n", - " loss = criterion(output, target_batch)\n", - " if (epoch + 1) % 100 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - "inputs = [sen[:3] for sen in seq_data]\n", - "\n", - "predict = model(input_batch).data.max(1, keepdim=True)[1]\n", - "print(inputs, '->', [number_dict[n.item()] for n in predict.squeeze()])" - ] - } - ], - "metadata": { - "colab": { - "name": "TextLSTM-Torch.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/3-3.Bi-LSTM/Bi-LSTM-Torch.py b/3-3.Bi-LSTM/Bi-LSTM.py similarity index 100% rename from 3-3.Bi-LSTM/Bi-LSTM-Torch.py rename to 3-3.Bi-LSTM/Bi-LSTM.py diff --git a/3-3.Bi-LSTM/Bi_LSTM_Tensor.ipynb b/3-3.Bi-LSTM/Bi_LSTM_Tensor.ipynb deleted file mode 100644 index b8ff1c8..0000000 --- a/3-3.Bi-LSTM/Bi_LSTM_Tensor.ipynb +++ /dev/null @@ -1,127 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "nfzqnU3AbkeL", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 233.0 - }, - "outputId": "f003ad6e-a38d-42c1-be4b-08f3002d5405" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1000 cost = 1.657833\n", - "Epoch: 2000 cost = 1.411641\n", - "Epoch: 3000 cost = 1.115576\n", - "Epoch: 4000 cost = 0.927828\n", - "Epoch: 5000 cost = 0.696575\n", - "Epoch: 6000 cost = 0.410544\n", - "Epoch: 7000 cost = 0.316320\n", - "Epoch: 8000 cost = 0.196335\n", - "Epoch: 9000 cost = 0.147900\n", - "Epoch: 10000 cost = 0.119816\n", - "Lorem ipsum dolor sit amet consectetur adipisicing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Ut enim ad minim veniam quis nostrud exercitation\n", - "['ipsum', 'dolor', 'sit', 'amet', 'consectetur', 'adipisicing', 'elit', 'sed', 'eiusmod', 'eiusmod', 'tempor', 'incididunt', 'ut', 'labore', 'et', 'dolore', 'magna', 'aliqua', 'Ut', 'enim', 'ad', 'minim', 'veniam', 'quis', 'nostrud', 'exercitation']\n" - ] - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - "'''\n", - "import tensorflow as tf\n", - "import numpy as np\n", - "\n", - "tf.reset_default_graph()\n", - "\n", - "sentence = (\n", - " 'Lorem ipsum dolor sit amet consectetur adipisicing elit '\n", - " 'sed do eiusmod tempor incididunt ut labore et dolore magna '\n", - " 'aliqua Ut enim ad minim veniam quis nostrud exercitation'\n", - ")\n", - "\n", - "word_dict = {w: i for i, w in enumerate(list(set(sentence.split())))}\n", - "number_dict = {i: w for i, w in enumerate(list(set(sentence.split())))}\n", - "n_class = len(word_dict)\n", - "n_step = len(sentence.split())\n", - "n_hidden = 5\n", - "\n", - "def make_batch(sentence):\n", - " input_batch = []\n", - " target_batch = []\n", - "\n", - " words = sentence.split()\n", - " for i, word in enumerate(words[:-1]):\n", - " input = [word_dict[n] for n in words[:(i + 1)]]\n", - " input = input + [0] * (n_step - len(input))\n", - " target = word_dict[words[i + 1]]\n", - " input_batch.append(np.eye(n_class)[input])\n", - " target_batch.append(np.eye(n_class)[target])\n", - "\n", - " return input_batch, target_batch\n", - "\n", - "# Bi-LSTM Model\n", - "X = tf.placeholder(tf.float32, [None, n_step, n_class])\n", - "Y = tf.placeholder(tf.float32, [None, n_class])\n", - "\n", - "W = tf.Variable(tf.random_normal([n_hidden * 2, n_class]))\n", - "b = tf.Variable(tf.random_normal([n_class]))\n", - "\n", - "lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(n_hidden)\n", - "lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(n_hidden)\n", - "\n", - "# outputs : [batch_size, len_seq, n_hidden], states : [batch_size, n_hidden]\n", - "outputs, _ = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell,lstm_bw_cell, X, dtype=tf.float32)\n", - "\n", - "outputs = tf.concat([outputs[0], outputs[1]], 2) # output[0] : lstm_fw, output[1] : lstm_bw\n", - "outputs = tf.transpose(outputs, [1, 0, 2]) # [n_step, batch_size, n_hidden]\n", - "outputs = outputs[-1] # [batch_size, n_hidden]\n", - "\n", - "model = tf.matmul(outputs, W) + b\n", - "\n", - "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=model, labels=Y))\n", - "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n", - "\n", - "prediction = tf.cast(tf.argmax(model, 1), tf.int32)\n", - "\n", - "# Training\n", - "init = tf.global_variables_initializer()\n", - "sess = tf.Session()\n", - "sess.run(init)\n", - "\n", - "input_batch, target_batch = make_batch(sentence)\n", - "\n", - "for epoch in range(10000):\n", - " _, loss = sess.run([optimizer, cost], feed_dict={X: input_batch, Y: target_batch})\n", - " if (epoch + 1)%1000 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - "predict = sess.run([prediction], feed_dict={X: input_batch})\n", - "print(sentence)\n", - "print([number_dict[n] for n in [pre for pre in predict[0]]])" - ] - } - ], - "metadata": { - "colab": { - "name": "Bi-LSTM-Tensor.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/3-3.Bi-LSTM/Bi_LSTM_Torch.ipynb b/3-3.Bi-LSTM/Bi_LSTM_Torch.ipynb deleted file mode 100644 index 7af2de5..0000000 --- a/3-3.Bi-LSTM/Bi_LSTM_Torch.ipynb +++ /dev/null @@ -1,150 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "id": "ITN5GnDEeeZJ", - "colab_type": "code", - "colab": {} - }, - "outputs": [], - "source": [ - "# http://pytorch.org/\n", - "from os.path import exists\n", - "from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag\n", - "platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())\n", - "cuda_output = !ldconfig -p|grep cudart.so|sed -e 's/.*\\.\\([0-9]*\\)\\.\\([0-9]*\\)$/cu\\1\\2/'\n", - "accelerator = cuda_output[0] if exists('/dev/nvidia0') else 'cpu'" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "xRIwwk0SeXNg", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 233.0 - }, - "outputId": "2a4b4ba1-ddb7-4bc7-df24-1782844a11c6" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1000 cost = 1.372447\n", - "Epoch: 2000 cost = 0.822247\n", - "Epoch: 3000 cost = 0.682714\n", - "Epoch: 4000 cost = 1.896184\n", - "Epoch: 5000 cost = 1.262253\n", - "Epoch: 6000 cost = 0.943720\n", - "Epoch: 7000 cost = 0.710601\n", - "Epoch: 8000 cost = 0.431383\n", - "Epoch: 9000 cost = 0.292406\n", - "Epoch: 10000 cost = 0.196790\n", - "Lorem ipsum dolor sit amet consectetur adipisicing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Ut enim ad minim veniam quis nostrud exercitation\n", - "['ipsum', 'dolor', 'sit', 'amet', 'consectetur', 'adipisicing', 'elit', 'sed', 'do', 'eiusmod', 'eiusmod', 'incididunt', 'ut', 'labore', 'et', 'dolore', 'magna', 'aliqua', 'Ut', 'enim', 'ad', 'minim', 'veniam', 'quis', 'nostrud', 'exercitation']\n" - ] - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - "'''\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from torch.autograd import Variable\n", - "import torch.nn.functional as F\n", - "\n", - "dtype = torch.FloatTensor\n", - "\n", - "sentence = (\n", - " 'Lorem ipsum dolor sit amet consectetur adipisicing elit '\n", - " 'sed do eiusmod tempor incididunt ut labore et dolore magna '\n", - " 'aliqua Ut enim ad minim veniam quis nostrud exercitation'\n", - ")\n", - "\n", - "word_dict = {w: i for i, w in enumerate(list(set(sentence.split())))}\n", - "number_dict = {i: w for i, w in enumerate(list(set(sentence.split())))}\n", - "n_class = len(word_dict)\n", - "max_len = len(sentence.split())\n", - "n_hidden = 5\n", - "\n", - "def make_batch(sentence):\n", - " input_batch = []\n", - " target_batch = []\n", - "\n", - " words = sentence.split()\n", - " for i, word in enumerate(words[:-1]):\n", - " input = [word_dict[n] for n in words[:(i + 1)]]\n", - " input = input + [0] * (max_len - len(input))\n", - " target = word_dict[words[i + 1]]\n", - " input_batch.append(np.eye(n_class)[input])\n", - " target_batch.append(target)\n", - "\n", - " return Variable(torch.Tensor(input_batch)), Variable(torch.LongTensor(target_batch))\n", - "\n", - "class BiLSTM(nn.Module):\n", - " def __init__(self):\n", - " super(BiLSTM, self).__init__()\n", - "\n", - " self.lstm = nn.LSTM(input_size=n_class, hidden_size=n_hidden, bidirectional=True)\n", - " self.W = nn.Parameter(torch.randn([n_hidden * 2, n_class]).type(dtype))\n", - " self.b = nn.Parameter(torch.randn([n_class]).type(dtype))\n", - "\n", - " def forward(self, X):\n", - " input = X.transpose(0, 1) # input : [n_step, batch_size, n_class]\n", - "\n", - " hidden_state = Variable(torch.zeros(1*2, len(X), n_hidden)) # [num_layers(=1) * num_directions(=1), batch_size, n_hidden]\n", - " cell_state = Variable(torch.zeros(1*2, len(X), n_hidden)) # [num_layers(=1) * num_directions(=1), batch_size, n_hidden]\n", - "\n", - " outputs, (_, _) = self.lstm(input, (hidden_state, cell_state))\n", - " outputs = outputs[-1] # [batch_size, n_hidden]\n", - " model = torch.mm(outputs, self.W) + self.b # model : [batch_size, n_class]\n", - " return model\n", - "\n", - "input_batch, target_batch = make_batch(sentence)\n", - "\n", - "model = BiLSTM()\n", - "\n", - "criterion = nn.CrossEntropyLoss()\n", - "optimizer = optim.Adam(model.parameters(), lr=0.001)\n", - "\n", - "# Training\n", - "for epoch in range(10000):\n", - " optimizer.zero_grad()\n", - " output = model(input_batch)\n", - " loss = criterion(output, target_batch)\n", - " if (epoch + 1) % 1000 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - "predict = model(input_batch).data.max(1, keepdim=True)[1]\n", - "print(sentence)\n", - "print([number_dict[n.item()] for n in predict.squeeze()])" - ] - } - ], - "metadata": { - "colab": { - "name": "Bi-LSTM-Torch.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/4-1.Seq2Seq/Seq2Seq-Torch.py b/4-1.Seq2Seq/Seq2Seq.py similarity index 100% rename from 4-1.Seq2Seq/Seq2Seq-Torch.py rename to 4-1.Seq2Seq/Seq2Seq.py diff --git a/4-1.Seq2Seq/Seq2Seq_Tensor.ipynb b/4-1.Seq2Seq/Seq2Seq_Tensor.ipynb deleted file mode 100644 index 659af03..0000000 --- a/4-1.Seq2Seq/Seq2Seq_Tensor.ipynb +++ /dev/null @@ -1,149 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "jKQFhU-jfraL", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 269.0 - }, - "outputId": "b8bf4d8b-3a7e-4d49-91ae-2a31942c5c0a" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:From :47: BasicRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "This class is equivalent as tf.keras.layers.SimpleRNNCell, and will be replaced by that in Tensorflow 2.0.\n", - "Epoch: 1000 cost = 0.001075\n", - "Epoch: 2000 cost = 0.000416\n", - "Epoch: 3000 cost = 0.000238\n", - "Epoch: 4000 cost = 0.000062\n", - "Epoch: 5000 cost = 0.000021\n", - "test\n", - "man -> women\n", - "mans -> women\n", - "king -> queen\n", - "black -> white\n", - "upp -> down\n" - ] - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - " reference : https://github.com/golbin/TensorFlow-Tutorials/blob/master/10%20-%20RNN/03%20-%20Seq2Seq.py\n", - "'''\n", - "import tensorflow as tf\n", - "import numpy as np\n", - "\n", - "tf.reset_default_graph()\n", - "# S: Symbol that shows starting of decoding input\n", - "# E: Symbol that shows starting of decoding output\n", - "# P: Symbol that will fill in blank sequence if current batch data size is short than time steps\n", - "\n", - "char_arr = [c for c in 'SEPabcdefghijklmnopqrstuvwxyz']\n", - "num_dic = {n: i for i, n in enumerate(char_arr)}\n", - "\n", - "seq_data = [['man', 'women'], ['black', 'white'], ['king', 'queen'], ['girl', 'boy'], ['up', 'down'], ['high', 'low']]\n", - "\n", - "# Seq2Seq Parameter\n", - "n_step = 5\n", - "n_hidden = 128\n", - "n_class = len(num_dic) # number of class(=number of vocab)\n", - "\n", - "def make_batch(seq_data):\n", - " input_batch, output_batch, target_batch = [], [], []\n", - "\n", - " for seq in seq_data:\n", - " for i in range(2):\n", - " seq[i] = seq[i] + 'P' * (n_step - len(seq[i]))\n", - "\n", - " input = [num_dic[n] for n in seq[0]]\n", - " output = [num_dic[n] for n in ('S' + seq[1])]\n", - " target = [num_dic[n] for n in (seq[1] + 'E')]\n", - "\n", - " input_batch.append(np.eye(n_class)[input])\n", - " output_batch.append(np.eye(n_class)[output])\n", - "\n", - " target_batch.append(target)\n", - "\n", - " return input_batch, output_batch, target_batch\n", - "\n", - "# Model\n", - "enc_input = tf.placeholder(tf.float32, [None, None, n_class]) # [batch_size, max_len(=encoder_step), n_class]\n", - "dec_input = tf.placeholder(tf.float32, [None, None, n_class]) # [batch_size, max_len+1(=decoder_step) (becase of 'S' or 'E'), n_class]\n", - "targets = tf.placeholder(tf.int64, [None, None]) # [batch_size, max_len+1], not one-hot\n", - "\n", - "with tf.variable_scope('encode'):\n", - " enc_cell = tf.nn.rnn_cell.BasicRNNCell(n_hidden)\n", - " enc_cell = tf.nn.rnn_cell.DropoutWrapper(enc_cell, output_keep_prob=0.5)\n", - " _, enc_states = tf.nn.dynamic_rnn(enc_cell, enc_input, dtype=tf.float32)\n", - " # encoder state will go to decoder initial_state, enc_states : [batch_size, n_hidden(=128)]\n", - "\n", - "with tf.variable_scope('decode'):\n", - " dec_cell = tf.nn.rnn_cell.BasicRNNCell(n_hidden)\n", - " dec_cell = tf.nn.rnn_cell.DropoutWrapper(dec_cell, output_keep_prob=0.5)\n", - " outputs, _ = tf.nn.dynamic_rnn(dec_cell, dec_input, initial_state=enc_states, dtype=tf.float32)\n", - " # outputs : [batch_size, max_len+1, n_hidden(=128)]\n", - "\n", - "model = tf.layers.dense(outputs, n_class, activation=None) # model : [batch_size, max_len+1, n_class]\n", - "\n", - "cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=model, labels=targets))\n", - "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n", - "\n", - "# Training\n", - "sess = tf.Session()\n", - "sess.run(tf.global_variables_initializer())\n", - "input_batch, output_batch, target_batch = make_batch(seq_data)\n", - "\n", - "for epoch in range(5000):\n", - " _, loss = sess.run([optimizer, cost], feed_dict={enc_input: input_batch, dec_input: output_batch, targets: target_batch})\n", - " if (epoch + 1)%1000 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - "# Test\n", - "def translate(word):\n", - " seq_data = [word, 'P' * len(word)]\n", - "\n", - " input_batch, output_batch, _ = make_batch([seq_data])\n", - " prediction = tf.argmax(model, 2)\n", - "\n", - " result = sess.run(prediction, feed_dict={enc_input: input_batch, dec_input: output_batch})\n", - "\n", - " decoded = [char_arr[i] for i in result[0]]\n", - " end = decoded.index('E')\n", - " translated = ''.join(decoded[:end])\n", - "\n", - " return translated.replace('P','')\n", - "\n", - "print('test')\n", - "print('man ->', translate('man'))\n", - "print('mans ->', translate('mans'))\n", - "print('king ->', translate('king'))\n", - "print('black ->', translate('black'))\n", - "print('upp ->', translate('upp'))" - ] - } - ], - "metadata": { - "colab": { - "name": "Seq2Seq-Tensor.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/4-1.Seq2Seq/Seq2Seq_Torch.ipynb b/4-1.Seq2Seq/Seq2Seq_Torch.ipynb deleted file mode 100644 index e1914e2..0000000 --- a/4-1.Seq2Seq/Seq2Seq_Torch.ipynb +++ /dev/null @@ -1,172 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "5_Xm1E_kf5FQ", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 251.0 - }, - "outputId": "f9c547c4-7834-451a-a4d0-6b5e7d8c8f0f" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:38: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n", - " \"num_layers={}\".format(dropout, num_layers))\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1000 cost = 0.003078\n", - "Epoch: 2000 cost = 0.000852\n", - "Epoch: 3000 cost = 0.000365\n", - "Epoch: 4000 cost = 0.000183\n", - "Epoch: 5000 cost = 0.000099\n", - "test\n", - "man -> women\n", - "mans -> women\n", - "king -> queen\n", - "black -> white\n", - "upp -> down\n" - ] - } - ], - "source": [ - "# code by Tae Hwan Jung(Jeff Jung) @graykode\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "from torch.autograd import Variable\n", - "\n", - "dtype = torch.FloatTensor\n", - "# S: Symbol that shows starting of decoding input\n", - "# E: Symbol that shows starting of decoding output\n", - "# P: Symbol that will fill in blank sequence if current batch data size is short than time steps\n", - "\n", - "char_arr = [c for c in 'SEPabcdefghijklmnopqrstuvwxyz']\n", - "num_dic = {n: i for i, n in enumerate(char_arr)}\n", - "\n", - "seq_data = [['man', 'women'], ['black', 'white'], ['king', 'queen'], ['girl', 'boy'], ['up', 'down'], ['high', 'low']]\n", - "\n", - "# Seq2Seq Parameter\n", - "n_step = 5\n", - "n_hidden = 128\n", - "n_class = len(num_dic)\n", - "batch_size = len(seq_data)\n", - "\n", - "def make_batch(seq_data):\n", - " input_batch, output_batch, target_batch = [], [], []\n", - "\n", - " for seq in seq_data:\n", - " for i in range(2):\n", - " seq[i] = seq[i] + 'P' * (n_step - len(seq[i]))\n", - "\n", - " input = [num_dic[n] for n in seq[0]]\n", - " output = [num_dic[n] for n in ('S' + seq[1])]\n", - " target = [num_dic[n] for n in (seq[1] + 'E')]\n", - "\n", - " input_batch.append(np.eye(n_class)[input])\n", - " output_batch.append(np.eye(n_class)[output])\n", - " target_batch.append(target) # not one-hot\n", - "\n", - " # make tensor\n", - " return Variable(torch.Tensor(input_batch)), Variable(torch.Tensor(output_batch)), Variable(torch.LongTensor(target_batch))\n", - "\n", - "# Model\n", - "class Seq2Seq(nn.Module):\n", - " def __init__(self):\n", - " super(Seq2Seq, self).__init__()\n", - "\n", - " self.enc_cell = nn.RNN(input_size=n_class, hidden_size=n_hidden, dropout=0.5)\n", - " self.dec_cell = nn.RNN(input_size=n_class, hidden_size=n_hidden, dropout=0.5)\n", - " self.fc = nn.Linear(n_hidden, n_class)\n", - "\n", - " def forward(self, enc_input, enc_hidden, dec_input):\n", - " enc_input = enc_input.transpose(0, 1) # enc_input: [max_len(=n_step, time step), batch_size, n_class]\n", - " dec_input = dec_input.transpose(0, 1) # dec_input: [max_len(=n_step, time step), batch_size, n_class]\n", - "\n", - " # enc_states : [num_layers(=1) * num_directions(=1), batch_size, n_hidden]\n", - " _, enc_states = self.enc_cell(enc_input, enc_hidden)\n", - " # outputs : [max_len+1(=6), batch_size, num_directions(=1) * n_hidden(=128)]\n", - " outputs, _ = self.dec_cell(dec_input, enc_states)\n", - "\n", - " model = self.fc(outputs) # model : [max_len+1(=6), batch_size, n_class]\n", - " return model\n", - "\n", - "\n", - "input_batch, output_batch, target_batch = make_batch(seq_data)\n", - "\n", - "model = Seq2Seq()\n", - "criterion = nn.CrossEntropyLoss()\n", - "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n", - "\n", - "for epoch in range(5000):\n", - " # make hidden shape [num_layers * num_directions, batch_size, n_hidden]\n", - " hidden = Variable(torch.zeros(1, batch_size, n_hidden))\n", - "\n", - " optimizer.zero_grad()\n", - " # input_batch : [batch_size, max_len(=n_step, time step), n_class]\n", - " # output_batch : [batch_size, max_len+1(=n_step, time step) (becase of 'S' or 'E'), n_class]\n", - " # target_batch : [batch_size, max_len+1(=n_step, time step)], not one-hot\n", - " output = model(input_batch, hidden, output_batch)\n", - " # output : [max_len+1, batch_size, num_directions(=1) * n_hidden]\n", - " output = output.transpose(0, 1) # [batch_size, max_len+1(=6), num_directions(=1) * n_hidden]\n", - " loss = 0\n", - " for i in range(0, len(target_batch)):\n", - " # output[i] : [max_len+1, num_directions(=1) * n_hidden, target_batch[i] : max_len+1]\n", - " loss += criterion(output[i], target_batch[i])\n", - " if (epoch + 1) % 1000 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - "\n", - "# Test\n", - "def translate(word):\n", - " input_batch, output_batch, _ = make_batch([[word, 'P' * len(word)]])\n", - "\n", - " # make hidden shape [num_layers * num_directions, batch_size, n_hidden]\n", - " hidden = Variable(torch.zeros(1, 1, n_hidden))\n", - " output = model(input_batch, hidden, output_batch)\n", - " # output : [max_len+1(=6), batch_size(=1), n_class]\n", - "\n", - " predict = output.data.max(2, keepdim=True)[1] # select n_class dimension\n", - " decoded = [char_arr[i] for i in predict]\n", - " end = decoded.index('E')\n", - " translated = ''.join(decoded[:end])\n", - "\n", - " return translated.replace('P', '')\n", - "\n", - "print('test')\n", - "print('man ->', translate('man'))\n", - "print('mans ->', translate('mans'))\n", - "print('king ->', translate('king'))\n", - "print('black ->', translate('black'))\n", - "print('upp ->', translate('upp'))" - ] - } - ], - "metadata": { - "colab": { - "name": "Seq2Seq-Torch.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/4-2.Seq2Seq(Attention)/Seq2Seq(Attention)-Torch.py b/4-2.Seq2Seq(Attention)/Seq2Seq(Attention).py similarity index 100% rename from 4-2.Seq2Seq(Attention)/Seq2Seq(Attention)-Torch.py rename to 4-2.Seq2Seq(Attention)/Seq2Seq(Attention).py diff --git a/4-2.Seq2Seq(Attention)/Seq2Seq(Attention)_Tensor.ipynb b/4-2.Seq2Seq(Attention)/Seq2Seq(Attention)_Tensor.ipynb deleted file mode 100644 index 2a57d0b..0000000 --- a/4-2.Seq2Seq(Attention)/Seq2Seq(Attention)_Tensor.ipynb +++ /dev/null @@ -1,171 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "Nt432JiJgboz", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 486.0 - }, - "outputId": "df4c9ae6-9df1-4a6d-f240-5123f887bd9d" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "WARNING:tensorflow:From :53: BasicRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "This class is equivalent as tf.keras.layers.SimpleRNNCell, and will be replaced by that in Tensorflow 2.0.\n", - "Epoch: 0400 cost = 0.000000\n", - "Epoch: 0800 cost = 0.000000\n", - "Epoch: 1200 cost = 0.000000\n", - "Epoch: 1600 cost = 0.000000\n", - "Epoch: 2000 cost = 0.000000\n", - "['ich', 'mochte', 'ein', 'bier', 'P'] -> ['i', 'want', 'a', 'beer', 'E']\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAEzCAYAAABE0wr4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFY9JREFUeJzt3XlwVfX5x/FPLtmsAlZWA4SJFaKO\nCUkgBMpmE2hoYJggRRonyFIhBUsFHBAYK9YyLDO4FCIZLLak2KJFkA5YYYK4FSgQgTEBDBBDExI2\nWcRgICb3/v7g5y0pIAmSc0Ke9+svcu7yfb658c05944kwOfz+QQAhnncHgAA3EYIAZhHCAGYRwgB\nmEcIAZhHCAGYZzqEpaWlioqK0qFDh77zfomJiXr99dcdmsp5ixcv1sMPP+z2GN/L2rVr1bdvX7fH\nqJMjR44oMjJSBw4cuOK2W3E/t7JAtwdwU7t27ZSXl+f2GA3euXPn9O6772rEiBFuj3JNqampSk1N\ndXuMm6ax7edyiYmJOn78uDyeS+dhwcHB6tSpkyZNmqRevXq5MpPpM0LUzrZt2/Tmm2+6PQYakZkz\nZyovL095eXnasmWLBg0apIyMjOtendUX0yG8/NLk9OnTmjx5srp27apevXpp/vz5qq6u9t+3oqJC\nU6dOVUxMjPr27astW7bU+3yRkZFat26dhg0bpujoaI0ZM0ZHjx5VRkaGYmNjNXToUJWUlPjvv3nz\nZqWmpiomJkb9+vXTK6+8osv/x6Hly5crMTFRsbGxGjVqlIqKimqst2rVKvXp00cxMTGaM2eOJGn9\n+vWaMmWK9u3bp6ioKBUVFcnr9SozM1MDBgxQly5dlJqaqm3bttX790OSjh49qgkTJqhHjx7q2rWr\npkyZojNnzmjNmjVKSEiQ9N/XdcuWLf7vR1pamo4dO+bIjHW1b98+DRkyRLGxsUpPT1dpaWmN/UhS\nQUGBRo8erfj4eCUkJOjZZ5/VxYsXJUlr1qzRwIEDtXDhQsXGxtb4mbgVhIaGauTIkYqIiND777/v\nygymQ3i5Z555RlVVVfrggw/01ltvadOmTVq+fLn/9rfeekuPPfaYtm/frl69eun55593ZK6VK1dq\nyZIleuedd7Rnzx6NHj1aTzzxhD7++GNVVVX5Zzxw4IB+/etfKyMjQzt37tTLL7+s7OxsrV69WpK0\nadMmZWVlKTMzU9u3b9ePfvQjTZo0yb9OaWmpTp48qU2bNikzM1MrVqzQrl27NHjwYE2YMEEPPPCA\n8vLyFBERob/85S/6xz/+oaVLlyo3N1dpaWmaOHGizp49W6/fC5/PpwkTJqhVq1Z67733lJOTo/Ly\ncv3+97+/6v2zs7P16quvavPmzTp79qz+/Oc/1+t8N+qNN95QVlaWPvzwQwUFBenpp5+ucXtFRYUe\nf/xxxcfHa8uWLXr77beVn5+vzMxM/32++OILBQQEaMeOHWrfvr3TW7gpqqurFRjozrt1hFDS2bNn\n9f7772v8+PFq2rSp7r77br344ouKi4vz3+ehhx5STEyMQkJCNHDgQP3nP//RN998U++zDRo0SG3a\ntFGHDh3UqVMn3X///YqOjtYdd9yh+Ph4HT58WNKlUHfv3l0/+9nPFBQUpNjYWKWkpOjdd9+VJK1e\nvVopKSl64IEHFBwcrEmTJmnixIn+PQQEBCgjI0MhISHq3bu3WrRoocLCwqvOtGrVKo0aNUr33HOP\ngoKCNGLECLVv314bNmyo1+9FXl6eCgoKNH36dN1+++266667NHnyZG3YsEEVFRVX3P+RRx5R69at\ndddddykhIeGa+3FbWlqa2rVrp2bNmmns2LHKzc3V119/7b/9gw8+0DfffKMnnnhCwcHBCgsL069+\n9Su9/fbb/vuUl5dr3LhxCgoKUkBAgBvbuGFff/21VqxYobKyMvXv39+VGUx/WPItj8cjr9db42/S\n6OjoGve5/LbQ0FD5fD5VVlYqKCioXmdr27at/88hISFq06ZNja8rKyslSSUlJbr33ntrPLZjx47+\nS9aSkhJ169bNf9sPf/hDpaSk+L8OCwtTkyZN/F+Hhob6L73+V3FxsebPn68FCxb4j/l8Ph09evRG\ntlhrJSUl8nq96tmz5xW3Xe11uPw1u+222665H7dd/rqFh4fL5/PV2E9JSYnOnj2rqKioGo/zer3+\n1/+OO+5Qs2bNnBn4Jpg3b57/5yc0NFSRkZFatmyZOnTo4Mo8hFDyv4/2Xf8Qj1t/y377ydq1vv7W\nt/9B/K9v5w4ICJDX670pM4WGhup3v/tdjZA6ISQkRCEhIfr000+vuG3NmjVXHLvW96qhuXzOb38G\nT5486T8WEhKiiIgI/9n91Vz+l9itYObMmUpPT3d7DL9b4yfFAR6Pp8aHB7m5ufV+qXczhYeHX3Hp\n9/nnn6tjx46SpA4dOtTY37lz5/Taa6/p/PnzN7RWQUFBjWNHjhy5ganrpmPHjrp48aL/7QDp0vtn\np06dqve169Plr0txcbGaNGmiVq1a+Y917NhRpaWlKi8v9x/78ssv9dVXXzk6Z2NGCCU1b95cSUlJ\neuWVV3TmzBkdP35cs2fPVnFxsduj1drQoUO1fft25eTkqKqqSrm5uVq/fr2GDh0qSRo2bJg2bNig\n3NxcVVZWKisrS6tXr9btt99+3ecOCQnRF198oTNnzqiyslJpaWlauXKlcnNzVV1drffee0+DBw/W\n559/Xq977NSpk7p166Y5c+bo9OnT/g9KfvOb39TruvVt5cqVOnHihMrLy5Wdna1+/frVuDTu3bu3\nWrVqpblz5+qrr77S6dOnNW3atGt+SIS6I4T/b/78+brzzjuVmJioYcOGqU+fPho7dqzbY9VadHS0\n5s2bp0WLFik+Pl6zZ8/WM888o4EDB0qSkpKSNG3aNE2dOlUJCQnav39/jU8dv0v//v3l8Xj0k5/8\nRJ9++qmGDRumxx57TFOmTFFcXJwWLVqkF154Qffcc099blGStHDhQgUGBiopKUlJSUk6d+6cXnzx\nxXpftz6lpaVp7Nix6tOnj6qqqvTcc8/VuD0wMFBLlixRSUmJevfurcGDB6tFixZ69tln3Rm4EQrg\nX6gGYB1nhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMy75f71mQGe4Y6t9eqnL2h8\n9FOOrLWxbI8j60hSQIt35Ds1yLH1ksNiHFvLydfMSY11X5Kze8vxrrrqcc4Iv0PEg+Fuj1AvAoI6\nuz1CvWmsr1lj3ZfUMPZGCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgB\nmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmNegQlhaWqqoqCgdOnTI7VEAGNKg\nfp1nu3btlJeX5/YYAIxpUGeEAOCGBhXCI0eOKDIyUgcOHHB7FACGNKgQAoAbAnw+n8/tIb515MgR\nJSUlad26dercufNV71OUX6yIB8MdngxAY9agPiypjfHRTzm2Vo53lQZ4hjuy1sayPY6sI0metgfl\nPdbJsfWSw2IcW8vJ18xJjXVfkrN7y/GuuupxLo0BmEcIAZhHCAGYRwgBmNegPixp3769CgoK3B4D\ngDGcEQIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwjxAC\nMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMK9B/fKm2thYtqdRrpccFuPIOpKU43V2PaCh44wQgHmE\nEIB5hBCAeYQQgHmEEIB5hBCAeYQQgHmEEIB5hBCAeYQQgHmEEIB5hBCAeYQQgHmEEIB5hBCAeYQQ\ngHmEEIB5hBCAeYQQgHmEEIB5hBCAeYQQgHkNJoRr1qzRqVOn3B4DgEENIoTV1dWaN28eIQTgiuuG\n8KGHHlJOTo7/68cff1wpKSn+rz/77DNFRUWpsLBQGRkZSkhIUHx8vCZMmKATJ0747xcZGamNGzcq\nLS1NMTExGjJkiAoKCiRJXbt21blz5/Twww/r5Zdfvpn7A4Drum4IExIStGvXLkmXztz27t2rCxcu\n6MyZM5Kk3NxcxcbG6vnnn1fTpk318ccfa/PmzSovL9eCBQtqPNeyZcs0d+5cbd26Vc2bN9fixYsl\nSevXr5d06fJ48uTJN3WDAHA91w1hjx49tHv3bknS3r171bFjR0VHR+uTTz6RdCmEPXv21NKlSzVn\nzhwFBweradOmSkxMVH5+fo3nGjx4sCIiIvSDH/xAffv2VWFhYT1sCQDqJvB6d+jRo4d++9vf6uLF\ni9q5c6e6deum1q1b65NPPlH//v2Vm5urMWPGKD8/Xy+99JI+++wzVVZWyuv1qk2bNjWeq3379v4/\n33bbbbp48WKdBw5o8Y4CgjrX+XE3ytP2oCPr5HgdWeay9VY5u6CDGuveGuu+JPf3dt0Q3n333QoL\nC1NeXp527typESNGqGXLllq/fr2Ki4t14cIFhYeHKykpScOHD1dWVpaaNWum7OxsZWdn13guj+f7\nfzbjOzVIvu/9LLXjaXtQ3mOdHFkrOSzGkXWkSz90AzzDHVvPSY11b411X5Kze7tWcGtVph49eig3\nN1e7d+9WXFyc7r//fhUVFelf//qXunfvrsOHD+v8+fP65S9/qWbNmkm6dBkNALeCWodw7dq1at26\ntZo3b67AwEDdd999+utf/6qePXsqLCxMHo9Hu3fvVkVFhd58800VFRXpyy+/1IULF677/KGhoZKk\nw4cPq7y8/PvtCADqqFYhTEhI0OHDh9W1a1f/sbi4OB06dEg//vGP1aZNG02fPl2zZ89Wv379VFhY\nqEWLFunOO+/UT3/60+s+f8uWLZWcnKypU6dq4cKFN74bALgBAT6fz6m33G4Kp96zk3iP8FbUWPfW\nWPcl3ULvEQJAY0YIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgB\nmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYF+j2AHXl7G97c3a9xmpj2Z5GuR4/G40H\nZ4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwj\nhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMczWEe/fu1ciRIxUfH68ePXpo\n+vTpKi8vd3MkAAa5GsLJkyerS5cu+ve//63169crPz9ff/zjH90cCYBBAT6fz+fW4ufPn1dQUJCC\ng4MlSXPmzFFRUZFee+21az6mKL9YEQ+GOzUiAAMC3Vx827ZtWrJkiYqKilRVVaXq6mp17dr1Ox8z\nPvoph6aTcryrNMAz3LH1nOL0vjaW7XFsLU/bg/Ie6+TIWslhMY6sIzXen0XJ2b3leFdd9bhrl8aF\nhYV68sknNXjwYG3dulV5eXlKT093axwAhrl2Rrh//341adJEY8aMUUBAgKRLH554PHyQDcBZrlWn\nQ4cOqqysVH5+vsrLy5WZmamKigqdPHlS1dXVbo0FwCDXQtilSxeNHj1aY8aMUXJysoKCgjR37lyd\nO3eOS2QAjnL1w5IZM2ZoxowZNY5t3brVpWkAWMUbcgDMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADM\nI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzHP1lzfB\nhuSwGMfWyvE6t97Gsj2OrOPGek6+Zg0BZ4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwj\nhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMq3UIjxw5\nosjISB04cKA+5wEAx3FGCMA8QgjAvDqHcN++fRoyZIhiY2OVnp6u0tJSSdKOHTv0i1/8QnFxcerd\nu7deeukleb1e/+P+9re/KSUlRV26dFFycrL++c9/+m8bOXKkFixYoNTUVI0aNeombAsAaq/OIXzj\njTeUlZWlDz/8UEFBQXr66ad17NgxZWRk6Oc//7l27Nih5cuXa926dfr73/8uSdq0aZP+8Ic/aP78\n+dq1a5dmzJih6dOnq7Cw0P+877zzjmbPnq3ly5fftM0BQK34aqmkpMTXuXNn39q1a/3HPvroI19k\nZKQvMzPTN3To0Br3/9Of/uR75JFHfD6fzzdu3Djf/Pnza9yekZHhW7hwoc/n8/nS09N9EydOrNUc\nn+f9p7YjA0CtBNY1nPfee6//z+Hh4fL5fNq+fbv279+vqKioywOrli1bSpKKi4u1ZcsWvf766zVu\nb9q0qf/rsLCwWq0/Pvqpuo58w3K8qzTAM9yx9ZzSWPclObu3jWV7HFlHkjxtD8p7rJNj6yWHxTi2\nlpOvWY531VWP1zmEHs9/r6Z9Pp8kqV27dgoODtayZcuu+pjQ0FA9+eSTGj9+/DWfNzCwzqMAwE1R\n5/cIi4qK/H8uLi5WkyZNdN999+ngwYM1Phw5deqULly4IOnSmWNBQUGN5ykrK6txfwBwS51DuHLl\nSp04cULl5eXKzs5Wv379lJqaqvLyci1evFgVFRUqKyvTuHHjtHTpUklSWlqaNm7cqE2bNqmqqkq7\ndu1Samqqtm/fftM3BAB1VecQpqWlaezYserTp4+qqqr03HPPqXnz5srKytJHH32khIQEjRgxQvHx\n8Zo4caIkqWfPnpo1a5bmzZunuLg4zZo1S9OmTVPPnj1v+oYAoK5q/cZc+/bt/Ze3KSkpV9zevXt3\nrV69+pqPf/TRR/Xoo49e9bYVK1bUdgwAuOn4P0sAmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcI\nAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZjHb1UHblBy\nWIxja+V4nV1vY9kex9ZyY73/xRkhAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wgh\nAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPNc+XWeiYmJ\nOn78uDyeKzs8a9YspaWluTAVAKtc+73GM2fOVHp6ulvLA4Afl8YAzCOEAMxz7dJ43rx5WrBgwRXH\n9+zZoyZNmrgwEQCrAnw+n8/pRRMTEzV27Ngbeo+wKL9YEQ+G18NUAKxy7YzwRo2PfsqxtXK8qzTA\nM9yx9ZzSWPclNd69Ob2vjWV7HFvL0/agvMc6ObbWVY87sjoANGCEEIB5De7Dkn79+ikzM9OFiQBY\n5UoIN2/e7MayAHBVXBoDMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMI8QAjCP\nEAIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwL8Dn8/ncHgIA3MQZIQDzCCEA\n8wghAPMIIQDzCCEA8wghAPP+D11w97+YehQoAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "# code by Tae Hwan Jung(Jeff Jung) @graykode\n", - "import tensorflow as tf\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "tf.reset_default_graph()\n", - "# S: Symbol that shows starting of decoding input\n", - "# E: Symbol that shows starting of decoding output\n", - "# P: Symbol that will fill in blank sequence if current batch data size is short than time steps\n", - "sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E']\n", - "\n", - "word_list = \" \".join(sentences).split()\n", - "word_list = list(set(word_list))\n", - "word_dict = {w: i for i, w in enumerate(word_list)}\n", - "number_dict = {i: w for i, w in enumerate(word_list)}\n", - "n_class = len(word_dict) # vocab list\n", - "\n", - "# Parameter\n", - "n_step = 5 # maxium number of words in one sentence(=number of time steps)\n", - "n_hidden = 128\n", - "\n", - "def make_batch(sentences):\n", - " input_batch = [np.eye(n_class)[[word_dict[n] for n in sentences[0].split()]]]\n", - " output_batch = [np.eye(n_class)[[word_dict[n] for n in sentences[1].split()]]]\n", - " target_batch = [[word_dict[n] for n in sentences[2].split()]]\n", - " return input_batch, output_batch, target_batch\n", - "\n", - "# Model\n", - "enc_inputs = tf.placeholder(tf.float32, [None, None, n_class]) # [batch_size, n_step, n_class]\n", - "dec_inputs = tf.placeholder(tf.float32, [None, None, n_class]) # [batch_size, n_step, n_class]\n", - "targets = tf.placeholder(tf.int64, [1, n_step]) # [batch_size, n_step], not one-hot\n", - "\n", - "# Linear for attention\n", - "attn = tf.Variable(tf.random_normal([n_hidden, n_hidden]))\n", - "out = tf.Variable(tf.random_normal([n_hidden * 2, n_class]))\n", - "\n", - "def get_att_score(dec_output, enc_output): # enc_output [n_step, n_hidden]\n", - " score = tf.squeeze(tf.matmul(enc_output, attn), 0) # score : [n_hidden]\n", - " dec_output = tf.squeeze(dec_output, [0, 1]) # dec_output : [n_hidden]\n", - " return tf.tensordot(dec_output, score, 1) # inner product make scalar value\n", - "\n", - "def get_att_weight(dec_output, enc_outputs):\n", - " attn_scores = [] # list of attention scalar : [n_step]\n", - " enc_outputs = tf.transpose(enc_outputs, [1, 0, 2]) # enc_outputs : [n_step, batch_size, n_hidden]\n", - " for i in range(n_step):\n", - " attn_scores.append(get_att_score(dec_output, enc_outputs[i]))\n", - "\n", - " # Normalize scores to weights in range 0 to 1\n", - " return tf.reshape(tf.nn.softmax(attn_scores), [1, 1, -1]) # [1, 1, n_step]\n", - "\n", - "model = []\n", - "Attention = []\n", - "with tf.variable_scope('encode'):\n", - " enc_cell = tf.nn.rnn_cell.BasicRNNCell(n_hidden)\n", - " enc_cell = tf.nn.rnn_cell.DropoutWrapper(enc_cell, output_keep_prob=0.5)\n", - " # enc_outputs : [batch_size(=1), n_step(=decoder_step), n_hidden(=128)]\n", - " # enc_hidden : [batch_size(=1), n_hidden(=128)]\n", - " enc_outputs, enc_hidden = tf.nn.dynamic_rnn(enc_cell, enc_inputs, dtype=tf.float32)\n", - "\n", - "with tf.variable_scope('decode'):\n", - " dec_cell = tf.nn.rnn_cell.BasicRNNCell(n_hidden)\n", - " dec_cell = tf.nn.rnn_cell.DropoutWrapper(dec_cell, output_keep_prob=0.5)\n", - "\n", - " inputs = tf.transpose(dec_inputs, [1, 0, 2])\n", - " hidden = enc_hidden\n", - " for i in range(n_step):\n", - " # time_major True mean inputs shape: [max_time, batch_size, ...]\n", - " dec_output, hidden = tf.nn.dynamic_rnn(dec_cell, tf.expand_dims(inputs[i], 1),\n", - " initial_state=hidden, dtype=tf.float32, time_major=True)\n", - " attn_weights = get_att_weight(dec_output, enc_outputs) # attn_weights : [1, 1, n_step]\n", - " Attention.append(tf.squeeze(attn_weights))\n", - "\n", - " # matrix-matrix product of matrices [1, 1, n_step] x [1, n_step, n_hidden] = [1, 1, n_hidden]\n", - " context = tf.matmul(attn_weights, enc_outputs)\n", - " dec_output = tf.squeeze(dec_output, 0) # [1, n_step]\n", - " context = tf.squeeze(context, 1) # [1, n_hidden]\n", - "\n", - " model.append(tf.matmul(tf.concat((dec_output, context), 1), out)) # [n_step, batch_size(=1), n_class]\n", - "\n", - "trained_attn = tf.stack([Attention[0], Attention[1], Attention[2], Attention[3], Attention[4]], 0) # to show attention matrix\n", - "model = tf.transpose(model, [1, 0, 2]) # model : [n_step, n_class]\n", - "prediction = tf.argmax(model, 2)\n", - "cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=model, labels=targets))\n", - "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n", - "\n", - "# Training and Test\n", - "with tf.Session() as sess:\n", - " init = tf.global_variables_initializer()\n", - " sess.run(init)\n", - " for epoch in range(2000):\n", - " input_batch, output_batch, target_batch = make_batch(sentences)\n", - " _, loss, attention = sess.run([optimizer, cost, trained_attn],\n", - " feed_dict={enc_inputs: input_batch, dec_inputs: output_batch, targets: target_batch})\n", - "\n", - " if (epoch + 1) % 400 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - " predict_batch = [np.eye(n_class)[[word_dict[n] for n in 'P P P P P'.split()]]]\n", - " result = sess.run(prediction, feed_dict={enc_inputs: input_batch, dec_inputs: predict_batch})\n", - " print(sentences[0].split(), '->', [number_dict[n] for n in result[0]])\n", - "\n", - " # Show Attention\n", - " fig = plt.figure(figsize=(5, 5))\n", - " ax = fig.add_subplot(1, 1, 1)\n", - " ax.matshow(attention, cmap='viridis')\n", - " ax.set_xticklabels([''] + sentences[0].split(), fontdict={'fontsize': 14})\n", - " ax.set_yticklabels([''] + sentences[2].split(), fontdict={'fontsize': 14})\n", - " plt.show()" - ] - } - ], - "metadata": { - "colab": { - "name": "Seq2Seq(Attention)-Tensor.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/4-2.Seq2Seq(Attention)/Seq2Seq(Attention)_Torch.ipynb b/4-2.Seq2Seq(Attention)/Seq2Seq(Attention)_Torch.ipynb deleted file mode 100644 index 410a07b..0000000 --- a/4-2.Seq2Seq(Attention)/Seq2Seq(Attention)_Torch.ipynb +++ /dev/null @@ -1,194 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "g1lbXXSDgync", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 576.0 - }, - "outputId": "e1ff3a6e-a10e-4ed5-b337-16ebc63b3e60" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:38: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n", - " \"num_layers={}\".format(dropout, num_layers))\n", - "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:78: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 0400 cost = 0.000539\n", - "Epoch: 0800 cost = 0.000178\n", - "Epoch: 1200 cost = 0.000089\n", - "Epoch: 1600 cost = 0.000053\n", - "Epoch: 2000 cost = 0.000035\n", - "Epoch: 2400 cost = 0.000024\n", - "Epoch: 2800 cost = 0.000017\n", - "Epoch: 3200 cost = 0.000013\n", - "Epoch: 3600 cost = 0.000009\n", - "Epoch: 4000 cost = 0.000007\n", - "ich mochte ein bier P -> ['i', 'want', 'a', 'beer', 'E']\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAEzCAYAAABE0wr4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFfdJREFUeJzt3X1U1vX9x/EX97RSK+8Woo7KKE8g\noIjOuwY2HXk8mCtHw7xZyrS51I6mrmVrzptz7GZK8rPZJtlmZZo70rKD2d3UqaQdQQ0n4bjR1LzJ\nMJTg+v7+8Nf1k9QEJ9+v8n4+/uK6fX8+QE+/3+s6cQU4juMIAAwL9HoBAOA1QgjAPEIIwDxCCMA8\nQgjAPEIIwDzTIayoqFBMTIz27t37nfdLTk7Wyy+/7NKq3Ldw4ULde++9Xi/jv7J69Wr17dvX62U0\nSHl5uaKjo7Vnz55zbrsa93M1C/Z6AV5q166dCgoKvF7GFe/EiRN66623NGzYMK+XckFpaWlKS0vz\nehmXTVPbz9mSk5N18OBBBQaeOQ4LDQ1Vp06dNGHCBPXq1cuTNZk+IkT9bNq0Sa+++qrXy0ATMn36\ndBUUFKigoEAbNmzQPffco8zMzIuenTUW0yE8+9Tk6NGjmjhxorp27apevXpp7ty5qq2t9d+3qqpK\nkydPVlxcnPr27asNGzY0+vqio6O1Zs0aDR06VLGxsRo1apQOHDigzMxMxcfHa8iQISorK/Pff/36\n9UpLS1NcXJz69eun559/Xmf/j0NLly5VcnKy4uPjNWLECJWUlNSZt2LFCvXp00dxcXGaNWuWJCk3\nN1eTJk3Srl27FBMTo5KSEvl8PmVlZenuu+9Wly5dlJaWpk2bNjX690OSDhw4oHHjxqlHjx7q2rWr\nJk2apGPHjmnVqlVKSkqS9P8/1w0bNvi/H+np6frss89cWWND7dq1S4MHD1Z8fLwyMjJUUVFRZz+S\nVFRUpJEjRyoxMVFJSUl64okndPr0aUnSqlWrNHDgQM2fP1/x8fF1fieuBuHh4Ro+fLiioqL07rvv\nerIG0yE82+OPP66amhq99957ev3117Vu3TotXbrUf/vrr7+uBx98UJs3b1avXr301FNPubKu5cuX\na9GiRXrzzTf18ccfa+TIkXr44Yf14Ycfqqamxr/GPXv26Fe/+pUyMzO1detWPffcc8rJydHKlSsl\nSevWrVN2draysrK0efNm3XLLLZowYYJ/TkVFhQ4fPqx169YpKytLy5Yt07Zt2zRo0CCNGzdOnTt3\nVkFBgaKiovTSSy/p73//uxYvXqz8/Hylp6dr/PjxOn78eKN+LxzH0bhx49S6dWu98847ysvLU2Vl\npX7/+9+f9/45OTl64YUXtH79eh0/flx/+ctfGnV9l+qVV15Rdna23n//fYWEhOixxx6rc3tVVZUe\neughJSYmasOGDXrjjTdUWFiorKws/30+//xzBQQEaMuWLYqMjHR7C5dFbW2tgoO9ebWOEEo6fvy4\n3n33XY0dO1bNmjXTTTfdpGeeeUYJCQn++9x1112Ki4tTWFiYBg4cqP/85z/6+uuvG31t99xzj9q2\nbav27durU6dOuuOOOxQbG6vrrrtOiYmJ2rdvn6Qzoe7evbt+8pOfKCQkRPHx8UpNTdVbb70lSVq5\ncqVSU1PVuXNnhYaGasKECRo/frx/DwEBAcrMzFRYWJh69+6tli1bqri4+LxrWrFihUaMGKGbb75Z\nISEhGjZsmCIjI7V27dpG/V4UFBSoqKhIU6dO1bXXXqsbb7xREydO1Nq1a1VVVXXO/e+//361adNG\nN954o5KSki64H6+lp6erXbt2at68uUaPHq38/Hx99dVX/tvfe+89ff3113r44YcVGhqqiIgI/fKX\nv9Qbb7zhv09lZaXGjBmjkJAQBQQEeLGNS/bVV19p2bJl2r9/v/r37+/JGky/WfKNwMBA+Xy+Ov+S\nxsbG1rnP2beFh4fLcRxVV1crJCSkUdf2/e9/3/91WFiY2rZtW+dydXW1JKmsrEy33nprncd27NjR\nf8paVlambt26+W+74YYblJqa6r8cERGhoKAg/+Xw8HD/qde3lZaWau7cuZo3b57/OsdxdODAgUvZ\nYr2VlZXJ5/OpZ8+e59x2vp/D2T+za6655oL78drZP7cOHTrIcZw6+ykrK9Px48cVExNT53E+n8//\n87/uuuvUvHlzdxZ8GcyZM8f/+xMeHq7o6GgtWbJE7du392Q9hFDyv472XX+Ix6t/Zb95Z+1Cl7/x\nzX8Q3/bNugMCAuTz+S7LmsLDw/W73/2uTkjdEBYWprCwMO3YseOc21atWnXOdRf6Xl1pzl7nN7+D\nhw8f9l8XFhamqKgo/9H9+Zz9j9jVYPr06crIyPB6GX5Xx2+KCwIDA+u8eZCfn9/op3qXU4cOHc45\n9fv000/VsWNHSVL79u3r7O/EiRN68cUXdfLkyUuaVVRUVOe68vLyS1h1w3Ts2FGnT5/2vxwgnXn9\n7MiRI40+uzGd/XMpLS1VUFCQWrdu7b+uY8eOqqioUGVlpf+6L774Ql9++aWr62zKCKGkFi1aKCUl\nRc8//7yOHTumgwcPaubMmSotLfV6afU2ZMgQbd68WXl5eaqpqVF+fr5yc3M1ZMgQSdLQoUO1du1a\n5efnq7q6WtnZ2Vq5cqWuvfbaiz53WFiYPv/8cx07dkzV1dVKT0/X8uXLlZ+fr9raWr3zzjsaNGiQ\nPv3000bdY6dOndStWzfNmjVLR48e9b9R8utf/7pR5za25cuX69ChQ6qsrFROTo769etX59S4d+/e\nat26tWbPnq0vv/xSR48e1ZQpUy74JhEajhD+n7lz5+r6669XcnKyhg4dqj59+mj06NFeL6veYmNj\nNWfOHC1YsECJiYmaOXOmHn/8cQ0cOFCSlJKSoilTpmjy5MlKSkrS7t2767zr+F369++vwMBA/ehH\nP9KOHTs0dOhQPfjgg5o0aZISEhK0YMECPf3007r55psbc4uSpPnz5ys4OFgpKSlKSUnRiRMn9Mwz\nzzT63MaUnp6u0aNHq0+fPqqpqdGTTz5Z5/bg4GAtWrRIZWVl6t27twYNGqSWLVvqiSee8GbBTVAA\nf6EagHUcEQIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMI8QAjDvqvvrMwOuG+HarMVb/qDM\n7r9xZdZbeze6MkeSAlq+KefIPa7NGxAR59qsF3Y8rbGxj7o2zy1NdV+Su3vL86047/UcEX6HH3S+\nOv/S78UEhNzm9RIaTdSdHbxeQqNoqvuSroy9EUIA5hFCAOYRQgDmEUIA5hFCAOYRQgDmEUIA5hFC\nAOYRQgDmEUIA5hFCAOYRQgDmEUIA5hFCAOYRQgDmEUIA5hFCAOYRQgDmEUIA5hFCAOZdUSGsqKhQ\nTEyM9u7d6/VSABhyRX2cZ7t27VRQUOD1MgAYc0UdEQKAF66oEJaXlys6Olp79uzxeikADLmiQggA\nXghwHMfxehHfKC8vV0pKitasWaPbbrvtvPfZt6tcP+gc6fLKADRlV9SbJfWR2f03rs16uzJHA64b\n4cqst/ZudGWOJAV+/9/yfdbJtXkDIuJcm5XnW6G7A+9zbZ5bmuq+JHf3ludbcd7rOTUGYB4hBGAe\nIQRgHiEEYN4V9WZJZGSkioqKvF4GAGM4IgRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4h\nBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYN4V9eFN9eH76qsm\nOW9ARJwrcyQpz+fuvKYqqFXLJjtv+ce5rs2SpNfKN7k679s4IgRgHiEEYB4hBGAeIQRgHiEEYB4h\nBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEE\nYB4hBGAeIQRg3hUTwlWrVunIkSNeLwOAQVdECGtrazVnzhxCCMATFw3hXXfdpby8PP/lhx56SKmp\nqf7Ln3zyiWJiYlRcXKzMzEwlJSUpMTFR48aN06FDh/z3i46O1ttvv6309HTFxcVp8ODBKioqkiR1\n7dpVJ06c0L333qvnnnvucu4PAC7qoiFMSkrStm3bJJ05ctu5c6dOnTqlY8eOSZLy8/MVHx+vp556\nSs2aNdOHH36o9evXq7KyUvPmzavzXEuWLNHs2bO1ceNGtWjRQgsXLpQk5ebmSjpzejxx4sTLukEA\nuJiLhrBHjx7avn27JGnnzp3q2LGjYmNj9dFHH0k6E8KePXtq8eLFmjVrlkJDQ9WsWTMlJyersLCw\nznMNGjRIUVFR+t73vqe+ffuquLi4EbYEAA0TfLE79OjRQ7/97W91+vRpbd26Vd26dVObNm300Ucf\nqX///srPz9eoUaNUWFioZ599Vp988omqq6vl8/nUtm3bOs8VGRnp//qaa67R6dOnG7zgF3Y8rag7\nOzT4cZcqz7fCtVluaqr7kpru3tYe+h+vl9Bobogo93T+RUN40003KSIiQgUFBdq6dauGDRumVq1a\nKTc3V6WlpTp16pQ6dOiglJQU3XfffcrOzlbz5s2Vk5OjnJycOs8VGPjfvzczNvbR//o56ivPt0J3\nB97n2jy3NNV9Se7uLahVS1fmSGciOLDNL12bt/zjXNdm3RBRrmP7Iy9+x8s063zqVaYePXooPz9f\n27dvV0JCgu644w6VlJTon//8p7p37659+/bp5MmT+sUvfqHmzZtLOnMaDQBXg3qHcPXq1WrTpo1a\ntGih4OBg3X777frrX/+qnj17KiIiQoGBgdq+fbuqqqr06quvqqSkRF988YVOnTp10ecPDw+XJO3b\nt0+VlZX/3Y4AoIHqFcKkpCTt27dPXbt29V+XkJCgvXv36oc//KHatm2rqVOnaubMmerXr5+Ki4u1\nYMECXX/99frxj3980edv1aqVBgwYoMmTJ2v+/PmXvhsAuAQBjuM4Xi+iIdx8baupvpbWVPcl8Rrh\n5cJrhABgDCEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEE\nYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGBesNcLAK5abdz7FDu35xVWh7k2q4+L8/pc4HqO\nCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcI\nAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZjnaQh37typ4cOHKzExUT169NDU\nqVNVWVnp5ZIAGORpCCdOnKguXbroX//6l3Jzc1VYWKg//elPXi4JgEEBjuM4Xg0/efKkQkJCFBoa\nKkmaNWuWSkpK9OKLL17wMSWFpYq6s4NbSwRgQLCXwzdt2qRFixappKRENTU1qq2tVdeuXb/zMWNj\nH3VpdVKeb4XuDrzPtXluaar7ktzdW1Dn21yZI0lrC/+ggXf+xrV5v8l9xbVZfX5QrA/33eLarPPx\n7NS4uLhYjzzyiAYNGqSNGzeqoKBAGRkZXi0HgGGeHRHu3r1bQUFBGjVqlAICAiSdefMkMJA3sgG4\ny7PqtG/fXtXV1SosLFRlZaWysrJUVVWlw4cPq7a21qtlATDIsxB26dJFI0eO1KhRozRgwACFhIRo\n9uzZOnHiBKfIAFzl6Zsl06ZN07Rp0+pct3HjRo9WA8AqXpADYB4hBGAeIQRgHiEEYB4hBGAeIQRg\nHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGCe\npx/eBFzNHJc/g9vNeV877qbB7XnfxhEhAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA\n8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8+odwvLyckVH\nR2vPnj2NuR4AcB1HhADMI4QAzGtwCHft2qXBgwcrPj5eGRkZqqiokCRt2bJFP/vZz5SQkKDevXvr\n2Weflc/n8z/ub3/7m1JTU9WlSxcNGDBA//jHP/y3DR8+XPPmzVNaWppGjBhxGbYFAPXX4BC+8sor\nys7O1vvvv6+QkBA99thj+uyzz5SZmamf/vSn2rJli5YuXao1a9botddekyStW7dOf/zjHzV37lxt\n27ZN06ZN09SpU1VcXOx/3jfffFMzZ87U0qVLL9vmAKBenHoqKytzbrvtNmf16tX+6z744AMnOjra\nycrKcoYMGVLn/n/+85+d+++/33EcxxkzZowzd+7cOrdnZmY68+fPdxzHcTIyMpzx48fXax2fFvyn\nvksGgHoJbmg4b731Vv/XHTp0kOM42rx5s3bv3q2YmJizA6tWrVpJkkpLS7Vhwwa9/PLLdW5v1qyZ\n/3JERES95o+NfbShS75keb4VujvwPtfmuaWp7ktyd2+Bd97uyhxJenvH7zUg9reuzXvs76+5Nis5\nqkjrS6Jdm3U+DQ5hYOD/n007jiNJateunUJDQ7VkyZLzPiY8PFyPPPKIxo4de8HnDQ5u8FIA4LJo\n8GuEJSUl/q9LS0sVFBSk22+/Xf/+97/rvDly5MgRnTp1StKZI8eiorol3r9/f537A4BXGhzC5cuX\n69ChQ6qsrFROTo769euntLQ0VVZWauHChaqqqtL+/fs1ZswYLV68WJKUnp6ut99+W+vWrVNNTY22\nbdumtLQ0bd68+bJvCAAaqsEhTE9P1+jRo9WnTx/V1NToySefVIsWLZSdna0PPvhASUlJGjZsmBIT\nEzV+/HhJUs+ePTVjxgzNmTNHCQkJmjFjhqZMmaKePXte9g0BQEPV+4W5yMhI/+ltamrqObd3795d\nK1euvODjH3jgAT3wwAPnvW3ZsmX1XQYAXHb8nyUAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOE\nAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMzjU9WvEEGt\nWzfZeS9tW+3aLEn6a9kGV+b8vL0rY/x8hZ+4NmvOLbGuzUr2uTcv+QIfpc4RIQDzCCEA8wghAPMI\nIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wgh\nAPMIIQDzCCEA8wghAPMIIQDzPPk4z+TkZB08eFCBged2eMaMGUpPT/dgVQCs8uxzjadPn66MjAyv\nxgOAH6fGAMwjhADM8+zUeM6cOZo3b94513/88ccKCgryYEUArApwHMdxe2hycrJGjx59Sa8RlhSW\nKurODo2wKgBWeXZEeKnGxj7q2qw83wrdHXifK7OCWrd2ZY4krT24SAPbjndt3kvbVrs2q027/TpU\nEeHKrJ+37+XKHMnd30W3ubm3PN+K817Pa4QAzCOEAMy74t4s6devn7KysjxYEQCrPAnh+vXrvRgL\nAOfFqTEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wgh\nAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzAhzHcbxeBAB4iSNCAOYRQgDmEUIA5hFCAOYR\nQgDmEUIA5v0vuCX2qTfk+RcAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "# code by Tae Hwan Jung(Jeff Jung) @graykode\n", - "# Reference : https://github.com/hunkim/PyTorchZeroToAll/blob/master/14_2_seq2seq_att.py\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "from torch.autograd import Variable\n", - "import torch.nn.functional as F\n", - "import matplotlib.pyplot as plt\n", - "\n", - "dtype = torch.FloatTensor\n", - "# S: Symbol that shows starting of decoding input\n", - "# E: Symbol that shows starting of decoding output\n", - "# P: Symbol that will fill in blank sequence if current batch data size is short than time steps\n", - "sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E']\n", - "\n", - "word_list = \" \".join(sentences).split()\n", - "word_list = list(set(word_list))\n", - "word_dict = {w: i for i, w in enumerate(word_list)}\n", - "number_dict = {i: w for i, w in enumerate(word_list)}\n", - "n_class = len(word_dict) # vocab list\n", - "\n", - "# Parameter\n", - "n_hidden = 128\n", - "\n", - "def make_batch(sentences):\n", - " input_batch = [np.eye(n_class)[[word_dict[n] for n in sentences[0].split()]]]\n", - " output_batch = [np.eye(n_class)[[word_dict[n] for n in sentences[1].split()]]]\n", - " target_batch = [[word_dict[n] for n in sentences[2].split()]]\n", - "\n", - " # make tensor\n", - " return Variable(torch.Tensor(input_batch)), Variable(torch.Tensor(output_batch)), Variable(torch.LongTensor(target_batch))\n", - "\n", - "class Attention(nn.Module):\n", - " def __init__(self):\n", - " super(Attention, self).__init__()\n", - " self.enc_cell = nn.RNN(input_size=n_class, hidden_size=n_hidden, dropout=0.5)\n", - " self.dec_cell = nn.RNN(input_size=n_class, hidden_size=n_hidden, dropout=0.5)\n", - "\n", - " # Linear for attention\n", - " self.attn = nn.Linear(n_hidden, n_hidden)\n", - " self.out = nn.Linear(n_hidden * 2, n_class)\n", - "\n", - " def forward(self, enc_inputs, hidden, dec_inputs):\n", - " enc_inputs = enc_inputs.transpose(0, 1) # enc_inputs: [n_step(=n_step, time step), batch_size, n_class]\n", - " dec_inputs = dec_inputs.transpose(0, 1) # dec_inputs: [n_step(=n_step, time step), batch_size, n_class]\n", - "\n", - " # enc_outputs : [n_step, batch_size, num_directions(=1) * n_hidden], matrix F\n", - " # enc_hidden : [num_layers(=1) * num_directions(=1), batch_size, n_hidden]\n", - " enc_outputs, enc_hidden = self.enc_cell(enc_inputs, hidden)\n", - "\n", - " trained_attn = []\n", - " hidden = enc_hidden\n", - " n_step = len(dec_inputs)\n", - " model = Variable(torch.empty([n_step, 1, n_class]))\n", - "\n", - " for i in range(n_step): # each time step\n", - " # dec_output : [n_step(=1), batch_size(=1), num_directions(=1) * n_hidden]\n", - " # hidden : [num_layers(=1) * num_directions(=1), batch_size(=1), n_hidden]\n", - " dec_output, hidden = self.dec_cell(dec_inputs[i].unsqueeze(0), hidden)\n", - " attn_weights = self.get_att_weight(dec_output, enc_outputs) # attn_weights : [1, 1, n_step]\n", - " trained_attn.append(attn_weights.squeeze().data.numpy())\n", - "\n", - " # matrix-matrix product of matrices [1,1,n_step] x [1,n_step,n_hidden] = [1,1,n_hidden]\n", - " context = attn_weights.bmm(enc_outputs.transpose(0, 1))\n", - " dec_output = dec_output.squeeze(0) # dec_output : [batch_size(=1), num_directions(=1) * n_hidden]\n", - " context = context.squeeze(1) # [1, num_directions(=1) * n_hidden]\n", - " model[i] = self.out(torch.cat((dec_output, context), 1))\n", - "\n", - " # make model shape [n_step, n_class]\n", - " return model.transpose(0, 1).squeeze(0), trained_attn\n", - "\n", - " def get_att_weight(self, dec_output, enc_outputs): # get attention weight one 'dec_output' with 'enc_outputs'\n", - " n_step = len(enc_outputs)\n", - " attn_scores = Variable(torch.zeros(n_step)) # attn_scores : [n_step]\n", - "\n", - " for i in range(n_step):\n", - " attn_scores[i] = self.get_att_score(dec_output, enc_outputs[i])\n", - "\n", - " # Normalize scores to weights in range 0 to 1\n", - " return F.softmax(attn_scores).view(1, 1, -1)\n", - "\n", - " def get_att_score(self, dec_output, enc_output): # enc_outputs [batch_size, num_directions(=1) * n_hidden]\n", - " score = self.attn(enc_output) # score : [batch_size, n_hidden]\n", - " return torch.dot(dec_output.view(-1), score.view(-1)) # inner product make scalar value\n", - "\n", - "input_batch, output_batch, target_batch = make_batch(sentences)\n", - "\n", - "# hidden : [num_layers(=1) * num_directions(=1), batch_size, n_hidden]\n", - "hidden = Variable(torch.zeros(1, 1, n_hidden))\n", - "\n", - "model = Attention()\n", - "criterion = nn.CrossEntropyLoss()\n", - "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n", - "\n", - "# Train\n", - "for epoch in range(2000):\n", - " optimizer.zero_grad()\n", - " output, _ = model(input_batch, hidden, output_batch)\n", - "\n", - " loss = criterion(output, target_batch.squeeze(0))\n", - " if (epoch + 1) % 400 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - "# Test\n", - "test_batch = [np.eye(n_class)[[word_dict[n] for n in 'SPPPP']]]\n", - "test_batch = Variable(torch.Tensor(test_batch))\n", - "predict, trained_attn = model(input_batch, hidden, test_batch)\n", - "predict = predict.data.max(1, keepdim=True)[1]\n", - "print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()])\n", - "\n", - "# Show Attention\n", - "fig = plt.figure(figsize=(5, 5))\n", - "ax = fig.add_subplot(1, 1, 1)\n", - "ax.matshow(trained_attn, cmap='viridis')\n", - "ax.set_xticklabels([''] + sentences[0].split(), fontdict={'fontsize': 14})\n", - "ax.set_yticklabels([''] + sentences[2].split(), fontdict={'fontsize': 14})\n", - "plt.show()" - ] - } - ], - "metadata": { - "colab": { - "name": "Seq2Seq(Attention)-Torch.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention)-Torch.py b/4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention).py similarity index 100% rename from 4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention)-Torch.py rename to 4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention).py diff --git a/4-3.Bi-LSTM(Attention)/Bi_LSTM(Attention)_Tensor.ipynb b/4-3.Bi-LSTM(Attention)/Bi_LSTM(Attention)_Tensor.ipynb deleted file mode 100644 index b6fc105..0000000 --- a/4-3.Bi-LSTM(Attention)/Bi_LSTM(Attention)_Tensor.ipynb +++ /dev/null @@ -1,152 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "aenHhwSWhMGu", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 393.0 - }, - "outputId": "7296d0cd-fd1d-4ff7-9312-57d52b05e64d" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 001000 cost = 0.001100\n", - "Epoch: 002000 cost = 0.000286\n", - "Epoch: 003000 cost = 0.000123\n", - "Epoch: 004000 cost = 0.000062\n", - "Epoch: 005000 cost = 0.000034\n", - "sorry hate you is Bad Mean...\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJkAAAEMCAYAAADXgBT+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XlUVdUeB/DvBQREzUeOTM4C5uPK\nRQzIAbNwIEeQFMQizCkHJLW0Z+HTVMrKFEvp6RPLwhic0kyhVByAGBQxRwYVSCC4gDJP+/3h4y5u\n14t3OJvJ32ct14LD2ft3Lv44496/I2KMMRDCkU5LbwBp/yjJCHeUZIQ7SjLCHSUZ4Y6SjHBHSUa4\noyQj3FGSEe70WnoDyGN+fn4qr7t9+3aOWyI82pO1EkZGRrJ/BgYGiImJQXZ2Njp06AA9PT3cvXsX\n58+fx/PPP9/Sm6o2ET27bH0CAgJgZ2eHadOmyS2PiIhASkoKNm7c2EJbphlKslbI3t4ecXFx0NOT\nP5upqamBk5MTEhMTW2jLNEOHy1aoa9euOHPmjMLymJgYdOnSpQW2SDt04t8KLVq0CMuXL4eVlRXM\nzc1RW1uL3Nxc3Lp1C+vWrWvpzVMbHS5bqczMTERHRyMvLw/V1dXo2bMnxowZA7FY3NKbpjZKslZo\n586dWLp0aUtvhmDonKwVCgsLg1QqbenNEAztyVqhkJAQnD59Gq6urjAxMVG4ynR2dm6hLdMMJVkr\nZG1trfRnIpEIN27caMat0R4lGeGObmG0UowxXL16FTk5OQCAfv364YUXXmjhrdIMJVkrlJaWhkWL\nFiE7OxudOnUCAJSVlcHKygp79uxBjx49WngL1UOHy1borbfegqmpKVauXCl7IJ6Xl4etW7eipqam\nzY3CoCRrhSQSCWJjY2FoaCi3vLS0FBMmTMDFixdbaMs0Q/fJWqEuXbqgvLxcYXlNTQ1EIlELbJF2\n6JysFRo5ciRWrFgBf39/DBw4EACQnp6O7du3Y8SIEYLEqKioUHndjh07ahWLDpet0KNHj/Cvf/0L\nUVFRsmWMMTg7OyMwMBDGxsZax7C2tlZ5r6jtfTlKslaouroa+vr6ePjwIXJyclBdXQ0LCwtBR8X+\n/vvvsq8zMjIQGhqK119/Hf3790d9fT3S0tJw6NAhzJs3T2HwpNoYaXVsbGzYm2++yXbv3s1SUlJY\nfX0913gzZ85k9+7dU1ielpbG3NzctO6fzslaobCwMCQlJSExMREHDhxAVVUVXnzxRTg5OcHJyQkD\nBgwQNF5aWhp69eqlsNzU1BQZGRla90+HyyakpaWpvO6gQYO4bUdWVhYuXLiA77//Hunp6YI/u/T0\n9ISJiQkWLFggN0hyz549uH//PsLCwrTqn5KsCQ0nx4wxuZPkv38PaH9y/HclJSVISkqS7dEyMzNh\nZWUFOzs7+Pv7Cxrr3r17eP/993HlyhXZ52KMYdCgQdi+fbvsCldTlGRNaHhuCADJycmIjIyEj4+P\n3Mnxd999hzfeeAOvvvqqYHFfe+01VFVVQSwWw87ODra2thgyZAh0dXUFi/EkBQUFciNxzczMBOmX\nzsma0PiXvHDhQoSEhKB79+6yZf3794eNjQ18fX0FTTITExNcv34dDx48QH5+PgoLC1FeXs5tEsmE\nCRNw6tQpdO/eXe7zCYXu+Kvozz//hIGBgcJyIyMj5ObmChprz549uHDhAj744AP84x//wMGDB+Hi\n4oJp06ZxmXNpYmLyxNlRQqHDpYrefvttVFZW4q233oKZmRnq6urw4MEDfPvtt+jQoQP27t3LJW51\ndTVSU1ORmJiII0eO4N69e7h+/bqgMdauXYszZ87A1NQUpqamCodlbR/I0+FSRZ9++ik2bdqElStX\norKyEgCgp6cHJycnbNq0SdBY0dHRuHz5MpKTk/HHH3/AxMQEjo6OWLZsGRwdHQWN1eDll1/m0i9A\nezKVNb6iLC4uRnV1NZ5//nmF8fdCePnll+Hg4ABHR0c4Ojqid+/eT1zv9u3bsLS0FDy+0CjJVCSR\nSHD58uWW3gw5w4YNQ0pKiiB9xcfH4+eff0ZOTg5EIhH69u2L6dOn45///KfWfdOJv4peeeUV/PDD\nDy29GXKE2j/8+OOPmDdvHh48eIB+/fqhb9++yMzMxOzZsxETE6N1/3ROpqLi4mJs374dQUFB6N27\nt8LJcURERLNvk1Bjy/773/9i586dGDt2rNzyqKgofPnllxgzZoxW/VOSqcjW1ha2trYtvRlc5Obm\nPjGRxo0bhzVr1mjdPyWZitpT2YC/Mzc3R1JSksKAyJSUFEEmrVCSqeHQoUNyJ8d9+vSBu7s7XFxc\nWnrTtOLj44MFCxZg8uTJsueUGRkZOH78OJYvX651/5RkKvr666+xf/9+uLq6YtSoUQAe/0esWbMG\nZWVlmD59egtvoeY8PDzQo0cPRERE4PDhw6iurkafPn2wefNmTJw4UfsAWo9Ie0aMHTuWXb58WWF5\nQkICmzx5cgtsEWP29vaC9BMZGckePHggSF9PQvfJVCSRSJCYmKhwVVlXV4cRI0YgOTlZq/7PnTun\n8rpCF1yZMWMGbt++DXNzc7z00ktwcnKCo6MjnnvuOUH6pyRT0YwZM7B48WKMHz9ebnl0dDR27NiB\nY8eOadX/34usNIxja0xHRwcdO3ZEUlKSVrGepLS0FElJSUhOTkZiYiKuX7+OAQMGYOTIkXj33Xe1\n6puSTEW//fYbli1bBgcHB7mT4/j4eGzZsgVTpkwRLFZ0dDROnjyJRYsWyY1d++abb+Dq6qqQ6ELL\nyMhAbGwsQkNDBRmJS0mmhjt37iAiIgJZWVmyk+Pp06cLXmLTxcUFhw8fRufOneWWFxcXw93dHb/+\n+qug8VJTU2UjcK9cuQJjY2NIJBJIJBLY2dmhb9++WvVPV5cqCgoKwksvvYTVq1dzeSjeWHFxMcrK\nyhSSrKqqCiUlJYLH8/DwwIABA+Dl5YXAwECFuNqiPZmKli5diqSkJFRVVcHe3l42c6ipgnWaWrly\nJW7cuIFZs2bB3NxcNnYtPDwcAwcOFLzgys8//4yEhAQkJCSgpKQEdnZ2sLe3h729vVqTgJWhJFNT\nenq67NCSlJSEiooKODk54fPPPxcsRkVFBXbt2oXo6Gjk5ubKVb9etWqV4HuaxoqKipCYmIj4+Hhc\nuHABhYWFSEhI0KpPSjINlJWVITk5GUlJSYiOjsbdu3dx7dq1lt4srd27dw/Jycmyz5afn4/hw4cj\nODhYq34pyVT0yy+/yKao5eXlwcbGRnZyLBaLFco8aYMxhrNnzyI9PV02CrcxoZ+jvvPOO0hJSUFZ\nWRlsbW1lgyVtbGwEmSFFSaYia2tr2cmxm5sbjIyMuMVavXo1Tp48ib59+yokr0gkEnxY0fbt2+Ho\n6AiJRAJ9fX2l623duhWrV69Wu39KMhVdvXoVv//+OxISEpCSkgJzc3PZybGdnZ2gxVCGDx+O77//\nnstFhTY0HYlLSaYBxhhu3LiB2NhYRERE4O7du4LOIB83bhx+/vlnQQ/BQhCLxbh69ara7eg+mRrK\nysqQkpIiOzm+cuUKevfuDU9PT0Hj+Pn54bPPPsOKFSu4XkmqS9NbGbQnU9G0adOQlpYGExMTuZlE\nPCpRT506FX/++SfKysrw3HPPQUdHfipGbGys4DFVoenhkvZkKpo7dy4cHR1hbm7e5Hr+/v7Ytm2b\nVrHeeustrdq3NrQnE5iQ09RaG9qTtRJC/M3W1tZi165dTxzq7ePjo/1GakjTz0ZJJjAhpql98skn\n+O233+Dp6SkbAZGeno59+/ahrq4O8+bN0zqGJl5//XWN2tHhUmBCHC5HjRqF/fv3KxSfu3nzJvz8\n/HDq1Cmt+gceX8GqStsH8jSDvBWqqKhAnz59FJYPGjQIhYWFgsQwMjKS/TMwMEBMTAyys7PRoUMH\n6Onp4e7duzh//rwgN5npcNkKDR48GKGhoXjjjTfklh88eBD9+/cXJMaWLVtkXwcEBGD9+vUKpdQj\nIiIEuYihw6XANL0r3tjly5fh6+uLnj17yg31zs3NxVdffYWRI0cKsaky9vb2iIuLUxiMWVNTAycn\nJyQmJmrVPx0uVbRjx44nLi8rK8OGDRtk369YsULrWBKJBL/++ivmzJkDMzMz9OjRAx4eHjh58qTg\nCQYAXbt2fWKlxZiYGEFKiNKe7CmkUikKCwvh7u6Ow4cPK1zG3717F++++67We6+/y8vLg66urqyG\na0ZGBgwNDWFqaipoHAAIDw/HRx99BCsrK7kS67du3cK6deswZ84crfqnJHuKw4cPY8uWLXj06JHS\n+0Tjx49XuqfTxLlz5+Dn54fAwEDZDO4ff/wRgYGB2LFjB0aPHi1YrAaZmZmIjo6Wq349ZswYYSbJ\ncJs23I7U1tYyW1tblp2drfCvoKBA8HhTpkxhx48fV1geFRXFpk6dKni8oKAgwftsjJJMDXV1dXJf\n//HHH0wqlQoex9bWltXW1iosr66uZra2toLHGz16NCssLBS83wZ0C0NFcXFxeO+99xATE4Pa2lp4\ne3vjypUr0NfXR1BQkKClA/r164dTp07B1dVVbnlERMRTH9BrwtfXF0uXLoWrqytMTEwUrjK1/Wx0\nTqYid3d3zJ49Gx4eHjh69Ci2bt2KsLAwXLlyBXv37kVkZKRgsS5evIilS5fC3NwcZmZmYIwhMzMT\n+fn52LdvHyQSiWCxAMUSCY2JRCKaQd5c7OzskJSUBJFIBH9/f/To0QMffPABGGOwt7cXvD5FXl4e\nTp48iaysLIhEIlhYWGDy5Mno1q2boHGaAx0uVWRoaIiHDx/CwMAAly5dwpdffgngcaESHu8F79Wr\nF2bPno38/PwnPmLSVmVlpWx499NeFa3t66EpyVTk7OyMN998E7q6ujA2NoajoyOqqqqwadMmDB8+\nXNBYxcXF2LhxI3755ReIRCJcu3YNUqkUfn5++Pzzz9GzZ0+tYzg4OMgeGUkkkif+obD/v7tA28Ml\nJZmKAgICEBISgkePHsHLywsikQj19fX466+/sHnzZkFjbdiwAWVlZQgLC5PNHzAyMoKFhQU+/vhj\nQe7JNX5Nz7fffqt1f03idt3azvC+l9SYvb09KyoqYowxJhaLZcsfPXrEHBwcmm07hEJ7MhWFhYXB\ny8tL0PmVyujp6T1xOlx1dTWqqqoEj3fz5k1s3boV6enpT+xf24krlGQq4n0vqTGJRIJPPvkEq1at\nki27f/8+Pv74Yzg5OQkWp8GKFStgZmaGhQsXcpnrSbcwVMT7XlJjubm5WLx4MW7fvo26ujoYGhrK\nSlZt3bpV6Qu9NDVs2DDExcVpfRWpDO3JVHTz5s1mi9W7d298+umnqKysRFZWFgoKClBeXg5LS0vB\nEwx4PJ4sPz9f64qKytCerAnNeS+psQMHDiAoKAjx8fEoLi6Gq6srunTpguLiYixYsECQiSSNq20X\nFBQgLCwMU6dOhZmZmcLtDHqsxFHjSSHKKg4yge4lNebi4oIvvvgCNjY22LdvHw4dOoQjR44gIyMD\nS5YswenTp7WO0fjw/6RK241/RvfJOGp8pdW9e3ds27ZNsNf/NaWgoAA2NjYAgAsXLsDV1RW6uroY\nPHgw/vrrL0FiND78z507F999953COg33BLVFSdYEExMTLFu2DBYWFiguLm7yZfAvvviiYHGNjY2R\nlpYGQ0NDxMfH47333gMA5OTkCFoXLTU1FVevXsXly5fxww8/KPwBZWVlITs7W+s4lGRN+PTTT7F/\n/35cu3YNdXV1SE1NfeJ6Qj+79PT0hLu7O0QiEUaNGgUrKys8evQIS5YsEeZdR/9XUVGB8+fPo7a2\nFnv27FH4uaGhoVrzM5VqqbvAbY23t3ezxktOTmZnz55lVVVVjLHHo3ODg4NZdXW14LHmz58veJ+N\n0Yk/4Y6mxBHuKMkId5RkhDtKMsIdJRnh7pm/T3b27mCN2o0wPYGEP19Tq03gpJkaxdp9zA+Lpqpf\nI6w+477abb5JDsQCuzVqtztd9b3Sn9GeTEOd9C2bLVa/wcKPvFAaa6iF4H1SkhHuKMkId5RkhDtK\nMsIdJRnhjpKMcEdJRrh7apJlZ2fDysoKt2/f5rohhw4dgoODA9cYpGVw35OFhISgurqadxgAjwuV\nLF26FFZWVoIMGybC4JpkUqkUgYGBqKmp4RkGAJCWloYZM2YIUhKcCEvlJLt+/TqmTp0KiUQCb29v\n5OTkAABOnDiBKVOmQCKRwNnZGbt37wbwuIjbmDFjwBiDo6MjwsPDAQDHjh3DxIkTYWtrCw8PD4U3\nXpw5cwYuLi6wsbGBv7+/ynvBgoICBAYGtrt3RbYLTxufnZWVxSwtLdmsWbNYdnY2KykpYT4+PmzO\nnDksKyuLDRkyhJ05c4YxxlhKSgobOnQou3DhAmOMsbi4OGZpaclKS0sZY4ylpqYysVjMLl26xGpq\nalhwcDBzcHBgFRUVLDIykonFYrZp0yZWVlbGbty4wWxsbNixY8fUGk9+69YtZmlpybKyslRav7Tq\nllr9E/WpPArD09MTZmZmAB4XH5k/fz6MjY0RGxuLrl27Anj8ypf+/fvj2rVrT3xzxpEjR+Dg4CAr\nGuLj4wNTU1PZ4bSqqgrLli2DkZERrK2tYWVlhbS0NK3/kJqi7kiKBmP73VF7BIemozB+ubEFE4es\nVbudJqMwTld9j/EG6r8coqlRGCon2aBBg2Rf9+nTB4wx5Ofn49SpU4iMjEReXh4YY6ipqVF6iMvK\nypKr3qyvr4/JkyfLvu/atavcOZWhoWGzXTQQflROssYvW2f/n+B04sQJ7N27F0FBQXB0dISenh6m\nT5+utI+G6oTk2aLyiX9mZqbs6/v370NXVxc5OTmws7PDqFGjoKenh9LSUty7d09pHxYWFnL91NfX\nY9++fcjLy9Nw80lboHKShYaGIj8/H6Wlpdi/fz+cnZ3Rv39/ZGZmoqioCLm5ufjwww9hYmIiS5qG\nijiZmZkoLy+Hm5sbkpKSEB0djZqaGhw4cADBwcHo3Lkzn09HWgW1Tvx9fX2Rk5MDsViMzZs3w8jI\nCAkJCRg3bhx69uyJtWvXQiqVYsOGDejWrRuWLFkCOzs7zJ49G8uXL8eCBQvw5ZdfIjAwEKtWrYKl\npSWCg4PRqVMnrT/IunXrcPToUdmhfOLEiRCJRNi4cWOTh3DC3zM/g1zTMf50danYThl6QE64axOz\nlRYtWoSLFy8q/fn7778Pb2/vZtwioo42kWQNj6pI20SHS8IdJRnhrk0cLnkqrtO8PKbabXW0qMio\nQVuRoYFGoTRtpwztyQh3lGSEO0oywh0lGeGOkoxwR0lGuKMkI9xRkhHuKMkId1SmgHDXbsoUVFZW\nYsuWLXB2dsbw4cPh5eWFK1eucI9Lnq7dlCnYunUrYmNjceDAAVy6dAnDhg3D4sWLUVlZyT02aVq7\nKVPQuXNnrF27FhYWFjAwMICvry+kUinS09NV/YiEl6dNMW9rZQoapKSkMGtra5aXl9fkeiVVdzTq\nn6iuXZYpKC8vR0BAADw8PNCzZ88m1/0ty13t/gFg+sAUHEkfplab3VM1K4nwyx+bMXHoB2q3Y9m5\narc5VfJfTOjqq1E7ZdpdmQKpVIqFCxeiW7duWLdunVptCR8qn5MpK1MQHByMgIAAJCcnIzU1FdbW\n1kr74F2mIDs7G7NmzcLAgQOxa9cu6Ovrc4tFVNduyhQUFRVh3rx5GD9+PAIDA9GhQwet+yTCaDdl\nCrZt24bBgwdj9erVWvdFhNVuyhSEh4dDV1cXNjY2csupTEHLe+bLFKh7hdiAri4V2ylDD8gJd21i\nShyVKWjb2kSSUZmCto0Ol4Q7SjLCHSUZ4a5NnJPx1E23tEXaNof6svJmbacM7ckId5RkhDtKMsId\nJRnhjpKMcEdJRrijJCPcUZIR7qhMAeGu3ZQpyM/Ph7+/PxwcHDBixAh4e3vj6tWr3OOSp2s3ZQr8\n/Pygq6uLqKgonD9/HkOHDsXChQubJTZpWrsoU1BfX4+ZM2dizZo1eO6552BoaIgZM2ZAKpUiPz9f\n5V8G4UPlJDt48CB27dqFc+fOoUOHDnj//feRnZ2N1atXY+XKlbh8+TKCgoKwc+dOXLx4Eb169cLe\nvXsBAHFxcfDw8MC1a9fw4YcfIiAgAImJiXBxccHChQtlRVEqKysRGxuLo0ePIjw8HL/++itOnTr1\n9A+howN3d3d0794dAFBYWIh9+/ZBIpHA1NRUk98LEdLT6hg01MI4cuSIbFlMTAyzsrJipaWlrLi4\nWG79yZMns927dzPGFGthbNy4kc2fP1+2blVVFfvpp5/Yw4cPWWRkJLOysmIPHz6U/XzmzJnsiy++\nUKvugpOTE7O0tGTe3t4sPz//qeuXVt1Sq3+ivnZXpuDSpUsoKCjA7t274enpiaNHjzY55S75z0lq\n9d9gdL90nL87UK02m16bpVEsTWcr1d3KULtNVO1BuOjN1qidMu2qTEGD7t27Y82aNSgsLMTZs2e5\nxyNNaxdlCnJycjB27FhkZWXJluno6IAxBj29Z35cZotrF2UKTE1N0a1bN2zevBlSqRSVlZXYvn07\n9PX1YW9vr1XfRHvtokyBSCTC119/jS1btuDVV1+Fjo4OrK2t8c0336Bbt25a9U2098yXKVD35L0B\nnfgrtlOGHpAT7trEWTGVKWjb2kSSUZmCto0Ol4Q7SjLCHSUZ4a5NnJPxtGGAnUbtourVb6vTRf3K\nhw00qpqYnaRRLE3bKUN7MsIdJRnhjpKMcEdJRrijJCPcUZIR7ijJCHeUZIQ7KlNAuGs3ZQoai46O\nhpWVFeLj45s1LnmydlOmoEF5eTkCAwNhZGTUbDFJ09pFmYLGgoKC4OTkBGNjY7XaEX7aRZmCBrdu\n3cJPP/2Ed999V81fA+FJrdlKZmZmAABfX1/Mnz8fxsbGiI2NRdeuXQEAYrEY/fv3x7Vr1zBy5EiF\nPo4cOQIHBwc4OTkBAHx8fGBqaio7nFZVVWHZsmUwMjKCtbU1rKyskJaWptL2McYQEBAAf39/tfZi\n31z9HP3/2Ufl9RuLqg/XqJ0mmnqfpNB0et8RtL92U6ag4c29bm5uqn4kAMAC8Uq11m8QVR8OFx0P\ntdroNPps6tD0Racnb51Xu41O7zuozx2sUTtlVE4yZWUK9u7di6CgIDg6OkJPT6/JVzHzKlMglUqx\nY8cOhISEQCQSCd4/0U67KFNw7tw5FBUVYc6cOXBwcICDgwMePHiAd955Bxs3btSqb6I9lfdkoaGh\nsLe3h5GRkVyZgvj4eBQVFaGqqgqffPKJ0jIFAwYMgJubGzw8PBAdHQ1nZ2eEhoYiODgYr7/+ulYf\nYuLEibLzvAazZs3CmjVr8NJLL2nVN9FeuyhT0LFjR3Ts2FFuma6uLp5//nnZRQlpOc98mQJ1T94b\n0Im/YjulP1O7N0LU1CZmK1GZgratTSQZlSlo2+hwSbijJCPcUZIR7trEORlPog76zda29NUXNI6l\nSdt3ctQfx7e7N/BOjqNG7ZShPRnhjpKMcEdJRrijJCPcUZIR7ijJCHeUZIQ7SjLCHSUZ4Y5qYRDu\nuD9WCgkJgZeXF/T1NX98o4q5c+ciKSkJurq6smV9+vTBiRMnuMYlT8c1yRpqYXh4eHBPMgBYvHgx\nli1bxj0OUU+7q4VBWiH2FFlZWczS0pLNmjWLZWdns5KSEubj48PmzJnDsrKy2JAhQ9iZM2cYY4yl\npKSwoUOHsgsXLjDGGIuLi2OWlpastLSUMcZYamoqE4vF7NKlS6ympoYFBwczBwcHVlFRwSIjI5lY\nLGabNm1iZWVl7MaNG8zGxoYdO3bsaZvIGGPM29ubeXl5sSlTpjBbW1vm5eXF0tLSntou89p9lfon\nmms3tTAGDhyI+vp6+Pn5QU9PD5s2bcL8+fNx8uRJGBgYKG23wG6Nqr8COaervsd4gzlqtSmbLNEo\n1sXIVRjp/pna7Ww+SHn6Sn+ze/h3WJQ0V6N2yrSbWhjr16+X+/6jjz6Cg4MDfv/9d4wePVqlPggf\n7aIWxpN07twZXbt2RX5+frPEI8q1i1oYpaWlWL9+vVw/UqkUUqkUFhYWWvVNtKdykoWGhiI/Px+l\npaVytTAyMzNRVFSE3NxcfPjhh0prYZSXl8PNzQ1JSUmIjo5GTU0NDhw4gODgYHTu3FmrD9G5c2ek\npKTg448/RnFxMUpKSvDvf/8blpaWsLe316pvoj2Vk6yhFsbo0aNRW1uL9evXw9PTEwMHDsS4cePw\n5ptvYtq0aXj77bdx/PhxbNu2DUOGDJHVwjhw4ACGDBkiq4UxYsQIHD9+XJBaGADw1VdfgTGGCRMm\nYOzYsaipqcF//vMfucM8aRnPfC0Mda8QG9DVpWI7ZejPnHDXJqbEUS2Mtq1NJBnVwmjb6HBJuKMk\nI9xRkhHu2sQ5GU+srq7Z2uqX1GocS5O2gzpq9khN03bK0J6McEdJRrijJCPcUZIR7ijJCHeUZIQ7\nSjLCHSUZ4Y7KFBDuuO/JQkJCmm2CbkxMDKZMmQKxWIxJkybh9OnTzRKXNI1rkjWUKWiYV8nTzZs3\nsXLlSqxYsQIJCQlYvnw5vvrqK5SVlXGPTZrWbsoU7N+/H5MmTcIrr7wCAwMDTJo0CUePHhVk/gDR\n0tOmmLeVMgUuLi4sKCiI+fj4MIlEwtzc3FhCQsJT22WkUpkC3tpNmYLc3FyEh4djx44dGDRoEIKD\ng7Fo0SJERUXB2NhYabsFtqtV/RXIiao9CBe92Wq1qR1rq1GsM9Fr8PKrgWq3e23HGbXbrHrhFD67\nPkGjdsqofLhUVqYgNDQULi4uEIvFsLGxwZ07d9QuU9BQmkCbMgWMMUyZMgXDhg1Dp06dsHz5cujo\n6CAmJkbVj0g4aTdlCnr06CH3vnE9PT306tWLyhS0Au2iTAHweE9748YN2fe1tbXIzc2VHeJJy2kX\nZQoAwNvbG6dPn0Z0dDQqKyuxc+dO6OvrY+zYsVr3TbSj1om/r68vcnJyIBaLsXnzZhgZGSEhIQHj\nxo1Dz549sXbtWkilUmzYsAHdunXDkiVLZGUKli9fjgULFsjKFKxatQqWlpaClSkYM2YMAgICsHnz\nZvz111+wtrbG3r17YWRkpHXchSXpAAAAVElEQVTfRDvPfJkCda8QG9DVpWI7ZegBOeGuTcxWojIF\nbVubSDIqU9C20eGScEdJRrijJCPcPfO3MAh/tCcj3FGSEe4oyQh3lGSEO0oywh0lGeHufxMp6FYg\nfNQPAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - " Reference : https://github.com/prakashpandey9/Text-Classification-Pytorch/blob/master/models/LSTM_Attn.py\n", - "'''\n", - "import tensorflow as tf\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "tf.reset_default_graph()\n", - "\n", - "# Bi-LSTM(Attention) Parameters\n", - "embedding_dim = 2\n", - "n_hidden = 5 # number of hidden units in one cell\n", - "n_step = 3 # all sentence is consist of 3 words\n", - "n_class = 2 # 0 or 1\n", - "\n", - "# 3 words sentences (=sequence_length is 3)\n", - "sentences = [\"i love you\", \"he loves me\", \"she likes baseball\", \"i hate you\", \"sorry for that\", \"this is awful\"]\n", - "labels = [1, 1, 1, 0, 0, 0] # 1 is good, 0 is not good.\n", - "\n", - "word_list = \" \".join(sentences).split()\n", - "word_list = list(set(word_list))\n", - "word_dict = {w: i for i, w in enumerate(word_list)}\n", - "vocab_size = len(word_dict)\n", - "\n", - "input_batch = []\n", - "for sen in sentences:\n", - " input_batch.append(np.asarray([word_dict[n] for n in sen.split()]))\n", - "\n", - "target_batch = []\n", - "for out in labels:\n", - " target_batch.append(np.eye(n_class)[out]) # ONE-HOT : To using Tensor Softmax Loss function\n", - "\n", - "# LSTM Model\n", - "X = tf.placeholder(tf.int32, [None, n_step])\n", - "Y = tf.placeholder(tf.int32, [None, n_class])\n", - "out = tf.Variable(tf.random_normal([n_hidden * 2, n_class]))\n", - "\n", - "embedding = tf.Variable(tf.random_uniform([vocab_size, embedding_dim]))\n", - "input = tf.nn.embedding_lookup(embedding, X) # [batch_size, len_seq, embedding_dim]\n", - "\n", - "lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(n_hidden)\n", - "lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(n_hidden)\n", - "\n", - "# output : [batch_size, len_seq, n_hidden], states : [batch_size, n_hidden]\n", - "output, final_state = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell,lstm_bw_cell, input, dtype=tf.float32)\n", - "\n", - "# Attention\n", - "output = tf.concat([output[0], output[1]], 2) # output[0] : lstm_fw, output[1] : lstm_bw\n", - "final_hidden_state = tf.concat([final_state[1][0], final_state[1][1]], 1) # final_hidden_state : [batch_size, n_hidden * num_directions(=2)]\n", - "final_hidden_state = tf.expand_dims(final_hidden_state, 2) # final_hidden_state : [batch_size, n_hidden * num_directions(=2), 1]\n", - "\n", - "attn_weights = tf.squeeze(tf.matmul(output, final_hidden_state), 2) # attn_weights : [batch_size, n_step]\n", - "soft_attn_weights = tf.nn.softmax(attn_weights, 1)\n", - "context = tf.matmul(tf.transpose(output, [0, 2, 1]), tf.expand_dims(soft_attn_weights, 2)) # context : [batch_size, n_hidden * num_directions(=2), 1]\n", - "context = tf.squeeze(context, 2) # [batch_size, n_hidden * num_directions(=2)]\n", - "\n", - "model = tf.matmul(context, out)\n", - "\n", - "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=model, labels=Y))\n", - "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n", - "\n", - "# Model-Predict\n", - "hypothesis = tf.nn.softmax(model)\n", - "predictions = tf.argmax(hypothesis, 1)\n", - "\n", - "# Training\n", - "with tf.Session() as sess:\n", - " init = tf.global_variables_initializer()\n", - " sess.run(init)\n", - " for epoch in range(5000):\n", - " _, loss, attention = sess.run([optimizer, cost, soft_attn_weights], feed_dict={X: input_batch, Y: target_batch})\n", - " if (epoch + 1)%1000 == 0:\n", - " print('Epoch:', '%06d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - " # Test\n", - " test_text = 'sorry hate you'\n", - " tests = [np.asarray([word_dict[n] for n in test_text.split()])]\n", - "\n", - " predict = sess.run([predictions], feed_dict={X: tests})\n", - " result = predict[0][0]\n", - " if result == 0:\n", - " print(test_text,\"is Bad Mean...\")\n", - " else:\n", - " print(test_text,\"is Good Mean!!\")\n", - "\n", - " fig = plt.figure(figsize=(6, 3)) # [batch_size, n_step]\n", - " ax = fig.add_subplot(1, 1, 1)\n", - " ax.matshow(attention, cmap='viridis')\n", - " ax.set_xticklabels([''] + ['first_word', 'second_word', 'third_word'], fontdict={'fontsize': 14}, rotation=90)\n", - " ax.set_yticklabels([''] + ['batch_1', 'batch_2', 'batch_3', 'batch_4', 'batch_5', 'batch_6'], fontdict={'fontsize': 14})\n", - " plt.show()" - ] - } - ], - "metadata": { - "colab": { - "name": "Bi-LSTM(Attention)-Tensor.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/4-3.Bi-LSTM(Attention)/Bi_LSTM(Attention)_Torch.ipynb b/4-3.Bi-LSTM(Attention)/Bi_LSTM(Attention)_Torch.ipynb deleted file mode 100644 index 006c7bf..0000000 --- a/4-3.Bi-LSTM(Attention)/Bi_LSTM(Attention)_Torch.ipynb +++ /dev/null @@ -1,164 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "0WS_AKj4hR22", - "colab_type": "code", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 393.0 - }, - "outputId": "dbc2130f-0972-4f66-edbc-fce1c94b65cd" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 1000 cost = 0.003327\n", - "Epoch: 2000 cost = 0.000716\n", - "Epoch: 3000 cost = 0.000275\n", - "Epoch: 4000 cost = 0.000129\n", - "Epoch: 5000 cost = 0.000067\n", - "sorry hate you is Bad Mean...\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJkAAAEMCAYAAADXgBT+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XlcVOX+B/DPAAKSxo9wY1MRBbxd\nRgaxgVwwCxeuK4YKUhHmlgviUloWXhUkveWCpZResSwMwS3NVErFBYhFERMXFg1IQBhQ2UGe3x9e\n5sU0DszMOUcWv+/Xy9cLDud5vmfwy1mf53tEjDEGQgSk09obQDo+SjIiOEoyIjhKMiI4SjIiOEoy\nIjhKMiI4SjIiOEoyIji91t4A8kRAQIDa627dulXALeEf7cnaCCMjI/k/AwMDxMXFIS8vD506dYKe\nnh7u3LmD8+fP46WXXmrtTdWYiJ5dtj1BQUFwcnLCpEmTFJZHR0cjLS0N69ata6Ut0w4lWRvk7OyM\nhIQE6Okpns3U1dXB1dUVycnJrbRl2qHDZRtkbGyMM2fOKC2Pi4tD165dW2GLuKET/zZo3rx5WLx4\nMezs7GBpaYn6+noUFBTg5s2bWL16dWtvnsbocNlG5eTkIDY2FoWFhaitrUWPHj0wYsQIiMXi1t40\njVGStUHbt2/HwoULW3szeEPnZG1QVFQUZDJZa28Gb2hP1gZFRETg1KlT8PDwgJmZmdJVppubWytt\nmXYoydoge3t7lT8TiUTIyMh4hlvDHSUZERzdwmijGGO4evUq8vPzAQB9+/bFP/7xj1beKu1QkrVB\nmZmZmDdvHvLy8vDCCy8AACoqKmBnZ4ddu3ahe/furbyFmqHDZRv07rvvwtzcHMuWLZM/EC8sLMSm\nTZtQV1fX7kZhUJK1QRKJBPHx8TA0NFRYXl5ejjFjxuDixYuttGXaoftkbVDXrl1RWVmptLyurg4i\nkagVtogbOidrg4YOHYolS5YgMDAQNjY2AICsrCxs3boVQ4YM4SVGVVWV2ut27tyZUyw6XLZBjx49\nwscff4zTp0/LlzHG4ObmhtDQUJiYmHCOYW9vr/Zeket9OUqyNqi2thb6+vp4+PAh8vPzUVtbCysr\nK15Hxf7+++/yr7OzsxEZGYlp06bB2toaDQ0NyMzMxMGDBzFr1iylwZMaY6TNcXBwYO+88w7buXMn\nS0tLYw0NDYLGe/PNN9ndu3eVlmdmZjJPT0/O/dM5WRsUFRWFlJQUJCcnY9++faipqcErr7wCV1dX\nuLq6ol+/frzGy8zMRM+ePZWWm5ubIzs7m3P/dLhsRmZmptrr9u/fX7DtyM3NxYULF/D9998jKyuL\n92eX3t7eMDMzw5w5cxQGSe7atQt//vknoqKiOPVPSdaMxpNjxpjCSfLfvwe4nxz/3YMHD5CSkiLf\no+Xk5MDOzg5OTk4IDAzkNdbdu3fx4Ycf4sqVK/LPxRhD//79sXXrVvkVrrYoyZrR+NwQAFJTUxET\nEwM/Pz+Fk+PvvvsOb7/9Nt544w3e4v7rX/9CTU0NxGIxnJyc4OjoiIEDB0JXV5e3GE9TXFysMBLX\nwsKCl37pnKwZTX/Jc+fORUREBLp16yZfZm1tDQcHB/j7+/OaZGZmZrh+/Tru3buHoqIilJSUoLKy\nUrBJJGPGjMHJkyfRrVs3hc/HF7rjr6a//voLBgYGSsuNjIxQUFDAa6xdu3bhwoUL+Oijj/B///d/\n2L9/P9zd3TFp0iRB5lyamZk9dXYUX+hwqab33nsP1dXVePfdd2FhYYHHjx/j3r17+Pbbb9GpUyfs\n3r1bkLi1tbVIT09HcnIyDh8+jLt37+L69eu8xli1ahXOnDkDc3NzmJubKx2WuT6Qp8OlmjZu3Ijg\n4GAsW7YM1dXVAAA9PT24uroiODiY11ixsbG4fPkyUlNT8ccff8DMzAwuLi5YtGgRXFxceI3V6LXX\nXhOkX4D2ZGprekVZVlaG2tpavPTSS0rj7/nw2muvQSqVwsXFBS4uLujVq9dT17t16xZsbW15j883\nSjI1SSQSXL58ubU3Q8GgQYOQlpbGS1+JiYn4+eefkZ+fD5FIhD59+mDy5Mn45z//yblvOvFX0+uv\nv44ffvihtTdDAV/7hx9//BGzZs3CvXv30LdvX/Tp0wc5OTmYMWMG4uLiOPdP52RqKisrw9atWxEW\nFoZevXopnRxHR0c/823ia2zZf//7X2zfvh0jR45UWH769Gls2bIFI0aM4NQ/JZmaHB0d4ejo2Nqb\nIYiCgoKnJtKoUaOwcuVKzv1TkqmpI5UN+DtLS0ukpKQoDYhMS0vjZdIKJZkGDh48qHBy3Lt3b0yd\nOhXu7u6tvWmc+Pn5Yc6cORg/frz8OWV2djaOHTuGxYsXc+6fkkxNX331Ffbu3QsPDw8MGzYMwJP/\niJUrV6KiogKTJ09u5S3UnpeXF7p3747o6GgcOnQItbW16N27N0JCQjB27FjuATiPSHtOjBw5kl2+\nfFlpeVJSEhs/fnwrbBFjzs7OvPQTExPD7t27x0tfT0P3ydQkkUiQnJysdFX5+PFjDBkyBKmpqZz6\nP3funNrr8l1wZcqUKbh16xYsLS3x6quvwtXVFS4uLnjxxRd56Z+STE1TpkzB/PnzMXr0aIXlsbGx\n2LZtG44ePcqp/78XWWkcx9aUjo4OOnfujJSUFE6xnqa8vBwpKSlITU1FcnIyrl+/jn79+mHo0KFY\nunQpp74pydT022+/YdGiRZBKpQonx4mJidiwYQMmTJjAW6zY2FicOHEC8+bNUxi79vXXX8PDw0Mp\n0fmWnZ2N+Ph4REZG8jISl5JMA7dv30Z0dDRyc3PlJ8eTJ0/mvcSmu7s7Dh06hC5duigsLysrw9Sp\nU/Hrr7/yGi89PV0+AvfKlSswMTGBRCKBRCKBk5MT+vTpw6l/urpUU1hYGF599VWsWLFCkIfiTZWV\nlaGiokIpyWpqavDgwQPe43l5eaFfv37w8fFBaGioUlyuaE+mpoULFyIlJQU1NTVwdnaWzxxqrmCd\ntpYtW4aMjAxMnz4dlpaW8rFrBw4cgI2NDe8FV37++WckJSUhKSkJDx48gJOTE5ydneHs7KzRJGBV\nKMk0lJWVJT+0pKSkoKqqCq6urvj88895i1FVVYUdO3YgNjYWBQUFCtWvly9fzvuepqnS0lIkJycj\nMTERFy5cQElJCZKSkjj1SUmmhYqKCqSmpiIlJQWxsbG4c+cOrl271tqbxdndu3eRmpoq/2xFRUUY\nPHgwwsPDOfVLSaamX375RT5FrbCwEA4ODvKTY7FYrFTmiQvGGM6ePYusrCz5KNym+H6O+v777yMt\nLQ0VFRVwdHSUD5Z0cHDgZYYUJZma7O3t5SfHnp6eMDIyEizWihUrcOLECfTp00cpeUUiEe/DirZu\n3QoXFxdIJBLo6+urXG/Tpk1YsWKFxv1Tkqnp6tWr+P3335GUlIS0tDRYWlrKT46dnJx4LYYyePBg\nfP/994JcVHCh7UhcSjItMMaQkZGB+Ph4REdH486dO7zOIB81ahR+/vlnXg/BfBCLxbh69arG7eg+\nmQYqKiqQlpYmPzm+cuUKevXqBW9vb17jBAQE4D//+Q+WLFki6JWkprS9lUF7MjVNmjQJmZmZMDMz\nU5hJJEQl6okTJ+Kvv/5CRUUFXnzxRejoKE7FiI+P5z2mOrQ9XNKeTE1vvfUWXFxcYGlp2ex6gYGB\n2Lx5M6dY7777Lqf2bQ3tyXjG5zS1tob2ZG0EH3+z9fX12LFjx1OHevv5+XHfSC1p+9koyXjGxzS1\nzz77DL/99hu8vb3lIyCysrKwZ88ePH78GLNmzeIcQxvTpk3Tqh0dLnnGx+Fy2LBh2Lt3r1LxuRs3\nbiAgIAAnT57k1D/w5ApWXVwfyNMM8jaoqqoKvXv3Vlrev39/lJSU8BLDyMhI/s/AwABxcXHIy8tD\np06doKenhzt37uD8+fO83GSmw2UbNGDAAERGRuLtt99WWL5//35YW1vzEmPDhg3yr4OCgrBmzRql\nUurR0dG8XMTQ4ZJn2t4Vb+ry5cvw9/dHjx49FIZ6FxQU4Msvv8TQoUP52FQ5Z2dnJCQkKA3GrKur\ng6urK5KTkzn1T4dLNW3btu2pyysqKrB27Vr590uWLOEcSyKR4Ndff8XMmTNhYWGB7t27w8vLCydO\nnOA9wQDA2Nj4qZUW4+LieCkhSnuyFshkMpSUlGDq1Kk4dOiQ0mX8nTt3sHTpUs57r78rLCyErq6u\nvIZrdnY2DA0NYW5uzmscADhw4AA+/fRT2NnZKZRYv3nzJlavXo2ZM2dy6p+SrAWHDh3Chg0b8OjR\nI5X3iUaPHq1yT6eNc+fOISAgAKGhofIZ3D/++CNCQ0Oxbds2DB8+nLdYjXJychAbG6tQ/XrEiBH8\nTJIRbNpwB1JfX88cHR1ZXl6e0r/i4mLe402YMIEdO3ZMafnp06fZxIkTeY8XFhbGe59NUZJp4PHj\nxwpf//HHH0wmk/Eex9HRkdXX1ystr62tZY6OjrzHGz58OCspKeG930Z0C0NNCQkJ+OCDDxAXF4f6\n+nr4+vriypUr0NfXR1hYGK+lA/r27YuTJ0/Cw8NDYXl0dHSLD+i14e/vj4ULF8LDwwNmZmZKV5lc\nPxudk6lp6tSpmDFjBry8vHDkyBFs2rQJUVFRuHLlCnbv3o2YmBjeYl28eBELFy6EpaUlLCwswBhD\nTk4OioqKsGfPHkgkEt5iAcolEpoSiUQ0g/xZcXJyQkpKCkQiEQIDA9G9e3d89NFHYIzB2dmZ9/oU\nhYWFOHHiBHJzcyESiWBlZYXx48fD1NSU1zjPAh0u1WRoaIiHDx/CwMAAly5dwpYtWwA8KVQixHvB\ne/bsiRkzZqCoqOipj5i4qq6ulg/vbulV0VxfD01JpiY3Nze888470NXVhYmJCVxcXFBTU4Pg4GAM\nHjyY11hlZWVYt24dfvnlF4hEIly7dg0ymQwBAQH4/PPP0aNHD84xpFKp/JGRRCJ56h8K+9+7C7ge\nLinJ1BQUFISIiAg8evQIPj4+EIlEaGhowP379xESEsJrrLVr16KiogJRUVHy+QNGRkawsrLC+vXr\nebkn1/Q1Pd9++y3n/pol2HVrByP0vaSmnJ2dWWlpKWOMMbFYLF/+6NEjJpVKn9l28IX2ZGqKioqC\nj48Pr/MrVdHT03vqdLja2lrU1NTwHu/GjRvYtGkTsrKynto/14krlGRqEvpeUlMSiQSfffYZli9f\nLl/2559/Yv369XB1deUtTqMlS5bAwsICc+fOFWSuJ93CUJPQ95KaKigowPz583Hr1i08fvwYhoaG\n8pJVmzZtUvlCL20NGjQICQkJnK8iVaE9mZpu3LjxzGL16tULGzduRHV1NXJzc1FcXIzKykrY2try\nnmDAk/FkRUVFnCsqqkJ7smY8y3tJTe3btw9hYWFITExEWVkZPDw80LVrV5SVlWHOnDm8TCRpWm27\nuLgYUVFRmDhxIiwsLJRuZ9BjJQE1nRSiquIg4+leUlPu7u744osv4ODggD179uDgwYM4fPgwsrOz\nsWDBApw6dYpzjKaH/6dV2m76M7pPJqCmV1rdunXD5s2beXv9X3OKi4vh4OAAALhw4QI8PDygq6uL\nAQMG4P79+7zEaHr4f+utt/Ddd98prdN4T5ArSrJmmJmZYdGiRbCyskJZWVmzL4N/5ZVXeItrYmKC\nzMxMGBoaIjExER988AEAID8/n9e6aOnp6bh69SouX76MH374QekPKDc3F3l5eZzjUJI1Y+PGjdi7\ndy+uXbuGx48fIz09/anr8f3s0tvbG1OnToVIJMKwYcNgZ2eHR48eYcGCBfy86+h/qqqqcP78edTX\n12PXrl1KPzc0NNRofqZKrXUXuL3x9fV9pvFSU1PZ2bNnWU1NDWPsyejc8PBwVltby3us2bNn895n\nU3TiTwRHU+KI4CjJiOAoyYjgKMmI4CjJiOCe+/tk7jpeWrX7+urnmCNeplEbHS3rSoTHr8Vc1081\nbnfi5nmN24hMj4OV/Evjdjq9bqv+mca9EQCA9T/5n9yhSt9/8D/XUhVRJ1ve+6QkI4KjJCOCoyQj\ngqMkI4KjJCOCoyQjgqMkI4JrMcny8vJgZ2eHW7duCbohBw8ehFQqFTQGaR2C78kiIiJQW1srdBgA\nTwqVLFy4EHZ2drwMGyb8EDTJZDIZQkNDUVdXJ2QYAEBmZiamTJnCS0lwwi+1k+z69euYOHEiJBIJ\nfH19kZ+fDwA4fvw4JkyYAIlEAjc3N+zcuRPAkyJuI0aMAGMMLi4uOHDgAADg6NGjGDt2LBwdHeHl\n5aX0xoszZ87A3d0dDg4OCAwMVHsvWFxcjNDQ0A73rsgOoaXx2bm5uczW1pZNnz6d5eXlsQcPHjA/\nPz82c+ZMlpubywYOHMjOnDnDGGMsLS2Nvfzyy+zChQuMMcYSEhKYra0tKy8vZ4wxlp6ezsRiMbt0\n6RKrq6tj4eHhTCqVsqqqKhYTE8PEYjELDg5mFRUVLCMjgzk4OLCjR49qNJ785s2bzNbWluXm5qq1\nfnb6XY36J5pTexSGt7c3LCwsADwpPjJ79myYmJggPj4exsbGAJ688sXa2hrXrl176pszDh8+DKlU\nKi8a4ufnB3Nzc/nhtKamBosWLYKRkRHs7e1hZ2eHzMxMzn9IzdF0JEWj0w0HNB7Boe0ojJMP/osx\nxv4at9NmFIZOr9toKBigVTtV1E6y/v37y7/u3bs3GGMoKirCyZMnERMTg8LCQjDGUFdXp/IQl5ub\nq1C9WV9fH+PHj5d/b2xsrHBOZWho+MwuGohw1E6ypi9bZ/+b4HT8+HHs3r0bYWFhcHFxgZ6eHiZP\nnqyyj8bqhOT5ovaJf05OjvzrP//8E7q6usjPz4eTkxOGDRsGPT09lJeX4+7duyr7sLKyUuinoaEB\ne/bsQWFhoZabT9oDtZMsMjISRUVFKC8vx969e+Hm5gZra2vk5OSgtLQUBQUF+OSTT2BmZiZPmsaK\nODk5OaisrISnpydSUlIQGxuLuro67Nu3D+Hh4ejSpYswn460CRqd+Pv7+yM/Px9isRghISEwMjJC\nUlISRo0ahR49emDVqlWQyWRYu3YtTE1NsWDBAjg5OWHGjBlYvHgx5syZgy1btiA0NBTLly+Hra0t\nwsPD8cILL3D+IKtXr8aRI0fkh/KxY8dCJBJh3bp1zR7CifCe+xnk2o7xp6tL5XYqf6Zxb4RoqF3M\nVpo3bx4uXryo8ucffvghfH19n+EWEU20iyRrfFRF2ic6XBLBUZIRwbWLw6WQ9Hr1fGZt86bbaB3r\nL38HjdsMOKt5vKwZwICzflq1U4X2ZERwlGREcJRkRHCUZERwlGREcJRkRHCUZERwlGREcJRkRHBU\npoAIrsOUKaiursaGDRvg5uaGwYMHw8fHB1euXBE8LmlZhylTsGnTJsTHx2Pfvn24dOkSBg0ahPnz\n56O6ulrw2KR5HaZMQZcuXbBq1SpYWVnBwMAA/v7+kMlkyMrKUvcjEqG0NMW8vZUpaJSWlsbs7e1Z\nYWFhs+vlZORr1T9RX4csU1BZWYmgoCB4eXmhR48eza47f1SIxv0DwIm/tmOc+UKN2mg71Cd9cyAc\nAjdr3K5SWqlxm6wZH8Nmf7BW7VTpcGUKZDIZ5s6dC1NTU6xevVqjtkQYap+TqSpTEB4ejqCgIKSm\npiI9PR329vYq+xC6TEFeXh6mT58OGxsb7NixA/r6+oLFIurrMGUKSktLMWvWLIwePRqhoaHo1KkT\n5z4JPzpMmYLNmzdjwIABWLFiBee+CL86TJmCAwcOQFdXFw4OimPhqUxB63vuyxRoeoXYiK4uldup\nQg/IieDaxZQ4KlPQvrWLJKMyBe0bHS6J4CjJiOAoyYjg2sU5mZBub9G+FoambU2PaP9IzahI87Zm\nm7QYxzcDsNGynSq0JyOCoyQjgqMkI4KjJCOCoyQjgqMkI4KjJCOCoyQjgqMyBURwHaZMQVFREQID\nAyGVSjFkyBD4+vri6tWrgsclLeswZQoCAgKgq6uL06dP4/z583j55Zcxd+7cZxKbNK9DlCloaGjA\nm2++iZUrV+LFF1+EoaEhpkyZAplMhqKiIrV/GUQYaifZ/v37sWPHDpw7dw6dOnXChx9+iLy8PKxY\nsQLLli3D5cuXERYWhu3bt+PixYvo2bMndu/eDQBISEiAl5cXrl27hk8++QRBQUFITk6Gu7s75s6d\nKy+KUl1djfj4eBw5cgQHDhzAr7/+ipMnT7b8IXR0MHXqVHTr1g0AUFJSgj179kAikcDc3Fyb3wvh\nU0t1DBprYRw+fFi+LC4ujtnZ2bHy8nJWVlamsP748ePZzp07GWPKtTDWrVvHZs+eLV+3pqaG/fTT\nT+zhw4csJiaG2dnZsYcPH8p//uabb7IvvvhCo7oLrq6uzNbWlvn6+rKioqIW179Z1nytDMJdhytT\ncOnSJRQXF2Pnzp3w9vbGkSNHmp1y53EyXKP+G2VO+wT9o9Zp1Mb0iJFWsRK/XwbpzM81bmd845HG\nbU6lrMHowWu0aqdKhypT0Khbt25YuXIlSkpKcPbsWcHjkeZ1iDIF+fn5GDlyJHJzc+XLdHR0wBiD\nnt5zPy6z1XWIMgXm5uYwNTVFSEgIZDIZqqursXXrVujr68PZ2ZlT34S7DlGmQCQS4auvvsKGDRvw\nxhtvQEdHB/b29vj6669hamrKqW/C3XNfpkDTk/dGdOKv3E4VekBOBNcuzoqpTEH71i6SjMoUtG90\nuCSCoyQjgqMkI4JrF+dkQhqwrFi7htM0b7s0ruURJU+3DOvXf6Nxq89dRmkVTSeP3+FRtCcjgqMk\nI4KjJCOCoyQjgqMkI4KjJCOCoyQjgqMkI4KjMgVEcB2mTEFTsbGxsLOzQ2Ji4jONS56uw5QpaFRZ\nWYnQ0FAYGWk3CpXwr0OUKWgqLCwMrq6uMDEx0agdEU6HKFPQ6ObNm/jpp5+wdOlSDX8NRFAtTTFv\nL2UKGhoa2PTp01l0dDRjjLHXXnuNJSQktNgu58ZfavVPtNdhyhQ0vrnX09NT3Y8EAJj/RqhG6zc6\nkbsV46wCNGqj7VAfd+sbOJ2jema+KtoM9fml8CuM7fm+Vu1UUTvJVJUp2L17N8LCwuDi4gI9Pb1m\nX8UsVJkCmUyGbdu2ISIiAiKRiPf+CTcdokzBuXPnUFpaipkzZ0IqlUIqleLevXt4//33sW6ddvMq\nCX/U3pNFRkbC2dkZRkZGCmUKEhMTUVpaipqaGnz22WcqyxT069cPnp6e8PLyQmxsLNzc3BAZGYnw\n8HBMmzaN04cYO3YsXF1dFZZNnz4dK1euxKuvvsqpb8JdhyhT0LlzZ3Tu3Flhma6uLl566SUYGxtz\n6ptw99yXKdD05L0Rnfgrt1OFHpATwbWL2UpUpqB9axdJRmUK2jc6XBLBUZIRwVGSEcG1i3MyId1e\n1OeZtX1//xzt4qzSrq2NcYFW8WDMrYbv39GejAiOkowIjpKMCI6SjAiOkowIjpKMCI6SjAiOkowI\njpKMCI5qYRDBCf5YKSIiAj4+PtDX1xc0zltvvYWUlBTo6urKl/Xu3RvHjx8XNC5pmaBJ1lgLw8vL\nS/AkA4D58+dj0aJFgschmulwtTBIG9TSFPPGMgXTp09neXl57MGDB8zPz4/NnDmT5ebmsoEDB7Iz\nZ84wxhhLS0tjL7/8Mrtw4QJjTLlMQXp6OhOLxezSpUusrq6OhYeHM6lUyqqqqlhMTAwTi8UsODiY\nVVRUsIyMDObg4MCOHj2q1lR4X19f5uPjwyZMmMAcHR2Zj48Py8zMbLHdzaL7avVPtKfRlDgLCwsA\ngL+/P2bPng0TExPEx8fLp52JxWJYW1vj2rVrGDp0qFIfhw8fhlQqlc+R9PPzg7m5uby0VE1NDRYt\nWgQjIyPY29vDzs4OmZmZam2fjY0NGhoaEBAQAD09PQQHB2P27Nk4ceIEDAwMVLbz2POtur8CBZkf\nLEX/jV9o1Eb0WKtQuL1qKQZs0CwWANjs0Xyozy+3NmKs7QdatVOlw9TCWLNmjcL3n376KaRSKX7/\n/XcMHz5crT6IMDpELYyn6dKlC4yNjVFUxO97gojmOkQtjPLycqxZs0ahH5lMBplMBisrK059E+7U\nTrLIyEgUFRWhvLxcoRZGTk4OSktLUVBQgE8++URlLYzKykp4enoiJSUFsbGxqKurw759+xAeHo4u\nXbgN9+3SpQvS0tKwfv16lJWV4cGDB/j3v/8NW1tbODs7c+qbcKd2kjXWwhg+fDjq6+uxZs0aeHt7\nw8bGBqNGjcI777yDSZMm4b333sOxY8ewefNmDBw4UF4LY9++fRg4cKC8FsaQIUNw7NgxXmphAMCX\nX34JxhjGjBmDkSNHoq6uDt98843CYZ60jue+FoamV4iN6OpSuZ0q9GdOBNcupsRRLYz2rV0kGdXC\naN/ocEkER0lGBEdJRgTXLs7JhNRgVf3M2hpc69zySirolWteOp7d0+6RmrbtVKE9GREcJRkRHCUZ\nERwlGREcJRkRHCUZERwlGREcJRkRHJUpIIITfE8WERHxzCboxsXFYcKECRCLxRg3bhxOnTr1TOKS\n5gmaZI1lChrnVQrpxo0bWLZsGZYsWYKkpCQsXrwYX375JSoqKgSPTZrXYcoU7N27F+PGjcPrr78O\nAwMDjBs3DkeOHOFl/gDhqKUp5u2lTIG7uzsLCwtjfn5+TCKRME9PT5aUlNRiuxulRWr1T7TXYcoU\nFBQU4MCBA9i2bRv69++P8PBwzJs3D6dPn4aJiYnKduNOfKPur0BBtvdH6BcZolEbbUdhZAQHYuDH\nmzVu13tbWssr/c3JRxEY09VPq3aqqH24VFWmIDIyEu7u7hCLxXBwcMDt27c1LlPQWJqAS5kCxhgm\nTJiAQYMG4YUXXsDixYuho6ODuLg4dT8iEUiHKVPQvXt3hfeN6+npoWfPnlSmoA3oEGUKgCd72oyM\nDPn39fX1KCgokB/iSevpEGUKAMDX1xenTp1CbGwsqqursX37dujr62PkyJGc+ybcaHTi7+/vj/z8\nfIjFYoSEhMDIyAhJSUkYNWoUevTogVWrVkEmk2Ht2rUwNTXFggUL5GUKFi9ejDlz5sjLFCxfvhy2\ntra8lSkYMWIEgoKCEBISgvsC/tmLAAAAZElEQVT378Pe3h67d++GkZER574JN899mQJNrxAb0dWl\ncjtV6AE5EVy7mK1EZQrat3aRZFSmoH2jwyURHCUZERwlGRHcc38LgwiP9mREcJRkRHCUZERwlGRE\ncJRkRHCUZERw/w+BHfskq/G6tgAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - " Reference : https://github.com/prakashpandey9/Text-Classification-Pytorch/blob/master/models/LSTM_Attn.py\n", - "'''\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from torch.autograd import Variable\n", - "import torch.nn.functional as F\n", - "import matplotlib.pyplot as plt\n", - "\n", - "dtype = torch.FloatTensor\n", - "\n", - "# Bi-LSTM(Attention) Parameters\n", - "embedding_dim = 2\n", - "n_hidden = 5 # number of hidden units in one cell\n", - "num_classes = 2 # 0 or 1\n", - "\n", - "# 3 words sentences (=sequence_length is 3)\n", - "sentences = [\"i love you\", \"he loves me\", \"she likes baseball\", \"i hate you\", \"sorry for that\", \"this is awful\"]\n", - "labels = [1, 1, 1, 0, 0, 0] # 1 is good, 0 is not good.\n", - "\n", - "word_list = \" \".join(sentences).split()\n", - "word_list = list(set(word_list))\n", - "word_dict = {w: i for i, w in enumerate(word_list)}\n", - "vocab_size = len(word_dict)\n", - "\n", - "inputs = []\n", - "for sen in sentences:\n", - " inputs.append(np.asarray([word_dict[n] for n in sen.split()]))\n", - "\n", - "targets = []\n", - "for out in labels:\n", - " targets.append(out) # To using Torch Softmax Loss function\n", - "\n", - "input_batch = Variable(torch.LongTensor(inputs))\n", - "target_batch = Variable(torch.LongTensor(targets))\n", - "\n", - "class BiLSTM_Attention(nn.Module):\n", - " def __init__(self):\n", - " super(BiLSTM_Attention, self).__init__()\n", - "\n", - " self.embedding = nn.Embedding(vocab_size, embedding_dim)\n", - " self.lstm = nn.LSTM(embedding_dim, n_hidden, bidirectional=True)\n", - " self.out = nn.Linear(n_hidden * 2, num_classes)\n", - "\n", - " # lstm_output : [batch_size, n_step, n_hidden * num_directions(=2)], F matrix\n", - " def attention_net(self, lstm_output, final_state):\n", - " hidden = final_state.view(-1, n_hidden * 2, 1) # hidden : [batch_size, n_hidden * num_directions(=2), 1(=n_layer)]\n", - " attn_weights = torch.bmm(lstm_output, hidden).squeeze(2) # attn_weights : [batch_size, n_step]\n", - " soft_attn_weights = F.softmax(attn_weights, 1)\n", - " # [batch_size, n_hidden * num_directions(=2), n_step] * [batch_size, n_step, 1] = [batch_size, n_hidden * num_directions(=2), 1]\n", - " context = torch.bmm(lstm_output.transpose(1, 2), soft_attn_weights.unsqueeze(2)).squeeze(2)\n", - " return context, soft_attn_weights.data.numpy() # context : [batch_size, n_hidden * num_directions(=2)]\n", - "\n", - " def forward(self, X):\n", - " input = self.embedding(X) # input : [batch_size, len_seq, embedding_dim]\n", - " input = input.permute(1, 0, 2) # input : [len_seq, batch_size, embedding_dim]\n", - "\n", - " hidden_state = Variable(torch.zeros(1*2, len(X), n_hidden)) # [num_layers(=1) * num_directions(=2), batch_size, n_hidden]\n", - " cell_state = Variable(torch.zeros(1*2, len(X), n_hidden)) # [num_layers(=1) * num_directions(=2), batch_size, n_hidden]\n", - "\n", - " # final_hidden_state, final_cell_state : [num_layers(=1) * num_directions(=2), batch_size, n_hidden]\n", - " output, (final_hidden_state, final_cell_state) = self.lstm(input, (hidden_state, cell_state))\n", - " output = output.permute(1, 0, 2) # output : [batch_size, len_seq, n_hidden]\n", - " attn_output, attention = self.attention_net(output, final_hidden_state)\n", - " return self.out(attn_output), attention # model : [batch_size, num_classes], attention : [batch_size, n_step]\n", - "\n", - "model = BiLSTM_Attention()\n", - "\n", - "criterion = nn.CrossEntropyLoss()\n", - "optimizer = optim.Adam(model.parameters(), lr=0.001)\n", - "\n", - "# Training\n", - "for epoch in range(5000):\n", - " optimizer.zero_grad()\n", - " output, attention = model(input_batch)\n", - " loss = criterion(output, target_batch)\n", - " if (epoch + 1) % 1000 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - "\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - "# Test\n", - "test_text = 'sorry hate you'\n", - "tests = [np.asarray([word_dict[n] for n in test_text.split()])]\n", - "test_batch = Variable(torch.LongTensor(tests))\n", - "\n", - "# Predict\n", - "predict, _ = model(test_batch)\n", - "predict = predict.data.max(1, keepdim=True)[1]\n", - "if predict[0][0] == 0:\n", - " print(test_text,\"is Bad Mean...\")\n", - "else:\n", - " print(test_text,\"is Good Mean!!\")\n", - " \n", - "fig = plt.figure(figsize=(6, 3)) # [batch_size, n_step]\n", - "ax = fig.add_subplot(1, 1, 1)\n", - "ax.matshow(attention, cmap='viridis')\n", - "ax.set_xticklabels(['']+['first_word', 'second_word', 'third_word'], fontdict={'fontsize': 14}, rotation=90)\n", - "ax.set_yticklabels(['']+['batch_1', 'batch_2', 'batch_3', 'batch_4', 'batch_5', 'batch_6'], fontdict={'fontsize': 14})\n", - "plt.show()" - ] - } - ], - "metadata": { - "colab": { - "name": "Bi-LSTM(Attention)-Torch.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/5-1.Transformer/Transformer(Greedy_decoder)-Torch.py b/5-1.Transformer/Transformer(Greedy_decoder).py similarity index 100% rename from 5-1.Transformer/Transformer(Greedy_decoder)-Torch.py rename to 5-1.Transformer/Transformer(Greedy_decoder).py diff --git a/5-1.Transformer/Transformer(Greedy_decoder)_Torch.ipynb b/5-1.Transformer/Transformer(Greedy_decoder)_Torch.ipynb deleted file mode 100644 index 8f2f293..0000000 --- a/5-1.Transformer/Transformer(Greedy_decoder)_Torch.ipynb +++ /dev/null @@ -1,391 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1971 - }, - "colab_type": "code", - "id": "qR7gG-dBsCgk", - "outputId": "775d3a1a-99f7-48cd-aa97-48f9e7a2aef9" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 0001 cost = 2.107983\n", - "Epoch: 0002 cost = 0.133446\n", - "Epoch: 0003 cost = 0.155777\n", - "Epoch: 0004 cost = 0.011864\n", - "Epoch: 0005 cost = 0.010045\n", - "Epoch: 0006 cost = 0.406183\n", - "Epoch: 0007 cost = 0.003001\n", - "Epoch: 0008 cost = 0.022612\n", - "Epoch: 0009 cost = 0.112861\n", - "Epoch: 0010 cost = 0.003003\n", - "Epoch: 0011 cost = 0.000775\n", - "Epoch: 0012 cost = 0.017682\n", - "Epoch: 0013 cost = 0.000679\n", - "Epoch: 0014 cost = 0.002996\n", - "Epoch: 0015 cost = 0.002428\n", - "Epoch: 0016 cost = 0.001328\n", - "Epoch: 0017 cost = 0.004184\n", - "Epoch: 0018 cost = 0.005840\n", - "Epoch: 0019 cost = 0.009313\n", - "Epoch: 0020 cost = 0.028476\n", - "ich mochte ein bier P -> ['i', 'want', 'a', 'beer', 'E']\n", - "first head of last state enc_self_attns\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeYAAAH2CAYAAAClRS9UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGzFJREFUeJzt3XmMVfX5+PFnhkVMVUhURFDQuqci\nmwi0KEhQoliCilWq2GLdU/eKgLF8sSrS2liXYvxqDVatC6KmVaOi4lYtCmgEKyIIshUXKppB9rm/\nP4zz/RFQBpnhPHd8vf4azp2Z+9xPyH3PPefccytKpVIpAIAUKoseAAD4P8IMAIkIMwAkIswAkIgw\nA0AiwgwAiQgzACQizACQiDBTNr744ot46KGH4qabbqrZNn/+/OIGAqgHwkxZeO2116J3795x7733\nxp133hkREYsXL47jjz8+XnjhhWKHA6hDwkxZ+MMf/hAjRoyIv//971FRUREREW3atIkbbrhhg1fQ\nAOVOmCkLH3zwQZxwwgkRETVhjog48sgj7c4GGhRhpiy0bNkyFi1atNH2N998M3bccccCJgKoH42L\nHgBqY8CAAXH22WfH6aefHtXV1fHUU0/FrFmz4v7774/TTz+96PEA6kyFj32kHJRKpbj77rvj4Ycf\njgULFkSzZs2ibdu2MXjw4DjxxBOLHg+gzggzZeHDDz+Mdu3abbR9zZo1MWPGjOjSpUsBUwHUPceY\nKQsDBgzY5PYvv/wyzjzzzG08DUD9cYyZ1B566KF48MEHY+3atTFo0KCNbv/000+jRYsWBUwGUD+E\nmdSOOeaYaN68eVx66aXRu3fvjW7fbrvtom/fvtt+MIB64hgzZeGJJ56I/v37Fz0GQL0TZsrGW2+9\nFXPmzInVq1dvdNupp55awEQAdU+YKQvXXntt3HPPPbHzzjvHdtttt8FtFRUV8dxzzxU0GUDdEmbK\nQqdOneLWW2+Nn/zkJ0WPAlCvvF2KsrDjjjtG165dix4DoN4JM2XhwgsvjLvuuivs4AEaOruySevE\nE0/c4JOkFi5cGI0aNYrdd999g+0REQ8//PC2Hg+gXngfM2kdeeSRRY8AsM15xUxZWb9+fTRq1Cgi\nIlauXBnbb799wRMB1C3HmCkL8+fPj5/+9KcxadKkmm33339/HHfccfHhhx8WOBlA3RJmysLo0aPj\nsMMOix49etRsO+mkk+Lwww+P0aNHFzgZQN2yK5uy0KVLl5gyZUo0brzhaRFr166N7t27x7Rp0wqa\nDL7ZokWLYo899ih6DMqMV8yUhebNm8ecOXM22j5jxozYYYcdCpgINm/AgAGxfv36osegzDgrm7Jw\n+umnx9ChQ+PYY4+NPfbYI6qrq2PevHnx1FNPxeWXX170eLBJp556atx8881x1lln+QOSWrMrm7Lx\n7LPPxiOPPBILFy6MioqK2HPPPePEE0+MPn36FD0abNIxxxwTn376aaxYsSJ22GGHmncUfO21114r\naDIyE2aAevLoo49+6+3HH3/8NpqEciLMlI1HHnkknnzyyVi8eHFUVFRE27Zt48QTT4yjjjqq6NFg\ns9auXRtNmjQpegzKgGPM9WD9+vWxZs2ajba7GMZ3N27cuLj77rvj2GOPjZ49e0ZExAcffBDDhw+P\nFStWxMCBAwuesHx99NFHMX78+Jg7d26sWrVqo9v/+te/FjBVw7BmzZr485//HBMnTozPP/88ZsyY\nEVVVVXHNNdfEVVddFT/4wQ+KHpGEhLkOvfbaazF69OhYsGDBJj9s4d133y1gqoZhwoQJcfvtt0fH\njh032D5gwIAYPXq0MG+FSy65JJYvXx7dunWLZs2aFT1Og3LttdfGO++8E7/97W/jN7/5TUREVFdX\nx2effRbXXXddXHvttQVPSEbCXIdGjBgRPXr0iJEjR3qCq2PLly+P9u3bb7S9U6dOsXjx4gImajje\nfffdmDx5crRo0aLoURqcZ555Jh599NFo1apVzQev7LTTTjFmzJgYMGBAwdORlTDXoc8//zxGjx4d\nTZs2LXqUBmevvfaK5557Lo4++ugNtk+ePNkFHLbSXnvt5b229WT9+vWx6667brS9adOmsWLFigIm\nohwIcx3q06dPzJo1Kw455JCiR2lwLrjggrjggguiW7dusc8++0TEV8eYp0yZEmPGjCl4uvJ2+eWX\nx5VXXhknn3xytGnTJiorN7zu0L777lvQZOXvRz/6Udxxxx1x7rnn1mxbsWJFXH/99Z4n+EbOyt5K\n9913X83XK1eujIkTJ0bv3r03+Sru1FNP3ZajNTizZ8+OiRMnxsKFC2PNmjXRtm3bGDhwoCe4rXTg\ngQdutK2ioiJKpVJUVFQ4N2IrzJ49O84888xYt25dfPbZZ/HDH/4wFi9eHLvuumuMGzcu9ttvv6JH\nJCFh3kq1vbhFRUVFPPfcc/U8DWy5zR2jb9OmzTaapGFatWpVTJ48ORYuXBjNmjWLdu3aRc+ePTe6\n2Ah8TZgpC4sWLYrx48fHhx9+GKtXr97odm/pARoKx5jrUKlUivHjx0fnzp2jQ4cOERHx9NNPx6JF\ni2Lo0KEbHbuj9i688MJYu3ZtHHbYYU6uqwO9e/eOF154ISIiunfvXnPG8Ka4bOSWsbZsLWGuQ7//\n/e/j2WefjUMPPbRm2y677BK33HJLLFu2LIYNG1bgdOVt3rx58corr7ggQx255JJLar6+4oorIiLi\nyy+/jKZNm2700ZpsmW9a2yZNmsQnn3wSLVu2tMZbae7cufHCCy9Eo0aNom/fvg3unRl2Zdehnj17\nxsSJE2O33XbbYPtHH30UgwYNipdffrmgycrfueeeG7/+9a/j4IMPLnqUBmf58uXxu9/9Lp566qmo\nqKiImTNnxn//+9+46KKL4o9//GO0bNmy6BHL1pIlS+KKK66IadOmRalUilKpFI0bN45evXrFqFGj\nrO138Oqrr8Y555wTe+21V1RXV8eSJUvirrvuik6dOhU9Wp0R5jp06KGHxosvvrjRq7rly5dHnz59\nYvr06QVNVv6WLl0aQ4cOjYMOOih22223jXYP2hvx3V122WVRVVUVF154YQwePDjefvvtWLVqVVx9\n9dVRVVUVN998c9Ejlq0hQ4ZEkyZN4owzzoi2bdtGqVSKDz/8MO6+++4olUpx1113FT1i2Rk8eHD0\n798/TjvttIiIuOeee+KZZ56Je+65p+DJ6o79KXWoZ8+eMWLEiDjnnHOiTZs2NZ8ZPG7cuOjdu3fR\n45W1K6+8MpYuXRo77bRTfPLJJxvc9m3H8Ni8l156KSZNmhQtWrSoWctmzZrFyJEjo2/fvgVPV95m\nzpwZL7/88gafxdyuXbvo3LlzHHHEEQVOVr7mzJkTP/vZz2r+PWjQoLj11lsLnKjuCXMdGjVqVFx5\n5ZVx0kkn1ey2qqysjL59+8bvfve7oscra1OnTo0nn3zSW3fqQePGjTd5Cdk1a9Zs8gx4aq9t27ZR\nVVW1QZgjvrrmgf/L382aNWs2OAF0++233+SHr5QzYd5Kc+fOrbkS1bJly+LSSy+tOQP766MELVq0\niP/85z+uoLQV9ttvv9huu+2KHqNB6tSpU4wdO7bmQxYiIhYsWBDXXHNN9OjRo8DJytOcOXNqvh46\ndGhceuml8fOf/zz22WefqKioiHnz5sX9998f559/foFTkpljzFvpkEMOibfffjsivrqC0qZ2q7qC\n0tZ7/PHH44EHHoj+/ftHq1atNnrrWa9evQqarPwtXbo0zjvvvJg9e3asX78+mjVrFqtXr45DDz00\nbrjhho1OZuTbff08sLmnVs8J383BBx8cI0eO3GB9x4wZs9G2cr7SojBvpSVLlkTr1q0jwhWU6tOm\nLhv5NU9wdWPGjBmxcOHC2G677aJdu3b28HxHW/JpZ54TtlxtrrZY7ldaFGYASMSlqAAgEWEGgESE\nGQASEWYASESYASCRsrvASPXS/YoeodYqdn4iSsv6Fz1Gg2Rt64+1rT/ltrb9WncseoRa+9+3/xhn\nH3JZ0WPU2qTqCd94m1fM9aiiyf5Fj9BgWdv6Y23rj7WtP3sf3LboEeqMMANAIsIMAIkIMwAkIswA\nkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0Ai\nwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgz\nACQizACQiDADQCLCDACJCDMAJCLMAJBIqjAvXrw42rdvH3PmzCl6FAAoROOiB/j/tWnTJmbMmFH0\nGABQmFSvmAHg+y5VmBctWhQHHHBAzJ49u+hRAKAQqcIMAN93qY4x10bFzk9ERZP9ix6j1ipbvV/0\nCA2Wta0/1rb+lNPaTqoueoItM6l6QtEj1ImyC3NpWf8oFT1ELVW2ej+ql+5X9BgNkrWtP9a2/pTb\n2vZr3bHoEWptUvWEOKrypKLHqLVv+yPCrmwASESYASARYQaARIQZABJJdfLXHnvsEe+9917RYwBA\nYbxiBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBE\nhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFm\nAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEg\nEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESE\nGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYA\nSESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgkTRhfuSRR2LZsmVFjwEAhUoR\n5vXr18eYMWOEGYDvvc2GuXfv3jFp0qSaf5955plx7LHH1vx71qxZ0b59+5g7d26cc8450a1bt+ja\ntWucd9558fHHH9d83wEHHBBPP/10DB48ODp27BgDBgyI9957LyIiunTpEl988UWccMIJ8ac//aku\nHx8AlJXNhrlbt24xffr0iPjqle0777wTq1atis8++ywiIqZOnRqdOnWKq6++Onbcccd4+eWX4/nn\nn4+qqqoYO3bsBr/rzjvvjOuuuy5effXVaN68edxyyy0REfH4449HxFe7sy+++OI6fYAAUE4ab+4b\nunfvHg8++GBERLzzzjvRrl27aNWqVUybNi369u0bU6dOjR49esTQoUMjIqJp06bRtGnT6NOnTzzw\nwAMb/K7jjjsu9t5774iIOOKII+KRRx7Z4oErdn4iKprsv8U/V5TKVu8XPUKDZW3rj7WtP+W0tpOq\ni55gy0yqnlD0CHWiVmG+6qqrYvXq1fHGG2/EoYceGi1bttwgzEOHDo2ZM2fGjTfeGLNmzYo1a9ZE\ndXV17Lbbbhv8rj322KPm6+233z5Wr169xQOXlvWP0hb/VDEqW70f1Uv3K3qMBsna1h9rW3/KbW37\nte5Y9Ai1Nql6QhxVeVLRY9Tat/0Rsdld2bvvvnu0bt06ZsyYEW+88UZ06dIlOnXqFNOmTYsFCxbE\nqlWrom3btnH22WfHwQcfHJMnT44ZM2bEsGHDNr6zyhTnmgFAWrUqZffu3WPq1Knx5ptvRufOneOg\ngw6KefPmxSuvvBKHHXZYzJ8/P1asWBG/+tWvYqeddoqIr3Z7AwBbptZhfuyxx6Jly5bRvHnzaNy4\ncRx44IFx3333RY8ePaJ169ZRWVkZb775ZqxcuTIefPDBmDdvXnz++eexatWqzf7+Zs2aRUTE/Pnz\no6qqauseEQCUsVqFuVu3bjF//vzo0qVLzbbOnTvHnDlz4sc//nHstttuMWzYsBg1alT06tUr5s6d\nGzfffHO0aNEijj766M3+/l122SX69esXl156adxwww3f/dEAQJmrKJVK5XIuVUREWZ04UW4nepQT\na1t/rG39Kbe1dfJX/dmqk78AgG1HmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgB\nIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBE\nhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFm\nAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEg\nEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESE\nGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYA\nSESYASCRQsP8zjvvxJAhQ6Jr167RvXv3GDZsWFRVVRU5EgAUqtAwX3zxxdGhQ4f417/+FY8//njM\nnDkz7rjjjiJHAoBCVZRKpVJRd75ixYpo0qRJNG3aNCIirrnmmpg3b1785S9/+cafKa2dHRVN9t9W\nIwLANtW4yDt/7bXXYty4cTFv3rxYt25drF+/Prp06fKtP1Na1j8K+0tiC1W2ej+ql+5X9BgNkrWt\nP9a2/pTb2vZr3bHoEWptUvWEOKrypKLHqLVJ1RO+8bbCdmXPnTs3LrroojjuuOPi1VdfjRkzZsRp\np51W1DgAkEJhr5jffffdaNSoUQwdOjQqKioi4quTwSornSgOwPdXYRXcc889Y82aNTFz5syoqqqK\nW2+9NVauXBmffPJJrF+/vqixAKBQhYW5Q4cO8ctf/jKGDh0a/fr1iyZNmsR1110XX3zxhV3aAHxv\nFXry1/Dhw2P48OEbbHv11VcLmgYAiueALgAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJ\nCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLM\nAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANA\nIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkI\nMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswA\nkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0Ai\ntQ7zokWL4oADDojZs2fX5zwA8L3mFTMAJCLMAJDIFof53//+dwwYMCA6deoUp512WixevDgiIl5/\n/fU45ZRTonPnztGzZ8+48cYbo7q6uubn/va3v8Wxxx4bHTp0iH79+sWTTz5Zc9uQIUNi7NixMXDg\nwPjFL35RBw8LAMrTFof5gQceiNtuuy1efPHFaNKkSVxxxRWxdOnSOOecc2LQoEHx+uuvx/jx4+Mf\n//hHPPTQQxER8eyzz8ZNN90U119/fUyfPj2GDx8ew4YNi7lz59b83ieeeCJGjRoV48ePr7MHBwDl\npvGW/sDgwYOjTZs2ERFxxhlnxFlnnRUTJ06MvffeOwYNGhQREfvuu28MGTIkHn300TjllFPioYce\nihNOOCEOOeSQiIg48sgjo2fPnvHYY4/FZZddFhER7du3j06dOm32/it2fiIqmuy/pWMXprLV+0WP\n0GBZ2/pjbetPOa3tpOrNf08mk6onFD1CndjiMO+77741X7dt2zZKpVJMmTIl3n333Wjfvn3NbaVS\nKXbZZZeIiFiwYEH885//jHvvvXeD23fccceaf7du3bpW919a1j9KWzp0QSpbvR/VS/creowGydrW\nH2tbf8ptbfu17lj0CLU2qXpCHFV5UtFj1Nq3/RGxxWGurPy/vd+l0leJbNOmTTRt2jTuvPPOTf5M\ns2bN4qKLLoqzzz77mwdpvMWjAECDs8XHmOfNm1fz9YIFC6JRo0Zx4IEHxvvvv7/ByV7Lli2LVatW\nRcRXr6zfe++9DX7PkiVLNvh+AOA7hPn++++Pjz/+OKqqquLuu++OXr16xcCBA6OqqipuueWWWLly\nZSxZsiTOOuusuP322yPiq+PSTz/9dDz77LOxbt26mD59egwcODCmTJlS5w8IAMrZFod58ODBccYZ\nZ8Thhx8e69ati//5n/+J5s2bx2233RYvvfRSdOvWLU4++eTo2rVrnH/++RER0aNHjxg5cmSMGTMm\nOnfuHCNHjozLL788evToUecPCADKWUXp6wPFZaKcTpwotxM9yom1rT/Wtv6U29o6+av+fNvJX678\nBQCJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMA\nJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCI\nMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIM\nAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAk\nIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgw\nA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIo2LuNM+ffrERx99FJWVG/9dMHLk\nyBg8eHABUwFA8QoJc0TEiBEj4rTTTivq7gEgJbuyASARYQaARCpKpVJpW9/ptx1jfuutt6JRo0bf\n+LOltbOjosn+9TkeABSm7I4xl5b1j23+l8R3VNnq/aheul/RYzRI1rb+WNv6U25r2691x6JHqLVJ\n1RPiqMqTih6j1iZVT/jG2+zKBoBEhBkAEhFmAEiksGPMY8aMibFjx260vVevXnHrrbcWMBEAFK+Q\nMD///PNF3C0ApGdXNgAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANA\nIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkI\nMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkUlEq\nlUpFDwEAfMUrZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgkf8HhgybVjMLh/kAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "first head of last state dec_self_attns\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeYAAAH2CAYAAAClRS9UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHIlJREFUeJzt3XuQlYV9//HvLleTcJmJggKCCRr1\nVxUB5WJIIARlREtRMAYFG1IFk6kaUVFxUkq8EKMZEzWmqdZiMUVEsJOoRdFoooVqFK1gVAJypxil\nor9FbrLn94fj9scA4azs+nzP+nr9tZ6zl88+4+yb85xnz1aVSqVSAAApVBc9AAD4X8IMAIkIMwAk\nIswAkIgwA0AiwgwAiQgzACQizACQiDBTMd577724//7746c//WndbStXrixuEEAjEGYqwsKFC2PQ\noEFx7733xl133RUREevWrYszzjgjnnrqqWLHATQgYaYi3HTTTXH11VfHr371q6iqqoqIiM6dO8fN\nN9+8yyNogEonzFSEN954I84888yIiLowR0R87WtfczobaFKEmYrQoUOHWLt27W63v/jii9GmTZsC\nFgE0juZFD4ByDB8+PMaPHx/nnXde1NbWxrx58+K1116LmTNnxnnnnVf0PIAGU+XPPlIJSqVS3HPP\nPfHAAw/E6tWro3Xr1tG1a9cYPXp0jBw5suh5AA1GmKkIq1atim7duu12+/bt22Px4sXRu3fvAlYB\nNDzPMVMRhg8fvsfb33///Tj//PM/4TUAjcdzzKR2//33x6xZs2LHjh0xatSo3e5/++23o3379gUs\nA2gcwkxqp556arRr1y4mTpwYgwYN2u3+Vq1axZAhQz75YQCNxHPMVISHH344TjvttKJnADQ6YaZi\nvPTSS7Fs2bLYtm3bbvede+65BSwCaHjCTEW4/vrrY8aMGfH5z38+WrVqtct9VVVV8cQTTxS0DKBh\nCTMVoWfPnnH77bfHl7/85aKnADQqvy5FRWjTpk2ceOKJRc8AaHTCTEW4+OKL4+677w4neICmzqls\n0ho5cuQuf0lqzZo10axZszjkkEN2uT0i4oEHHvik5wE0Cr/HTFpf+9rXip4A8InziJmKsnPnzmjW\nrFlERGzZsiUOOOCAghcBNCzPMVMRVq5cGX/5l38Z8+fPr7tt5syZcfrpp8eqVasKXAbQsISZijB1\n6tTo06dP9O/fv+62s846K77yla/E1KlTC1wG0LCcyqYi9O7dO5599tlo3nzXyyJ27NgR/fr1ixde\neKGgZbB3a9eujS5duhQ9gwrjETMVoV27drFs2bLdbl+8eHF87nOfK2AR7Nvw4cNj586dRc+gwrgq\nm4pw3nnnxbhx42LYsGHRpUuXqK2tjRUrVsS8efPiiiuuKHoe7NG5554bt956a1xwwQX+AUnZnMqm\nYjz++OMxd+7cWLNmTVRVVcWhhx4aI0eOjMGDBxc9Dfbo1FNPjbfffjs2b94cn/vc5+p+o+AjCxcu\nLGgZmQkzQCN58MEH/+z9Z5xxxie0hEoizFSMuXPnxiOPPBLr1q2Lqqqq6Nq1a4wcOTJOPvnkoqfB\nPu3YsSNatGhR9AwqgOeYG8HOnTtj+/btu93uxTA+vjvuuCPuueeeGDZsWAwYMCAiIt5444246qqr\nYvPmzTFixIiCF1auN998M6ZPnx7Lly+PrVu37nb/v/zLvxSwqmnYvn17/OxnP4s5c+bEu+++G4sX\nL46ampq47rrr4vvf/3589rOfLXoiCQlzA1q4cGFMnTo1Vq9evcc/tvDqq68WsKppmD17dvziF7+I\n448/fpfbhw8fHlOnThXm/XDppZfGpk2bom/fvtG6deui5zQp119/fbzyyivxd3/3d3H55ZdHRERt\nbW288847ccMNN8T1119f8EIyEuYGdPXVV0f//v1j8uTJfsA1sE2bNsWxxx672+09e/aMdevWFbCo\n6Xj11VfjySefjPbt2xc9pcl57LHH4sEHH4yDDz647g+vtG3bNqZNmxbDhw8veB1ZCXMDevfdd2Pq\n1KnRsmXLoqc0OYcddlg88cQTccopp+xy+5NPPukFHPbTYYcd5ndtG8nOnTvjoIMO2u32li1bxubN\nmwtYRCUQ5gY0ePDgeO211+K4444rekqTc9FFF8VFF10Uffv2je7du0fEh88xP/vsszFt2rSC11W2\nK664Iq655po4++yzo3PnzlFdvevrDh1++OEFLat8f/EXfxF33nlnXHjhhXW3bd68OX74wx/6OcFe\nuSp7P/3yl7+se3vLli0xZ86cGDRo0B4fxZ177rmf5LQmZ+nSpTFnzpxYs2ZNbN++Pbp27RojRozw\nA24/HXXUUbvdVlVVFaVSKaqqqlwbsR+WLl0a559/fnzwwQfxzjvvxBe/+MVYt25dHHTQQXHHHXfE\nEUccUfREEhLm/VTui1tUVVXFE0880chroP729Rx9586dP6ElTdPWrVvjySefjDVr1kTr1q2jW7du\nMWDAgN1ebAQ+IsxUhLVr18b06dNj1apVsW3btt3u9ys9QFPhOeYGVCqVYvr06dGrV6/o0aNHREQ8\n+uijsXbt2hg3btxuz91Rvosvvjh27NgRffr0cXFdAxg0aFA89dRTERHRr1+/uiuG98TLRtaPY8v+\nEuYG9KMf/Sgef/zxOOGEE+puO/DAA+O2226LjRs3xqRJkwpcV9lWrFgRzzzzjBdkaCCXXnpp3dtX\nXnllRES8//770bJly93+tCb1s7dj26JFi3jrrbeiQ4cOjvF+Wr58eTz11FPRrFmzGDJkSJP7zQyn\nshvQgAEDYs6cOdGxY8ddbn/zzTdj1KhR8fTTTxe0rPJdeOGF8bd/+7dxzDHHFD2lydm0aVNce+21\nMW/evKiqqoolS5bE//zP/8Qll1wSP/7xj6NDhw5FT6xY69evjyuvvDJeeOGFKJVKUSqVonnz5jFw\n4MCYMmWKY/sxLFiwICZMmBCHHXZY1NbWxvr16+Puu++Onj17Fj2twQhzAzrhhBPit7/97W6P6jZt\n2hSDBw+ORYsWFbSs8m3YsCHGjRsXRx99dHTs2HG304PORnx8l112WdTU1MTFF18co0ePjpdffjm2\nbt0aP/jBD6KmpiZuvfXWoidWrLFjx0aLFi3i29/+dnTt2jVKpVKsWrUq7rnnniiVSnH33XcXPbHi\njB49Ok477bQYM2ZMRETMmDEjHnvssZgxY0bByxqO8ykNaMCAAXH11VfHhAkTonPnznV/M/iOO+6I\nQYMGFT2vol1zzTWxYcOGaNu2bbz11lu73PfnnsNj3373u9/F/Pnzo3379nXHsnXr1jF58uQYMmRI\nwesq25IlS+Lpp5/e5W8xd+vWLXr16hVf/epXC1xWuZYtWxbf+MY36v571KhRcfvttxe4qOEJcwOa\nMmVKXHPNNXHWWWfVnbaqrq6OIUOGxLXXXlv0vIr2/PPPxyOPPOJXdxpB8+bN9/gSstu3b9/jFfCU\nr2vXrlFTU7NLmCM+fM0D/y9/PNu3b9/lAtADDjhgj398pZIJ835avnx53StRbdy4MSZOnFh3BfZH\nzxK0b98+/vu//9srKO2HI444Ilq1alX0jCapZ8+eceONN9b9kYWIiNWrV8d1110X/fv3L3BZZVq2\nbFnd2+PGjYuJEyfGOeecE927d4+qqqpYsWJFzJw5M7773e8WuJLMPMe8n4477rh4+eWXI+LDV1Da\n02lVr6C0/x566KG477774rTTTouDDz54t189GzhwYEHLKt+GDRviO9/5TixdujR27twZrVu3jm3b\ntsUJJ5wQN998824XM/LnffRzYF8/Wv1M+HiOOeaYmDx58i7Hd9q0abvdVsmvtCjM+2n9+vXRqVOn\niPAKSo1pTy8b+RE/4BrG4sWLY82aNdGqVavo1q2bMzwfU33+2pmfCfVXzqstVvorLQozACTipagA\nIBFhBoBEhBkAEhFmAEhEmAEgkYp7gZHaDUcUPaFsVZ9/OEobTyt6RtmGdjq+6All+8eXfxzjj7us\n6BlNkmPbeBzbxlNpx3Z+7ey93ucRcyOqavGloic0WV84pmvRE5osx7bxOLaNpykdW2EGgESEGQAS\nEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESY\nASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaA\nRIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgkVRhXrduXRx77LGxbNmyoqcAQCGaFz3g/9e5\nc+dYvHhx0TMAoDCpHjEDwKddqjCvXbs2jjzyyFi6dGnRUwCgEKnCDACfdlWlUqlU9IiPrF27Nr7+\n9a/Hr3/96/jSl760x/cp7VgaVS32fB8AVLpUF3+Vo7TxtEjzL4l9qD74j1G74YiiZ5RtaKfji55Q\ntvm1s+Pk6rOKntEkObaNx7FtPJV2bOfXzt7rfU5lA0AiwgwAiQgzACQizACQSKqLv7p06RKvv/56\n0TMAoDAeMQNAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCI\nMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIM\nAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIs2LHlBfX75k\nQtETyrZwVmXt7fzMsqIn1Eu7Zz5f9ISyvTtgY9ETgArhETMAJCLMAJCIMANAIsIMAIkIMwAkIswA\nkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0Ai\nwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgz\nACQizACQiDADQCLCDACJCDMAJCLMAJBImjDPnTs3Nm7cWPQMAChUijDv3Lkzpk2bJswAfOrtM8yD\nBg2K+fPn1/33+eefH8OGDav779deey2OPfbYWL58eUyYMCH69u0bJ554YnznO9+JP/3pT3Xvd+SR\nR8ajjz4ao0ePjuOPPz6GDx8er7/+ekRE9O7dO957770488wz4yc/+UlDfn8AUFH2Gea+ffvGokWL\nIuLDR7avvPJKbN26Nd55552IiHj++eejZ8+e8YMf/CDatGkTTz/9dPzmN7+JmpqauPHGG3f5XHfd\ndVfccMMNsWDBgmjXrl3cdtttERHx0EMPRcSHp7O/973vNeg3CACVpPm+3qFfv34xa9asiIh45ZVX\nolu3bnHwwQfHCy+8EEOGDInnn38++vfvH+PGjYuIiJYtW0bLli1j8ODBcd999+3yuU4//fT4whe+\nEBERX/3qV2Pu3Ln1Hnzvzd+K7oceWO+PK8rCWZcXPaHJeuCkfyh6Qvlqix5QP/NrZxc9oclybBtP\nUzm2ZYX5+9//fmzbti1+//vfxwknnBAdOnTYJczjxo2LJUuWxC233BKvvfZabN++PWpra6Njx467\nfK4uXbrUvX3AAQfEtm3b6j14zOXT6/0xRVk46/Lof/bNRc8oW+dLlhU9oWwPnPQPMWrBhUXPKNu7\nAyrn+on5tbPj5Oqzip7RJDm2jafSju2f+0fEPk9lH3LIIdGpU6dYvHhx/P73v4/evXtHz54944UX\nXojVq1fH1q1bo2vXrjF+/Pg45phj4sknn4zFixfHpEmTdv9i1SmuNQOAtMoqZb9+/eL555+PF198\nMXr16hVHH310rFixIp555pno06dPrFy5MjZv3hx/8zd/E23bto2ID097AwD1U3aY/+3f/i06dOgQ\n7dq1i+bNm8dRRx0Vv/zlL6N///7RqVOnqK6ujhdffDG2bNkSs2bNihUrVsS7774bW7du3efnb926\ndURErFy5MmpqavbvOwKAClZWmPv27RsrV66M3r17193Wq1evWLZsWZx00knRsWPHmDRpUkyZMiUG\nDhwYy5cvj1tvvTXat28fp5xyyj4//4EHHhhDhw6NiRMnxs03V85zsgDQ0KpKpVKp6BH1UUkXU7n4\nq/G4+KvxVNpFNJXEsW08lXZs9+viLwDgkyPMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgz\nACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQ\niDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLC\nDACJCDMAJNK86AH11fmSZUVPqJdK2rvhpu5FTyjfg5W19zOtaoqeUC9VrVoVPaFspW3bip4ADcoj\nZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgB\nIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBE\nhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEik0\nzK+88kqMHTs2TjzxxOjXr19MmjQpampqipwEAIUqNMzf+973okePHvGf//mf8dBDD8WSJUvizjvv\nLHISABSqqlQqlYr64ps3b44WLVpEy5YtIyLiuuuuixUrVsQ//dM/7fVjVr+/Lrp+pvMnNREAPlHN\ni/ziCxcujDvuuCNWrFgRH3zwQezcuTN69+79Zz9m4kvXfkLr9t8DJ/1DjFpwYdEzyrbhpu5FTyjb\nMw9eEQPOuKnoGWX7zLz/KnpC2R7bcm+ccsCYomeUrbRtW9ETyja/dnacXH1W0TOapEo7tvNrZ+/1\nvsJOZS9fvjwuueSSOP3002PBggWxePHiGDOmcn4YAEBjKOwR86uvvhrNmjWLcePGRVVVVUR8eDFY\ndbULxQH49Cqsgoceemhs3749lixZEjU1NXH77bfHli1b4q233oqdO3cWNQsAClVYmHv06BHf+ta3\nYty4cTF06NBo0aJF3HDDDfHee+85pQ3Ap1ahF39dddVVcdVVV+1y24IFCwpaAwDF84QuACQizACQ\niDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLC\nDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMA\nJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACTSvOgB9bVoVdeiJ5TvpMrae3DzqqIn\n1EttBe0tbd9e9IR6qaS91cf/n6In1Esl7a196Q9FT/hU8ogZABIRZgBIRJgBIBFhBoBEhBkAEhFm\nAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEg\nEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESE\nGQASEWYASESYASARYQaARIQZABIpO8xr166NI488MpYuXdqYewDgU80jZgBIRJgBIJF6h/kPf/hD\nDB8+PHr27BljxoyJdevWRUTEc889F9/85jejV69eMWDAgLjllluitra27uP+9V//NYYNGxY9evSI\noUOHxiOPPFJ339ixY+PGG2+MESNGxF//9V83wLcFAJWp3mG+77774uc//3n89re/jRYtWsSVV14Z\nGzZsiAkTJsSoUaPiueeei+nTp8evf/3ruP/++yMi4vHHH4+f/vSn8cMf/jAWLVoUV111VUyaNCmW\nL19e93kffvjhmDJlSkyfPr3BvjkAqDRVpVKpVM47rl27Nr7+9a/Hj370o/irv/qriIh4+umn44IL\nLoiLLroonnjiiZg7d27d+//zP/9zzJs3L2bNmhXjx4+P7t27x5VXXll3/4UXXhhHHHFEXHbZZTF2\n7Nho27Zt/OxnP9vnjtc3vRVHtj+ovt8nAFSE5vX9gMMPP7zu7a5du0apVIpnn302Xn311Tj22GPr\n7iuVSnHggQdGRMTq1avjP/7jP+Lee+/d5f42bdrU/XenTp3K+vqn/vud9Z1cmDdGT44vzryh6Bll\nO3huy6InlG3B7MvjpLNuLnpG2T4797miJ5Rt/s774+Rm3yh6Rtmqexxd9ISyPbpoagztNaXoGWWr\nfekPRU8o2/za2XFy9VlFzyjb/NrZe72v3mGurv7fs98fPdju3LlztGzZMu666649fkzr1q3jkksu\nifHjx+99SPN6TwGAJqfezzGvWLGi7u3Vq1dHs2bN4qijjoo//vGPu1zstXHjxti6dWtEfPjI+vXX\nX9/l86xfv36X9wcAPkaYZ86cGX/605+ipqYm7rnnnhg4cGCMGDEiampq4rbbbostW7bE+vXr44IL\nLohf/OIXERExevToePTRR+Pxxx+PDz74IBYtWhQjRoyIZ599tsG/IQCoZPUO8+jRo+Pb3/52fOUr\nX4kPPvgg/v7v/z7atWsXP//5z+N3v/td9O3bN84+++w48cQT47vf/W5ERPTv3z8mT54c06ZNi169\nesXkyZPjiiuuiP79+zf4NwQAlazsJ3a7dOlSdzp62LBhu93fp0+fmDNnzl4//pxzzolzzjlnj/fN\nmDGj3BkA0KR55S8ASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESE\nGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYA\nSESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASKR50QPq\nq82CA4qeUL7RlbW31Ky26An1UmpWVfSEslW3alX0hHqppL3V724uekK9VNLezSP6FD2hXrZU2N69\n8YgZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIR\nZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgB\nIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIpHkRX3Tw4MHx\n5ptvRnX17v8umDx5cowePbqAVQBQvELCHBFx9dVXx5gxY4r68gCQklPZAJCIMANAIlWlUqn0SX/R\nP/cc80svvRTNmjXb68cuW/92HN7pwMacBwCFqbjnmEdOm9EIaxrHf912afS46JaiZ5TtM2/VFj2h\nbAvvuyz6f/PHRc8oW9tfvVT0hLI9+v6MGPqZsUXPKFv1IR2LnlC2f19+c5za/fKiZ5Tt//aonGP7\nzNwrYsCZNxU9o2zPzL1ir/c5lQ0AiQgzACQizACQSGHPMU+bNi1uvPHG3W4fOHBg3H777QUsAoDi\nFRLm3/zmN0V8WQBIz6lsAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFh\nBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkA\nEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEik\nqlQqlYoeAQB8yCNmAEhEmAEgEWEGgESEGQASEWYASESYASCR/we3F9fK+8uDiAAAAABJRU5ErkJg\ngg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "first head of last state dec_enc_attns\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeYAAAH2CAYAAAClRS9UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGzFJREFUeJzt3XmMVfX5+PFnhkVMVUhURFDQuqci\nmwi0KEhQoliCilWq2GLdU/eKgLF8sSrS2liXYvxqDVatC6KmVaOi4lYtCmgEKyIIshUXKppB9rm/\nP4zz/RFQBpnhPHd8vf4azp2Z+9xPyH3PPefccytKpVIpAIAUKoseAAD4P8IMAIkIMwAkIswAkIgw\nA0AiwgwAiQgzACQizACQiDBTNr744ot46KGH4qabbqrZNn/+/OIGAqgHwkxZeO2116J3795x7733\nxp133hkREYsXL47jjz8+XnjhhWKHA6hDwkxZ+MMf/hAjRoyIv//971FRUREREW3atIkbbrhhg1fQ\nAOVOmCkLH3zwQZxwwgkRETVhjog48sgj7c4GGhRhpiy0bNkyFi1atNH2N998M3bccccCJgKoH42L\nHgBqY8CAAXH22WfH6aefHtXV1fHUU0/FrFmz4v7774/TTz+96PEA6kyFj32kHJRKpbj77rvj4Ycf\njgULFkSzZs2ibdu2MXjw4DjxxBOLHg+gzggzZeHDDz+Mdu3abbR9zZo1MWPGjOjSpUsBUwHUPceY\nKQsDBgzY5PYvv/wyzjzzzG08DUD9cYyZ1B566KF48MEHY+3atTFo0KCNbv/000+jRYsWBUwGUD+E\nmdSOOeaYaN68eVx66aXRu3fvjW7fbrvtom/fvtt+MIB64hgzZeGJJ56I/v37Fz0GQL0TZsrGW2+9\nFXPmzInVq1dvdNupp55awEQAdU+YKQvXXntt3HPPPbHzzjvHdtttt8FtFRUV8dxzzxU0GUDdEmbK\nQqdOneLWW2+Nn/zkJ0WPAlCvvF2KsrDjjjtG165dix4DoN4JM2XhwgsvjLvuuivs4AEaOruySevE\nE0/c4JOkFi5cGI0aNYrdd999g+0REQ8//PC2Hg+gXngfM2kdeeSRRY8AsM15xUxZWb9+fTRq1Cgi\nIlauXBnbb799wRMB1C3HmCkL8+fPj5/+9KcxadKkmm33339/HHfccfHhhx8WOBlA3RJmysLo0aPj\nsMMOix49etRsO+mkk+Lwww+P0aNHFzgZQN2yK5uy0KVLl5gyZUo0brzhaRFr166N7t27x7Rp0wqa\nDL7ZokWLYo899ih6DMqMV8yUhebNm8ecOXM22j5jxozYYYcdCpgINm/AgAGxfv36osegzDgrm7Jw\n+umnx9ChQ+PYY4+NPfbYI6qrq2PevHnx1FNPxeWXX170eLBJp556atx8881x1lln+QOSWrMrm7Lx\n7LPPxiOPPBILFy6MioqK2HPPPePEE0+MPn36FD0abNIxxxwTn376aaxYsSJ22GGHmncUfO21114r\naDIyE2aAevLoo49+6+3HH3/8NpqEciLMlI1HHnkknnzyyVi8eHFUVFRE27Zt48QTT4yjjjqq6NFg\ns9auXRtNmjQpegzKgGPM9WD9+vWxZs2ajba7GMZ3N27cuLj77rvj2GOPjZ49e0ZExAcffBDDhw+P\nFStWxMCBAwuesHx99NFHMX78+Jg7d26sWrVqo9v/+te/FjBVw7BmzZr485//HBMnTozPP/88ZsyY\nEVVVVXHNNdfEVVddFT/4wQ+KHpGEhLkOvfbaazF69OhYsGDBJj9s4d133y1gqoZhwoQJcfvtt0fH\njh032D5gwIAYPXq0MG+FSy65JJYvXx7dunWLZs2aFT1Og3LttdfGO++8E7/97W/jN7/5TUREVFdX\nx2effRbXXXddXHvttQVPSEbCXIdGjBgRPXr0iJEjR3qCq2PLly+P9u3bb7S9U6dOsXjx4gImajje\nfffdmDx5crRo0aLoURqcZ555Jh599NFo1apVzQev7LTTTjFmzJgYMGBAwdORlTDXoc8//zxGjx4d\nTZs2LXqUBmevvfaK5557Lo4++ugNtk+ePNkFHLbSXnvt5b229WT9+vWx6667brS9adOmsWLFigIm\nohwIcx3q06dPzJo1Kw455JCiR2lwLrjggrjggguiW7dusc8++0TEV8eYp0yZEmPGjCl4uvJ2+eWX\nx5VXXhknn3xytGnTJiorN7zu0L777lvQZOXvRz/6Udxxxx1x7rnn1mxbsWJFXH/99Z4n+EbOyt5K\n9913X83XK1eujIkTJ0bv3r03+Sru1FNP3ZajNTizZ8+OiRMnxsKFC2PNmjXRtm3bGDhwoCe4rXTg\ngQdutK2ioiJKpVJUVFQ4N2IrzJ49O84888xYt25dfPbZZ/HDH/4wFi9eHLvuumuMGzcu9ttvv6JH\nJCFh3kq1vbhFRUVFPPfcc/U8DWy5zR2jb9OmzTaapGFatWpVTJ48ORYuXBjNmjWLdu3aRc+ePTe6\n2Ah8TZgpC4sWLYrx48fHhx9+GKtXr97odm/pARoKx5jrUKlUivHjx0fnzp2jQ4cOERHx9NNPx6JF\ni2Lo0KEbHbuj9i688MJYu3ZtHHbYYU6uqwO9e/eOF154ISIiunfvXnPG8Ka4bOSWsbZsLWGuQ7//\n/e/j2WefjUMPPbRm2y677BK33HJLLFu2LIYNG1bgdOVt3rx58corr7ggQx255JJLar6+4oorIiLi\nyy+/jKZNm2700ZpsmW9a2yZNmsQnn3wSLVu2tMZbae7cufHCCy9Eo0aNom/fvg3unRl2Zdehnj17\nxsSJE2O33XbbYPtHH30UgwYNipdffrmgycrfueeeG7/+9a/j4IMPLnqUBmf58uXxu9/9Lp566qmo\nqKiImTNnxn//+9+46KKL4o9//GO0bNmy6BHL1pIlS+KKK66IadOmRalUilKpFI0bN45evXrFqFGj\nrO138Oqrr8Y555wTe+21V1RXV8eSJUvirrvuik6dOhU9Wp0R5jp06KGHxosvvrjRq7rly5dHnz59\nYvr06QVNVv6WLl0aQ4cOjYMOOih22223jXYP2hvx3V122WVRVVUVF154YQwePDjefvvtWLVqVVx9\n9dVRVVUVN998c9Ejlq0hQ4ZEkyZN4owzzoi2bdtGqVSKDz/8MO6+++4olUpx1113FT1i2Rk8eHD0\n798/TjvttIiIuOeee+KZZ56Je+65p+DJ6o79KXWoZ8+eMWLEiDjnnHOiTZs2NZ8ZPG7cuOjdu3fR\n45W1K6+8MpYuXRo77bRTfPLJJxvc9m3H8Ni8l156KSZNmhQtWrSoWctmzZrFyJEjo2/fvgVPV95m\nzpwZL7/88gafxdyuXbvo3LlzHHHEEQVOVr7mzJkTP/vZz2r+PWjQoLj11lsLnKjuCXMdGjVqVFx5\n5ZVx0kkn1ey2qqysjL59+8bvfve7oscra1OnTo0nn3zSW3fqQePGjTd5Cdk1a9Zs8gx4aq9t27ZR\nVVW1QZgjvrrmgf/L382aNWs2OAF0++233+SHr5QzYd5Kc+fOrbkS1bJly+LSSy+tOQP766MELVq0\niP/85z+uoLQV9ttvv9huu+2KHqNB6tSpU4wdO7bmQxYiIhYsWBDXXHNN9OjRo8DJytOcOXNqvh46\ndGhceuml8fOf/zz22WefqKioiHnz5sX9998f559/foFTkpljzFvpkEMOibfffjsivrqC0qZ2q7qC\n0tZ7/PHH44EHHoj+/ftHq1atNnrrWa9evQqarPwtXbo0zjvvvJg9e3asX78+mjVrFqtXr45DDz00\nbrjhho1OZuTbff08sLmnVs8J383BBx8cI0eO3GB9x4wZs9G2cr7SojBvpSVLlkTr1q0jwhWU6tOm\nLhv5NU9wdWPGjBmxcOHC2G677aJdu3b28HxHW/JpZ54TtlxtrrZY7ldaFGYASMSlqAAgEWEGgESE\nGQASEWYASESYASCRsrvASPXS/YoeodYqdn4iSsv6Fz1Gg2Rt64+1rT/ltrb9WncseoRa+9+3/xhn\nH3JZ0WPU2qTqCd94m1fM9aiiyf5Fj9BgWdv6Y23rj7WtP3sf3LboEeqMMANAIsIMAIkIMwAkIswA\nkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0Ai\nwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgz\nACQizACQiDADQCLCDACJCDMAJCLMAJBIqjAvXrw42rdvH3PmzCl6FAAoROOiB/j/tWnTJmbMmFH0\nGABQmFSvmAHg+y5VmBctWhQHHHBAzJ49u+hRAKAQqcIMAN93qY4x10bFzk9ERZP9ix6j1ipbvV/0\nCA2Wta0/1rb+lNPaTqoueoItM6l6QtEj1ImyC3NpWf8oFT1ELVW2ej+ql+5X9BgNkrWtP9a2/pTb\n2vZr3bHoEWptUvWEOKrypKLHqLVv+yPCrmwASESYASARYQaARIQZABJJdfLXHnvsEe+9917RYwBA\nYbxiBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBE\nhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFm\nAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEg\nEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESE\nGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYA\nSESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgkTRhfuSRR2LZsmVFjwEAhUoR\n5vXr18eYMWOEGYDvvc2GuXfv3jFp0qSaf5955plx7LHH1vx71qxZ0b59+5g7d26cc8450a1bt+ja\ntWucd9558fHHH9d83wEHHBBPP/10DB48ODp27BgDBgyI9957LyIiunTpEl988UWccMIJ8ac//aku\nHx8AlJXNhrlbt24xffr0iPjqle0777wTq1atis8++ywiIqZOnRqdOnWKq6++Onbcccd4+eWX4/nn\nn4+qqqoYO3bsBr/rzjvvjOuuuy5effXVaN68edxyyy0REfH4449HxFe7sy+++OI6fYAAUE4ab+4b\nunfvHg8++GBERLzzzjvRrl27aNWqVUybNi369u0bU6dOjR49esTQoUMjIqJp06bRtGnT6NOnTzzw\nwAMb/K7jjjsu9t5774iIOOKII+KRRx7Z4oErdn4iKprsv8U/V5TKVu8XPUKDZW3rj7WtP+W0tpOq\ni55gy0yqnlD0CHWiVmG+6qqrYvXq1fHGG2/EoYceGi1bttwgzEOHDo2ZM2fGjTfeGLNmzYo1a9ZE\ndXV17Lbbbhv8rj322KPm6+233z5Wr169xQOXlvWP0hb/VDEqW70f1Uv3K3qMBsna1h9rW3/KbW37\nte5Y9Ai1Nql6QhxVeVLRY9Tat/0Rsdld2bvvvnu0bt06ZsyYEW+88UZ06dIlOnXqFNOmTYsFCxbE\nqlWrom3btnH22WfHwQcfHJMnT44ZM2bEsGHDNr6zyhTnmgFAWrUqZffu3WPq1Knx5ptvRufOneOg\ngw6KefPmxSuvvBKHHXZYzJ8/P1asWBG/+tWvYqeddoqIr3Z7AwBbptZhfuyxx6Jly5bRvHnzaNy4\ncRx44IFx3333RY8ePaJ169ZRWVkZb775ZqxcuTIefPDBmDdvXnz++eexatWqzf7+Zs2aRUTE/Pnz\no6qqauseEQCUsVqFuVu3bjF//vzo0qVLzbbOnTvHnDlz4sc//nHstttuMWzYsBg1alT06tUr5s6d\nGzfffHO0aNEijj766M3+/l122SX69esXl156adxwww3f/dEAQJmrKJVK5XIuVUREWZ04UW4nepQT\na1t/rG39Kbe1dfJX/dmqk78AgG1HmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgB\nIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBE\nhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFm\nAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEg\nEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESE\nGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYA\nSESYASCRQsP8zjvvxJAhQ6Jr167RvXv3GDZsWFRVVRU5EgAUqtAwX3zxxdGhQ4f417/+FY8//njM\nnDkz7rjjjiJHAoBCVZRKpVJRd75ixYpo0qRJNG3aNCIirrnmmpg3b1785S9/+cafKa2dHRVN9t9W\nIwLANtW4yDt/7bXXYty4cTFv3rxYt25drF+/Prp06fKtP1Na1j8K+0tiC1W2ej+ql+5X9BgNkrWt\nP9a2/pTb2vZr3bHoEWptUvWEOKrypKLHqLVJ1RO+8bbCdmXPnTs3LrroojjuuOPi1VdfjRkzZsRp\np51W1DgAkEJhr5jffffdaNSoUQwdOjQqKioi4quTwSornSgOwPdXYRXcc889Y82aNTFz5syoqqqK\nW2+9NVauXBmffPJJrF+/vqixAKBQhYW5Q4cO8ctf/jKGDh0a/fr1iyZNmsR1110XX3zxhV3aAHxv\nFXry1/Dhw2P48OEbbHv11VcLmgYAiueALgAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJ\nCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLM\nAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANA\nIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkI\nMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswA\nkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0Ai\ntQ7zokWL4oADDojZs2fX5zwA8L3mFTMAJCLMAJDIFof53//+dwwYMCA6deoUp512WixevDgiIl5/\n/fU45ZRTonPnztGzZ8+48cYbo7q6uubn/va3v8Wxxx4bHTp0iH79+sWTTz5Zc9uQIUNi7NixMXDg\nwPjFL35RBw8LAMrTFof5gQceiNtuuy1efPHFaNKkSVxxxRWxdOnSOOecc2LQoEHx+uuvx/jx4+Mf\n//hHPPTQQxER8eyzz8ZNN90U119/fUyfPj2GDx8ew4YNi7lz59b83ieeeCJGjRoV48ePr7MHBwDl\npvGW/sDgwYOjTZs2ERFxxhlnxFlnnRUTJ06MvffeOwYNGhQREfvuu28MGTIkHn300TjllFPioYce\nihNOOCEOOeSQiIg48sgjo2fPnvHYY4/FZZddFhER7du3j06dOm32/it2fiIqmuy/pWMXprLV+0WP\n0GBZ2/pjbetPOa3tpOrNf08mk6onFD1CndjiMO+77741X7dt2zZKpVJMmTIl3n333Wjfvn3NbaVS\nKXbZZZeIiFiwYEH885//jHvvvXeD23fccceaf7du3bpW919a1j9KWzp0QSpbvR/VS/creowGydrW\nH2tbf8ptbfu17lj0CLU2qXpCHFV5UtFj1Nq3/RGxxWGurPy/vd+l0leJbNOmTTRt2jTuvPPOTf5M\ns2bN4qKLLoqzzz77mwdpvMWjAECDs8XHmOfNm1fz9YIFC6JRo0Zx4IEHxvvvv7/ByV7Lli2LVatW\nRcRXr6zfe++9DX7PkiVLNvh+AOA7hPn++++Pjz/+OKqqquLuu++OXr16xcCBA6OqqipuueWWWLly\nZSxZsiTOOuusuP322yPiq+PSTz/9dDz77LOxbt26mD59egwcODCmTJlS5w8IAMrZFod58ODBccYZ\nZ8Thhx8e69ati//5n/+J5s2bx2233RYvvfRSdOvWLU4++eTo2rVrnH/++RER0aNHjxg5cmSMGTMm\nOnfuHCNHjozLL788evToUecPCADKWUXp6wPFZaKcTpwotxM9yom1rT/Wtv6U29o6+av+fNvJX678\nBQCJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMA\nJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCI\nMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIM\nAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAk\nIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgw\nA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIo2LuNM+ffrERx99FJWVG/9dMHLk\nyBg8eHABUwFA8QoJc0TEiBEj4rTTTivq7gEgJbuyASARYQaARCpKpVJpW9/ptx1jfuutt6JRo0bf\n+LOltbOjosn+9TkeABSm7I4xl5b1j23+l8R3VNnq/aheul/RYzRI1rb+WNv6U25r2691x6JHqLVJ\n1RPiqMqTih6j1iZVT/jG2+zKBoBEhBkAEhFmAEiksGPMY8aMibFjx260vVevXnHrrbcWMBEAFK+Q\nMD///PNF3C0ApGdXNgAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANA\nIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkI\nMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkUlEq\nlUpFDwEAfMUrZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgkf8HhgybVjMLh/kAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode, Derek Miller @dmmiller612\n", - " Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch\n", - " https://github.com/JayParks/transformer\n", - "'''\n", - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode, Derek Miller @dmmiller612\n", - " Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch\n", - " https://github.com/JayParks/transformer\n", - "'''\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from torch.autograd import Variable\n", - "import matplotlib.pyplot as plt\n", - "\n", - "dtype = torch.FloatTensor\n", - "# S: Symbol that shows starting of decoding input\n", - "# E: Symbol that shows starting of decoding output\n", - "# P: Symbol that will fill in blank sequence if current batch data size is short than time steps\n", - "sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E']\n", - "\n", - "# Transformer Parameters\n", - "# Padding Should be Zero index\n", - "src_vocab = {'P' : 0, 'ich' : 1, 'mochte' : 2, 'ein' : 3, 'bier' : 4}\n", - "src_vocab_size = len(src_vocab)\n", - "\n", - "tgt_vocab = {'P' : 0, 'i' : 1, 'want' : 2, 'a' : 3, 'beer' : 4, 'S' : 5, 'E' : 6}\n", - "number_dict = {i: w for i, w in enumerate(tgt_vocab)}\n", - "tgt_vocab_size = len(tgt_vocab)\n", - "\n", - "src_len = 5\n", - "tgt_len = 5\n", - "\n", - "d_model = 512 # Embedding Size\n", - "d_ff = 2048 # FeedForward dimension\n", - "d_k = d_v = 64 # dimension of K(=Q), V\n", - "n_layers = 6 # number of Encoder of Decoder Layer\n", - "n_heads = 8 # number of heads in Multi-Head Attention\n", - "\n", - "def make_batch(sentences):\n", - " input_batch = [[src_vocab[n] for n in sentences[0].split()]]\n", - " output_batch = [[tgt_vocab[n] for n in sentences[1].split()]]\n", - " target_batch = [[tgt_vocab[n] for n in sentences[2].split()]]\n", - " return Variable(torch.LongTensor(input_batch)), Variable(torch.LongTensor(output_batch)), Variable(torch.LongTensor(target_batch))\n", - "\n", - "def get_sinusoid_encoding_table(n_position, d_model):\n", - " def cal_angle(position, hid_idx):\n", - " return position / np.power(10000, 2 * (hid_idx // 2) / d_model)\n", - " def get_posi_angle_vec(position):\n", - " return [cal_angle(position, hid_j) for hid_j in range(d_model)]\n", - "\n", - " sinusoid_table = np.array([get_posi_angle_vec(pos_i) for pos_i in range(n_position)])\n", - " sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # dim 2i\n", - " sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # dim 2i+1\n", - " return torch.FloatTensor(sinusoid_table)\n", - "\n", - "def get_attn_pad_mask(seq_q, seq_k):\n", - " # print(seq_q)\n", - " batch_size, len_q = seq_q.size()\n", - " batch_size, len_k = seq_k.size()\n", - " # eq(zero) is PAD token\n", - " pad_attn_mask = seq_k.data.eq(0).unsqueeze(1) # batch_size x 1 x len_k(=len_q), one is masking\n", - " return pad_attn_mask.expand(batch_size, len_q, len_k) # batch_size x len_q x len_k\n", - "\n", - "def get_attn_subsequent_mask(seq):\n", - " attn_shape = [seq.size(0), seq.size(1), seq.size(1)]\n", - " subsequent_mask = np.triu(np.ones(attn_shape), k=1)\n", - " subsequent_mask = torch.from_numpy(subsequent_mask).byte()\n", - " return subsequent_mask\n", - "\n", - "class ScaledDotProductAttention(nn.Module):\n", - " def __init__(self):\n", - " super(ScaledDotProductAttention, self).__init__()\n", - "\n", - " def forward(self, Q, K, V, attn_mask):\n", - " scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k) # scores : [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", - " scores.masked_fill_(attn_mask, -1e9) # Fills elements of self tensor with value where mask is one.\n", - " attn = nn.Softmax(dim=-1)(scores)\n", - " context = torch.matmul(attn, V)\n", - " return context, attn\n", - "\n", - "class MultiHeadAttention(nn.Module):\n", - " def __init__(self):\n", - " super(MultiHeadAttention, self).__init__()\n", - " self.W_Q = nn.Linear(d_model, d_k * n_heads)\n", - " self.W_K = nn.Linear(d_model, d_k * n_heads)\n", - " self.W_V = nn.Linear(d_model, d_v * n_heads)\n", - " self.linear = nn.Linear(n_heads * d_v, d_model)\n", - " self.layer_norm = nn.LayerNorm(d_model)\n", - "\n", - " def forward(self, Q, K, V, attn_mask):\n", - " # q: [batch_size x len_q x d_model], k: [batch_size x len_k x d_model], v: [batch_size x len_k x d_model]\n", - " residual, batch_size = Q, Q.size(0)\n", - " # (B, S, D) -proj-> (B, S, D) -split-> (B, S, H, W) -trans-> (B, H, S, W)\n", - " q_s = self.W_Q(Q).view(batch_size, -1, n_heads, d_k).transpose(1,2) # q_s: [batch_size x n_heads x len_q x d_k]\n", - " k_s = self.W_K(K).view(batch_size, -1, n_heads, d_k).transpose(1,2) # k_s: [batch_size x n_heads x len_k x d_k]\n", - " v_s = self.W_V(V).view(batch_size, -1, n_heads, d_v).transpose(1,2) # v_s: [batch_size x n_heads x len_k x d_v]\n", - "\n", - " attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1, 1) # attn_mask : [batch_size x n_heads x len_q x len_k]\n", - "\n", - " # context: [batch_size x n_heads x len_q x d_v], attn: [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", - " context, attn = ScaledDotProductAttention()(q_s, k_s, v_s, attn_mask)\n", - " context = context.transpose(1, 2).contiguous().view(batch_size, -1, n_heads * d_v) # context: [batch_size x len_q x n_heads * d_v]\n", - " output = self.linear(context)\n", - " return self.layer_norm(output + residual), attn # output: [batch_size x len_q x d_model]\n", - "\n", - "class PoswiseFeedForwardNet(nn.Module):\n", - " def __init__(self):\n", - " super(PoswiseFeedForwardNet, self).__init__()\n", - " self.conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_ff, kernel_size=1)\n", - " self.conv2 = nn.Conv1d(in_channels=d_ff, out_channels=d_model, kernel_size=1)\n", - " self.layer_norm = nn.LayerNorm(d_model)\n", - "\n", - " def forward(self, inputs):\n", - " residual = inputs # inputs : [batch_size, len_q, d_model]\n", - " output = nn.ReLU()(self.conv1(inputs.transpose(1, 2)))\n", - " output = self.conv2(output).transpose(1, 2)\n", - " return self.layer_norm(output + residual)\n", - "\n", - "class EncoderLayer(nn.Module):\n", - " def __init__(self):\n", - " super(EncoderLayer, self).__init__()\n", - " self.enc_self_attn = MultiHeadAttention()\n", - " self.pos_ffn = PoswiseFeedForwardNet()\n", - "\n", - " def forward(self, enc_inputs, enc_self_attn_mask):\n", - " enc_outputs, attn = self.enc_self_attn(enc_inputs, enc_inputs, enc_inputs, enc_self_attn_mask) # enc_inputs to same Q,K,V\n", - " enc_outputs = self.pos_ffn(enc_outputs) # enc_outputs: [batch_size x len_q x d_model]\n", - " return enc_outputs, attn\n", - "\n", - "class DecoderLayer(nn.Module):\n", - " def __init__(self):\n", - " super(DecoderLayer, self).__init__()\n", - " self.dec_self_attn = MultiHeadAttention()\n", - " self.dec_enc_attn = MultiHeadAttention()\n", - " self.pos_ffn = PoswiseFeedForwardNet()\n", - "\n", - " def forward(self, dec_inputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask):\n", - " dec_outputs, dec_self_attn = self.dec_self_attn(dec_inputs, dec_inputs, dec_inputs, dec_self_attn_mask)\n", - " dec_outputs, dec_enc_attn = self.dec_enc_attn(dec_outputs, enc_outputs, enc_outputs, dec_enc_attn_mask)\n", - " dec_outputs = self.pos_ffn(dec_outputs)\n", - " return dec_outputs, dec_self_attn, dec_enc_attn\n", - "\n", - "class Encoder(nn.Module):\n", - " def __init__(self):\n", - " super(Encoder, self).__init__()\n", - " self.src_emb = nn.Embedding(src_vocab_size, d_model)\n", - " self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(src_len+1, d_model),freeze=True)\n", - " self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])\n", - "\n", - " def forward(self, enc_inputs): # enc_inputs : [batch_size x source_len]\n", - " enc_outputs = self.src_emb(enc_inputs) + self.pos_emb(torch.LongTensor([[1,2,3,4,0]]))\n", - " enc_self_attn_mask = get_attn_pad_mask(enc_inputs, enc_inputs)\n", - " enc_self_attns = []\n", - " for layer in self.layers:\n", - " enc_outputs, enc_self_attn = layer(enc_outputs, enc_self_attn_mask)\n", - " enc_self_attns.append(enc_self_attn)\n", - " return enc_outputs, enc_self_attns\n", - "\n", - "class Decoder(nn.Module):\n", - " def __init__(self):\n", - " super(Decoder, self).__init__()\n", - " self.tgt_emb = nn.Embedding(tgt_vocab_size, d_model)\n", - " self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(tgt_len+1, d_model),freeze=True)\n", - " self.layers = nn.ModuleList([DecoderLayer() for _ in range(n_layers)])\n", - "\n", - " def forward(self, dec_inputs, enc_inputs, enc_outputs): # dec_inputs : [batch_size x target_len]\n", - " dec_outputs = self.tgt_emb(dec_inputs) + self.pos_emb(torch.LongTensor([[5,1,2,3,4]]))\n", - " dec_self_attn_pad_mask = get_attn_pad_mask(dec_inputs, dec_inputs)\n", - " dec_self_attn_subsequent_mask = get_attn_subsequent_mask(dec_inputs)\n", - " dec_self_attn_mask = torch.gt((dec_self_attn_pad_mask + dec_self_attn_subsequent_mask), 0)\n", - "\n", - " dec_enc_attn_mask = get_attn_pad_mask(dec_inputs, enc_inputs)\n", - "\n", - " dec_self_attns, dec_enc_attns = [], []\n", - " for layer in self.layers:\n", - " dec_outputs, dec_self_attn, dec_enc_attn = layer(dec_outputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask)\n", - " dec_self_attns.append(dec_self_attn)\n", - " dec_enc_attns.append(dec_enc_attn)\n", - " return dec_outputs, dec_self_attns, dec_enc_attns\n", - "\n", - "class Transformer(nn.Module):\n", - " def __init__(self):\n", - " super(Transformer, self).__init__()\n", - " self.encoder = Encoder()\n", - " self.decoder = Decoder()\n", - " self.projection = nn.Linear(d_model, tgt_vocab_size, bias=False)\n", - " def forward(self, enc_inputs, dec_inputs):\n", - " enc_outputs, enc_self_attns = self.encoder(enc_inputs)\n", - " dec_outputs, dec_self_attns, dec_enc_attns = self.decoder(dec_inputs, enc_inputs, enc_outputs)\n", - " dec_logits = self.projection(dec_outputs) # dec_logits : [batch_size x src_vocab_size x tgt_vocab_size]\n", - " return dec_logits.view(-1, dec_logits.size(-1)), enc_self_attns, dec_self_attns, dec_enc_attns\n", - "\n", - "def greedy_decoder(model, enc_input, start_symbol):\n", - " \"\"\"\n", - " For simplicity, a Greedy Decoder is Beam search when K=1. This is necessary for inference as we don't know the\n", - " target sequence input. Therefore we try to generate the target input word by word, then feed it into the transformer.\n", - " Starting Reference: http://nlp.seas.harvard.edu/2018/04/03/attention.html#greedy-decoding\n", - " :param model: Transformer Model\n", - " :param enc_input: The encoder input\n", - " :param start_symbol: The start symbol. In this example it is 'S' which corresponds to index 4\n", - " :return: The target input\n", - " \"\"\"\n", - " enc_outputs, enc_self_attns = model.encoder(enc_input)\n", - " dec_input = torch.zeros(1, 5).type_as(enc_input.data)\n", - " next_symbol = start_symbol\n", - " for i in range(0, 5):\n", - " dec_input[0][i] = next_symbol\n", - " dec_outputs, _, _ = model.decoder(dec_input, enc_input, enc_outputs)\n", - " projected = model.projection(dec_outputs)\n", - " prob = projected.squeeze(0).max(dim=-1, keepdim=False)[1]\n", - " next_word = prob.data[i]\n", - " next_symbol = next_word.item()\n", - " return dec_input\n", - "\n", - "def showgraph(attn):\n", - " attn = attn[-1].squeeze(0)[0]\n", - " attn = attn.squeeze(0).data.numpy()\n", - " fig = plt.figure(figsize=(n_heads, n_heads)) # [n_heads, n_heads]\n", - " ax = fig.add_subplot(1, 1, 1)\n", - " ax.matshow(attn, cmap='viridis')\n", - " ax.set_xticklabels(['']+sentences[0].split(), fontdict={'fontsize': 14}, rotation=90)\n", - " ax.set_yticklabels(['']+sentences[2].split(), fontdict={'fontsize': 14})\n", - " plt.show()\n", - "\n", - "model = Transformer()\n", - "\n", - "criterion = nn.CrossEntropyLoss()\n", - "optimizer = optim.Adam(model.parameters(), lr=0.001)\n", - "\n", - "for epoch in range(20):\n", - " optimizer.zero_grad()\n", - " enc_inputs, dec_inputs, target_batch = make_batch(sentences)\n", - " outputs, enc_self_attns, dec_self_attns, dec_enc_attns = model(enc_inputs, dec_inputs)\n", - " loss = criterion(outputs, target_batch.contiguous().view(-1))\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - "# Test\n", - "greedy_dec_input = greedy_decoder(model, enc_inputs, start_symbol=tgt_vocab[\"S\"])\n", - "predict, _, _, _ = model(enc_inputs, greedy_dec_input)\n", - "predict = predict.data.max(1, keepdim=True)[1]\n", - "print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()])\n", - "\n", - "print('first head of last state enc_self_attns')\n", - "showgraph(enc_self_attns)\n", - "\n", - "print('first head of last state dec_self_attns')\n", - "showgraph(dec_self_attns)\n", - "\n", - "print('first head of last state dec_enc_attns')\n", - "showgraph(dec_enc_attns)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "Transformer(Greedy_decoder)-Torch.ipynb", - "provenance": [], - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.9" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "metadata": { - "collapsed": false - }, - "source": [] - } - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/5-1.Transformer/Transformer-Torch.py b/5-1.Transformer/Transformer.py similarity index 100% rename from 5-1.Transformer/Transformer-Torch.py rename to 5-1.Transformer/Transformer.py diff --git a/5-1.Transformer/Transformer_Torch.ipynb b/5-1.Transformer/Transformer_Torch.ipynb deleted file mode 100644 index e481e8e..0000000 --- a/5-1.Transformer/Transformer_Torch.ipynb +++ /dev/null @@ -1,380 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "g831xANXh2HY" - }, - "outputs": [], - "source": [ - "# http://pytorch.org/\n", - "from os.path import exists\n", - "from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag\n", - "platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())\n", - "cuda_output = !ldconfig -p|grep cudart.so|sed -e 's/.*\\.\\([0-9]*\\)\\.\\([0-9]*\\)$/cu\\1\\2/'\n", - "accelerator = cuda_output[0] if exists('/dev/nvidia0') else 'cpu'" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1971 - }, - "colab_type": "code", - "id": "nNemnO18h6PV", - "outputId": "a770e281-5e28-466d-9e04-db845e0ba90e" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 0001 cost = 2.092809\n", - "Epoch: 0002 cost = 0.255108\n", - "Epoch: 0003 cost = 0.273580\n", - "Epoch: 0004 cost = 0.072622\n", - "Epoch: 0005 cost = 0.023565\n", - "Epoch: 0006 cost = 0.034211\n", - "Epoch: 0007 cost = 0.003267\n", - "Epoch: 0008 cost = 0.003564\n", - "Epoch: 0009 cost = 0.024341\n", - "Epoch: 0010 cost = 0.000667\n", - "Epoch: 0011 cost = 0.000384\n", - "Epoch: 0012 cost = 0.000739\n", - "Epoch: 0013 cost = 0.000474\n", - "Epoch: 0014 cost = 0.000306\n", - "Epoch: 0015 cost = 0.003650\n", - "Epoch: 0016 cost = 0.000577\n", - "Epoch: 0017 cost = 0.001067\n", - "Epoch: 0018 cost = 0.000163\n", - "Epoch: 0019 cost = 0.000262\n", - "Epoch: 0020 cost = 0.000920\n", - "ich mochte ein bier P -> ['i', 'want', 'a', 'beer', 'E']\n", - "first head of last state enc_self_attns\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeYAAAH2CAYAAAClRS9UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAG6RJREFUeJzt3XmMVfX98PHPDAxgqkKiIjgKWsUl\nFRFQgRYFCWoES1CxSl1arHvqXhEx1h9uaGtjVYrx0RqsWhdETas+KipuxbobwarICLKJC3XJIOvM\nff4wzu8hqAwyw/nc8fX6azh3Zu7nfkPmfe85555bUSqVSgEApFBZ9AAAwP8SZgBIRJgBIBFhBoBE\nhBkAEhFmAEhEmAEgEWEGgESEmbLxxRdfxD333BPXXnttw7a5c+cWNxBAMxBmysLzzz8fAwcOjNtv\nvz1uvvnmiIhYuHBhHHroofHUU08VOxxAExJmysIf//jHuOCCC+If//hHVFRUREREdXV1XH311Wu8\nggYod8JMWXjvvffisMMOi4hoCHNExP777293NtCiCDNloWPHjrFgwYK1tr/22mux2WabFTARQPNo\nXfQA0BjDhg2Lk046KY477rior6+PRx55JN5+++24884747jjjit6PIAmU+FjHykHpVIpbr311rj3\n3ntj3rx50a5du+jSpUuMHDkyDj/88KLHA2gywkxZeP/996Nr165rbV+5cmXMmDEjevfuXcBUAE3P\nMWbKwrBhw75x+5dffhknnHDCRp4GoPk4xkxq99xzT9x9992xatWqGDFixFq3f/LJJ9GhQ4cCJgNo\nHsJMagcffHC0b98+zjnnnBg4cOBat7dt2zYGDx688QcDaCaOMVMWHnrooRg6dGjRYwA0O2GmbLz+\n+usxe/bsWLFixVq3HX300QVMBND0hJmycPnll8dtt90WW2yxRbRt23aN2yoqKuKJJ54oaDKApiXM\nlIWePXvGhAkT4mc/+1nRowA0K2+Xoixsttlmsffeexc9BkCzE2bKwhlnnBG33HJL2MEDtHR2ZZPW\n4YcfvsYnSc2fPz9atWoVnTt3XmN7RMS99967sccDaBbex0xa+++/f9EjAGx0XjFTVurq6qJVq1YR\nEbFs2bLYZJNNCp4IoGk5xkxZmDt3bvz85z+PqVOnNmy7884745BDDon333+/wMkAmpYwUxbGjRsX\n++yzT/Tr169h2xFHHBH77rtvjBs3rsDJAJqWXdmUhd69e8cLL7wQrVuveVrEqlWrom/fvvHKK68U\nNBl8uwULFsS2225b9BiUGa+YKQvt27eP2bNnr7V9xowZsemmmxYwEazbsGHDoq6urugxKDPOyqYs\nHHfccTFq1KgYMmRIbLvttlFfXx9z5syJRx55JM4777yix4NvdPTRR8d1110XJ554oieQNJpd2ZSN\nxx9/PO67776YP39+VFRUxHbbbReHH354DBo0qOjR4BsdfPDB8cknn8TSpUtj0003bXhHwdeef/75\ngiYjM2EGaCb333//d95+6KGHbqRJKCfCTNm477774uGHH46FCxdGRUVFdOnSJQ4//PA44IADih4N\n1mnVqlVRVVVV9BiUAceYm0FdXV2sXLlyre0uhvH9TZw4MW699dYYMmRI9O/fPyIi3nvvvRgzZkws\nXbo0hg8fXvCE5evDDz+MSZMmRU1NTSxfvnyt2//2t78VMFXLsHLlyvjLX/4SU6ZMic8//zxmzJgR\ntbW1cdlll8VFF10UP/rRj4oekYSEuQk9//zzMW7cuJg3b943ftjCW2+9VcBULcPkyZPjxhtvjD33\n3HON7cOGDYtx48YJ8wY4++yz47PPPos+ffpEu3btih6nRbn88svjzTffjN///vfxu9/9LiIi6uvr\n49NPP40rrrgiLr/88oInJCNhbkIXXHBB9OvXL8aOHesPXBP77LPPonv37mtt79mzZyxcuLCAiVqO\nt956K6ZNmxYdOnQoepQW57HHHov7778/OnXq1PDBK5tvvnmMHz8+hg0bVvB0ZCXMTejzzz+PcePG\nRZs2bYoepcXZfvvt44knnogDDzxwje3Tpk1zAYcNtP3223uvbTOpq6uLrbbaaq3tbdq0iaVLlxYw\nEeVAmJvQoEGD4u2334499tij6FFanNNPPz1OP/306NOnT+y4444R8dUx5hdeeCHGjx9f8HTl7bzz\nzosLL7wwjjzyyKiuro7KyjWvO7TTTjsVNFn5+8lPfhI33XRTnHLKKQ3bli5dGldeeaW/E3wrZ2Vv\noDvuuKPh62XLlsWUKVNi4MCB3/gq7uijj96Yo7U4s2bNiilTpsT8+fNj5cqV0aVLlxg+fLg/cBto\n1113XWtbRUVFlEqlqKiocG7EBpg1a1accMIJsXr16vj000/jxz/+cSxcuDC22mqrmDhxYnTr1q3o\nEUlImDdQYy9uUVFREU888UQzTwPrb13H6KurqzfSJC3T8uXLY9q0aTF//vxo165ddO3aNfr377/W\nxUbga8JMWViwYEFMmjQp3n///VixYsVat3tLD9BSOMbchEqlUkyaNCl69eoVPXr0iIiIRx99NBYs\nWBCjRo1a69gdjXfGGWfEqlWrYp999nFyXRMYOHBgPPXUUxER0bdv34Yzhr+Jy0auH2vLhhLmJvSH\nP/whHn/88dhrr70atm255ZZx/fXXx5IlS2L06NEFTlfe5syZE88995wLMjSRs88+u+Hr888/PyIi\nvvzyy2jTps1aH63J+vm2ta2qqoqPP/44OnbsaI03UE1NTTz11FPRqlWrGDx4cIt7Z4Zd2U2of//+\nMWXKlNh6663X2P7hhx/GiBEj4tlnny1osvJ3yimnxG9/+9vYfffdix6lxfnss8/i0ksvjUceeSQq\nKipi5syZ8d///jfOPPPM+NOf/hQdO3YsesSytWjRojj//PPjlVdeiVKpFKVSKVq3bh0DBgyIiy++\n2Np+D9OnT4+TTz45tt9++6ivr49FixbFLbfcEj179ix6tCYjzE1or732iqeffnqtV3WfffZZDBo0\nKF599dWCJit/ixcvjlGjRsVuu+0WW2+99Vq7B+2N+P7OPffcqK2tjTPOOCNGjhwZb7zxRixfvjwu\nueSSqK2tjeuuu67oEcvWscceG1VVVXH88cdHly5dolQqxfvvvx+33nprlEqluOWWW4oeseyMHDky\nhg4dGsccc0xERNx2223x2GOPxW233VbwZE3H/pQm1L9//7jgggvi5JNPjurq6obPDJ44cWIMHDiw\n6PHK2oUXXhiLFy+OzTffPD7++OM1bvuuY3is2zPPPBNTp06NDh06NKxlu3btYuzYsTF48OCCpytv\nM2fOjGeffXaNz2Lu2rVr9OrVK/bbb78CJytfs2fPjl/84hcN/x4xYkRMmDChwImanjA3oYsvvjgu\nvPDCOOKIIxp2W1VWVsbgwYPj0ksvLXq8svbyyy/Hww8/7K07zaB169bfeAnZlStXfuMZ8DRely5d\nora2do0wR3x1zQP/l7+flStXrnEC6CabbPKNH75SzoR5A9XU1DRciWrJkiVxzjnnNJyB/fVRgg4d\nOsQHH3zgCkoboFu3btG2bduix2iRevbsGVdddVXDhyxERMybNy8uu+yy6NevX4GTlafZs2c3fD1q\n1Kg455xz4pe//GXsuOOOUVFREXPmzIk777wzTjvttAKnJDPHmDfQHnvsEW+88UZEfHUFpW/areoK\nShvuwQcfjLvuuiuGDh0anTp1WuutZwMGDChosvK3ePHiOPXUU2PWrFlRV1cX7dq1ixUrVsRee+0V\nV1999VonM/Ldvv47sK4/rf4mfD+77757jB07do31HT9+/FrbyvlKi8K8gRYtWhTbbLNNRLiCUnP6\npstGfs0fuKYxY8aMmD9/frRt2za6du1qD8/3tD6fduZvwvprzNUWy/1Ki8IMAIm4FBUAJCLMAJCI\nMANAIsIMAIkIMwAkUnYXGKlf3K3oERqtYouHorRkaNFjtEjWtvmU29rWleqLHqHRWm/5f2P1JwcX\nPUajDanuVfQIjfZ/3vhTnLTHuUWP0WhT6yd/621eMTejiqqdix6hxbK2zcfaNh9r23x22L1L0SM0\nGWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESE\nGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYA\nSESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgkVRhXrhwYXTv3j1mz55d9CgA\nUIjWRQ/w/6uuro4ZM2YUPQYAFCbVK2YA+KFLFeYFCxbELrvsErNmzSp6FAAoRKowA8APXapjzI1R\nscVDUVG1c9FjNFplp3eLHqHFsrbNp5zWttxeXVR1ril6hEabWl/0BOtnav3kokdoEmUX5tKSoVEq\neohGquz0btQv7lb0GC2StW0+5ba2daXyqUdV55pY9cGORY/RaEOqexU9QqNNrZ8cB1QeUfQYjfZd\nTyLK7ckmALRowgwAiQgzACQizACQSKqTv7bddtt45513ih4DAArjFTMAJCLMAJCIMANAIsIMAIkI\nMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswA\nkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0Ai\nwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgz\nACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQ\niDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLC\nDACJCDMAJCLMAJCIMANAIsIMAImkCfN9990XS5YsKXoMAChUijDX1dXF+PHjhRmAH7x1hnngwIEx\nderUhn+fcMIJMWTIkIZ/v/3229G9e/eoqamJk08+Ofr06RN77713nHrqqfHRRx81fN8uu+wSjz76\naIwcOTL23HPPGDZsWLzzzjsREdG7d+/44osv4rDDDos///nPTfn4AKCsrDPMffr0iVdffTUivnpl\n++abb8by5cvj008/jYiIl19+OXr27BmXXHJJbLbZZvHss8/Gk08+GbW1tXHVVVet8btuvvnmuOKK\nK2L69OnRvn37uP766yMi4sEHH4yIr3Znn3XWWU36AAGgnLRe1zf07ds37r777oiIePPNN6Nr167R\nqVOneOWVV2Lw4MHx8ssvR79+/WLUqFEREdGmTZto06ZNDBo0KO666641ftchhxwSO+ywQ0RE7Lff\nfnHfffet98AVWzwUFVU7r/fPFaWy07tFj9BiWdvmU05rm+J43Hqo6lxT9AiNNrW+6AnWz9T6yUWP\n0CQaFeaLLrooVqxYES+99FLstdde0bFjxzXCPGrUqJg5c2Zcc8018fbbb8fKlSujvr4+tt566zV+\n17bbbtvw9SabbBIrVqxY74FLS4ZGab1/qhiVnd6N+sXdih6jRbK2zafc1rauVD71qOpcE6s+2LHo\nMRptSHWvokdotKn1k+OAyiOKHqPRvutJxDqfbHbu3Dm22WabmDFjRrz00kvRu3fv6NmzZ7zyyisx\nb968WL58eXTp0iVOOumk2H333WPatGkxY8aMGD169Np3Vlluz20BYONqVCn79u0bL7/8crz22mvR\nq1ev2G233WLOnDnx3HPPxT777BNz586NpUuXxm9+85vYfPPNI+Kr3d4AwPppdJgfeOCB6NixY7Rv\n3z5at24du+66a9xxxx3Rr1+/2GabbaKysjJee+21WLZsWdx9990xZ86c+Pzzz2P58uXr/P3t2rWL\niIi5c+dGbW3thj0iAChjjQpznz59Yu7cudG7d++Gbb169YrZs2fHT3/609h6661j9OjRcfHFF8eA\nAQOipqYmrrvuuujQoUMceOCB6/z9W265ZRx00EFxzjnnxNVXX/39Hw0AlLmKUqlULudSRUSU1Ukp\n5XYSTTmxts2n3NbWyV/Nx8lfzWeDTv4CADYeYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhE\nmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEG\ngESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQAS\nEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESY\nASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaA\nRIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIR\nZgBIRJgBIBFhBoBECg3zm2++Gccee2zsvffe0bdv3xg9enTU1tYWORIAFKrQMJ911lnRo0eP+Pe/\n/x0PPvhgzJw5M2666aYiRwKAQlWUSqVSUXe+dOnSqKqqijZt2kRExGWXXRZz5syJv/71r9/6M6VV\ns6KiaueNNSIAbFSti7zz559/PiZOnBhz5syJ1atXR11dXfTu3fs7f6a0ZGgU9kxiPVV2ejfqF3cr\neowWydo2n3Jb27pSfdEjNFpV55pY9cGORY/RaEOqexU9QqNNrZ8cB1QeUfQYjTa1fvK33lbYruya\nmpo488wz45BDDonp06fHjBkz4phjjilqHABIobBXzG+99Va0atUqRo0aFRUVFRHx1clglZVOFAfg\nh6uwCm633XaxcuXKmDlzZtTW1saECRNi2bJl8fHHH0ddXV1RYwFAoQoLc48ePeLXv/51jBo1Kg46\n6KCoqqqKK664Ir744gu7tAH4wSr05K8xY8bEmDFj1tg2ffr0gqYBgOI5oAsAiQgzACQizACQiDAD\nQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJ\nCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLM\nAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANA\nIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkI\nMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswA\nkIgwA0AiwgwAiQgzACQizACQSKPDvGDBgthll11i1qxZzTkPAPygecUMAIkIMwAkst5h/s9//hPD\nhg2Lnj17xjHHHBMLFy6MiIgXX3wxjjrqqOjVq1f0798/rrnmmqivr2/4ub///e8xZMiQ6NGjRxx0\n0EHx8MMPN9x27LHHxlVXXRXDhw+PX/3qV03wsACgPK13mO+666644YYb4umnn46qqqo4//zzY/Hi\nxXHyySfHiBEj4sUXX4xJkybFP//5z7jnnnsiIuLxxx+Pa6+9Nq688sp49dVXY8yYMTF69Oioqalp\n+L0PPfRQXHzxxTFp0qQme3AAUG5ar+8PjBw5MqqrqyMi4vjjj48TTzwxpkyZEjvssEOMGDEiIiJ2\n2mmnOPbYY+P++++Po446Ku6555447LDDYo899oiIiP333z/69+8fDzzwQJx77rkREdG9e/fo2bPn\nOu+/YouHoqJq5/UduzCVnd4teoQWy9o2n3Ja23I7HlfVuWbd35TE1Pp1f08mU+snFz1Ck1jvMO+0\n004NX3fp0iVKpVK88MIL8dZbb0X37t0bbiuVSrHllltGRMS8efPiX//6V9x+++1r3L7ZZps1/Hub\nbbZp1P2XlgyN0voOXZDKTu9G/eJuRY/RIlnb5lNua1tXKp96VHWuiVUf7Fj0GI02pLpX0SM02tT6\nyXFA5RFFj9Fo3/UkYr3DXFn5v89PS6WvElldXR1t2rSJm2+++Rt/pl27dnHmmWfGSSed9O2DtF7v\nUQCgxVnvvUBz5sxp+HrevHnRqlWr2HXXXePdd99d42SvJUuWxPLlyyPiq1fW77zzzhq/Z9GiRWt8\nPwDwPcJ85513xkcffRS1tbVx6623xoABA2L48OFRW1sb119/fSxbtiwWLVoUJ554Ytx4440R8dVx\n6UcffTQef/zxWL16dbz66qsxfPjweOGFF5r8AQFAOVvvMI8cOTKOP/742HfffWP16tXxP//zP9G+\nffu44YYb4plnnok+ffrEkUceGXvvvXecdtppERHRr1+/GDt2bIwfPz569eoVY8eOjfPOOy/69evX\n5A8IAMpZRenrA8VlopxOSim3k2jKibVtPuW2tk7+aj5O/mo+33XyV7m90wAAWjRhBoBEhBkAEhFm\nAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEg\nEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESE\nGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYA\nSESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASAR\nYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZ\nABIRZgBIRJgBIBFhBoBEhBkAEhFmAEikdRF3OmjQoPjwww+jsnLt5wVjx46NkSNHFjAVABSvkDBH\nRFxwwQVxzDHHFHX3AJCSXdkAkIgwA0AiFaVSqbSx7/S7jjG//vrr0apVq2/92dKqWVFRtXNzjgcA\nhSm7Y8ylJUNjoz+T+J4qO70b9Yu7FT1Gi2Rtm0+5rW1dqb7oERqtqnNNrPpgx6LHaLQh1b2KHqHR\nptZPjgMqjyh6jEabWj/5W2+zKxsAEhFmAEhEmAEgkcKOMY8fPz6uuuqqtbYPGDAgJkyYUMBEAFC8\nQsL85JNPFnG3AJCeXdkAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIM\nAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAk\nIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkEhF\nqVQqFT0EAPAVr5gBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgET+Hy6qs27DXv9XAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "first head of last state dec_self_attns\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeYAAAH2CAYAAAClRS9UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHD9JREFUeJzt3XlwVYXd8PFfwmqrwEwVFRBs1apT\nFQFlaVEQUR7RUlSsRdEWq7hM3SsqjvXBqqi101atfX21FqrWFezUZVDAvVh3R7AqBUG24kJFnyAQ\nSO77h2OelwElkcTzu/Hz+Ss5Jzf53TNwvzlLzq0olUqlAABSqCx6AADgfwkzACQizACQiDADQCLC\nDACJCDMAJCLMAJCIMANAIsJM2fjoo4/i7rvvjt/97nd1yxYsWFDcQABNQJgpC88880wMHDgwbrvt\ntrj55psjImLJkiVx+OGHx+OPP17scACNSJgpC7/61a/iwgsvjL/97W9RUVERERGdO3eOa665Zr09\naIByJ8yUhbfeeiuOOOKIiIi6MEdEHHDAAQ5nA82KMFMWOnbsGIsXL95g+csvvxxbbbVVARMBNI2W\nRQ8A9TFs2LAYM2ZMHH/88VFbWxtTp06NN954I+644444/vjjix4PoNFUeNtHykGpVIpJkybFvffe\nGwsXLoy2bdtG165dY+TIkXHkkUcWPR5AoxFmysLbb78d3bp122B5dXV1zJo1K3r16lXAVACNzzlm\nysKwYcM2uvzjjz+OE0888UueBqDpOMdManfffXfcddddsXbt2hgxYsQG699///3o0KFDAZMBNA1h\nJrVDDjkk2rdvH+ecc04MHDhwg/Vt2rSJwYMHf/mDATQR55gpCw8++GAceuihRY8B0OSEmbLxyiuv\nxNy5c2PNmjUbrDv22GMLmAig8QkzZeHyyy+PW2+9Nb7xjW9EmzZt1ltXUVERM2bMKGgygMYlzJSF\nHj16xPXXXx/f+973ih4FoEn5cynKwlZbbRX77rtv0WMANDlhpiycccYZccstt4QDPEBz51A2aR15\n5JHrvZPUokWLokWLFrH99tuvtzwi4t577/2yxwNoEv6OmbQOOOCAokcA+NLZY6as1NTURIsWLSIi\nYtWqVbHFFlsUPBFA43KOmbKwYMGC+P73vx/Tpk2rW3bHHXfEYYcdFm+//XaBkwE0LmGmLIwfPz56\n9+4d/fr1q1t21FFHxX777Rfjx48vcDKAxuVQNmWhV69e8eyzz0bLlutfFrF27dro27dvvPjiiwVN\nBp9t8eLF0aVLl6LHoMzYY6YstG/fPubOnbvB8lmzZsWWW25ZwESwacOGDYuampqix6DMuCqbsnD8\n8cfH6NGjY+jQodGlS5eora2N+fPnx9SpU+O8884rejzYqGOPPTauvfbaOOmkk/wCSb05lE3ZmD59\nekyZMiUWLVoUFRUVscMOO8SRRx4ZgwYNKno02KhDDjkk3n///Vi5cmVsueWWdX9R8KlnnnmmoMnI\nTJgBmsh99933uesPP/zwL2kSyokwUzamTJkSDz30UCxZsiQqKiqia9euceSRR8ZBBx1U9GiwSWvX\nro1WrVoVPQZlwDnmJlBTUxPV1dUbLHczjC/uhhtuiEmTJsXQoUOjf//+ERHx1ltvxQUXXBArV66M\n4cOHFzxh+XrnnXdi4sSJMW/evFi9evUG6//85z8XMFXzUF1dHb///e9j8uTJ8eGHH8asWbOiqqoq\nLrvssrj44ovj61//etEjkpAwN6Jnnnkmxo8fHwsXLtzomy28/vrrBUzVPNxzzz1x4403xt57773e\n8mHDhsX48eOFeTOcffbZsWLFiujTp0+0bdu26HGalcsvvzxee+21+MUvfhE///nPIyKitrY2Pvjg\ng7jiiivi8ssvL3hCMhLmRnThhRdGv379Yty4cV7gGtmKFStizz333GB5jx49YsmSJQVM1Hy8/vrr\n8dhjj0WHDh2KHqXZeeSRR+K+++6L7bbbru6NV9q1axcTJkyIYcOGFTwdWQlzI/rwww9j/Pjx0bp1\n66JHaXZ23HHHmDFjRhx88MHrLX/sscfcwGEz7bjjjv7WtonU1NTENttss8Hy1q1bx8qVKwuYiHIg\nzI1o0KBB8cYbb8Ree+1V9CjNzumnnx6nn3569OnTJ3baaaeI+OQc87PPPhsTJkwoeLrydt5558VF\nF10URx99dHTu3DkqK9e/79DOO+9c0GTl7zvf+U7cdNNNccopp9QtW7lyZVx55ZVeJ/hMrsreTLff\nfnvdx6tWrYrJkyfHwIEDN7oXd+yxx36ZozU7c+bMicmTJ8eiRYuiuro6unbtGsOHD/cCt5l22223\nDZZVVFREqVSKiooK10Zshjlz5sSJJ54Y69atiw8++CC+9a1vxZIlS2KbbbaJG264IXbZZZeiRyQh\nYd5M9b25RUVFRcyYMaOJp4GG29Q5+s6dO39JkzRPq1evjsceeywWLVoUbdu2jW7dukX//v03uNkI\nfEqYKQuLFy+OiRMnxttvvx1r1qzZYL0/6QGaC+eYG1GpVIqJEydGz549o3v37hER8fDDD8fixYtj\n9OjRG5y7o/7OOOOMWLt2bfTu3dvFdY1g4MCB8fjjj0dERN++feuuGN4Yt41sGNuWzSXMjejqq6+O\n6dOnxz777FO3bOutt47rrrsuli9fHmPHji1wuvI2f/78ePrpp92QoZGcffbZdR+ff/75ERHx8ccf\nR+vWrTd4a00a5rO2batWreK9996Ljh072sabad68efH4449HixYtYvDgwc3uLzMcym5E/fv3j8mT\nJ8e222673vJ33nknRowYEU899VRBk5W/U045JX72s5/FHnvsUfQozc6KFSvil7/8ZUydOjUqKipi\n9uzZ8Z///CfOPPPM+PWvfx0dO3YsesSytXTp0jj//PPjxRdfjFKpFKVSKVq2bBkDBgyISy65xLb9\nAmbOnBknn3xy7LjjjlFbWxtLly6NW265JXr06FH0aI1GmBvRPvvsE0888cQGe3UrVqyIQYMGxUsv\nvVTQZOVv2bJlMXr06Nh9991j22233eDwoKMRX9y5554bVVVVccYZZ8TIkSPj1VdfjdWrV8ell14a\nVVVVce211xY9Ytk67rjjolWrVnHCCSdE165do1Qqxdtvvx2TJk2KUqkUt9xyS9Ejlp2RI0fGoYce\nGqNGjYqIiFtvvTUeeeSRuPXWWwuerPE4ntKI+vfvHxdeeGGcfPLJ0blz57r3DL7hhhti4MCBRY9X\n1i666KJYtmxZtGvXLt5777311n3eOTw27cknn4xp06ZFhw4d6rZl27ZtY9y4cTF48OCCpytvs2fP\njqeeemq992Lu1q1b9OzZM/bff/8CJytfc+fOjR/+8Id1n48YMSKuv/76AidqfMLciC655JK46KKL\n4qijjqo7bFVZWRmDBw+OX/7yl0WPV9ZeeOGFeOihh/zpThNo2bLlRm8hW11dvdEr4Km/rl27RlVV\n1Xphjvjkngf+LX8x1dXV610AusUWW2z0zVfKmTBvpnnz5tXdiWr58uVxzjnn1F2B/elZgg4dOsS/\n//1vd1DaDLvssku0adOm6DGapR49esRVV11V9yYLERELFy6Myy67LPr161fgZOVp7ty5dR+PHj06\nzjnnnDjmmGNip512ioqKipg/f37ccccdcdpppxU4JZk5x7yZ9tprr3j11Vcj4pM7KG3ssKo7KG2+\nBx54IO6888449NBDY7vtttvgT88GDBhQ0GTlb9myZXHqqafGnDlzoqamJtq2bRtr1qyJffbZJ665\n5poNLmbk8336OrCpl1avCV/MHnvsEePGjVtv+06YMGGDZeV8p0Vh3kxLly6NTp06RYQ7KDWljd02\n8lNe4BrHrFmzYtGiRdGmTZvo1q2bIzxfUEPe7cxrQsPV526L5X6nRWEGgETcigoAEhFmAEhEmAEg\nEWEGgESEGQASKbsbjNQu26XoEeqt4hsPRmn5oUWPUW9DOu1d9Aj19n9f/XWM2evcosdolmzbpmPb\nNp1y27bTau/5zHX2mJtQRatvFz1Cs/XNPboWPUKzZds2Hdu26TSnbSvMAJCIMANAIsIMAIkIMwAk\nIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgw\nA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwA\niQgzACQizACQiDADQCLCDACJCDMAJJIqzEuWLIk999wz5s6dW/QoAFCIlkUP8P/r3LlzzJo1q+gx\nAKAwqfaYAeCrLlWYFy9eHLvuumvMmTOn6FEAoBCpwgwAX3UVpVKpVPQQn1q8eHEceOCBcf/998e3\nv/3tjX5Nae2cqGi18XUAUO5SXfxVH6Xlh0aa3yQ2oXK7f0Xtsl2KHqPehnTau+gR6m1a7T1xUOVR\nRY/RLNm2Tce2bTrltm2n1d7zmescygaARIQZABIRZgBIRJgBIJFUF3916dIl3nzzzaLHAIDC2GMG\ngESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQAS\nEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESY\nASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgERaFj1AQ1XVri56hHprF+U1\nb0WbNkWP0CDlNG9pzZqiRwDKhD1mAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBI\nRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFh\nBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkA\nEhFmAEhEmAEgkTRhnjJlSixfvrzoMQCgUCnCXFNTExMmTBBmAL7yNhnmgQMHxrRp0+o+P/HEE2Po\n0KF1n7/xxhux5557xrx58+Lkk0+OPn36xL777hunnnpqvPvuu3Vft+uuu8bDDz8cI0eOjL333juG\nDRsWb775ZkRE9OrVKz766KM44ogj4re//W1jPj8AKCubDHOfPn3ipZdeiohP9mxfe+21WL16dXzw\nwQcREfHCCy9Ejx494tJLL42tttoqnnrqqXj00UejqqoqrrrqqvW+18033xxXXHFFzJw5M9q3bx/X\nXXddREQ88MADEfHJ4eyzzjqrUZ8gAJSTlpv6gr59+8Zdd90VERGvvfZadOvWLbbbbrt48cUXY/Dg\nwfHCCy9Ev379YvTo0RER0bp162jdunUMGjQo7rzzzvW+12GHHRbf/OY3IyJi//33jylTpjR44K9v\nMz1atNq1wY8rSrtOi4oeod4eWVX0BA3zyKrbih6h2ZpWe0/RIzRbtm3TaS7btl5hvvjii2PNmjXx\n/PPPxz777BMdO3ZcL8yjR4+O2bNnx29+85t44403orq6Ompra2Pbbbdd73t16dKl7uMtttgi1qxZ\n0+CBV743uMGPKUq7Tovio6U7FD1GvY3YaUDRI9TbI6tui4O3GFX0GPVW+gL/1osyrfaeOKjyqKLH\naJZs26ZTbtv2836J2OSh7O233z46deoUs2bNiueffz569eoVPXr0iBdffDEWLlwYq1evjq5du8aY\nMWNijz32iMceeyxmzZoVY8eO3fCHVaa41gwA0qpXKfv27RsvvPBCvPzyy9GzZ8/YfffdY/78+fH0\n009H7969Y8GCBbFy5cr46U9/Gu3atYuITw57AwANU+8w//Wvf42OHTtG+/bto2XLlrHbbrvF7bff\nHv369YtOnTpFZWVlvPzyy7Fq1aq46667Yv78+fHhhx/G6tWrN/n927ZtGxERCxYsiKqqqs17RgBQ\nxuoV5j59+sSCBQuiV69edct69uwZc+fOje9+97ux7bbbxtixY+OSSy6JAQMGxLx58+Laa6+NDh06\nxMEHH7zJ77/11lvHkCFD4pxzzolrrrnmiz8bAChzFaVSqVT0EA1RThdTufir6bj4q+mU20U05cS2\nbTrltm036+IvAODLI8wAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIM\nAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAk\nIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAk0rLoARpq\ny8q2RY/QIOU0b2nNmqJHaJBymxegPuwxA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCI\nMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIM\nAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAk\nIswAkIgwA0AiwgwAiQgzACQizACQSKFhfu211+K4446LfffdN/r27Rtjx46NqqqqIkcCgEIVGuaz\nzjorunfvHv/4xz/igQceiNmzZ8dNN91U5EgAUKiKUqlUKuqHr1y5Mlq1ahWtW7eOiIjLLrss5s+f\nH3/84x8/8zGltXOiotW3v6wRAeBL1bLIH/7MM8/EDTfcEPPnz49169ZFTU1N9OrV63MfU1p+aBT2\nm0QDVW73r6hdtkvRY9TbkE57Fz1CvU2rvScOqjyq6DGaJdu26di2Tafctu202ns+c11hh7LnzZsX\nZ555Zhx22GExc+bMmDVrVowaNaqocQAghcL2mF9//fVo0aJFjB49OioqKiLik4vBKitdKA7AV1dh\nFdxhhx2iuro6Zs+eHVVVVXH99dfHqlWr4r333ouampqixgKAQhUW5u7du8dPfvKTGD16dAwZMiRa\ntWoVV1xxRXz00UcOaQPwlVXoxV8XXHBBXHDBBestmzlzZkHTAEDxnNAFgESEGQASEWYASESYASAR\nYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZ\nABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBI\nRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgERaFj1AQ/1jdU3RI9Tbd6O85m2x9TeKHqFBymne2g//\np+gRGqSiVeuiR6i3Uk35/B+LiIjKFkVPUH+1ZbZtmwl7zACQiDADQCLCDACJCDMAJCLMAJCIMANA\nIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkI\nMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswA\nkIgwA0AiwgwAiQgzACQizACQSL3DvHjx4th1111jzpw5TTkPAHyl2WMGgESEGQASaXCY//nPf8aw\nYcOiR48eMWrUqFiyZElERDz33HPxox/9KHr27Bn9+/eP3/zmN1FbW1v3uL/85S8xdOjQ6N69ewwZ\nMiQeeuihunXHHXdcXHXVVTF8+PD48Y9/3AhPCwDKU4PDfOedd8Yf/vCHeOKJJ6JVq1Zx/vnnx7Jl\ny+Lkk0+OESNGxHPPPRcTJ06M+++/P+6+++6IiJg+fXr87ne/iyuvvDJeeumluOCCC2Ls2LExb968\nuu/74IMPxiWXXBITJ05stCcHAOWmolQqlerzhYsXL44DDzwwrr766vjBD34QERFPPfVUnHTSSXH6\n6afHjBkzYsqUKXVf/6c//SmmTp0ad911V4wZMyZ22mmnOP/88+vWn3LKKbHLLrvEueeeG8cdd1y0\na9cufv/7329yjo+r34yvtd61oc8TAMpCy4Y+YOedd677uGvXrlEqleLZZ5+N119/Pfbcc8+6daVS\nKbbeeuuIiFi4cGH8/e9/j9tuu2299VtttVXd5506darXz39l6SENHbkw393xrZi54FtFj1Fvl/Y+\nuOgR6m3qu/8n/qvjKUWPUW+1H/5P0SPU2yNrbo+D2xxb9Bj1VqqpKXqEepu27s44qOWPih6j/mrL\naNvW3hMHVR5V9Bj1Nq32ns9c1+AwV1b+79HvT3e2O3fuHK1bt46bb755o49p27ZtnHnmmTFmzJjP\nHqRlg0cBgGanweeY58+fX/fxwoULo0WLFrHbbrvFv/71r/Uu9lq+fHmsXr06Ij7Zs37zzTfX+z5L\nly5d7+sBgC8Q5jvuuCPefffdqKqqikmTJsWAAQNi+PDhUVVVFdddd12sWrUqli5dGieddFLceOON\nERExcuTIePjhh2P69Omxbt26eOmll2L48OHx7LPPNvoTAoBy1uAwjxw5Mk444YTYb7/9Yt26dfHf\n//3f0b59+/jDH/4QTz75ZPTp0yeOPvro2HfffeO0006LiIh+/frFuHHjYsKECdGzZ88YN25cnHfe\nedGvX79Gf0IAUM7qfWK3S5cudYejhw4dusH63r17x+TJkz/z8cccc0wcc8wxG11366231ncMAGjW\n3PkLABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIR\nZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgB\nIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABJpWfQADbWspn3RIzRI\nOc1b+nhV0SM0SDnNW7lF26JHaJBymrd2zZqiR2iQilbl87Jb0aJN0SM0SOXXvlb0CI3CHjMAJCLM\nAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANA\nIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkI\nMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJBIyyJ+6KBBg+Kdd96JysoN\nfy8YN25cjBw5soCpAKB4hYQ5IuLCCy+MUaNGFfXjASAlh7IBIBFhBoBEKkqlUunL/qGfd475lVde\niRYtWnzmYz+snhvtW+/clOMBQGHK7hzzjEVHNcE0TeOInV6OKfN6FD1Gvd3Yfa+iR6i3h6smxZAt\nf1z0GPVW0bKw/2oNNnXFH+O/Ovy06DHqrXbNmqJHqLdHVt0WB29RPtfWVHzOTlI25faa8HDVpM9c\n51A2ACQizACQiDADQCKFnfiaMGFCXHXVVRssHzBgQFx//fUFTAQAxSskzI8++mgRPxYA0nMoGwAS\nEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESY\nASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaA\nRIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASqSiVSqWihwAAPmGPGQASEWYA\nSESYASARYQaARIQZABIRZgBI5P8B1EjV+06cOJIAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "first head of last state dec_enc_attns\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeYAAAH2CAYAAAClRS9UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGzFJREFUeJzt3XmMVfX5+PFnhkVMVUhURFDQuqci\nmwi0KEhQoliCilWq2GLdU/eKgLF8sSrS2liXYvxqDVatC6KmVaOi4lYtCmgEKyIIshUXKppB9rm/\nP4zz/RFQBpnhPHd8vf4azp2Z+9xPyH3PPefccytKpVIpAIAUKoseAAD4P8IMAIkIMwAkIswAkIgw\nA0AiwgwAiQgzACQizACQiDBTNr744ot46KGH4qabbqrZNn/+/OIGAqgHwkxZeO2116J3795x7733\nxp133hkREYsXL47jjz8+XnjhhWKHA6hDwkxZ+MMf/hAjRoyIv//971FRUREREW3atIkbbrhhg1fQ\nAOVOmCkLH3zwQZxwwgkRETVhjog48sgj7c4GGhRhpiy0bNkyFi1atNH2N998M3bccccCJgKoH42L\nHgBqY8CAAXH22WfH6aefHtXV1fHUU0/FrFmz4v7774/TTz+96PEA6kyFj32kHJRKpbj77rvj4Ycf\njgULFkSzZs2ibdu2MXjw4DjxxBOLHg+gzggzZeHDDz+Mdu3abbR9zZo1MWPGjOjSpUsBUwHUPceY\nKQsDBgzY5PYvv/wyzjzzzG08DUD9cYyZ1B566KF48MEHY+3atTFo0KCNbv/000+jRYsWBUwGUD+E\nmdSOOeaYaN68eVx66aXRu3fvjW7fbrvtom/fvtt+MIB64hgzZeGJJ56I/v37Fz0GQL0TZsrGW2+9\nFXPmzInVq1dvdNupp55awEQAdU+YKQvXXntt3HPPPbHzzjvHdtttt8FtFRUV8dxzzxU0GUDdEmbK\nQqdOneLWW2+Nn/zkJ0WPAlCvvF2KsrDjjjtG165dix4DoN4JM2XhwgsvjLvuuivs4AEaOruySevE\nE0/c4JOkFi5cGI0aNYrdd999g+0REQ8//PC2Hg+gXngfM2kdeeSRRY8AsM15xUxZWb9+fTRq1Cgi\nIlauXBnbb799wRMB1C3HmCkL8+fPj5/+9KcxadKkmm33339/HHfccfHhhx8WOBlA3RJmysLo0aPj\nsMMOix49etRsO+mkk+Lwww+P0aNHFzgZQN2yK5uy0KVLl5gyZUo0brzhaRFr166N7t27x7Rp0wqa\nDL7ZokWLYo899ih6DMqMV8yUhebNm8ecOXM22j5jxozYYYcdCpgINm/AgAGxfv36osegzDgrm7Jw\n+umnx9ChQ+PYY4+NPfbYI6qrq2PevHnx1FNPxeWXX170eLBJp556atx8881x1lln+QOSWrMrm7Lx\n7LPPxiOPPBILFy6MioqK2HPPPePEE0+MPn36FD0abNIxxxwTn376aaxYsSJ22GGHmncUfO21114r\naDIyE2aAevLoo49+6+3HH3/8NpqEciLMlI1HHnkknnzyyVi8eHFUVFRE27Zt48QTT4yjjjqq6NFg\ns9auXRtNmjQpegzKgGPM9WD9+vWxZs2ajba7GMZ3N27cuLj77rvj2GOPjZ49e0ZExAcffBDDhw+P\nFStWxMCBAwuesHx99NFHMX78+Jg7d26sWrVqo9v/+te/FjBVw7BmzZr485//HBMnTozPP/88ZsyY\nEVVVVXHNNdfEVVddFT/4wQ+KHpGEhLkOvfbaazF69OhYsGDBJj9s4d133y1gqoZhwoQJcfvtt0fH\njh032D5gwIAYPXq0MG+FSy65JJYvXx7dunWLZs2aFT1Og3LttdfGO++8E7/97W/jN7/5TUREVFdX\nx2effRbXXXddXHvttQVPSEbCXIdGjBgRPXr0iJEjR3qCq2PLly+P9u3bb7S9U6dOsXjx4gImajje\nfffdmDx5crRo0aLoURqcZ555Jh599NFo1apVzQev7LTTTjFmzJgYMGBAwdORlTDXoc8//zxGjx4d\nTZs2LXqUBmevvfaK5557Lo4++ugNtk+ePNkFHLbSXnvt5b229WT9+vWx6667brS9adOmsWLFigIm\nohwIcx3q06dPzJo1Kw455JCiR2lwLrjggrjggguiW7dusc8++0TEV8eYp0yZEmPGjCl4uvJ2+eWX\nx5VXXhknn3xytGnTJiorN7zu0L777lvQZOXvRz/6Udxxxx1x7rnn1mxbsWJFXH/99Z4n+EbOyt5K\n9913X83XK1eujIkTJ0bv3r03+Sru1FNP3ZajNTizZ8+OiRMnxsKFC2PNmjXRtm3bGDhwoCe4rXTg\ngQdutK2ioiJKpVJUVFQ4N2IrzJ49O84888xYt25dfPbZZ/HDH/4wFi9eHLvuumuMGzcu9ttvv6JH\nJCFh3kq1vbhFRUVFPPfcc/U8DWy5zR2jb9OmzTaapGFatWpVTJ48ORYuXBjNmjWLdu3aRc+ePTe6\n2Ah8TZgpC4sWLYrx48fHhx9+GKtXr97odm/pARoKx5jrUKlUivHjx0fnzp2jQ4cOERHx9NNPx6JF\ni2Lo0KEbHbuj9i688MJYu3ZtHHbYYU6uqwO9e/eOF154ISIiunfvXnPG8Ka4bOSWsbZsLWGuQ7//\n/e/j2WefjUMPPbRm2y677BK33HJLLFu2LIYNG1bgdOVt3rx58corr7ggQx255JJLar6+4oorIiLi\nyy+/jKZNm2700ZpsmW9a2yZNmsQnn3wSLVu2tMZbae7cufHCCy9Eo0aNom/fvg3unRl2Zdehnj17\nxsSJE2O33XbbYPtHH30UgwYNipdffrmgycrfueeeG7/+9a/j4IMPLnqUBmf58uXxu9/9Lp566qmo\nqKiImTNnxn//+9+46KKL4o9//GO0bNmy6BHL1pIlS+KKK66IadOmRalUilKpFI0bN45evXrFqFGj\nrO138Oqrr8Y555wTe+21V1RXV8eSJUvirrvuik6dOhU9Wp0R5jp06KGHxosvvrjRq7rly5dHnz59\nYvr06QVNVv6WLl0aQ4cOjYMOOih22223jXYP2hvx3V122WVRVVUVF154YQwePDjefvvtWLVqVVx9\n9dVRVVUVN998c9Ejlq0hQ4ZEkyZN4owzzoi2bdtGqVSKDz/8MO6+++4olUpx1113FT1i2Rk8eHD0\n798/TjvttIiIuOeee+KZZ56Je+65p+DJ6o79KXWoZ8+eMWLEiDjnnHOiTZs2NZ8ZPG7cuOjdu3fR\n45W1K6+8MpYuXRo77bRTfPLJJxvc9m3H8Ni8l156KSZNmhQtWrSoWctmzZrFyJEjo2/fvgVPV95m\nzpwZL7/88gafxdyuXbvo3LlzHHHEEQVOVr7mzJkTP/vZz2r+PWjQoLj11lsLnKjuCXMdGjVqVFx5\n5ZVx0kkn1ey2qqysjL59+8bvfve7oscra1OnTo0nn3zSW3fqQePGjTd5Cdk1a9Zs8gx4aq9t27ZR\nVVW1QZgjvrrmgf/L382aNWs2OAF0++233+SHr5QzYd5Kc+fOrbkS1bJly+LSSy+tOQP766MELVq0\niP/85z+uoLQV9ttvv9huu+2KHqNB6tSpU4wdO7bmQxYiIhYsWBDXXHNN9OjRo8DJytOcOXNqvh46\ndGhceuml8fOf/zz22WefqKioiHnz5sX9998f559/foFTkpljzFvpkEMOibfffjsivrqC0qZ2q7qC\n0tZ7/PHH44EHHoj+/ftHq1atNnrrWa9evQqarPwtXbo0zjvvvJg9e3asX78+mjVrFqtXr45DDz00\nbrjhho1OZuTbff08sLmnVs8J383BBx8cI0eO3GB9x4wZs9G2cr7SojBvpSVLlkTr1q0jwhWU6tOm\nLhv5NU9wdWPGjBmxcOHC2G677aJdu3b28HxHW/JpZ54TtlxtrrZY7ldaFGYASMSlqAAgEWEGgESE\nGQASEWYASESYASCRsrvASPXS/YoeodYqdn4iSsv6Fz1Gg2Rt64+1rT/ltrb9WncseoRa+9+3/xhn\nH3JZ0WPU2qTqCd94m1fM9aiiyf5Fj9BgWdv6Y23rj7WtP3sf3LboEeqMMANAIsIMAIkIMwAkIswA\nkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0Ai\nwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgz\nACQizACQiDADQCLCDACJCDMAJCLMAJBIqjAvXrw42rdvH3PmzCl6FAAoROOiB/j/tWnTJmbMmFH0\nGABQmFSvmAHg+y5VmBctWhQHHHBAzJ49u+hRAKAQqcIMAN93qY4x10bFzk9ERZP9ix6j1ipbvV/0\nCA2Wta0/1rb+lNPaTqoueoItM6l6QtEj1ImyC3NpWf8oFT1ELVW2ej+ql+5X9BgNkrWtP9a2/pTb\n2vZr3bHoEWptUvWEOKrypKLHqLVv+yPCrmwASESYASARYQaARIQZABJJdfLXHnvsEe+9917RYwBA\nYbxiBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBE\nhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFm\nAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEg\nEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESE\nGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYA\nSESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgkTRhfuSRR2LZsmVFjwEAhUoR\n5vXr18eYMWOEGYDvvc2GuXfv3jFp0qSaf5955plx7LHH1vx71qxZ0b59+5g7d26cc8450a1bt+ja\ntWucd9558fHHH9d83wEHHBBPP/10DB48ODp27BgDBgyI9957LyIiunTpEl988UWccMIJ8ac//aku\nHx8AlJXNhrlbt24xffr0iPjqle0777wTq1atis8++ywiIqZOnRqdOnWKq6++Onbcccd4+eWX4/nn\nn4+qqqoYO3bsBr/rzjvvjOuuuy5effXVaN68edxyyy0REfH4449HxFe7sy+++OI6fYAAUE4ab+4b\nunfvHg8++GBERLzzzjvRrl27aNWqVUybNi369u0bU6dOjR49esTQoUMjIqJp06bRtGnT6NOnTzzw\nwAMb/K7jjjsu9t5774iIOOKII+KRRx7Z4oErdn4iKprsv8U/V5TKVu8XPUKDZW3rj7WtP+W0tpOq\ni55gy0yqnlD0CHWiVmG+6qqrYvXq1fHGG2/EoYceGi1bttwgzEOHDo2ZM2fGjTfeGLNmzYo1a9ZE\ndXV17Lbbbhv8rj322KPm6+233z5Wr169xQOXlvWP0hb/VDEqW70f1Uv3K3qMBsna1h9rW3/KbW37\nte5Y9Ai1Nql6QhxVeVLRY9Tat/0Rsdld2bvvvnu0bt06ZsyYEW+88UZ06dIlOnXqFNOmTYsFCxbE\nqlWrom3btnH22WfHwQcfHJMnT44ZM2bEsGHDNr6zyhTnmgFAWrUqZffu3WPq1Knx5ptvRufOneOg\ngw6KefPmxSuvvBKHHXZYzJ8/P1asWBG/+tWvYqeddoqIr3Z7AwBbptZhfuyxx6Jly5bRvHnzaNy4\ncRx44IFx3333RY8ePaJ169ZRWVkZb775ZqxcuTIefPDBmDdvXnz++eexatWqzf7+Zs2aRUTE/Pnz\no6qqauseEQCUsVqFuVu3bjF//vzo0qVLzbbOnTvHnDlz4sc//nHstttuMWzYsBg1alT06tUr5s6d\nGzfffHO0aNEijj766M3+/l122SX69esXl156adxwww3f/dEAQJmrKJVK5XIuVUREWZ04UW4nepQT\na1t/rG39Kbe1dfJX/dmqk78AgG1HmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgB\nIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBE\nhBkAEhFmAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFm\nAEhEmAEgEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEg\nEWEGgESEGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESE\nGQASEWYASESYASARYQaARIQZABIRZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgEWEGgESEGQASEWYA\nSESYASCRQsP8zjvvxJAhQ6Jr167RvXv3GDZsWFRVVRU5EgAUqtAwX3zxxdGhQ4f417/+FY8//njM\nnDkz7rjjjiJHAoBCVZRKpVJRd75ixYpo0qRJNG3aNCIirrnmmpg3b1785S9/+cafKa2dHRVN9t9W\nIwLANtW4yDt/7bXXYty4cTFv3rxYt25drF+/Prp06fKtP1Na1j8K+0tiC1W2ej+ql+5X9BgNkrWt\nP9a2/pTb2vZr3bHoEWptUvWEOKrypKLHqLVJ1RO+8bbCdmXPnTs3LrroojjuuOPi1VdfjRkzZsRp\np51W1DgAkEJhr5jffffdaNSoUQwdOjQqKioi4quTwSornSgOwPdXYRXcc889Y82aNTFz5syoqqqK\nW2+9NVauXBmffPJJrF+/vqixAKBQhYW5Q4cO8ctf/jKGDh0a/fr1iyZNmsR1110XX3zxhV3aAHxv\nFXry1/Dhw2P48OEbbHv11VcLmgYAiueALgAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJ\nCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLM\nAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANA\nIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkI\nMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswA\nkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0Ai\ntQ7zokWL4oADDojZs2fX5zwA8L3mFTMAJCLMAJDIFof53//+dwwYMCA6deoUp512WixevDgiIl5/\n/fU45ZRTonPnztGzZ8+48cYbo7q6uubn/va3v8Wxxx4bHTp0iH79+sWTTz5Zc9uQIUNi7NixMXDg\nwPjFL35RBw8LAMrTFof5gQceiNtuuy1efPHFaNKkSVxxxRWxdOnSOOecc2LQoEHx+uuvx/jx4+Mf\n//hHPPTQQxER8eyzz8ZNN90U119/fUyfPj2GDx8ew4YNi7lz59b83ieeeCJGjRoV48ePr7MHBwDl\npvGW/sDgwYOjTZs2ERFxxhlnxFlnnRUTJ06MvffeOwYNGhQREfvuu28MGTIkHn300TjllFPioYce\nihNOOCEOOeSQiIg48sgjo2fPnvHYY4/FZZddFhER7du3j06dOm32/it2fiIqmuy/pWMXprLV+0WP\n0GBZ2/pjbetPOa3tpOrNf08mk6onFD1CndjiMO+77741X7dt2zZKpVJMmTIl3n333Wjfvn3NbaVS\nKXbZZZeIiFiwYEH885//jHvvvXeD23fccceaf7du3bpW919a1j9KWzp0QSpbvR/VS/creowGydrW\nH2tbf8ptbfu17lj0CLU2qXpCHFV5UtFj1Nq3/RGxxWGurPy/vd+l0leJbNOmTTRt2jTuvPPOTf5M\ns2bN4qKLLoqzzz77mwdpvMWjAECDs8XHmOfNm1fz9YIFC6JRo0Zx4IEHxvvvv7/ByV7Lli2LVatW\nRcRXr6zfe++9DX7PkiVLNvh+AOA7hPn++++Pjz/+OKqqquLuu++OXr16xcCBA6OqqipuueWWWLly\nZSxZsiTOOuusuP322yPiq+PSTz/9dDz77LOxbt26mD59egwcODCmTJlS5w8IAMrZFod58ODBccYZ\nZ8Thhx8e69ati//5n/+J5s2bx2233RYvvfRSdOvWLU4++eTo2rVrnH/++RER0aNHjxg5cmSMGTMm\nOnfuHCNHjozLL788evToUecPCADKWUXp6wPFZaKcTpwotxM9yom1rT/Wtv6U29o6+av+fNvJX678\nBQCJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMA\nJCLMAJCIMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCI\nMANAIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIM\nAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAk\nIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkIswAkIgw\nA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIo2LuNM+ffrERx99FJWVG/9dMHLk\nyBg8eHABUwFA8QoJc0TEiBEj4rTTTivq7gEgJbuyASARYQaARCpKpVJpW9/ptx1jfuutt6JRo0bf\n+LOltbOjosn+9TkeABSm7I4xl5b1j23+l8R3VNnq/aheul/RYzRI1rb+WNv6U25r2691x6JHqLVJ\n1RPiqMqTih6j1iZVT/jG2+zKBoBEhBkAEhFmAEiksGPMY8aMibFjx260vVevXnHrrbcWMBEAFK+Q\nMD///PNF3C0ApGdXNgAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANA\nIsIMAIkIMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkI\nMwAkIswAkIgwA0AiwgwAiQgzACQizACQiDADQCLCDACJCDMAJCLMAJCIMANAIsIMAIkIMwAkUlEq\nlUpFDwEAfMUrZgBIRJgBIBFhBoBEhBkAEhFmAEhEmAEgkf8HhgybVjMLh/kAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode, Derek Miller @dmmiller612\n", - " Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch\n", - " https://github.com/JayParks/transformer\n", - "'''\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from torch.autograd import Variable\n", - "import matplotlib.pyplot as plt\n", - "\n", - "dtype = torch.FloatTensor\n", - "# S: Symbol that shows starting of decoding input\n", - "# E: Symbol that shows starting of decoding output\n", - "# P: Symbol that will fill in blank sequence if current batch data size is short than time steps\n", - "sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E']\n", - "\n", - "# Transformer Parameters\n", - "# Padding Should be Zero\n", - "src_vocab = {'P' : 0, 'ich' : 1, 'mochte' : 2, 'ein' : 3, 'bier' : 4}\n", - "src_vocab_size = len(src_vocab)\n", - "\n", - "tgt_vocab = {'P' : 0, 'i' : 1, 'want' : 2, 'a' : 3, 'beer' : 4, 'S' : 5, 'E' : 6}\n", - "number_dict = {i: w for i, w in enumerate(tgt_vocab)}\n", - "tgt_vocab_size = len(tgt_vocab)\n", - "\n", - "src_len = 5\n", - "tgt_len = 5\n", - "\n", - "d_model = 512 # Embedding Size\n", - "d_ff = 2048 # FeedForward dimension\n", - "d_k = d_v = 64 # dimension of K(=Q), V\n", - "n_layers = 6 # number of Encoder of Decoder Layer\n", - "n_heads = 8 # number of heads in Multi-Head Attention\n", - "\n", - "def make_batch(sentences):\n", - " input_batch = [[src_vocab[n] for n in sentences[0].split()]]\n", - " output_batch = [[tgt_vocab[n] for n in sentences[1].split()]]\n", - " target_batch = [[tgt_vocab[n] for n in sentences[2].split()]]\n", - " return Variable(torch.LongTensor(input_batch)), Variable(torch.LongTensor(output_batch)), Variable(torch.LongTensor(target_batch))\n", - "\n", - "def get_sinusoid_encoding_table(n_position, d_model):\n", - " def cal_angle(position, hid_idx):\n", - " return position / np.power(10000, 2 * (hid_idx // 2) / d_model)\n", - " def get_posi_angle_vec(position):\n", - " return [cal_angle(position, hid_j) for hid_j in range(d_model)]\n", - "\n", - " sinusoid_table = np.array([get_posi_angle_vec(pos_i) for pos_i in range(n_position)])\n", - " sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # dim 2i\n", - " sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # dim 2i+1\n", - " return torch.FloatTensor(sinusoid_table)\n", - "\n", - "def get_attn_pad_mask(seq_q, seq_k):\n", - " batch_size, len_q = seq_q.size()\n", - " batch_size, len_k = seq_k.size()\n", - " # eq(zero) is PAD token\n", - " pad_attn_mask = seq_k.data.eq(0).unsqueeze(1) # batch_size x 1 x len_k(=len_q), one is masking\n", - " return pad_attn_mask.expand(batch_size, len_q, len_k) # batch_size x len_q x len_k\n", - "\n", - "def get_attn_subsequent_mask(seq):\n", - " attn_shape = [seq.size(0), seq.size(1), seq.size(1)]\n", - " subsequent_mask = np.triu(np.ones(attn_shape), k=1)\n", - " subsequent_mask = torch.from_numpy(subsequent_mask).byte()\n", - " return subsequent_mask\n", - "\n", - "class ScaledDotProductAttention(nn.Module):\n", - " def __init__(self):\n", - " super(ScaledDotProductAttention, self).__init__()\n", - "\n", - " def forward(self, Q, K, V, attn_mask):\n", - " scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k) # scores : [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", - " scores.masked_fill_(attn_mask, -1e9) # Fills elements of self tensor with value where mask is one.\n", - " attn = nn.Softmax(dim=-1)(scores)\n", - " context = torch.matmul(attn, V)\n", - " return context, attn\n", - "\n", - "class MultiHeadAttention(nn.Module):\n", - " def __init__(self):\n", - " super(MultiHeadAttention, self).__init__()\n", - " self.W_Q = nn.Linear(d_model, d_k * n_heads)\n", - " self.W_K = nn.Linear(d_model, d_k * n_heads)\n", - " self.W_V = nn.Linear(d_model, d_v * n_heads)\n", - " self.linear = nn.Linear(n_heads * d_v, d_model)\n", - " self.layer_norm = nn.LayerNorm(d_model)\n", - "\n", - " def forward(self, Q, K, V, attn_mask):\n", - " # q: [batch_size x len_q x d_model], k: [batch_size x len_k x d_model], v: [batch_size x len_k x d_model]\n", - " residual, batch_size = Q, Q.size(0)\n", - " # (B, S, D) -proj-> (B, S, D) -split-> (B, S, H, W) -trans-> (B, H, S, W)\n", - " q_s = self.W_Q(Q).view(batch_size, -1, n_heads, d_k).transpose(1,2) # q_s: [batch_size x n_heads x len_q x d_k]\n", - " k_s = self.W_K(K).view(batch_size, -1, n_heads, d_k).transpose(1,2) # k_s: [batch_size x n_heads x len_k x d_k]\n", - " v_s = self.W_V(V).view(batch_size, -1, n_heads, d_v).transpose(1,2) # v_s: [batch_size x n_heads x len_k x d_v]\n", - "\n", - " attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1, 1) # attn_mask : [batch_size x n_heads x len_q x len_k]\n", - "\n", - " # context: [batch_size x n_heads x len_q x d_v], attn: [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", - " context, attn = ScaledDotProductAttention()(q_s, k_s, v_s, attn_mask)\n", - " context = context.transpose(1, 2).contiguous().view(batch_size, -1, n_heads * d_v) # context: [batch_size x len_q x n_heads * d_v]\n", - " output = self.linear(context)\n", - " return self.layer_norm(output + residual), attn # output: [batch_size x len_q x d_model]\n", - "\n", - "class PoswiseFeedForwardNet(nn.Module):\n", - " def __init__(self):\n", - " super(PoswiseFeedForwardNet, self).__init__()\n", - " self.conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_ff, kernel_size=1)\n", - " self.conv2 = nn.Conv1d(in_channels=d_ff, out_channels=d_model, kernel_size=1)\n", - " self.layer_norm = nn.LayerNorm(d_model)\n", - "\n", - " def forward(self, inputs):\n", - " residual = inputs # inputs : [batch_size, len_q, d_model]\n", - " output = nn.ReLU()(self.conv1(inputs.transpose(1, 2)))\n", - " output = self.conv2(output).transpose(1, 2)\n", - " return self.layer_norm(output + residual)\n", - "\n", - "class EncoderLayer(nn.Module):\n", - " def __init__(self):\n", - " super(EncoderLayer, self).__init__()\n", - " self.enc_self_attn = MultiHeadAttention()\n", - " self.pos_ffn = PoswiseFeedForwardNet()\n", - "\n", - " def forward(self, enc_inputs, enc_self_attn_mask):\n", - " enc_outputs, attn = self.enc_self_attn(enc_inputs, enc_inputs, enc_inputs, enc_self_attn_mask) # enc_inputs to same Q,K,V\n", - " enc_outputs = self.pos_ffn(enc_outputs) # enc_outputs: [batch_size x len_q x d_model]\n", - " return enc_outputs, attn\n", - "\n", - "class DecoderLayer(nn.Module):\n", - " def __init__(self):\n", - " super(DecoderLayer, self).__init__()\n", - " self.dec_self_attn = MultiHeadAttention()\n", - " self.dec_enc_attn = MultiHeadAttention()\n", - " self.pos_ffn = PoswiseFeedForwardNet()\n", - "\n", - " def forward(self, dec_inputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask):\n", - " dec_outputs, dec_self_attn = self.dec_self_attn(dec_inputs, dec_inputs, dec_inputs, dec_self_attn_mask)\n", - " dec_outputs, dec_enc_attn = self.dec_enc_attn(dec_outputs, enc_outputs, enc_outputs, dec_enc_attn_mask)\n", - " dec_outputs = self.pos_ffn(dec_outputs)\n", - " return dec_outputs, dec_self_attn, dec_enc_attn\n", - "\n", - "class Encoder(nn.Module):\n", - " def __init__(self):\n", - " super(Encoder, self).__init__()\n", - " self.src_emb = nn.Embedding(src_vocab_size, d_model)\n", - " self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(src_len+1, d_model),freeze=True)\n", - " self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])\n", - "\n", - " def forward(self, enc_inputs): # enc_inputs : [batch_size x source_len]\n", - " enc_outputs = self.src_emb(enc_inputs) + self.pos_emb(torch.LongTensor([[1,2,3,4,0]]))\n", - " enc_self_attn_mask = get_attn_pad_mask(enc_inputs, enc_inputs)\n", - " enc_self_attns = []\n", - " for layer in self.layers:\n", - " enc_outputs, enc_self_attn = layer(enc_outputs, enc_self_attn_mask)\n", - " enc_self_attns.append(enc_self_attn)\n", - " return enc_outputs, enc_self_attns\n", - "\n", - "class Decoder(nn.Module):\n", - " def __init__(self):\n", - " super(Decoder, self).__init__()\n", - " self.tgt_emb = nn.Embedding(tgt_vocab_size, d_model)\n", - " self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(tgt_len+1, d_model),freeze=True)\n", - " self.layers = nn.ModuleList([DecoderLayer() for _ in range(n_layers)])\n", - "\n", - " def forward(self, dec_inputs, enc_inputs, enc_outputs): # dec_inputs : [batch_size x target_len]\n", - " dec_outputs = self.tgt_emb(dec_inputs) + self.pos_emb(torch.LongTensor([[5,1,2,3,4]]))\n", - " dec_self_attn_pad_mask = get_attn_pad_mask(dec_inputs, dec_inputs)\n", - " dec_self_attn_subsequent_mask = get_attn_subsequent_mask(dec_inputs)\n", - " dec_self_attn_mask = torch.gt((dec_self_attn_pad_mask + dec_self_attn_subsequent_mask), 0)\n", - "\n", - " dec_enc_attn_mask = get_attn_pad_mask(dec_inputs, enc_inputs)\n", - "\n", - " dec_self_attns, dec_enc_attns = [], []\n", - " for layer in self.layers:\n", - " dec_outputs, dec_self_attn, dec_enc_attn = layer(dec_outputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask)\n", - " dec_self_attns.append(dec_self_attn)\n", - " dec_enc_attns.append(dec_enc_attn)\n", - " return dec_outputs, dec_self_attns, dec_enc_attns\n", - "\n", - "class Transformer(nn.Module):\n", - " def __init__(self):\n", - " super(Transformer, self).__init__()\n", - " self.encoder = Encoder()\n", - " self.decoder = Decoder()\n", - " self.projection = nn.Linear(d_model, tgt_vocab_size, bias=False)\n", - " def forward(self, enc_inputs, dec_inputs):\n", - " enc_outputs, enc_self_attns = self.encoder(enc_inputs)\n", - " dec_outputs, dec_self_attns, dec_enc_attns = self.decoder(dec_inputs, enc_inputs, enc_outputs)\n", - " dec_logits = self.projection(dec_outputs) # dec_logits : [batch_size x src_vocab_size x tgt_vocab_size]\n", - " return dec_logits.view(-1, dec_logits.size(-1)), enc_self_attns, dec_self_attns, dec_enc_attns\n", - "\n", - "model = Transformer()\n", - "\n", - "criterion = nn.CrossEntropyLoss()\n", - "optimizer = optim.Adam(model.parameters(), lr=0.001)\n", - "\n", - "def showgraph(attn):\n", - " attn = attn[-1].squeeze(0)[0]\n", - " attn = attn.squeeze(0).data.numpy()\n", - " fig = plt.figure(figsize=(n_heads, n_heads)) # [n_heads, n_heads]\n", - " ax = fig.add_subplot(1, 1, 1)\n", - " ax.matshow(attn, cmap='viridis')\n", - " ax.set_xticklabels(['']+sentences[0].split(), fontdict={'fontsize': 14}, rotation=90)\n", - " ax.set_yticklabels(['']+sentences[2].split(), fontdict={'fontsize': 14})\n", - " plt.show()\n", - "\n", - "for epoch in range(20):\n", - " optimizer.zero_grad()\n", - " enc_inputs, dec_inputs, target_batch = make_batch(sentences)\n", - " outputs, enc_self_attns, dec_self_attns, dec_enc_attns = model(enc_inputs, dec_inputs)\n", - " loss = criterion(outputs, target_batch.contiguous().view(-1))\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - "# Test\n", - "predict, _, _, _ = model(enc_inputs, dec_inputs)\n", - "predict = predict.data.max(1, keepdim=True)[1]\n", - "print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()])\n", - "\n", - "print('first head of last state enc_self_attns')\n", - "showgraph(enc_self_attns)\n", - "\n", - "print('first head of last state dec_self_attns')\n", - "showgraph(dec_self_attns)\n", - "\n", - "print('first head of last state dec_enc_attns')\n", - "showgraph(dec_enc_attns)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "Transformer-Torch.ipynb", - "provenance": [], - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.9" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "metadata": { - "collapsed": false - }, - "source": [] - } - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/5-2.BERT/BERT-Torch.py b/5-2.BERT/BERT.py similarity index 100% rename from 5-2.BERT/BERT-Torch.py rename to 5-2.BERT/BERT.py diff --git a/5-2.BERT/BERT_Torch.ipynb b/5-2.BERT/BERT_Torch.ipynb deleted file mode 100644 index c4c7f0a..0000000 --- a/5-2.BERT/BERT_Torch.ipynb +++ /dev/null @@ -1,305 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "id": "bqxycRhzia7r", - "colab_type": "code", - "outputId": "0d350e82-951e-4c85-f503-758ff7aa2aa5", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 409.0 - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch: 0010 cost = 52.300804\n", - "Epoch: 0020 cost = 26.399708\n", - "Epoch: 0030 cost = 12.202291\n", - "Epoch: 0040 cost = 7.295442\n", - "Epoch: 0050 cost = 6.520606\n", - "Epoch: 0060 cost = 4.038918\n", - "Epoch: 0070 cost = 2.668994\n", - "Epoch: 0080 cost = 2.948867\n", - "Epoch: 0090 cost = 1.921901\n", - "Epoch: 0100 cost = 2.681553\n", - "Hello, how are you? I am Romeo.\n", - "Hello, Romeo My name is Juliet. Nice to meet you.\n", - "Nice meet you too. How are you today?\n", - "Great. My baseball team won the competition.\n", - "Oh Congratulations, Juliet\n", - "Thanks you Romeo\n", - "['[CLS]', 'thanks', 'you', 'romeo', '[SEP]', '[MASK]', 'how', 'are', 'you', 'i', 'am', '[MASK]', '[SEP]']\n", - "masked tokens list : [15, 28]\n", - "predict masked tokens list : [15, 15]\n", - "isNext : False\n", - "predict isNext : False\n" - ] - } - ], - "source": [ - "'''\n", - " code by Tae Hwan Jung(Jeff Jung) @graykode\n", - " Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch\n", - " https://github.com/JayParks/transformer, https://github.com/dhlee347/pytorchic-bert\n", - "'''\n", - "import math\n", - "import re\n", - "from random import *\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from torch.autograd import Variable\n", - "\n", - "# BERT Parameters\n", - "maxlen = 30\n", - "batch_size = 6\n", - "max_pred = 5 # max tokens of prediction\n", - "n_layers = 6\n", - "n_heads = 12\n", - "d_model = 768\n", - "d_ff = 768*4 # 4*d_model, FeedForward dimension\n", - "d_k = d_v = 64 # dimension of K(=Q), V\n", - "n_segments = 2\n", - "\n", - "text = (\n", - " 'Hello, how are you? I am Romeo.\\n'\n", - " 'Hello, Romeo My name is Juliet. Nice to meet you.\\n'\n", - " 'Nice meet you too. How are you today?\\n'\n", - " 'Great. My baseball team won the competition.\\n'\n", - " 'Oh Congratulations, Juliet\\n'\n", - " 'Thanks you Romeo'\n", - ")\n", - "sentences = re.sub(\"[.,!?\\\\-]\", '', text.lower()).split('\\n') # filter '.', ',', '?', '!'\n", - "word_list = list(set(\" \".join(sentences).split()))\n", - "word_dict = {'[PAD]' : 0, '[CLS]' : 1, '[SEP]' : 2, '[MASK]' : 3}\n", - "for i, w in enumerate(word_list):\n", - " word_dict[w] = i + 4\n", - "number_dict = {i: w for i, w in enumerate(word_dict)}\n", - "vocab_size = len(word_dict)\n", - "\n", - "token_list = list()\n", - "for sentence in sentences:\n", - " arr = [word_dict[s] for s in sentence.split()]\n", - " token_list.append(arr)\n", - "\n", - "# sample IsNext and NotNext to be same in small batch size\n", - "def make_batch():\n", - " batch = []\n", - " positive = negative = 0\n", - " while positive != batch_size/2 or negative != batch_size/2:\n", - " tokens_a_index, tokens_b_index= randrange(len(sentences)), randrange(len(sentences)) # sample random index in sentences\n", - " tokens_a, tokens_b= token_list[tokens_a_index], token_list[tokens_b_index]\n", - " input_ids = [word_dict['[CLS]']] + tokens_a + [word_dict['[SEP]']] + tokens_b + [word_dict['[SEP]']]\n", - " segment_ids = [0] * (1 + len(tokens_a) + 1) + [1] * (len(tokens_b) + 1)\n", - "\n", - " # MASK LM\n", - " n_pred = min(max_pred, max(1, int(round(len(input_ids) * 0.15)))) # 15 % of tokens in one sentence\n", - " cand_maked_pos = [i for i, token in enumerate(input_ids)\n", - " if token != word_dict['[CLS]'] and token != word_dict['[SEP]']]\n", - " shuffle(cand_maked_pos)\n", - " masked_tokens, masked_pos = [], []\n", - " for pos in cand_maked_pos[:n_pred]:\n", - " masked_pos.append(pos)\n", - " masked_tokens.append(input_ids[pos])\n", - " if random() < 0.8: # 80%\n", - " input_ids[pos] = word_dict['[MASK]'] # make mask\n", - " elif random() < 0.5: # 10%\n", - " index = randint(0, vocab_size - 1) # random index in vocabulary\n", - " input_ids[pos] = word_dict[number_dict[index]] # replace\n", - "\n", - " # Zero Paddings\n", - " n_pad = maxlen - len(input_ids)\n", - " input_ids.extend([0] * n_pad)\n", - " segment_ids.extend([0] * n_pad)\n", - "\n", - " # Zero Padding (100% - 15%) tokens\n", - " if max_pred > n_pred:\n", - " n_pad = max_pred - n_pred\n", - " masked_tokens.extend([0] * n_pad)\n", - " masked_pos.extend([0] * n_pad)\n", - "\n", - " if tokens_a_index + 1 == tokens_b_index and positive < batch_size/2:\n", - " batch.append([input_ids, segment_ids, masked_tokens, masked_pos, True]) # IsNext\n", - " positive += 1\n", - " elif tokens_a_index + 1 != tokens_b_index and negative < batch_size/2:\n", - " batch.append([input_ids, segment_ids, masked_tokens, masked_pos, False]) # NotNext\n", - " negative += 1\n", - " return batch\n", - "# Proprecessing Finished\n", - "\n", - "def get_attn_pad_mask(seq_q, seq_k):\n", - " batch_size, len_q = seq_q.size()\n", - " batch_size, len_k = seq_k.size()\n", - " # eq(zero) is PAD token\n", - " pad_attn_mask = seq_k.data.eq(0).unsqueeze(1) # batch_size x 1 x len_k(=len_q), one is masking\n", - " return pad_attn_mask.expand(batch_size, len_q, len_k) # batch_size x len_q x len_k\n", - "\n", - "def gelu(x):\n", - " \"Implementation of the gelu activation function by Hugging Face\"\n", - " return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))\n", - "\n", - "class Embedding(nn.Module):\n", - " def __init__(self):\n", - " super(Embedding, self).__init__()\n", - " self.tok_embed = nn.Embedding(vocab_size, d_model) # token embedding\n", - " self.pos_embed = nn.Embedding(maxlen, d_model) # position embedding\n", - " self.seg_embed = nn.Embedding(n_segments, d_model) # segment(token type) embedding\n", - " self.norm = nn.LayerNorm(d_model)\n", - "\n", - " def forward(self, x, seg):\n", - " seq_len = x.size(1)\n", - " pos = torch.arange(seq_len, dtype=torch.long)\n", - " pos = pos.unsqueeze(0).expand_as(x) # (seq_len,) -> (batch_size, seq_len)\n", - " embedding = self.tok_embed(x) + self.pos_embed(pos) + self.seg_embed(seg)\n", - " return self.norm(embedding)\n", - "\n", - "class ScaledDotProductAttention(nn.Module):\n", - " def __init__(self):\n", - " super(ScaledDotProductAttention, self).__init__()\n", - "\n", - " def forward(self, Q, K, V, attn_mask):\n", - " scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k) # scores : [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", - " scores.masked_fill_(attn_mask, -1e9) # Fills elements of self tensor with value where mask is one.\n", - " attn = nn.Softmax(dim=-1)(scores)\n", - " context = torch.matmul(attn, V)\n", - " return context, attn\n", - "\n", - "class MultiHeadAttention(nn.Module):\n", - " def __init__(self):\n", - " super(MultiHeadAttention, self).__init__()\n", - " self.W_Q = nn.Linear(d_model, d_k * n_heads)\n", - " self.W_K = nn.Linear(d_model, d_k * n_heads)\n", - " self.W_V = nn.Linear(d_model, d_v * n_heads)\n", - " def forward(self, Q, K, V, attn_mask):\n", - " # q: [batch_size x len_q x d_model], k: [batch_size x len_k x d_model], v: [batch_size x len_k x d_model]\n", - " residual, batch_size = Q, Q.size(0)\n", - " # (B, S, D) -proj-> (B, S, D) -split-> (B, S, H, W) -trans-> (B, H, S, W)\n", - " q_s = self.W_Q(Q).view(batch_size, -1, n_heads, d_k).transpose(1,2) # q_s: [batch_size x n_heads x len_q x d_k]\n", - " k_s = self.W_K(K).view(batch_size, -1, n_heads, d_k).transpose(1,2) # k_s: [batch_size x n_heads x len_k x d_k]\n", - " v_s = self.W_V(V).view(batch_size, -1, n_heads, d_v).transpose(1,2) # v_s: [batch_size x n_heads x len_k x d_v]\n", - "\n", - " attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1, 1) # attn_mask : [batch_size x n_heads x len_q x len_k]\n", - "\n", - " # context: [batch_size x n_heads x len_q x d_v], attn: [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", - " context, attn = ScaledDotProductAttention()(q_s, k_s, v_s, attn_mask)\n", - " context = context.transpose(1, 2).contiguous().view(batch_size, -1, n_heads * d_v) # context: [batch_size x len_q x n_heads * d_v]\n", - " output = nn.Linear(n_heads * d_v, d_model)(context)\n", - " return nn.LayerNorm(d_model)(output + residual), attn # output: [batch_size x len_q x d_model]\n", - "\n", - "class PoswiseFeedForwardNet(nn.Module):\n", - " def __init__(self):\n", - " super(PoswiseFeedForwardNet, self).__init__()\n", - " self.fc1 = nn.Linear(d_model, d_ff)\n", - " self.fc2 = nn.Linear(d_ff, d_model)\n", - "\n", - " def forward(self, x):\n", - " # (batch_size, len_seq, d_model) -> (batch_size, len_seq, d_ff) -> (batch_size, len_seq, d_model)\n", - " return self.fc2(gelu(self.fc1(x)))\n", - "\n", - "class EncoderLayer(nn.Module):\n", - " def __init__(self):\n", - " super(EncoderLayer, self).__init__()\n", - " self.enc_self_attn = MultiHeadAttention()\n", - " self.pos_ffn = PoswiseFeedForwardNet()\n", - "\n", - " def forward(self, enc_inputs, enc_self_attn_mask):\n", - " enc_outputs, attn = self.enc_self_attn(enc_inputs, enc_inputs, enc_inputs, enc_self_attn_mask) # enc_inputs to same Q,K,V\n", - " enc_outputs = self.pos_ffn(enc_outputs) # enc_outputs: [batch_size x len_q x d_model]\n", - " return enc_outputs, attn\n", - "\n", - "class BERT(nn.Module):\n", - " def __init__(self):\n", - " super(BERT, self).__init__()\n", - " self.embedding = Embedding()\n", - " self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])\n", - " self.fc = nn.Linear(d_model, d_model)\n", - " self.activ1 = nn.Tanh()\n", - " self.linear = nn.Linear(d_model, d_model)\n", - " self.activ2 = gelu\n", - " self.norm = nn.LayerNorm(d_model)\n", - " self.classifier = nn.Linear(d_model, 2)\n", - " # decoder is shared with embedding layer\n", - " embed_weight = self.embedding.tok_embed.weight\n", - " n_vocab, n_dim = embed_weight.size()\n", - " self.decoder = nn.Linear(n_dim, n_vocab, bias=False)\n", - " self.decoder.weight = embed_weight\n", - " self.decoder_bias = nn.Parameter(torch.zeros(n_vocab))\n", - "\n", - " def forward(self, input_ids, segment_ids, masked_pos):\n", - " output = self.embedding(input_ids, segment_ids)\n", - " enc_self_attn_mask = get_attn_pad_mask(input_ids, input_ids)\n", - " for layer in self.layers:\n", - " output, enc_self_attn = layer(output, enc_self_attn_mask)\n", - " # output : [batch_size, len, d_model], attn : [batch_size, n_heads, d_mode, d_model]\n", - " # it will be decided by first token(CLS)\n", - " h_pooled = self.activ1(self.fc(output[:, 0])) # [batch_size, d_model]\n", - " logits_clsf = self.classifier(h_pooled) # [batch_size, 2]\n", - "\n", - " masked_pos = masked_pos[:, :, None].expand(-1, -1, output.size(-1)) # [batch_size, maxlen, d_model]\n", - " h_masked = torch.gather(output, 1, masked_pos) # masking position [batch_size, len, d_model]\n", - " h_masked = self.norm(self.activ2(self.linear(h_masked)))\n", - " logits_lm = self.decoder(h_masked) + self.decoder_bias # [batch_size, maxlen, n_vocab]\n", - "\n", - " return logits_lm, logits_clsf\n", - "\n", - "model = BERT()\n", - "criterion = nn.CrossEntropyLoss()\n", - "optimizer = optim.Adam(model.parameters(), lr=0.001)\n", - "\n", - "batch = make_batch()\n", - "input_ids, segment_ids, masked_tokens, masked_pos, isNext = zip(*batch)\n", - "input_ids, segment_ids, masked_tokens, masked_pos, isNext = \\\n", - " torch.LongTensor(input_ids), torch.LongTensor(segment_ids), torch.LongTensor(masked_tokens), \\\n", - " torch.LongTensor(masked_pos), torch.LongTensor(isNext)\n", - "\n", - "for epoch in range(100):\n", - " optimizer.zero_grad()\n", - " logits_lm, logits_clsf = model(input_ids, segment_ids, masked_pos)\n", - " loss_lm = criterion(logits_lm.transpose(1, 2), masked_tokens) # for masked LM\n", - " loss_lm = (loss_lm.float()).mean()\n", - " loss_clsf = criterion(logits_clsf, isNext) # for sentence classification\n", - " loss = loss_lm + loss_clsf\n", - " if (epoch + 1) % 10 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - "# Predict mask tokens ans isNext\n", - "input_ids, segment_ids, masked_tokens, masked_pos, isNext = batch[0]\n", - "print(text)\n", - "print([number_dict[w] for w in input_ids if number_dict[w] != '[PAD]'])\n", - "\n", - "logits_lm, logits_clsf = model(torch.LongTensor([input_ids]), \\\n", - " torch.LongTensor([segment_ids]), torch.LongTensor([masked_pos]))\n", - "logits_lm = logits_lm.data.max(2)[1][0].data.numpy()\n", - "print('masked tokens list : ',[pos for pos in masked_tokens if pos != 0])\n", - "print('predict masked tokens list : ',[pos for pos in logits_lm if pos != 0])\n", - "\n", - "logits_clsf = logits_clsf.data.max(1)[1].data.numpy()[0]\n", - "print('isNext : ', True if isNext else False)\n", - "print('predict isNext : ',True if logits_clsf else False)" - ] - } - ], - "metadata": { - "colab": { - "name": "BERT-Torch.ipynb", - "version": "0.3.2", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/1-1.NNLM/NNLM-Tensor.py b/archive/tensorflow/v1/1-1.NNLM/NNLM.py similarity index 100% rename from 1-1.NNLM/NNLM-Tensor.py rename to archive/tensorflow/v1/1-1.NNLM/NNLM.py diff --git a/1-2.Word2Vec/Word2Vec-Skipgram-Tensor(NCE_loss).py b/archive/tensorflow/v1/1-2.Word2Vec/Word2Vec-Skipgram(NCE_loss).py similarity index 100% rename from 1-2.Word2Vec/Word2Vec-Skipgram-Tensor(NCE_loss).py rename to archive/tensorflow/v1/1-2.Word2Vec/Word2Vec-Skipgram(NCE_loss).py diff --git a/1-2.Word2Vec/Word2Vec-Skipgram-Tensor(Softmax).py b/archive/tensorflow/v1/1-2.Word2Vec/Word2Vec-Skipgram(Softmax).py similarity index 100% rename from 1-2.Word2Vec/Word2Vec-Skipgram-Tensor(Softmax).py rename to archive/tensorflow/v1/1-2.Word2Vec/Word2Vec-Skipgram(Softmax).py diff --git a/2-1.TextCNN/TextCNN-Tensor.py b/archive/tensorflow/v1/2-1.TextCNN/TextCNN.py similarity index 100% rename from 2-1.TextCNN/TextCNN-Tensor.py rename to archive/tensorflow/v1/2-1.TextCNN/TextCNN.py diff --git a/3-1.TextRNN/TextRNN-Tensor.py b/archive/tensorflow/v1/3-1.TextRNN/TextRNN.py similarity index 100% rename from 3-1.TextRNN/TextRNN-Tensor.py rename to archive/tensorflow/v1/3-1.TextRNN/TextRNN.py diff --git a/3-2.TextLSTM/TextLSTM-Tensor.py b/archive/tensorflow/v1/3-2.TextLSTM/TextLSTM.py similarity index 100% rename from 3-2.TextLSTM/TextLSTM-Tensor.py rename to archive/tensorflow/v1/3-2.TextLSTM/TextLSTM.py diff --git a/3-3.Bi-LSTM/Bi-LSTM-Tensor.py b/archive/tensorflow/v1/3-3.Bi-LSTM/Bi-LSTM.py similarity index 100% rename from 3-3.Bi-LSTM/Bi-LSTM-Tensor.py rename to archive/tensorflow/v1/3-3.Bi-LSTM/Bi-LSTM.py diff --git a/4-1.Seq2Seq/Seq2Seq-Tensor.py b/archive/tensorflow/v1/4-1.Seq2Seq/Seq2Seq.py similarity index 100% rename from 4-1.Seq2Seq/Seq2Seq-Tensor.py rename to archive/tensorflow/v1/4-1.Seq2Seq/Seq2Seq.py diff --git a/4-2.Seq2Seq(Attention)/Seq2Seq(Attention)-Tensor.py b/archive/tensorflow/v1/4-2.Seq2Seq(Attention)/Seq2Seq(Attention).py similarity index 100% rename from 4-2.Seq2Seq(Attention)/Seq2Seq(Attention)-Tensor.py rename to archive/tensorflow/v1/4-2.Seq2Seq(Attention)/Seq2Seq(Attention).py diff --git a/4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention)-Tensor.py b/archive/tensorflow/v1/4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention).py similarity index 100% rename from 4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention)-Tensor.py rename to archive/tensorflow/v1/4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention).py From 5c4ed7f3cdec9f34baef489f967b91ebd885cc82 Mon Sep 17 00:00:00 2001 From: graykode Date: Fri, 14 Aug 2020 19:17:55 +0900 Subject: [PATCH 2/3] #57 (refactor) update above pytorch 1.0.0 with refactoring code --- 1-1.NNLM/NNLM.py | 94 ++++++------ 1-2.Word2Vec/Word2Vec-Skipgram(Softmax).py | 129 ++++++++-------- 2-1.TextCNN/TextCNN.py | 128 +++++++--------- 3-1.TextRNN/TextRNN.py | 100 ++++++------- 3-2.TextLSTM/TextLSTM.py | 78 +++++----- 3-3.Bi-LSTM/Bi-LSTM.py | 83 ++++++----- 4-1.Seq2Seq/Seq2Seq.py | 131 ++++++++--------- 4-2.Seq2Seq(Attention)/Seq2Seq(Attention).py | 105 ++++++------- 4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention).py | 126 +++++++--------- .../Transformer(Greedy_decoder).py | 99 ++++++------- 5-1.Transformer/Transformer.py | 98 ++++++------ 5-2.BERT/BERT.py | 139 +++++++++--------- 12 files changed, 625 insertions(+), 685 deletions(-) diff --git a/1-1.NNLM/NNLM.py b/1-1.NNLM/NNLM.py index 2a1a97c..6fcf3ee 100644 --- a/1-1.NNLM/NNLM.py +++ b/1-1.NNLM/NNLM.py @@ -1,33 +1,17 @@ +# %% # code by Tae Hwan Jung @graykode -import numpy as np import torch import torch.nn as nn import torch.optim as optim -from torch.autograd import Variable -dtype = torch.FloatTensor - -sentences = [ "i like dog", "i love coffee", "i hate milk"] - -word_list = " ".join(sentences).split() -word_list = list(set(word_list)) -word_dict = {w: i for i, w in enumerate(word_list)} -number_dict = {i: w for i, w in enumerate(word_list)} -n_class = len(word_dict) # number of Vocabulary - -# NNLM Parameter -n_step = 2 # n-1 in paper -n_hidden = 2 # h in paper -m = 2 # m in paper - -def make_batch(sentences): +def make_batch(): input_batch = [] target_batch = [] for sen in sentences: - word = sen.split() - input = [word_dict[n] for n in word[:-1]] - target = word_dict[word[-1]] + word = sen.split() # space tokenizer + input = [word_dict[n] for n in word[:-1]] # create (1~n-1) as input + target = word_dict[word[-1]] # create (n) as target, We usually call this 'casual language model' input_batch.append(input) target_batch.append(target) @@ -39,44 +23,56 @@ class NNLM(nn.Module): def __init__(self): super(NNLM, self).__init__() self.C = nn.Embedding(n_class, m) - self.H = nn.Parameter(torch.randn(n_step * m, n_hidden).type(dtype)) - self.W = nn.Parameter(torch.randn(n_step * m, n_class).type(dtype)) - self.d = nn.Parameter(torch.randn(n_hidden).type(dtype)) - self.U = nn.Parameter(torch.randn(n_hidden, n_class).type(dtype)) - self.b = nn.Parameter(torch.randn(n_class).type(dtype)) + self.H = nn.Linear(n_step * m, n_hidden, bias=False) + self.d = nn.Parameter(torch.ones(n_hidden)) + self.U = nn.Linear(n_hidden, n_class, bias=False) + self.W = nn.Linear(n_step * m, n_class, bias=False) + self.b = nn.Parameter(torch.ones(n_class)) def forward(self, X): - X = self.C(X) + X = self.C(X) # X : [batch_size, n_step, n_class] X = X.view(-1, n_step * m) # [batch_size, n_step * n_class] - tanh = torch.tanh(self.d + torch.mm(X, self.H)) # [batch_size, n_hidden] - output = self.b + torch.mm(X, self.W) + torch.mm(tanh, self.U) # [batch_size, n_class] + tanh = torch.tanh(self.d + self.H(X)) # [batch_size, n_hidden] + output = self.b + self.W(X) + self.U(tanh) # [batch_size, n_class] return output -model = NNLM() +if __name__ == '__main__': + n_step = 2 # number of steps, n-1 in paper + n_hidden = 2 # number of hidden size, h in paper + m = 2 # embedding size, m in paper + + sentences = ["i like dog", "i love coffee", "i hate milk"] + + word_list = " ".join(sentences).split() + word_list = list(set(word_list)) + word_dict = {w: i for i, w in enumerate(word_list)} + number_dict = {i: w for i, w in enumerate(word_list)} + n_class = len(word_dict) # number of Vocabulary -criterion = nn.CrossEntropyLoss() -optimizer = optim.Adam(model.parameters(), lr=0.001) + model = NNLM() -input_batch, target_batch = make_batch(sentences) -input_batch = Variable(torch.LongTensor(input_batch)) -target_batch = Variable(torch.LongTensor(target_batch)) + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.001) -# Training -for epoch in range(5000): + input_batch, target_batch = make_batch() + input_batch = torch.LongTensor(input_batch) + target_batch = torch.LongTensor(target_batch) - optimizer.zero_grad() - output = model(input_batch) + # Training + for epoch in range(5000): + optimizer.zero_grad() + output = model(input_batch) - # output : [batch_size, n_class], target_batch : [batch_size] (LongTensor, not one-hot) - loss = criterion(output, target_batch) - if (epoch + 1)%1000 == 0: - print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) + # output : [batch_size, n_class], target_batch : [batch_size] + loss = criterion(output, target_batch) + if (epoch + 1) % 1000 == 0: + print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) - loss.backward() - optimizer.step() + loss.backward() + optimizer.step() -# Predict -predict = model(input_batch).data.max(1, keepdim=True)[1] + # Predict + predict = model(input_batch).data.max(1, keepdim=True)[1] -# Test -print([sen.split()[:2] for sen in sentences], '->', [number_dict[n.item()] for n in predict.squeeze()]) + # Test + print([sen.split()[:2] for sen in sentences], '->', [number_dict[n.item()] for n in predict.squeeze()]) \ No newline at end of file diff --git a/1-2.Word2Vec/Word2Vec-Skipgram(Softmax).py b/1-2.Word2Vec/Word2Vec-Skipgram(Softmax).py index 3f2a8e7..4a582fb 100644 --- a/1-2.Word2Vec/Word2Vec-Skipgram(Softmax).py +++ b/1-2.Word2Vec/Word2Vec-Skipgram(Softmax).py @@ -1,93 +1,82 @@ -''' - code by Tae Hwan Jung(Jeff Jung) @graykode -''' +# %% +# code by Tae Hwan Jung @graykode import numpy as np import torch import torch.nn as nn import torch.optim as optim -from torch.autograd import Variable import matplotlib.pyplot as plt -dtype = torch.FloatTensor - -# 3 Words Sentence -sentences = [ "i like dog", "i like cat", "i like animal", - "dog cat animal", "apple cat dog like", "dog fish milk like", - "dog cat eyes like", "i like apple", "apple i hate", - "apple i movie book music like", "cat dog hate", "cat dog like"] - -word_sequence = " ".join(sentences).split() -word_list = " ".join(sentences).split() -word_list = list(set(word_list)) -word_dict = {w: i for i, w in enumerate(word_list)} - -# Word2Vec Parameter -batch_size = 20 # To show 2 dim embedding graph -embedding_size = 2 # To show 2 dim embedding graph -voc_size = len(word_list) - -def random_batch(data, size): +def random_batch(): random_inputs = [] random_labels = [] - random_index = np.random.choice(range(len(data)), size, replace=False) + random_index = np.random.choice(range(len(skip_grams)), batch_size, replace=False) for i in random_index: - random_inputs.append(np.eye(voc_size)[data[i][0]]) # target - random_labels.append(data[i][1]) # context word + random_inputs.append(np.eye(voc_size)[skip_grams[i][0]]) # target + random_labels.append(skip_grams[i][1]) # context word return random_inputs, random_labels -# Make skip gram of one size window -skip_grams = [] -for i in range(1, len(word_sequence) - 1): - target = word_dict[word_sequence[i]] - context = [word_dict[word_sequence[i - 1]], word_dict[word_sequence[i + 1]]] - - for w in context: - skip_grams.append([target, w]) - # Model class Word2Vec(nn.Module): def __init__(self): super(Word2Vec, self).__init__() - # W and WT is not Traspose relationship - self.W = nn.Parameter(-2 * torch.rand(voc_size, embedding_size) + 1).type(dtype) # voc_size > embedding_size Weight - self.WT = nn.Parameter(-2 * torch.rand(embedding_size, voc_size) + 1).type(dtype) # embedding_size > voc_size Weight + self.W = nn.Linear(voc_size, embedding_size, bias=False) # voc_size > embedding_size Weight + self.WT = nn.Linear(embedding_size, voc_size, bias=False) # embedding_size > voc_size Weight def forward(self, X): # X : [batch_size, voc_size] - hidden_layer = torch.matmul(X, self.W) # hidden_layer : [batch_size, embedding_size] - output_layer = torch.matmul(hidden_layer, self.WT) # output_layer : [batch_size, voc_size] + hidden_layer = self.W(X) # hidden_layer : [batch_size, embedding_size] + output_layer = self.WT(hidden_layer) # output_layer : [batch_size, voc_size] return output_layer -model = Word2Vec() - -criterion = nn.CrossEntropyLoss() -optimizer = optim.Adam(model.parameters(), lr=0.001) - -# Training -for epoch in range(5000): - - input_batch, target_batch = random_batch(skip_grams, batch_size) - - input_batch = Variable(torch.Tensor(input_batch)) - target_batch = Variable(torch.LongTensor(target_batch)) - - optimizer.zero_grad() - output = model(input_batch) - - # output : [batch_size, voc_size], target_batch : [batch_size] (LongTensor, not one-hot) - loss = criterion(output, target_batch) - if (epoch + 1)%1000 == 0: - print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) - - loss.backward() - optimizer.step() - -for i, label in enumerate(word_list): - W, WT = model.parameters() - x,y = float(W[i][0]), float(W[i][1]) - plt.scatter(x, y) - plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom') -plt.show() +if __name__ == '__main__': + batch_size = 2 # mini-batch size + embedding_size = 2 # embedding size + + sentences = ["apple banana fruit", "banana orange fruit", "orange banana fruit", + "dog cat animal", "cat monkey animal", "monkey dog animal"] + + word_sequence = " ".join(sentences).split() + word_list = " ".join(sentences).split() + word_list = list(set(word_list)) + word_dict = {w: i for i, w in enumerate(word_list)} + voc_size = len(word_list) + + # Make skip gram of one size window + skip_grams = [] + for i in range(1, len(word_sequence) - 1): + target = word_dict[word_sequence[i]] + context = [word_dict[word_sequence[i - 1]], word_dict[word_sequence[i + 1]]] + for w in context: + skip_grams.append([target, w]) + + model = Word2Vec() + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.001) + + # Training + for epoch in range(5000): + input_batch, target_batch = random_batch() + input_batch = torch.Tensor(input_batch) + target_batch = torch.LongTensor(target_batch) + + optimizer.zero_grad() + output = model(input_batch) + + # output : [batch_size, voc_size], target_batch : [batch_size] (LongTensor, not one-hot) + loss = criterion(output, target_batch) + if (epoch + 1) % 1000 == 0: + print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) + + loss.backward() + optimizer.step() + + for i, label in enumerate(word_list): + W, WT = model.parameters() + x, y = W[0][i].item(), W[1][i].item() + plt.scatter(x, y) + plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom') + plt.show() diff --git a/2-1.TextCNN/TextCNN.py b/2-1.TextCNN/TextCNN.py index 94baee6..f19c139 100644 --- a/2-1.TextCNN/TextCNN.py +++ b/2-1.TextCNN/TextCNN.py @@ -1,56 +1,22 @@ -''' - code by Tae Hwan Jung(Jeff Jung) @graykode -''' +# %% +# code by Tae Hwan Jung @graykode import numpy as np import torch import torch.nn as nn import torch.optim as optim -from torch.autograd import Variable import torch.nn.functional as F -dtype = torch.FloatTensor - -# Text-CNN Parameter -embedding_size = 2 # n-gram -sequence_length = 3 -num_classes = 2 # 0 or 1 -filter_sizes = [2, 2, 2] # n-gram window -num_filters = 3 - -# 3 words sentences (=sequence_length is 3) -sentences = ["i love you", "he loves me", "she likes baseball", "i hate you", "sorry for that", "this is awful"] -labels = [1, 1, 1, 0, 0, 0] # 1 is good, 0 is not good. - -word_list = " ".join(sentences).split() -word_list = list(set(word_list)) -word_dict = {w: i for i, w in enumerate(word_list)} -vocab_size = len(word_dict) - -inputs = [] -for sen in sentences: - inputs.append(np.asarray([word_dict[n] for n in sen.split()])) - -targets = [] -for out in labels: - targets.append(out) # To using Torch Softmax Loss function - -input_batch = Variable(torch.LongTensor(inputs)) -target_batch = Variable(torch.LongTensor(targets)) - - class TextCNN(nn.Module): def __init__(self): super(TextCNN, self).__init__() - self.num_filters_total = num_filters * len(filter_sizes) - self.W = nn.Parameter(torch.empty(vocab_size, embedding_size).uniform_(-1, 1)).type(dtype) - self.Weight = nn.Parameter(torch.empty(self.num_filters_total, num_classes).uniform_(-1, 1)).type(dtype) - self.Bias = nn.Parameter(0.1 * torch.ones([num_classes])).type(dtype) - self.filter_list = nn.ModuleList( - [nn.Conv2d(1, num_filters, (size, embedding_size), bias=True) for size in filter_sizes]) + self.W = nn.Embedding(vocab_size, embedding_size) + self.Weight = nn.Linear(self.num_filters_total, num_classes, bias=False) + self.Bias = nn.Parameter(torch.ones([num_classes])) + self.filter_list = nn.ModuleList([nn.Conv2d(1, num_filters, (size, embedding_size)) for size in filter_sizes]) def forward(self, X): - embedded_chars = self.W[X] # [batch_size, sequence_length, embedding_size] + embedded_chars = self.W(X) # [batch_size, sequence_length, sequence_length] embedded_chars = embedded_chars.unsqueeze(1) # add channel(=1) [batch, channel(=1), sequence_length, embedding_size] pooled_outputs = [] @@ -65,36 +31,54 @@ def forward(self, X): h_pool = torch.cat(pooled_outputs, len(filter_sizes)) # [batch_size(=6), output_height(=1), output_width(=1), output_channel(=3) * 3] h_pool_flat = torch.reshape(h_pool, [-1, self.num_filters_total]) # [batch_size(=6), output_height * output_width * (output_channel * 3)] - - model = torch.mm(h_pool_flat, self.Weight) + self.Bias # [batch_size, num_classes] + model = self.Weight(h_pool_flat) + self.Bias # [batch_size, num_classes] return model -model = TextCNN() - -criterion = nn.CrossEntropyLoss() -optimizer = optim.Adam(model.parameters(), lr=0.001) - -# Training -for epoch in range(5000): - optimizer.zero_grad() - output = model(input_batch) - - # output : [batch_size, num_classes], target_batch : [batch_size] (LongTensor, not one-hot) - loss = criterion(output, target_batch) - if (epoch + 1) % 1000 == 0: - print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) - - loss.backward() - optimizer.step() - -# Test -test_text = 'sorry hate you' -tests = [np.asarray([word_dict[n] for n in test_text.split()])] -test_batch = Variable(torch.LongTensor(tests)) - -# Predict -predict = model(test_batch).data.max(1, keepdim=True)[1] -if predict[0][0] == 0: - print(test_text,"is Bad Mean...") -else: - print(test_text,"is Good Mean!!") +if __name__ == '__main__': + embedding_size = 2 # embedding size + sequence_length = 3 # sequence length + num_classes = 2 # number of classes + filter_sizes = [2, 2, 2] # n-gram windows + num_filters = 3 # number of filters + + # 3 words sentences (=sequence_length is 3) + sentences = ["i love you", "he loves me", "she likes baseball", "i hate you", "sorry for that", "this is awful"] + labels = [1, 1, 1, 0, 0, 0] # 1 is good, 0 is not good. + + word_list = " ".join(sentences).split() + word_list = list(set(word_list)) + word_dict = {w: i for i, w in enumerate(word_list)} + vocab_size = len(word_dict) + + model = TextCNN() + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.001) + + inputs = torch.LongTensor([np.asarray([word_dict[n] for n in sen.split()]) for sen in sentences]) + targets = torch.LongTensor([out for out in labels]) # To using Torch Softmax Loss function + + # Training + for epoch in range(5000): + optimizer.zero_grad() + output = model(inputs) + + # output : [batch_size, num_classes], target_batch : [batch_size] (LongTensor, not one-hot) + loss = criterion(output, targets) + if (epoch + 1) % 1000 == 0: + print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) + + loss.backward() + optimizer.step() + + # Test + test_text = 'sorry hate you' + tests = [np.asarray([word_dict[n] for n in test_text.split()])] + test_batch = torch.LongTensor(tests) + + # Predict + predict = model(test_batch).data.max(1, keepdim=True)[1] + if predict[0][0] == 0: + print(test_text,"is Bad Mean...") + else: + print(test_text,"is Good Mean!!") \ No newline at end of file diff --git a/3-1.TextRNN/TextRNN.py b/3-1.TextRNN/TextRNN.py index cd6e25a..8bccb5f 100644 --- a/3-1.TextRNN/TextRNN.py +++ b/3-1.TextRNN/TextRNN.py @@ -1,53 +1,30 @@ -''' - code by Tae Hwan Jung(Jeff Jung) @graykode -''' +# %% +# code by Tae Hwan Jung @graykode import numpy as np import torch import torch.nn as nn import torch.optim as optim -from torch.autograd import Variable -dtype = torch.FloatTensor - -sentences = [ "i like dog", "i love coffee", "i hate milk"] - -word_list = " ".join(sentences).split() -word_list = list(set(word_list)) -word_dict = {w: i for i, w in enumerate(word_list)} -number_dict = {i: w for i, w in enumerate(word_list)} -n_class = len(word_dict) - -# TextRNN Parameter -batch_size = len(sentences) -n_step = 2 # number of cells(= number of Step) -n_hidden = 5 # number of hidden units in one cell - -def make_batch(sentences): +def make_batch(): input_batch = [] target_batch = [] for sen in sentences: - word = sen.split() - input = [word_dict[n] for n in word[:-1]] - target = word_dict[word[-1]] + word = sen.split() # space tokenizer + input = [word_dict[n] for n in word[:-1]] # create (1~n-1) as input + target = word_dict[word[-1]] # create (n) as target, We usually call this 'casual language model' input_batch.append(np.eye(n_class)[input]) target_batch.append(target) return input_batch, target_batch -# to Torch.Tensor -input_batch, target_batch = make_batch(sentences) -input_batch = Variable(torch.Tensor(input_batch)) -target_batch = Variable(torch.LongTensor(target_batch)) - class TextRNN(nn.Module): def __init__(self): super(TextRNN, self).__init__() - self.rnn = nn.RNN(input_size=n_class, hidden_size=n_hidden) - self.W = nn.Parameter(torch.randn([n_hidden, n_class]).type(dtype)) - self.b = nn.Parameter(torch.randn([n_class]).type(dtype)) + self.W = nn.Linear(n_hidden, n_class, bias=False) + self.b = nn.Parameter(torch.ones([n_class])) def forward(self, hidden, X): X = X.transpose(0, 1) # X : [n_step, batch_size, n_class] @@ -55,34 +32,51 @@ def forward(self, hidden, X): # outputs : [n_step, batch_size, num_directions(=1) * n_hidden] # hidden : [num_layers(=1) * num_directions(=1), batch_size, n_hidden] outputs = outputs[-1] # [batch_size, num_directions(=1) * n_hidden] - model = torch.mm(outputs, self.W) + self.b # model : [batch_size, n_class] + model = self.W(outputs) + self.b # model : [batch_size, n_class] return model -model = TextRNN() +if __name__ == '__main__': + n_step = 2 # number of cells(= number of Step) + n_hidden = 5 # number of hidden units in one cell + + sentences = ["i like dog", "i love coffee", "i hate milk"] + + word_list = " ".join(sentences).split() + word_list = list(set(word_list)) + word_dict = {w: i for i, w in enumerate(word_list)} + number_dict = {i: w for i, w in enumerate(word_list)} + n_class = len(word_dict) + batch_size = len(sentences) + + model = TextRNN() + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.001) -criterion = nn.CrossEntropyLoss() -optimizer = optim.Adam(model.parameters(), lr=0.001) + input_batch, target_batch = make_batch() + input_batch = torch.FloatTensor(input_batch) + target_batch = torch.LongTensor(target_batch) -# Training -for epoch in range(5000): - optimizer.zero_grad() + # Training + for epoch in range(5000): + optimizer.zero_grad() - # hidden : [num_layers * num_directions, batch, hidden_size] - hidden = Variable(torch.zeros(1, batch_size, n_hidden)) - # input_batch : [batch_size, n_step, n_class] - output = model(hidden, input_batch) + # hidden : [num_layers * num_directions, batch, hidden_size] + hidden = torch.zeros(1, batch_size, n_hidden) + # input_batch : [batch_size, n_step, n_class] + output = model(hidden, input_batch) - # output : [batch_size, n_class], target_batch : [batch_size] (LongTensor, not one-hot) - loss = criterion(output, target_batch) - if (epoch + 1) % 1000 == 0: - print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) + # output : [batch_size, n_class], target_batch : [batch_size] (LongTensor, not one-hot) + loss = criterion(output, target_batch) + if (epoch + 1) % 1000 == 0: + print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) - loss.backward() - optimizer.step() + loss.backward() + optimizer.step() -input = [sen.split()[:2] for sen in sentences] + input = [sen.split()[:2] for sen in sentences] -# Predict -hidden = Variable(torch.zeros(1, batch_size, n_hidden)) -predict = model(hidden, input_batch).data.max(1, keepdim=True)[1] -print([sen.split()[:2] for sen in sentences], '->', [number_dict[n.item()] for n in predict.squeeze()]) \ No newline at end of file + # Predict + hidden = torch.zeros(1, batch_size, n_hidden) + predict = model(hidden, input_batch).data.max(1, keepdim=True)[1] + print([sen.split()[:2] for sen in sentences], '->', [number_dict[n.item()] for n in predict.squeeze()]) \ No newline at end of file diff --git a/3-2.TextLSTM/TextLSTM.py b/3-2.TextLSTM/TextLSTM.py index 44525ff..29e15e4 100644 --- a/3-2.TextLSTM/TextLSTM.py +++ b/3-2.TextLSTM/TextLSTM.py @@ -1,26 +1,11 @@ -''' - code by Tae Hwan Jung(Jeff Jung) @graykode -''' +# %% +# code by Tae Hwan Jung @graykode import numpy as np import torch import torch.nn as nn import torch.optim as optim -from torch.autograd import Variable -dtype = torch.FloatTensor - -char_arr = [c for c in 'abcdefghijklmnopqrstuvwxyz'] -word_dict = {n: i for i, n in enumerate(char_arr)} -number_dict = {i: w for i, w in enumerate(char_arr)} -n_class = len(word_dict) # number of class(=number of vocab) - -seq_data = ['make', 'need', 'coal', 'word', 'love', 'hate', 'live', 'home', 'hash', 'star'] - -# TextLSTM Parameters -n_step = 3 -n_hidden = 128 - -def make_batch(seq_data): +def make_batch(): input_batch, target_batch = [], [] for seq in seq_data: @@ -29,49 +14,60 @@ def make_batch(seq_data): input_batch.append(np.eye(n_class)[input]) target_batch.append(target) - return Variable(torch.Tensor(input_batch)), Variable(torch.LongTensor(target_batch)) + return input_batch, target_batch class TextLSTM(nn.Module): def __init__(self): super(TextLSTM, self).__init__() self.lstm = nn.LSTM(input_size=n_class, hidden_size=n_hidden) - self.W = nn.Parameter(torch.randn([n_hidden, n_class]).type(dtype)) - self.b = nn.Parameter(torch.randn([n_class]).type(dtype)) + self.W = nn.Linear(n_hidden, n_class, bias=False) + self.b = nn.Parameter(torch.ones([n_class])) def forward(self, X): input = X.transpose(0, 1) # X : [n_step, batch_size, n_class] - hidden_state = Variable(torch.zeros(1, len(X), n_hidden)) # [num_layers(=1) * num_directions(=1), batch_size, n_hidden] - cell_state = Variable(torch.zeros(1, len(X), n_hidden)) # [num_layers(=1) * num_directions(=1), batch_size, n_hidden] + hidden_state = torch.zeros(1, len(X), n_hidden) # [num_layers(=1) * num_directions(=1), batch_size, n_hidden] + cell_state = torch.zeros(1, len(X), n_hidden) # [num_layers(=1) * num_directions(=1), batch_size, n_hidden] outputs, (_, _) = self.lstm(input, (hidden_state, cell_state)) outputs = outputs[-1] # [batch_size, n_hidden] - model = torch.mm(outputs, self.W) + self.b # model : [batch_size, n_class] + model = self.W(outputs) + self.b # model : [batch_size, n_class] return model -input_batch, target_batch = make_batch(seq_data) +if __name__ == '__main__': + n_step = 3 # number of cells(= number of Step) + n_hidden = 128 # number of hidden units in one cell + + char_arr = [c for c in 'abcdefghijklmnopqrstuvwxyz'] + word_dict = {n: i for i, n in enumerate(char_arr)} + number_dict = {i: w for i, w in enumerate(char_arr)} + n_class = len(word_dict) # number of class(=number of vocab) + + seq_data = ['make', 'need', 'coal', 'word', 'love', 'hate', 'live', 'home', 'hash', 'star'] -model = TextLSTM() + model = TextLSTM() -criterion = nn.CrossEntropyLoss() -optimizer = optim.Adam(model.parameters(), lr=0.001) + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.001) -output = model(input_batch) + input_batch, target_batch = make_batch() + input_batch = torch.FloatTensor(input_batch) + target_batch = torch.LongTensor(target_batch) -# Training -for epoch in range(1000): - optimizer.zero_grad() + # Training + for epoch in range(1000): + optimizer.zero_grad() - output = model(input_batch) - loss = criterion(output, target_batch) - if (epoch + 1) % 100 == 0: - print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) + output = model(input_batch) + loss = criterion(output, target_batch) + if (epoch + 1) % 100 == 0: + print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) - loss.backward() - optimizer.step() + loss.backward() + optimizer.step() -inputs = [sen[:3] for sen in seq_data] + inputs = [sen[:3] for sen in seq_data] -predict = model(input_batch).data.max(1, keepdim=True)[1] -print(inputs, '->', [number_dict[n.item()] for n in predict.squeeze()]) \ No newline at end of file + predict = model(input_batch).data.max(1, keepdim=True)[1] + print(inputs, '->', [number_dict[n.item()] for n in predict.squeeze()]) \ No newline at end of file diff --git a/3-3.Bi-LSTM/Bi-LSTM.py b/3-3.Bi-LSTM/Bi-LSTM.py index a4ee1d2..3b7e638 100644 --- a/3-3.Bi-LSTM/Bi-LSTM.py +++ b/3-3.Bi-LSTM/Bi-LSTM.py @@ -1,28 +1,11 @@ -''' - code by Tae Hwan Jung(Jeff Jung) @graykode -''' +# %% +# code by Tae Hwan Jung @graykode import numpy as np import torch import torch.nn as nn import torch.optim as optim -from torch.autograd import Variable -import torch.nn.functional as F -dtype = torch.FloatTensor - -sentence = ( - 'Lorem ipsum dolor sit amet consectetur adipisicing elit ' - 'sed do eiusmod tempor incididunt ut labore et dolore magna ' - 'aliqua Ut enim ad minim veniam quis nostrud exercitation' -) - -word_dict = {w: i for i, w in enumerate(list(set(sentence.split())))} -number_dict = {i: w for i, w in enumerate(list(set(sentence.split())))} -n_class = len(word_dict) -max_len = len(sentence.split()) -n_hidden = 5 - -def make_batch(sentence): +def make_batch(): input_batch = [] target_batch = [] @@ -34,45 +17,61 @@ def make_batch(sentence): input_batch.append(np.eye(n_class)[input]) target_batch.append(target) - return Variable(torch.Tensor(input_batch)), Variable(torch.LongTensor(target_batch)) + return input_batch, target_batch class BiLSTM(nn.Module): def __init__(self): super(BiLSTM, self).__init__() self.lstm = nn.LSTM(input_size=n_class, hidden_size=n_hidden, bidirectional=True) - self.W = nn.Parameter(torch.randn([n_hidden * 2, n_class]).type(dtype)) - self.b = nn.Parameter(torch.randn([n_class]).type(dtype)) + self.W = nn.Linear(n_hidden * 2, n_class, bias=False) + self.b = nn.Parameter(torch.ones([n_class])) def forward(self, X): input = X.transpose(0, 1) # input : [n_step, batch_size, n_class] - hidden_state = Variable(torch.zeros(1*2, len(X), n_hidden)) # [num_layers(=1) * num_directions(=2), batch_size, n_hidden] - cell_state = Variable(torch.zeros(1*2, len(X), n_hidden)) # [num_layers(=1) * num_directions(=2), batch_size, n_hidden] + hidden_state = torch.zeros(1*2, len(X), n_hidden) # [num_layers(=1) * num_directions(=2), batch_size, n_hidden] + cell_state = torch.zeros(1*2, len(X), n_hidden) # [num_layers(=1) * num_directions(=2), batch_size, n_hidden] outputs, (_, _) = self.lstm(input, (hidden_state, cell_state)) outputs = outputs[-1] # [batch_size, n_hidden] - model = torch.mm(outputs, self.W) + self.b # model : [batch_size, n_class] + model = self.W(outputs) + self.b # model : [batch_size, n_class] return model -input_batch, target_batch = make_batch(sentence) +if __name__ == '__main__': + n_hidden = 5 # number of hidden units in one cell + + sentence = ( + 'Lorem ipsum dolor sit amet consectetur adipisicing elit ' + 'sed do eiusmod tempor incididunt ut labore et dolore magna ' + 'aliqua Ut enim ad minim veniam quis nostrud exercitation' + ) + + word_dict = {w: i for i, w in enumerate(list(set(sentence.split())))} + number_dict = {i: w for i, w in enumerate(list(set(sentence.split())))} + n_class = len(word_dict) + max_len = len(sentence.split()) + + model = BiLSTM() -model = BiLSTM() + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.001) -criterion = nn.CrossEntropyLoss() -optimizer = optim.Adam(model.parameters(), lr=0.001) + input_batch, target_batch = make_batch() + input_batch = torch.FloatTensor(input_batch) + target_batch = torch.LongTensor(target_batch) -# Training -for epoch in range(10000): - optimizer.zero_grad() - output = model(input_batch) - loss = criterion(output, target_batch) - if (epoch + 1) % 1000 == 0: - print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) + # Training + for epoch in range(10000): + optimizer.zero_grad() + output = model(input_batch) + loss = criterion(output, target_batch) + if (epoch + 1) % 1000 == 0: + print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) - loss.backward() - optimizer.step() + loss.backward() + optimizer.step() -predict = model(input_batch).data.max(1, keepdim=True)[1] -print(sentence) -print([number_dict[n.item()] for n in predict.squeeze()]) + predict = model(input_batch).data.max(1, keepdim=True)[1] + print(sentence) + print([number_dict[n.item()] for n in predict.squeeze()]) diff --git a/4-1.Seq2Seq/Seq2Seq.py b/4-1.Seq2Seq/Seq2Seq.py index ad9055d..8740484 100644 --- a/4-1.Seq2Seq/Seq2Seq.py +++ b/4-1.Seq2Seq/Seq2Seq.py @@ -1,26 +1,15 @@ -# code by Tae Hwan Jung(Jeff Jung) @graykode +# %% +# code by Tae Hwan Jung @graykode +import argparse import numpy as np import torch import torch.nn as nn -from torch.autograd import Variable -dtype = torch.FloatTensor # S: Symbol that shows starting of decoding input # E: Symbol that shows starting of decoding output # P: Symbol that will fill in blank sequence if current batch data size is short than time steps -char_arr = [c for c in 'SEPabcdefghijklmnopqrstuvwxyz'] -num_dic = {n: i for i, n in enumerate(char_arr)} - -seq_data = [['man', 'women'], ['black', 'white'], ['king', 'queen'], ['girl', 'boy'], ['up', 'down'], ['high', 'low']] - -# Seq2Seq Parameter -n_step = 5 -n_hidden = 128 -n_class = len(num_dic) -batch_size = len(seq_data) - -def make_batch(seq_data): +def make_batch(): input_batch, output_batch, target_batch = [], [], [] for seq in seq_data: @@ -36,7 +25,7 @@ def make_batch(seq_data): target_batch.append(target) # not one-hot # make tensor - return Variable(torch.Tensor(input_batch)), Variable(torch.Tensor(output_batch)), Variable(torch.LongTensor(target_batch)) + return torch.FloatTensor(input_batch), torch.FloatTensor(output_batch), torch.LongTensor(target_batch) # Model class Seq2Seq(nn.Module): @@ -59,53 +48,63 @@ def forward(self, enc_input, enc_hidden, dec_input): model = self.fc(outputs) # model : [max_len+1(=6), batch_size, n_class] return model - -input_batch, output_batch, target_batch = make_batch(seq_data) - -model = Seq2Seq() -criterion = nn.CrossEntropyLoss() -optimizer = torch.optim.Adam(model.parameters(), lr=0.001) - -for epoch in range(5000): - # make hidden shape [num_layers * num_directions, batch_size, n_hidden] - hidden = Variable(torch.zeros(1, batch_size, n_hidden)) - - optimizer.zero_grad() - # input_batch : [batch_size, max_len(=n_step, time step), n_class] - # output_batch : [batch_size, max_len+1(=n_step, time step) (becase of 'S' or 'E'), n_class] - # target_batch : [batch_size, max_len+1(=n_step, time step)], not one-hot - output = model(input_batch, hidden, output_batch) - # output : [max_len+1, batch_size, n_class] - output = output.transpose(0, 1) # [batch_size, max_len+1(=6), n_class] - loss = 0 - for i in range(0, len(target_batch)): - # output[i] : [max_len+1, n_class, target_batch[i] : max_len+1] - loss += criterion(output[i], target_batch[i]) - if (epoch + 1) % 1000 == 0: - print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) - loss.backward() - optimizer.step() - - -# Test -def translate(word): - input_batch, output_batch, _ = make_batch([[word, 'P' * len(word)]]) - - # make hidden shape [num_layers * num_directions, batch_size, n_hidden] - hidden = Variable(torch.zeros(1, 1, n_hidden)) - output = model(input_batch, hidden, output_batch) - # output : [max_len+1(=6), batch_size(=1), n_class] - - predict = output.data.max(2, keepdim=True)[1] # select n_class dimension - decoded = [char_arr[i] for i in predict] - end = decoded.index('E') - translated = ''.join(decoded[:end]) - - return translated.replace('P', '') - -print('test') -print('man ->', translate('man')) -print('mans ->', translate('mans')) -print('king ->', translate('king')) -print('black ->', translate('black')) -print('upp ->', translate('upp')) \ No newline at end of file +if __name__ == '__main__': + n_step = 5 + n_hidden = 128 + + char_arr = [c for c in 'SEPabcdefghijklmnopqrstuvwxyz'] + num_dic = {n: i for i, n in enumerate(char_arr)} + seq_data = [['man', 'women'], ['black', 'white'], ['king', 'queen'], ['girl', 'boy'], ['up', 'down'], ['high', 'low']] + + n_class = len(num_dic) + batch_size = len(seq_data) + + model = Seq2Seq() + + criterion = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(model.parameters(), lr=0.001) + + input_batch, output_batch, target_batch = make_batch() + + for epoch in range(5000): + # make hidden shape [num_layers * num_directions, batch_size, n_hidden] + hidden = torch.zeros(1, batch_size, n_hidden) + + optimizer.zero_grad() + # input_batch : [batch_size, max_len(=n_step, time step), n_class] + # output_batch : [batch_size, max_len+1(=n_step, time step) (becase of 'S' or 'E'), n_class] + # target_batch : [batch_size, max_len+1(=n_step, time step)], not one-hot + output = model(input_batch, hidden, output_batch) + # output : [max_len+1, batch_size, n_class] + output = output.transpose(0, 1) # [batch_size, max_len+1(=6), n_class] + loss = 0 + for i in range(0, len(target_batch)): + # output[i] : [max_len+1, n_class, target_batch[i] : max_len+1] + loss += criterion(output[i], target_batch[i]) + if (epoch + 1) % 1000 == 0: + print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) + loss.backward() + optimizer.step() + + # Test + def translate(word, args): + input_batch, output_batch, _ = make_batch([[word, 'P' * len(word)]], args) + + # make hidden shape [num_layers * num_directions, batch_size, n_hidden] + hidden = torch.zeros(1, 1, args.n_hidden) + output = model(input_batch, hidden, output_batch) + # output : [max_len+1(=6), batch_size(=1), n_class] + + predict = output.data.max(2, keepdim=True)[1] # select n_class dimension + decoded = [char_arr[i] for i in predict] + end = decoded.index('E') + translated = ''.join(decoded[:end]) + + return translated.replace('P', '') + + print('test') + print('man ->', translate('man')) + print('mans ->', translate('mans')) + print('king ->', translate('king')) + print('black ->', translate('black')) + print('upp ->', translate('upp')) \ No newline at end of file diff --git a/4-2.Seq2Seq(Attention)/Seq2Seq(Attention).py b/4-2.Seq2Seq(Attention)/Seq2Seq(Attention).py index 2bbe1af..6337656 100644 --- a/4-2.Seq2Seq(Attention)/Seq2Seq(Attention).py +++ b/4-2.Seq2Seq(Attention)/Seq2Seq(Attention).py @@ -1,34 +1,23 @@ -# code by Tae Hwan Jung(Jeff Jung) @graykode +# %% +# code by Tae Hwan Jung @graykode # Reference : https://github.com/hunkim/PyTorchZeroToAll/blob/master/14_2_seq2seq_att.py import numpy as np import torch import torch.nn as nn -from torch.autograd import Variable import torch.nn.functional as F import matplotlib.pyplot as plt -dtype = torch.FloatTensor # S: Symbol that shows starting of decoding input # E: Symbol that shows starting of decoding output # P: Symbol that will fill in blank sequence if current batch data size is short than time steps -sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E'] -word_list = " ".join(sentences).split() -word_list = list(set(word_list)) -word_dict = {w: i for i, w in enumerate(word_list)} -number_dict = {i: w for i, w in enumerate(word_list)} -n_class = len(word_dict) # vocab list - -# Parameter -n_hidden = 128 - -def make_batch(sentences): +def make_batch(): input_batch = [np.eye(n_class)[[word_dict[n] for n in sentences[0].split()]]] output_batch = [np.eye(n_class)[[word_dict[n] for n in sentences[1].split()]]] target_batch = [[word_dict[n] for n in sentences[2].split()]] # make tensor - return Variable(torch.Tensor(input_batch)), Variable(torch.Tensor(output_batch)), Variable(torch.LongTensor(target_batch)) + return torch.FloatTensor(input_batch), torch.FloatTensor(output_batch), torch.LongTensor(target_batch) class Attention(nn.Module): def __init__(self): @@ -51,7 +40,7 @@ def forward(self, enc_inputs, hidden, dec_inputs): trained_attn = [] hidden = enc_hidden n_step = len(dec_inputs) - model = Variable(torch.empty([n_step, 1, n_class])) + model = torch.empty([n_step, 1, n_class]) for i in range(n_step): # each time step # dec_output : [n_step(=1), batch_size(=1), num_directions(=1) * n_hidden] @@ -71,7 +60,7 @@ def forward(self, enc_inputs, hidden, dec_inputs): def get_att_weight(self, dec_output, enc_outputs): # get attention weight one 'dec_output' with 'enc_outputs' n_step = len(enc_outputs) - attn_scores = Variable(torch.zeros(n_step)) # attn_scores : [n_step] + attn_scores = torch.zeros(n_step) # attn_scores : [n_step] for i in range(n_step): attn_scores[i] = self.get_att_score(dec_output, enc_outputs[i]) @@ -83,38 +72,50 @@ def get_att_score(self, dec_output, enc_output): # enc_outputs [batch_size, num score = self.attn(enc_output) # score : [batch_size, n_hidden] return torch.dot(dec_output.view(-1), score.view(-1)) # inner product make scalar value -input_batch, output_batch, target_batch = make_batch(sentences) - -# hidden : [num_layers(=1) * num_directions(=1), batch_size, n_hidden] -hidden = Variable(torch.zeros(1, 1, n_hidden)) - -model = Attention() -criterion = nn.CrossEntropyLoss() -optimizer = torch.optim.Adam(model.parameters(), lr=0.001) - -# Train -for epoch in range(2000): - optimizer.zero_grad() - output, _ = model(input_batch, hidden, output_batch) - - loss = criterion(output, target_batch.squeeze(0)) - if (epoch + 1) % 400 == 0: - print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) - - loss.backward() - optimizer.step() - -# Test -test_batch = [np.eye(n_class)[[word_dict[n] for n in 'SPPPP']]] -test_batch = Variable(torch.Tensor(test_batch)) -predict, trained_attn = model(input_batch, hidden, test_batch) -predict = predict.data.max(1, keepdim=True)[1] -print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()]) - -# Show Attention -fig = plt.figure(figsize=(5, 5)) -ax = fig.add_subplot(1, 1, 1) -ax.matshow(trained_attn, cmap='viridis') -ax.set_xticklabels([''] + sentences[0].split(), fontdict={'fontsize': 14}) -ax.set_yticklabels([''] + sentences[2].split(), fontdict={'fontsize': 14}) -plt.show() \ No newline at end of file +if __name__ == '__main__': + n_step = 5 # number of cells(= number of Step) + n_hidden = 128 # number of hidden units in one cell + + sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E'] + + word_list = " ".join(sentences).split() + word_list = list(set(word_list)) + word_dict = {w: i for i, w in enumerate(word_list)} + number_dict = {i: w for i, w in enumerate(word_list)} + n_class = len(word_dict) # vocab list + + # hidden : [num_layers(=1) * num_directions(=1), batch_size, n_hidden] + hidden = torch.zeros(1, 1, n_hidden) + + model = Attention() + criterion = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(model.parameters(), lr=0.001) + + input_batch, output_batch, target_batch = make_batch() + + # Train + for epoch in range(2000): + optimizer.zero_grad() + output, _ = model(input_batch, hidden, output_batch) + + loss = criterion(output, target_batch.squeeze(0)) + if (epoch + 1) % 400 == 0: + print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) + + loss.backward() + optimizer.step() + + # Test + test_batch = [np.eye(n_class)[[word_dict[n] for n in 'SPPPP']]] + test_batch = torch.FloatTensor(test_batch) + predict, trained_attn = model(input_batch, hidden, test_batch) + predict = predict.data.max(1, keepdim=True)[1] + print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()]) + + # Show Attention + fig = plt.figure(figsize=(5, 5)) + ax = fig.add_subplot(1, 1, 1) + ax.matshow(trained_attn, cmap='viridis') + ax.set_xticklabels([''] + sentences[0].split(), fontdict={'fontsize': 14}) + ax.set_yticklabels([''] + sentences[2].split(), fontdict={'fontsize': 14}) + plt.show() \ No newline at end of file diff --git a/4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention).py b/4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention).py index 1d066d9..7ff6365 100644 --- a/4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention).py +++ b/4-3.Bi-LSTM(Attention)/Bi-LSTM(Attention).py @@ -1,42 +1,13 @@ -''' - code by Tae Hwan Jung(Jeff Jung) @graykode - Reference : https://github.com/prakashpandey9/Text-Classification-Pytorch/blob/master/models/LSTM_Attn.py -''' +# %% +# code by Tae Hwan Jung(Jeff Jung) @graykode +# Reference : https://github.com/prakashpandey9/Text-Classification-Pytorch/blob/master/models/LSTM_Attn.py import numpy as np import torch import torch.nn as nn import torch.optim as optim -from torch.autograd import Variable import torch.nn.functional as F import matplotlib.pyplot as plt -dtype = torch.FloatTensor - -# Bi-LSTM(Attention) Parameters -embedding_dim = 2 -n_hidden = 5 # number of hidden units in one cell -num_classes = 2 # 0 or 1 - -# 3 words sentences (=sequence_length is 3) -sentences = ["i love you", "he loves me", "she likes baseball", "i hate you", "sorry for that", "this is awful"] -labels = [1, 1, 1, 0, 0, 0] # 1 is good, 0 is not good. - -word_list = " ".join(sentences).split() -word_list = list(set(word_list)) -word_dict = {w: i for i, w in enumerate(word_list)} -vocab_size = len(word_dict) - -inputs = [] -for sen in sentences: - inputs.append(np.asarray([word_dict[n] for n in sen.split()])) - -targets = [] -for out in labels: - targets.append(out) # To using Torch Softmax Loss function - -input_batch = Variable(torch.LongTensor(inputs)) -target_batch = Variable(torch.LongTensor(targets)) - class BiLSTM_Attention(nn.Module): def __init__(self): super(BiLSTM_Attention, self).__init__() @@ -58,8 +29,8 @@ def forward(self, X): input = self.embedding(X) # input : [batch_size, len_seq, embedding_dim] input = input.permute(1, 0, 2) # input : [len_seq, batch_size, embedding_dim] - hidden_state = Variable(torch.zeros(1*2, len(X), n_hidden)) # [num_layers(=1) * num_directions(=2), batch_size, n_hidden] - cell_state = Variable(torch.zeros(1*2, len(X), n_hidden)) # [num_layers(=1) * num_directions(=2), batch_size, n_hidden] + hidden_state = torch.zeros(1*2, len(X), n_hidden) # [num_layers(=1) * num_directions(=2), batch_size, n_hidden] + cell_state = torch.zeros(1*2, len(X), n_hidden) # [num_layers(=1) * num_directions(=2), batch_size, n_hidden] # final_hidden_state, final_cell_state : [num_layers(=1) * num_directions(=2), batch_size, n_hidden] output, (final_hidden_state, final_cell_state) = self.lstm(input, (hidden_state, cell_state)) @@ -67,38 +38,55 @@ def forward(self, X): attn_output, attention = self.attention_net(output, final_hidden_state) return self.out(attn_output), attention # model : [batch_size, num_classes], attention : [batch_size, n_step] -model = BiLSTM_Attention() - -criterion = nn.CrossEntropyLoss() -optimizer = optim.Adam(model.parameters(), lr=0.001) - -# Training -for epoch in range(5000): - optimizer.zero_grad() - output, attention = model(input_batch) - loss = criterion(output, target_batch) - if (epoch + 1) % 1000 == 0: - print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) - - loss.backward() - optimizer.step() - -# Test -test_text = 'sorry hate you' -tests = [np.asarray([word_dict[n] for n in test_text.split()])] -test_batch = Variable(torch.LongTensor(tests)) - -# Predict -predict, _ = model(test_batch) -predict = predict.data.max(1, keepdim=True)[1] -if predict[0][0] == 0: - print(test_text,"is Bad Mean...") -else: - print(test_text,"is Good Mean!!") - -fig = plt.figure(figsize=(6, 3)) # [batch_size, n_step] -ax = fig.add_subplot(1, 1, 1) -ax.matshow(attention, cmap='viridis') -ax.set_xticklabels(['']+['first_word', 'second_word', 'third_word'], fontdict={'fontsize': 14}, rotation=90) -ax.set_yticklabels(['']+['batch_1', 'batch_2', 'batch_3', 'batch_4', 'batch_5', 'batch_6'], fontdict={'fontsize': 14}) -plt.show() \ No newline at end of file +if __name__ == '__main__': + embedding_dim = 2 # embedding size + n_hidden = 5 # number of hidden units in one cell + num_classes = 2 # 0 or 1 + + # 3 words sentences (=sequence_length is 3) + sentences = ["i love you", "he loves me", "she likes baseball", "i hate you", "sorry for that", "this is awful"] + labels = [1, 1, 1, 0, 0, 0] # 1 is good, 0 is not good. + + word_list = " ".join(sentences).split() + word_list = list(set(word_list)) + word_dict = {w: i for i, w in enumerate(word_list)} + vocab_size = len(word_dict) + + model = BiLSTM_Attention() + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.001) + + inputs = torch.LongTensor([np.asarray([word_dict[n] for n in sen.split()]) for sen in sentences]) + targets = torch.LongTensor([out for out in labels]) # To using Torch Softmax Loss function + + # Training + for epoch in range(5000): + optimizer.zero_grad() + output, attention = model(inputs) + loss = criterion(output, targets) + if (epoch + 1) % 1000 == 0: + print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) + + loss.backward() + optimizer.step() + + # Test + test_text = 'sorry hate you' + tests = [np.asarray([word_dict[n] for n in test_text.split()])] + test_batch = torch.LongTensor(tests) + + # Predict + predict, _ = model(test_batch) + predict = predict.data.max(1, keepdim=True)[1] + if predict[0][0] == 0: + print(test_text,"is Bad Mean...") + else: + print(test_text,"is Good Mean!!") + + fig = plt.figure(figsize=(6, 3)) # [batch_size, n_step] + ax = fig.add_subplot(1, 1, 1) + ax.matshow(attention, cmap='viridis') + ax.set_xticklabels(['']+['first_word', 'second_word', 'third_word'], fontdict={'fontsize': 14}, rotation=90) + ax.set_yticklabels(['']+['batch_1', 'batch_2', 'batch_3', 'batch_4', 'batch_5', 'batch_6'], fontdict={'fontsize': 14}) + plt.show() \ No newline at end of file diff --git a/5-1.Transformer/Transformer(Greedy_decoder).py b/5-1.Transformer/Transformer(Greedy_decoder).py index ef674f4..35301ac 100644 --- a/5-1.Transformer/Transformer(Greedy_decoder).py +++ b/5-1.Transformer/Transformer(Greedy_decoder).py @@ -1,44 +1,22 @@ -''' - code by Tae Hwan Jung(Jeff Jung) @graykode, Derek Miller @dmmiller612 - Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch - https://github.com/JayParks/transformer -''' +# %% +# code by Tae Hwan Jung(Jeff Jung) @graykode, Derek Miller @dmmiller612 +# Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch +# https://github.com/JayParks/transformer import numpy as np import torch import torch.nn as nn import torch.optim as optim -from torch.autograd import Variable import matplotlib.pyplot as plt -dtype = torch.FloatTensor # S: Symbol that shows starting of decoding input # E: Symbol that shows starting of decoding output # P: Symbol that will fill in blank sequence if current batch data size is short than time steps -sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E'] -# Transformer Parameters -# Padding Should be Zero index -src_vocab = {'P' : 0, 'ich' : 1, 'mochte' : 2, 'ein' : 3, 'bier' : 4} -src_vocab_size = len(src_vocab) - -tgt_vocab = {'P' : 0, 'i' : 1, 'want' : 2, 'a' : 3, 'beer' : 4, 'S' : 5, 'E' : 6} -number_dict = {i: w for i, w in enumerate(tgt_vocab)} -tgt_vocab_size = len(tgt_vocab) - -src_len = 5 -tgt_len = 5 - -d_model = 512 # Embedding Size -d_ff = 2048 # FeedForward dimension -d_k = d_v = 64 # dimension of K(=Q), V -n_layers = 6 # number of Encoder of Decoder Layer -n_heads = 8 # number of heads in Multi-Head Attention - -def make_batch(sentences): +def make_batch(): input_batch = [[src_vocab[n] for n in sentences[0].split()]] output_batch = [[tgt_vocab[n] for n in sentences[1].split()]] target_batch = [[tgt_vocab[n] for n in sentences[2].split()]] - return Variable(torch.LongTensor(input_batch)), Variable(torch.LongTensor(output_batch)), Variable(torch.LongTensor(target_batch)) + return torch.LongTensor(input_batch), torch.LongTensor(output_batch), torch.LongTensor(target_batch) def get_sinusoid_encoding_table(n_position, d_model): def cal_angle(position, hid_idx): @@ -220,31 +198,52 @@ def showgraph(attn): ax.set_yticklabels(['']+sentences[2].split(), fontdict={'fontsize': 14}) plt.show() -model = Transformer() +if __name__ == '__main__': + sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E'] + # Transformer Parameters + # Padding Should be Zero index + src_vocab = {'P': 0, 'ich': 1, 'mochte': 2, 'ein': 3, 'bier': 4} + src_vocab_size = len(src_vocab) + + tgt_vocab = {'P': 0, 'i': 1, 'want': 2, 'a': 3, 'beer': 4, 'S': 5, 'E': 6} + number_dict = {i: w for i, w in enumerate(tgt_vocab)} + tgt_vocab_size = len(tgt_vocab) + + src_len = 5 # length of source + tgt_len = 5 # length of target + + d_model = 512 # Embedding Size + d_ff = 2048 # FeedForward dimension + d_k = d_v = 64 # dimension of K(=Q), V + n_layers = 6 # number of Encoder of Decoder Layer + n_heads = 8 # number of heads in Multi-Head Attention + + model = Transformer() + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.001) -criterion = nn.CrossEntropyLoss() -optimizer = optim.Adam(model.parameters(), lr=0.001) + enc_inputs, dec_inputs, target_batch = make_batch() -for epoch in range(20): - optimizer.zero_grad() - enc_inputs, dec_inputs, target_batch = make_batch(sentences) - outputs, enc_self_attns, dec_self_attns, dec_enc_attns = model(enc_inputs, dec_inputs) - loss = criterion(outputs, target_batch.contiguous().view(-1)) - print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) - loss.backward() - optimizer.step() + for epoch in range(20): + optimizer.zero_grad() + outputs, enc_self_attns, dec_self_attns, dec_enc_attns = model(enc_inputs, dec_inputs) + loss = criterion(outputs, target_batch.contiguous().view(-1)) + print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) + loss.backward() + optimizer.step() -# Test -greedy_dec_input = greedy_decoder(model, enc_inputs, start_symbol=tgt_vocab["S"]) -predict, _, _, _ = model(enc_inputs, greedy_dec_input) -predict = predict.data.max(1, keepdim=True)[1] -print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()]) + # Test + greedy_dec_input = greedy_decoder(model, enc_inputs, start_symbol=tgt_vocab["S"]) + predict, _, _, _ = model(enc_inputs, greedy_dec_input) + predict = predict.data.max(1, keepdim=True)[1] + print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()]) -print('first head of last state enc_self_attns') -showgraph(enc_self_attns) + print('first head of last state enc_self_attns') + showgraph(enc_self_attns) -print('first head of last state dec_self_attns') -showgraph(dec_self_attns) + print('first head of last state dec_self_attns') + showgraph(dec_self_attns) -print('first head of last state dec_enc_attns') -showgraph(dec_enc_attns) \ No newline at end of file + print('first head of last state dec_enc_attns') + showgraph(dec_enc_attns) \ No newline at end of file diff --git a/5-1.Transformer/Transformer.py b/5-1.Transformer/Transformer.py index c027fbf..fd1421e 100644 --- a/5-1.Transformer/Transformer.py +++ b/5-1.Transformer/Transformer.py @@ -1,44 +1,22 @@ -''' - code by Tae Hwan Jung(Jeff Jung) @graykode, Derek Miller @dmmiller612 - Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch - https://github.com/JayParks/transformer -''' +# %% +# code by Tae Hwan Jung(Jeff Jung) @graykode, Derek Miller @dmmiller612 +# Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch +# https://github.com/JayParks/transformer import numpy as np import torch import torch.nn as nn import torch.optim as optim -from torch.autograd import Variable import matplotlib.pyplot as plt -dtype = torch.FloatTensor # S: Symbol that shows starting of decoding input # E: Symbol that shows starting of decoding output # P: Symbol that will fill in blank sequence if current batch data size is short than time steps -sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E'] - -# Transformer Parameters -# Padding Should be Zero -src_vocab = {'P' : 0, 'ich' : 1, 'mochte' : 2, 'ein' : 3, 'bier' : 4} -src_vocab_size = len(src_vocab) - -tgt_vocab = {'P' : 0, 'i' : 1, 'want' : 2, 'a' : 3, 'beer' : 4, 'S' : 5, 'E' : 6} -number_dict = {i: w for i, w in enumerate(tgt_vocab)} -tgt_vocab_size = len(tgt_vocab) - -src_len = 5 -tgt_len = 5 - -d_model = 512 # Embedding Size -d_ff = 2048 # FeedForward dimension -d_k = d_v = 64 # dimension of K(=Q), V -n_layers = 6 # number of Encoder of Decoder Layer -n_heads = 8 # number of heads in Multi-Head Attention def make_batch(sentences): input_batch = [[src_vocab[n] for n in sentences[0].split()]] output_batch = [[tgt_vocab[n] for n in sentences[1].split()]] target_batch = [[tgt_vocab[n] for n in sentences[2].split()]] - return Variable(torch.LongTensor(input_batch)), Variable(torch.LongTensor(output_batch)), Variable(torch.LongTensor(target_batch)) + return torch.LongTensor(input_batch), torch.LongTensor(output_batch), torch.LongTensor(target_batch) def get_sinusoid_encoding_table(n_position, d_model): def cal_angle(position, hid_idx): @@ -187,11 +165,6 @@ def forward(self, enc_inputs, dec_inputs): dec_logits = self.projection(dec_outputs) # dec_logits : [batch_size x src_vocab_size x tgt_vocab_size] return dec_logits.view(-1, dec_logits.size(-1)), enc_self_attns, dec_self_attns, dec_enc_attns -model = Transformer() - -criterion = nn.CrossEntropyLoss() -optimizer = optim.Adam(model.parameters(), lr=0.001) - def showgraph(attn): attn = attn[-1].squeeze(0)[0] attn = attn.squeeze(0).data.numpy() @@ -202,25 +175,52 @@ def showgraph(attn): ax.set_yticklabels(['']+sentences[2].split(), fontdict={'fontsize': 14}) plt.show() -for epoch in range(20): - optimizer.zero_grad() +if __name__ == '__main__': + sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E'] + + # Transformer Parameters + # Padding Should be Zero + src_vocab = {'P': 0, 'ich': 1, 'mochte': 2, 'ein': 3, 'bier': 4} + src_vocab_size = len(src_vocab) + + tgt_vocab = {'P': 0, 'i': 1, 'want': 2, 'a': 3, 'beer': 4, 'S': 5, 'E': 6} + number_dict = {i: w for i, w in enumerate(tgt_vocab)} + tgt_vocab_size = len(tgt_vocab) + + src_len = 5 # length of source + tgt_len = 5 # length of target + + d_model = 512 # Embedding Size + d_ff = 2048 # FeedForward dimension + d_k = d_v = 64 # dimension of K(=Q), V + n_layers = 6 # number of Encoder of Decoder Layer + n_heads = 8 # number of heads in Multi-Head Attention + + model = Transformer() + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.001) + enc_inputs, dec_inputs, target_batch = make_batch(sentences) - outputs, enc_self_attns, dec_self_attns, dec_enc_attns = model(enc_inputs, dec_inputs) - loss = criterion(outputs, target_batch.contiguous().view(-1)) - print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) - loss.backward() - optimizer.step() -# Test -predict, _, _, _ = model(enc_inputs, dec_inputs) -predict = predict.data.max(1, keepdim=True)[1] -print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()]) + for epoch in range(20): + optimizer.zero_grad() + outputs, enc_self_attns, dec_self_attns, dec_enc_attns = model(enc_inputs, dec_inputs) + loss = criterion(outputs, target_batch.contiguous().view(-1)) + print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) + loss.backward() + optimizer.step() + + # Test + predict, _, _, _ = model(enc_inputs, dec_inputs) + predict = predict.data.max(1, keepdim=True)[1] + print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()]) -print('first head of last state enc_self_attns') -showgraph(enc_self_attns) + print('first head of last state enc_self_attns') + showgraph(enc_self_attns) -print('first head of last state dec_self_attns') -showgraph(dec_self_attns) + print('first head of last state dec_self_attns') + showgraph(dec_self_attns) -print('first head of last state dec_enc_attns') -showgraph(dec_enc_attns) \ No newline at end of file + print('first head of last state dec_enc_attns') + showgraph(dec_enc_attns) \ No newline at end of file diff --git a/5-2.BERT/BERT.py b/5-2.BERT/BERT.py index 7ae6be7..1a7f625 100644 --- a/5-2.BERT/BERT.py +++ b/5-2.BERT/BERT.py @@ -1,8 +1,7 @@ -''' - code by Tae Hwan Jung(Jeff Jung) @graykode - Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch - https://github.com/JayParks/transformer, https://github.com/dhlee347/pytorchic-bert -''' +# %% +# code by Tae Hwan Jung(Jeff Jung) @graykode +# Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch +# https://github.com/JayParks/transformer, https://github.com/dhlee347/pytorchic-bert import math import re from random import * @@ -10,39 +9,6 @@ import torch import torch.nn as nn import torch.optim as optim -from torch.autograd import Variable - -# BERT Parameters -maxlen = 30 -batch_size = 6 -max_pred = 5 # max tokens of prediction -n_layers = 6 -n_heads = 12 -d_model = 768 -d_ff = 768*4 # 4*d_model, FeedForward dimension -d_k = d_v = 64 # dimension of K(=Q), V -n_segments = 2 - -text = ( - 'Hello, how are you? I am Romeo.\n' - 'Hello, Romeo My name is Juliet. Nice to meet you.\n' - 'Nice meet you too. How are you today?\n' - 'Great. My baseball team won the competition.\n' - 'Oh Congratulations, Juliet\n' - 'Thanks you Romeo' -) -sentences = re.sub("[.,!?\\-]", '', text.lower()).split('\n') # filter '.', ',', '?', '!' -word_list = list(set(" ".join(sentences).split())) -word_dict = {'[PAD]' : 0, '[CLS]' : 1, '[SEP]' : 2, '[MASK]' : 3} -for i, w in enumerate(word_list): - word_dict[w] = i + 4 -number_dict = {i: w for i, w in enumerate(word_dict)} -vocab_size = len(word_dict) - -token_list = list() -for sentence in sentences: - arr = [word_dict[s] for s in sentence.split()] - token_list.append(arr) # sample IsNext and NotNext to be same in small batch size def make_batch(): @@ -205,39 +171,68 @@ def forward(self, input_ids, segment_ids, masked_pos): return logits_lm, logits_clsf -model = BERT() -criterion = nn.CrossEntropyLoss() -optimizer = optim.Adam(model.parameters(), lr=0.001) +if __name__ == '__main__': + # BERT Parameters + maxlen = 30 # maximum of length + batch_size = 6 + max_pred = 5 # max tokens of prediction + n_layers = 6 # number of Encoder of Encoder Layer + n_heads = 12 # number of heads in Multi-Head Attention + d_model = 768 # Embedding Size + d_ff = 768 * 4 # 4*d_model, FeedForward dimension + d_k = d_v = 64 # dimension of K(=Q), V + n_segments = 2 + + text = ( + 'Hello, how are you? I am Romeo.\n' + 'Hello, Romeo My name is Juliet. Nice to meet you.\n' + 'Nice meet you too. How are you today?\n' + 'Great. My baseball team won the competition.\n' + 'Oh Congratulations, Juliet\n' + 'Thanks you Romeo' + ) + sentences = re.sub("[.,!?\\-]", '', text.lower()).split('\n') # filter '.', ',', '?', '!' + word_list = list(set(" ".join(sentences).split())) + word_dict = {'[PAD]': 0, '[CLS]': 1, '[SEP]': 2, '[MASK]': 3} + for i, w in enumerate(word_list): + word_dict[w] = i + 4 + number_dict = {i: w for i, w in enumerate(word_dict)} + vocab_size = len(word_dict) + + token_list = list() + for sentence in sentences: + arr = [word_dict[s] for s in sentence.split()] + token_list.append(arr) + + model = BERT() + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.001) + + batch = make_batch() + input_ids, segment_ids, masked_tokens, masked_pos, isNext = map(torch.LongTensor, zip(*batch)) + + for epoch in range(100): + optimizer.zero_grad() + logits_lm, logits_clsf = model(input_ids, segment_ids, masked_pos) + loss_lm = criterion(logits_lm.transpose(1, 2), masked_tokens) # for masked LM + loss_lm = (loss_lm.float()).mean() + loss_clsf = criterion(logits_clsf, isNext) # for sentence classification + loss = loss_lm + loss_clsf + if (epoch + 1) % 10 == 0: + print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) + loss.backward() + optimizer.step() + + # Predict mask tokens ans isNext + input_ids, segment_ids, masked_tokens, masked_pos, isNext = map(torch.LongTensor, zip(batch[0])) + print(text) + print([number_dict[w.item()] for w in input_ids[0] if number_dict[w.item()] != '[PAD]']) -batch = make_batch() -input_ids, segment_ids, masked_tokens, masked_pos, isNext = zip(*batch) -input_ids, segment_ids, masked_tokens, masked_pos, isNext = \ - torch.LongTensor(input_ids), torch.LongTensor(segment_ids), torch.LongTensor(masked_tokens), \ - torch.LongTensor(masked_pos), torch.LongTensor(isNext) - -for epoch in range(100): - optimizer.zero_grad() logits_lm, logits_clsf = model(input_ids, segment_ids, masked_pos) - loss_lm = criterion(logits_lm.transpose(1, 2), masked_tokens) # for masked LM - loss_lm = (loss_lm.float()).mean() - loss_clsf = criterion(logits_clsf, isNext) # for sentence classification - loss = loss_lm + loss_clsf - if (epoch + 1) % 10 == 0: - print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss)) - loss.backward() - optimizer.step() - -# Predict mask tokens ans isNext -input_ids, segment_ids, masked_tokens, masked_pos, isNext = batch[0] -print(text) -print([number_dict[w] for w in input_ids if number_dict[w] != '[PAD]']) - -logits_lm, logits_clsf = model(torch.LongTensor([input_ids]), \ - torch.LongTensor([segment_ids]), torch.LongTensor([masked_pos])) -logits_lm = logits_lm.data.max(2)[1][0].data.numpy() -print('masked tokens list : ',[pos for pos in masked_tokens if pos != 0]) -print('predict masked tokens list : ',[pos for pos in logits_lm if pos != 0]) - -logits_clsf = logits_clsf.data.max(1)[1].data.numpy()[0] -print('isNext : ', True if isNext else False) -print('predict isNext : ',True if logits_clsf else False) + logits_lm = logits_lm.data.max(2)[1][0].data.numpy() + print('masked tokens list : ',[pos.item() for pos in masked_tokens[0] if pos.item() != 0]) + print('predict masked tokens list : ',[pos for pos in logits_lm if pos != 0]) + + logits_clsf = logits_clsf.data.max(1)[1].data.numpy()[0] + print('isNext : ', True if isNext else False) + print('predict isNext : ',True if logits_clsf else False) From d2515bb510288fbf9d257f79011a488aa6598809 Mon Sep 17 00:00:00 2001 From: graykode Date: Sat, 15 Aug 2020 17:02:05 +0900 Subject: [PATCH 3/3] (add) Contribution Guidelines, (edit) main README page --- CONTRIBUTING.md | 6 +++++ README.md | 68 ++++++++++++++++++------------------------------- 2 files changed, 31 insertions(+), 43 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..cc3d7bf --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,6 @@ +## Contribution Guidelines + +Thank you to everyone who contributes. Here are some rules to follow before contributing. +1. Contributions are open to the smallest details such as typos, comments and code refactors. +2. Do not commit the jupyter notebook file(*.ipynb). When the modified python code is merged into the master branch, the github action automatically generates an ipynb. +3. Please attach a commit message appropriate to the modified code. \ No newline at end of file diff --git a/README.md b/README.md index 5179838..ce30349 100644 --- a/README.md +++ b/README.md @@ -2,22 +2,23 @@

-`nlp-tutorial` is a tutorial for who is studying NLP(Natural Language Processing) using **TensorFlow** and **Pytorch**. Most of the models in NLP were implemented with less than **100 lines** of code.(except comments or blank lines) +`nlp-tutorial` is a tutorial for who is studying NLP(Natural Language Processing) using **Pytorch**. Most of the models in NLP were implemented with less than **100 lines** of code.(except comments or blank lines) +- [08-14-2020] Old TensorFlow v1 code is archived in [the archive folder](archive). For beginner readability, only pytorch version 1.0 or higher is supported. ## Curriculum - (Example Purpose) #### 1. Basic Embedding Model -- 1-1. [NNLM(Neural Network Language Model)](https://github.com/graykode/nlp-tutorial/tree/master/1-1.NNLM) - **Predict Next Word** +- 1-1. [NNLM(Neural Network Language Model)](1-1.NNLM) - **Predict Next Word** - Paper - [A Neural Probabilistic Language Model(2003)](http://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf) - - Colab - [NNLM_Tensor.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/1-1.NNLM/NNLM_Tensor.ipynb), [NNLM_Torch.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/1-1.NNLM/NNLM_Torch.ipynb) -- 1-2. [Word2Vec(Skip-gram)](https://github.com/graykode/nlp-tutorial/tree/master/1-2.Word2Vec) - **Embedding Words and Show Graph** + - Colab - [NNLM.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/1-1.NNLM/NNLM.ipynb) +- 1-2. [Word2Vec(Skip-gram)](1-2.Word2Vec) - **Embedding Words and Show Graph** - Paper - [Distributed Representations of Words and Phrases and their Compositionality(2013)](https://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf) - - Colab - [Word2Vec_Tensor(NCE_loss).ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/1-2.Word2Vec/Word2Vec_Skipgram_Tensor(NCE_loss).ipynb), [Word2Vec_Tensor(Softmax).ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/1-2.Word2Vec/Word2Vec_Skipgram_Tensor(Softmax).ipynb), [Word2Vec_Torch(Softmax).ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/1-2.Word2Vec/Word2Vec_Skipgram_Torch(Softmax).ipynb) -- 1-3. [FastText(Application Level)](https://github.com/graykode/nlp-tutorial/tree/master/1-3.FastText) - **Sentence Classification** + - Colab - [Word2Vec.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/1-2.Word2Vec/Word2Vec_Skipgram(Softmax).ipynb) +- 1-3. [FastText(Application Level)](1-3.FastText) - **Sentence Classification** - Paper - [Bag of Tricks for Efficient Text Classification(2016)](https://arxiv.org/pdf/1607.01759.pdf) - Colab - [FastText.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/1-3.FastText/FastText.ipynb) @@ -25,73 +26,54 @@ #### 2. CNN(Convolutional Neural Network) -- 2-1. [TextCNN](https://github.com/graykode/nlp-tutorial/tree/master/2-1.TextCNN) - **Binary Sentiment Classification** +- 2-1. [TextCNN](2-1.TextCNN) - **Binary Sentiment Classification** - Paper - [Convolutional Neural Networks for Sentence Classification(2014)](http://www.aclweb.org/anthology/D14-1181) - - Colab - [TextCNN_Tensor.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/2-1.TextCNN/TextCNN_Tensor.ipynb), [TextCNN_Torch.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/2-1.TextCNN/TextCNN_Torch.ipynb) -- 2-2. DCNN(Dynamic Convolutional Neural Network) + - [TextCNN.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/2-1.TextCNN/TextCNN.ipynb) #### 3. RNN(Recurrent Neural Network) -- 3-1. [TextRNN](https://github.com/graykode/nlp-tutorial/tree/master/3-1.TextRNN) - **Predict Next Step** +- 3-1. [TextRNN](3-1.TextRNN) - **Predict Next Step** - Paper - [Finding Structure in Time(1990)](http://psych.colorado.edu/~kimlab/Elman1990.pdf) - - Colab - [TextRNN_Tensor.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/3-1.TextRNN/TextRNN_Tensor.ipynb), [TextRNN_Torch.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/3-1.TextRNN/TextRNN_Torch.ipynb) + - Colab - [TextRNN.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/3-1.TextRNN/TextRNN.ipynb) - 3-2. [TextLSTM](https://github.com/graykode/nlp-tutorial/tree/master/3-2.TextLSTM) - **Autocomplete** - Paper - [LONG SHORT-TERM MEMORY(1997)](https://www.bioinf.jku.at/publications/older/2604.pdf) - - Colab - [TextLSTM_Tensor.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/3-2.TextLSTM/TextLSTM_Tensor.ipynb), [TextLSTM_Torch.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/3-2.TextLSTM/TextLSTM_Torch.ipynb) -- 3-3. [Bi-LSTM](https://github.com/graykode/nlp-tutorial/tree/master/3-3.Bi-LSTM) - **Predict Next Word in Long Sentence** - - Colab - [Bi_LSTM_Tensor.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/3-3.Bi-LSTM/Bi_LSTM_Tensor.ipynb), [Bi_LSTM_Torch.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/3-3.Bi-LSTM/Bi_LSTM_Torch.ipynb) + - Colab - [TextLSTM.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/3-2.TextLSTM/TextLSTM.ipynb) +- 3-3. [Bi-LSTM](3-3.Bi-LSTM) - **Predict Next Word in Long Sentence** + - Colab - [Bi_LSTM.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/3-3.Bi-LSTM/Bi_LSTM.ipynb) #### 4. Attention Mechanism -- 4-1. [Seq2Seq](https://github.com/graykode/nlp-tutorial/tree/master/4-1.Seq2Seq) - **Change Word** +- 4-1. [Seq2Seq](4-1.Seq2Seq) - **Change Word** - Paper - [Learning Phrase Representations using RNN Encoder–Decoder for Statistical Machine Translation(2014)](https://arxiv.org/pdf/1406.1078.pdf) - - Colab - [Seq2Seq_Tensor.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/4-1.Seq2Seq/Seq2Seq_Tensor.ipynb), [Seq2Seq_Torch.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/4-1.Seq2Seq/Seq2Seq_Torch.ipynb) -- 4-2. [Seq2Seq with Attention](https://github.com/graykode/nlp-tutorial/tree/master/4-2.Seq2Seq(Attention)) - **Translate** + - Colab - [Seq2Seq.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/4-1.Seq2Seq/Seq2Seq.ipynb) +- 4-2. [Seq2Seq with Attention](4-2.Seq2Seq(Attention)) - **Translate** - Paper - [Neural Machine Translation by Jointly Learning to Align and Translate(2014)](https://arxiv.org/abs/1409.0473) - - Colab - [Seq2Seq(Attention)_Tensor.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/4-2.Seq2Seq(Attention)/Seq2Seq(Attention)_Tensor.ipynb), [Seq2Seq(Attention)_Torch.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/4-2.Seq2Seq(Attention)/Seq2Seq(Attention)_Torch.ipynb) -- 4-3. [Bi-LSTM with Attention](https://github.com/graykode/nlp-tutorial/tree/master/4-3.Bi-LSTM(Attention)) - **Binary Sentiment Classification** - - Colab - [Bi_LSTM(Attention)_Tensor.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/4-3.Bi-LSTM(Attention)/Bi_LSTM(Attention)_Tensor.ipynb), [Bi_LSTM(Attention)_Torch.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/4-3.Bi-LSTM(Attention)/Bi_LSTM(Attention)_Torch.ipynb) + - Colab - [Seq2Seq(Attention).ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/4-2.Seq2Seq(Attention)/Seq2Seq(Attention).ipynb) +- 4-3. [Bi-LSTM with Attention](4-3.Bi-LSTM(Attention)) - **Binary Sentiment Classification** + - Colab - [Bi_LSTM(Attention).ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/4-3.Bi-LSTM(Attention)/Bi_LSTM(Attention).ipynb) #### 5. Model based on Transformer -- 5-1. [The Transformer](https://github.com/graykode/nlp-tutorial/tree/master/5-1.Transformer) - **Translate** +- 5-1. [The Transformer](5-1.Transformer) - **Translate** - Paper - [Attention Is All You Need(2017)](https://arxiv.org/abs/1706.03762) - - Colab - [Transformer_Torch.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/5-1.Transformer/Transformer_Torch.ipynb), [Transformer(Greedy_decoder)_Torch.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/5-1.Transformer/Transformer(Greedy_decoder)_Torch.ipynb) -- 5-2. [BERT](https://github.com/graykode/nlp-tutorial/tree/master/5-2.BERT) - **Classification Next Sentence & Predict Masked Tokens** + - Colab - [Transformer.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/5-1.Transformer/Transformer.ipynb), [Transformer(Greedy_decoder).ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/5-1.Transformer/Transformer(Greedy_decoder).ipynb) +- 5-2. [BERT](5-2.BERT) - **Classification Next Sentence & Predict Masked Tokens** - Paper - [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding(2018)](https://arxiv.org/abs/1810.04805) - - Colab - [BERT_Torch.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/5-2.BERT/BERT_Torch.ipynb) - -| Model | Example | Framework | Lines(torch/tensor) | -| :------------------------: | :--------------------------------: | :-----------: | :-----------------: | -| NNLM | Predict Next Word | Torch, Tensor | 67/83 | -| Word2Vec(Softmax) | Embedding Words and Show Graph | Torch, Tensor | 77/94 | -| TextCNN | Sentence Classification | Torch, Tensor | 94/99 | -| TextRNN | Predict Next Step | Torch, Tensor | 70/88 | -| TextLSTM | Autocomplete | Torch, Tensor | 73/78 | -| Bi-LSTM | Predict Next Word in Long Sentence | Torch, Tensor | 73/78 | -| Seq2Seq | Change Word | Torch, Tensor | 93/111 | -| Seq2Seq with Attention | Translate | Torch, Tensor | 108/118 | -| Bi-LSTM with Attention | Binary Sentiment Classification | Torch, Tensor | 92/104 | -| Transformer | Translate | Torch | 222/0 | -| Greedy Decoder Transformer | Translate | Torch | 246/0 | -| BERT | how to train | Torch | 242/0 | - + - Colab - [BERT.ipynb](https://colab.research.google.com/github/graykode/nlp-tutorial/blob/master/5-2.BERT/BERT.ipynb) ## Dependencies - Python 3.5+ -- Tensorflow 1.12.0+ -- Pytorch 0.4.1+ -- Plan to add Keras Version +- Pytorch 1.0.0+