4-2.Seq2Seq(Attention)/Seq2Seq(Attention)_Tensor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "Nt432JiJgboz",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 486.0
    },
    "outputId": "df4c9ae6-9df1-4a6d-f240-5123f887bd9d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-1-12c065d137be>:53: BasicRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.SimpleRNNCell, and will be replaced by that in Tensorflow 2.0.\n",
      "Epoch: 0400 cost = 0.000000\n",
      "Epoch: 0800 cost = 0.000000\n",
      "Epoch: 1200 cost = 0.000000\n",
      "Epoch: 1600 cost = 0.000000\n",
      "Epoch: 2000 cost = 0.000000\n",
      "['ich', 'mochte', 'ein', 'bier', 'P'] -> ['i', 'want', 'a', 'beer', 'E']\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAEzCAYAAABE0wr4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFY9JREFUeJzt3XlwVfX5x/FPLtmsAlZWA4SJFaKO\nCUkgBMpmE2hoYJggRRonyFIhBUsFHBAYK9YyLDO4FCIZLLak2KJFkA5YYYK4FSgQgTEBDBBDExI2\nWcRgICb3/v7g5y0pIAmSc0Ke9+svcu7yfb658c05944kwOfz+QQAhnncHgAA3EYIAZhHCAGYRwgB\nmEcIAZhHCAGYZzqEpaWlioqK0qFDh77zfomJiXr99dcdmsp5ixcv1sMPP+z2GN/L2rVr1bdvX7fH\nqJMjR44oMjJSBw4cuOK2W3E/t7JAtwdwU7t27ZSXl+f2GA3euXPn9O6772rEiBFuj3JNqampSk1N\ndXuMm6ax7edyiYmJOn78uDyeS+dhwcHB6tSpkyZNmqRevXq5MpPpM0LUzrZt2/Tmm2+6PQYakZkz\nZyovL095eXnasmWLBg0apIyMjOtendUX0yG8/NLk9OnTmjx5srp27apevXpp/vz5qq6u9t+3oqJC\nU6dOVUxMjPr27astW7bU+3yRkZFat26dhg0bpujoaI0ZM0ZHjx5VRkaGYmNjNXToUJWUlPjvv3nz\nZqWmpiomJkb9+vXTK6+8osv/x6Hly5crMTFRsbGxGjVqlIqKimqst2rVKvXp00cxMTGaM2eOJGn9\n+vWaMmWK9u3bp6ioKBUVFcnr9SozM1MDBgxQly5dlJqaqm3bttX790OSjh49qgkTJqhHjx7q2rWr\npkyZojNnzmjNmjVKSEiQ9N/XdcuWLf7vR1pamo4dO+bIjHW1b98+DRkyRLGxsUpPT1dpaWmN/UhS\nQUGBRo8erfj4eCUkJOjZZ5/VxYsXJUlr1qzRwIEDtXDhQsXGxtb4mbgVhIaGauTIkYqIiND777/v\nygymQ3i5Z555RlVVVfrggw/01ltvadOmTVq+fLn/9rfeekuPPfaYtm/frl69eun55593ZK6VK1dq\nyZIleuedd7Rnzx6NHj1aTzzxhD7++GNVVVX5Zzxw4IB+/etfKyMjQzt37tTLL7+s7OxsrV69WpK0\nadMmZWVlKTMzU9u3b9ePfvQjTZo0yb9OaWmpTp48qU2bNikzM1MrVqzQrl27NHjwYE2YMEEPPPCA\n8vLyFBERob/85S/6xz/+oaVLlyo3N1dpaWmaOHGizp49W6/fC5/PpwkTJqhVq1Z67733lJOTo/Ly\ncv3+97+/6v2zs7P16quvavPmzTp79qz+/Oc/1+t8N+qNN95QVlaWPvzwQwUFBenpp5+ucXtFRYUe\nf/xxxcfHa8uWLXr77beVn5+vzMxM/32++OILBQQEaMeOHWrfvr3TW7gpqqurFRjozrt1hFDS2bNn\n9f7772v8+PFq2rSp7r77br344ouKi4vz3+ehhx5STEyMQkJCNHDgQP3nP//RN998U++zDRo0SG3a\ntFGHDh3UqVMn3X///YqOjtYdd9yh+Ph4HT58WNKlUHfv3l0/+9nPFBQUpNjYWKWkpOjdd9+VJK1e\nvVopKSl64IEHFBwcrEmTJmnixIn+PQQEBCgjI0MhISHq3bu3WrRoocLCwqvOtGrVKo0aNUr33HOP\ngoKCNGLECLVv314bNmyo1+9FXl6eCgoKNH36dN1+++266667NHnyZG3YsEEVFRVX3P+RRx5R69at\ndddddykhIeGa+3FbWlqa2rVrp2bNmmns2LHKzc3V119/7b/9gw8+0DfffKMnnnhCwcHBCgsL069+\n9Su9/fbb/vuUl5dr3LhxCgoKUkBAgBvbuGFff/21VqxYobKyMvXv39+VGUx/WPItj8cjr9db42/S\n6OjoGve5/LbQ0FD5fD5VVlYqKCioXmdr27at/88hISFq06ZNja8rKyslSSUlJbr33ntrPLZjx47+\nS9aSkhJ169bNf9sPf/hDpaSk+L8OCwtTkyZN/F+Hhob6L73+V3FxsebPn68FCxb4j/l8Ph09evRG\ntlhrJSUl8nq96tmz5xW3Xe11uPw1u+222665H7dd/rqFh4fL5/PV2E9JSYnOnj2rqKioGo/zer3+\n1/+OO+5Qs2bNnBn4Jpg3b57/5yc0NFSRkZFatmyZOnTo4Mo8hFDyv4/2Xf8Qj1t/y377ydq1vv7W\nt/9B/K9v5w4ICJDX670pM4WGhup3v/tdjZA6ISQkRCEhIfr000+vuG3NmjVXHLvW96qhuXzOb38G\nT5486T8WEhKiiIgI/9n91Vz+l9itYObMmUpPT3d7DL9b4yfFAR6Pp8aHB7m5ufV+qXczhYeHX3Hp\n9/nnn6tjx46SpA4dOtTY37lz5/Taa6/p/PnzN7RWQUFBjWNHjhy5ganrpmPHjrp48aL/7QDp0vtn\np06dqve169Plr0txcbGaNGmiVq1a+Y917NhRpaWlKi8v9x/78ssv9dVXXzk6Z2NGCCU1b95cSUlJ\neuWVV3TmzBkdP35cs2fPVnFxsduj1drQoUO1fft25eTkqKqqSrm5uVq/fr2GDh0qSRo2bJg2bNig\n3NxcVVZWKisrS6tXr9btt99+3ecOCQnRF198oTNnzqiyslJpaWlauXKlcnNzVV1drffee0+DBw/W\n559/Xq977NSpk7p166Y5c+bo9OnT/g9KfvOb39TruvVt5cqVOnHihMrLy5Wdna1+/frVuDTu3bu3\nWrVqpblz5+qrr77S6dOnNW3atGt+SIS6I4T/b/78+brzzjuVmJioYcOGqU+fPho7dqzbY9VadHS0\n5s2bp0WLFik+Pl6zZ8/WM888o4EDB0qSkpKSNG3aNE2dOlUJCQnav39/jU8dv0v//v3l8Xj0k5/8\nRJ9++qmGDRumxx57TFOmTFFcXJwWLVqkF154Qffcc099blGStHDhQgUGBiopKUlJSUk6d+6cXnzx\nxXpftz6lpaVp7Nix6tOnj6qqqvTcc8/VuD0wMFBLlixRSUmJevfurcGDB6tFixZ69tln3Rm4EQrg\nX6gGYB1nhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMy75f71mQGe4Y6t9eqnL2h8\n9FOOrLWxbI8j60hSQIt35Ds1yLH1ksNiHFvLydfMSY11X5Kze8vxrrrqcc4Iv0PEg+Fuj1AvAoI6\nuz1CvWmsr1lj3ZfUMPZGCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgB\nmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmNegQlhaWqqoqCgdOnTI7VEAGNKg\nfp1nu3btlJeX5/YYAIxpUGeEAOCGBhXCI0eOKDIyUgcOHHB7FACGNKgQAoAbAnw+n8/tIb515MgR\nJSUlad26dercufNV71OUX6yIB8MdngxAY9agPiypjfHRTzm2Vo53lQZ4hjuy1sayPY6sI0metgfl\nPdbJsfWSw2IcW8vJ18xJjXVfkrN7y/GuuupxLo0BmEcIAZhHCAGYRwgBmNegPixp3769CgoK3B4D\ngDGcEQIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwjxAC\nMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMK9B/fKm2thYtqdRrpccFuPIOpKU43V2PaCh44wQgHmE\nEIB5hBCAeYQQgHmEEIB5hBCAeYQQgHmEEIB5hBCAeYQQgHmEEIB5hBCAeYQQgHmEEIB5hBCAeYQQ\ngHmEEIB5hBCAeYQQgHmEEIB5hBCAeYQQgHkNJoRr1qzRqVOn3B4DgEENIoTV1dWaN28eIQTgiuuG\n8KGHHlJOTo7/68cff1wpKSn+rz/77DNFRUWpsLBQGRkZSkhIUHx8vCZMmKATJ0747xcZGamNGzcq\nLS1NMTExGjJkiAoKCiRJXbt21blz5/Twww/r5Zdfvpn7A4Drum4IExIStGvXLkmXztz27t2rCxcu\n6MyZM5Kk3NxcxcbG6vnnn1fTpk318ccfa/PmzSovL9eCBQtqPNeyZcs0d+5cbd26Vc2bN9fixYsl\nSevXr5d06fJ48uTJN3WDAHA91w1hjx49tHv3bknS3r171bFjR0VHR+uTTz6RdCmEPXv21NKlSzVn\nzhwFBweradOmSkxMVH5+fo3nGjx4sCIiIvSDH/xAffv2VWFhYT1sCQDqJvB6d+jRo4d++9vf6uLF\ni9q5c6e6deum1q1b65NPPlH//v2Vm5urMWPGKD8/Xy+99JI+++wzVVZWyuv1qk2bNjWeq3379v4/\n33bbbbp48WKdBw5o8Y4CgjrX+XE3ytP2oCPr5HgdWeay9VY5u6CDGuveGuu+JPf3dt0Q3n333QoL\nC1NeXp527typESNGqGXLllq/fr2Ki4t14cIFhYeHKykpScOHD1dWVpaaNWum7OxsZWdn13guj+f7\nfzbjOzVIvu/9LLXjaXtQ3mOdHFkrOSzGkXWkSz90AzzDHVvPSY11b411X5Kze7tWcGtVph49eig3\nN1e7d+9WXFyc7r//fhUVFelf//qXunfvrsOHD+v8+fP65S9/qWbNmkm6dBkNALeCWodw7dq1at26\ntZo3b67AwEDdd999+utf/6qePXsqLCxMHo9Hu3fvVkVFhd58800VFRXpyy+/1IULF677/KGhoZKk\nw4cPq7y8/PvtCADqqFYhTEhI0OHDh9W1a1f/sbi4OB06dEg//vGP1aZNG02fPl2zZ89Wv379VFhY\nqEWLFunOO+/UT3/60+s+f8uWLZWcnKypU6dq4cKFN74bALgBAT6fz6m33G4Kp96zk3iP8FbUWPfW\nWPcl3ULvEQJAY0YIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgB\nmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYF+j2AHXl7G97c3a9xmpj2Z5GuR4/G40H\nZ4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwj\nhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMczWEe/fu1ciRIxUfH68ePXpo\n+vTpKi8vd3MkAAa5GsLJkyerS5cu+ve//63169crPz9ff/zjH90cCYBBAT6fz+fW4ufPn1dQUJCC\ng4MlSXPmzFFRUZFee+21az6mKL9YEQ+GOzUiAAMC3Vx827ZtWrJkiYqKilRVVaXq6mp17dr1Ox8z\nPvoph6aTcryrNMAz3LH1nOL0vjaW7XFsLU/bg/Ie6+TIWslhMY6sIzXen0XJ2b3leFdd9bhrl8aF\nhYV68sknNXjwYG3dulV5eXlKT093axwAhrl2Rrh//341adJEY8aMUUBAgKRLH554PHyQDcBZrlWn\nQ4cOqqysVH5+vsrLy5WZmamKigqdPHlS1dXVbo0FwCDXQtilSxeNHj1aY8aMUXJysoKCgjR37lyd\nO3eOS2QAjnL1w5IZM2ZoxowZNY5t3brVpWkAWMUbcgDMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADM\nI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzHP1lzfB\nhuSwGMfWyvE6t97Gsj2OrOPGek6+Zg0BZ4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwj\nhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMq3UIjxw5\nosjISB04cKA+5wEAx3FGCMA8QgjAvDqHcN++fRoyZIhiY2OVnp6u0tJSSdKOHTv0i1/8QnFxcerd\nu7deeukleb1e/+P+9re/KSUlRV26dFFycrL++c9/+m8bOXKkFixYoNTUVI0aNeombAsAaq/OIXzj\njTeUlZWlDz/8UEFBQXr66ad17NgxZWRk6Oc//7l27Nih5cuXa926dfr73/8uSdq0aZP+8Ic/aP78\n+dq1a5dmzJih6dOnq7Cw0P+877zzjmbPnq3ly5fftM0BQK34aqmkpMTXuXNn39q1a/3HPvroI19k\nZKQvMzPTN3To0Br3/9Of/uR75JFHfD6fzzdu3Djf/Pnza9yekZHhW7hwoc/n8/nS09N9EydOrNUc\nn+f9p7YjA0CtBNY1nPfee6//z+Hh4fL5fNq+fbv279+vqKioywOrli1bSpKKi4u1ZcsWvf766zVu\nb9q0qf/rsLCwWq0/Pvqpuo58w3K8qzTAM9yx9ZzSWPclObu3jWV7HFlHkjxtD8p7rJNj6yWHxTi2\nlpOvWY531VWP1zmEHs9/r6Z9Pp8kqV27dgoODtayZcuu+pjQ0FA9+eSTGj9+/DWfNzCwzqMAwE1R\n5/cIi4qK/H8uLi5WkyZNdN999+ngwYM1Phw5deqULly4IOnSmWNBQUGN5ykrK6txfwBwS51DuHLl\nSp04cULl5eXKzs5Wv379lJqaqvLyci1evFgVFRUqKyvTuHHjtHTpUklSWlqaNm7cqE2bNqmqqkq7\ndu1Samqqtm/fftM3BAB1VecQpqWlaezYserTp4+qqqr03HPPqXnz5srKytJHH32khIQEjRgxQvHx\n8Zo4caIkqWfPnpo1a5bmzZunuLg4zZo1S9OmTVPPnj1v+oYAoK5q/cZc+/bt/Ze3KSkpV9zevXt3\nrV69+pqPf/TRR/Xoo49e9bYVK1bUdgwAuOn4P0sAmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcI\nAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZjHb1UHblBy\nWIxja+V4nV1vY9kex9ZyY73/xRkhAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wgh\nAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPNc+XWeiYmJ\nOn78uDyeKzs8a9YspaWluTAVAKtc+73GM2fOVHp6ulvLA4Afl8YAzCOEAMxz7dJ43rx5WrBgwRXH\n9+zZoyZNmrgwEQCrAnw+n8/pRRMTEzV27Ngbeo+wKL9YEQ+G18NUAKxy7YzwRo2PfsqxtXK8qzTA\nM9yx9ZzSWPclNd69Ob2vjWV7HFvL0/agvMc6ObbWVY87sjoANGCEEIB5De7Dkn79+ikzM9OFiQBY\n5UoIN2/e7MayAHBVXBoDMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMI8QAjCP\nEAIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwL8Dn8/ncHgIA3MQZIQDzCCEA\n8wghAPMIIQDzCCEA8wghAPP+D11w97+YehQoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# code by Tae Hwan Jung(Jeff Jung) @graykode\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "tf.reset_default_graph()\n",
    "# S: Symbol that shows starting of decoding input\n",
    "# E: Symbol that shows starting of decoding output\n",
    "# P: Symbol that will fill in blank sequence if current batch data size is short than time steps\n",
    "sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E']\n",
    "\n",
    "word_list = \" \".join(sentences).split()\n",
    "word_list = list(set(word_list))\n",
    "word_dict = {w: i for i, w in enumerate(word_list)}\n",
    "number_dict = {i: w for i, w in enumerate(word_list)}\n",
    "n_class = len(word_dict)  # vocab list\n",
    "\n",
    "# Parameter\n",
    "n_step = 5  # maxium number of words in one sentence(=number of time steps)\n",
    "n_hidden = 128\n",
    "\n",
    "def make_batch(sentences):\n",
    "    input_batch = [np.eye(n_class)[[word_dict[n] for n in sentences[0].split()]]]\n",
    "    output_batch = [np.eye(n_class)[[word_dict[n] for n in sentences[1].split()]]]\n",
    "    target_batch = [[word_dict[n] for n in sentences[2].split()]]\n",
    "    return input_batch, output_batch, target_batch\n",
    "\n",
    "# Model\n",
    "enc_inputs = tf.placeholder(tf.float32, [None, None, n_class])  # [batch_size, n_step, n_class]\n",
    "dec_inputs = tf.placeholder(tf.float32, [None, None, n_class])  # [batch_size, n_step, n_class]\n",
    "targets = tf.placeholder(tf.int64, [1, n_step])  # [batch_size, n_step], not one-hot\n",
    "\n",
    "# Linear for attention\n",
    "attn = tf.Variable(tf.random_normal([n_hidden, n_hidden]))\n",
    "out = tf.Variable(tf.random_normal([n_hidden * 2, n_class]))\n",
    "\n",
    "def get_att_score(dec_output, enc_output):  # enc_output [n_step, n_hidden]\n",
    "    score = tf.squeeze(tf.matmul(enc_output, attn), 0)  # score : [n_hidden]\n",
    "    dec_output = tf.squeeze(dec_output, [0, 1])  # dec_output : [n_hidden]\n",
    "    return tf.tensordot(dec_output, score, 1)  # inner product make scalar value\n",
    "\n",
    "def get_att_weight(dec_output, enc_outputs):\n",
    "    attn_scores = []  # list of attention scalar : [n_step]\n",
    "    enc_outputs = tf.transpose(enc_outputs, [1, 0, 2])  # enc_outputs : [n_step, batch_size, n_hidden]\n",
    "    for i in range(n_step):\n",
    "        attn_scores.append(get_att_score(dec_output, enc_outputs[i]))\n",
    "\n",
    "    # Normalize scores to weights in range 0 to 1\n",
    "    return tf.reshape(tf.nn.softmax(attn_scores), [1, 1, -1])  # [1, 1, n_step]\n",
    "\n",
    "model = []\n",
    "Attention = []\n",
    "with tf.variable_scope('encode'):\n",
    "    enc_cell = tf.nn.rnn_cell.BasicRNNCell(n_hidden)\n",
    "    enc_cell = tf.nn.rnn_cell.DropoutWrapper(enc_cell, output_keep_prob=0.5)\n",
    "    # enc_outputs : [batch_size(=1), n_step(=decoder_step), n_hidden(=128)]\n",
    "    # enc_hidden : [batch_size(=1), n_hidden(=128)]\n",
    "    enc_outputs, enc_hidden = tf.nn.dynamic_rnn(enc_cell, enc_inputs, dtype=tf.float32)\n",
    "\n",
    "with tf.variable_scope('decode'):\n",
    "    dec_cell = tf.nn.rnn_cell.BasicRNNCell(n_hidden)\n",
    "    dec_cell = tf.nn.rnn_cell.DropoutWrapper(dec_cell, output_keep_prob=0.5)\n",
    "\n",
    "    inputs = tf.transpose(dec_inputs, [1, 0, 2])\n",
    "    hidden = enc_hidden\n",
    "    for i in range(n_step):\n",
    "        # time_major True mean inputs shape: [max_time, batch_size, ...]\n",
    "        dec_output, hidden = tf.nn.dynamic_rnn(dec_cell, tf.expand_dims(inputs[i], 1),\n",
    "                                               initial_state=hidden, dtype=tf.float32, time_major=True)\n",
    "        attn_weights = get_att_weight(dec_output, enc_outputs)  # attn_weights : [1, 1, n_step]\n",
    "        Attention.append(tf.squeeze(attn_weights))\n",
    "\n",
    "        # matrix-matrix product of matrices [1, 1, n_step] x [1, n_step, n_hidden] = [1, 1, n_hidden]\n",
    "        context = tf.matmul(attn_weights, enc_outputs)\n",
    "        dec_output = tf.squeeze(dec_output, 0)  # [1, n_step]\n",
    "        context = tf.squeeze(context, 1)  # [1, n_hidden]\n",
    "\n",
    "        model.append(tf.matmul(tf.concat((dec_output, context), 1), out))  # [n_step, batch_size(=1), n_class]\n",
    "\n",
    "trained_attn = tf.stack([Attention[0], Attention[1], Attention[2], Attention[3], Attention[4]], 0)  # to show attention matrix\n",
    "model = tf.transpose(model, [1, 0, 2])  # model : [n_step, n_class]\n",
    "prediction = tf.argmax(model, 2)\n",
    "cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=model, labels=targets))\n",
    "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n",
    "\n",
    "# Training and Test\n",
    "with tf.Session() as sess:\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "    for epoch in range(2000):\n",
    "        input_batch, output_batch, target_batch = make_batch(sentences)\n",
    "        _, loss, attention = sess.run([optimizer, cost, trained_attn],\n",
    "                                      feed_dict={enc_inputs: input_batch, dec_inputs: output_batch, targets: target_batch})\n",
    "\n",
    "        if (epoch + 1) % 400 == 0:\n",
    "            print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n",
    "\n",
    "    predict_batch = [np.eye(n_class)[[word_dict[n] for n in 'P P P P P'.split()]]]\n",
    "    result = sess.run(prediction, feed_dict={enc_inputs: input_batch, dec_inputs: predict_batch})\n",
    "    print(sentences[0].split(), '->', [number_dict[n] for n in result[0]])\n",
    "\n",
    "    # Show Attention\n",
    "    fig = plt.figure(figsize=(5, 5))\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.matshow(attention, cmap='viridis')\n",
    "    ax.set_xticklabels([''] + sentences[0].split(), fontdict={'fontsize': 14})\n",
    "    ax.set_yticklabels([''] + sentences[2].split(), fontdict={'fontsize': 14})\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Seq2Seq(Attention)-Tensor.ipynb",
   "version": "0.3.2",
   "provenance": [],
   "collapsed_sections": []
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "accelerator": "GPU"
 },
 "nbformat": 4,
 "nbformat_minor": 0
}