Deep Autoencoders in Keras - 784-256-128-64-32-16-8-16-32-64-128-256-784-tf1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import os\n",
    "#os.environ['KERAS_BACKEND'] = 'plaidml.keras.backend' # https://github.com/plaidml/plaidml/blob/master/docs/index.md\n",
    "\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Input, Dense\n",
    "from tensorflow.keras.datasets import mnist\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# keras version still end up import tf.keras version which is incompatible, import directly instead\n",
    "from tensorflow.keras.callbacks import TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoding_dim = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_img = Input(shape=(784,))\n",
    "encoded = Dense(256, activation='relu')(input_img) # apply to previous layer = input_img\n",
    "encoded = Dense(128, activation='relu')(encoded)\n",
    "encoded = Dense(64, activation='relu')(encoded)\n",
    "encoded = Dense(32, activation='relu')(encoded)\n",
    "encoded = Dense(16, activation='relu')(encoded)\n",
    "encoded = Dense(encoding_dim, activation='relu')(encoded) # the bottleneck = encoding_dim\n",
    "\n",
    "decoded = Dense(16, activation='relu')(encoded)\n",
    "decoded = Dense(32, activation='relu')(decoded)\n",
    "decoded = Dense(64, activation='relu')(decoded)\n",
    "decoded = Dense(128, activation='relu')(decoded)\n",
    "decoded = Dense(256, activation='relu')(decoded)\n",
    "decoded = Dense(784, activation='sigmoid')(decoded)\n",
    "\n",
    "autoencoder = Model(input_img, decoded)\n",
    "autoencoder.run_eagerly = True # plaidml / keras not running eager mode by default\n",
    "\n",
    "encoder = Model(input_img, encoded)\n",
    "\n",
    "encoded_input = Input(shape=(encoding_dim,))\n",
    "\n",
    "decoder_layer = autoencoder.layers[-6](encoded_input)\n",
    "decoder_layer = autoencoder.layers[-5](decoder_layer)\n",
    "decoder_layer = autoencoder.layers[-4](decoder_layer)\n",
    "decoder_layer = autoencoder.layers[-3](decoder_layer)\n",
    "decoder_layer = autoencoder.layers[-2](decoder_layer)\n",
    "decoder_layer = autoencoder.layers[-1](decoder_layer) # extract last layer\n",
    "\n",
    "decoder = Model(encoded_input, decoder_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "autoencoder.compile(optimizer='adam', loss='binary_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000\n",
      "10000\n",
      "x_train shape = (60000, 784)\n",
      "x_test shape = (10000, 784)\n"
     ]
    }
   ],
   "source": [
    "# Load mnist dataset\n",
    "(x_train, _),(x_test, _) = mnist.load_data()\n",
    "\n",
    "print(len(x_train))\n",
    "print(len(x_test))\n",
    "\n",
    "x_train = x_train.astype('float32')/255\n",
    "x_test = x_test.astype('float32')/255\n",
    "x_train = x_train.reshape(len(x_train), (np.prod(x_train.shape[1:])))\n",
    "x_test = x_test.reshape(len(x_test), (np.prod(x_test.shape[1:])))\n",
    "\n",
    "print(f'x_train shape = {x_train.shape}')\n",
    "print(f'x_test shape = {x_test.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1595257933\n",
      "Epoch 1/50\n",
      "  2/235 [..............................] - ETA: 16:54 - loss: 0.6928WARNING:tensorflow:Method (on_train_batch_end) is slow compared to the batch update (4.604496). Check your callbacks.\n",
      "  3/235 [..............................] - ETA: 11:17 - loss: 0.6923WARNING:tensorflow:Method (on_train_batch_end) is slow compared to the batch update (0.547999). Check your callbacks.\n",
      "235/235 [==============================] - 81s 344ms/step - loss: 0.2754 - val_loss: 0.2340\n",
      "Epoch 2/50\n",
      "235/235 [==============================] - 65s 277ms/step - loss: 0.2202 - val_loss: 0.2022\n",
      "Epoch 3/50\n",
      "235/235 [==============================] - 66s 281ms/step - loss: 0.1892 - val_loss: 0.1753\n",
      "Epoch 4/50\n",
      "235/235 [==============================] - 77s 328ms/step - loss: 0.1704 - val_loss: 0.1658\n",
      "Epoch 5/50\n",
      "235/235 [==============================] - 80s 339ms/step - loss: 0.1639 - val_loss: 0.1612\n",
      "Epoch 6/50\n",
      "235/235 [==============================] - 77s 329ms/step - loss: 0.1598 - val_loss: 0.1574\n",
      "Epoch 7/50\n",
      "235/235 [==============================] - 87s 369ms/step - loss: 0.1554 - val_loss: 0.1541\n",
      "Epoch 8/50\n",
      "235/235 [==============================] - 86s 365ms/step - loss: 0.1510 - val_loss: 0.1486\n",
      "Epoch 9/50\n",
      "235/235 [==============================] - 87s 371ms/step - loss: 0.1479 - val_loss: 0.1465\n",
      "Epoch 10/50\n",
      "235/235 [==============================] - 85s 361ms/step - loss: 0.1460 - val_loss: 0.1450\n",
      "Epoch 11/50\n",
      "235/235 [==============================] - 88s 372ms/step - loss: 0.1448 - val_loss: 0.1442\n",
      "Epoch 12/50\n",
      "235/235 [==============================] - 90s 383ms/step - loss: 0.1437 - val_loss: 0.1433\n",
      "Epoch 13/50\n",
      "235/235 [==============================] - 94s 400ms/step - loss: 0.1428 - val_loss: 0.1427\n",
      "Epoch 14/50\n",
      "235/235 [==============================] - 85s 361ms/step - loss: 0.1418 - val_loss: 0.1416\n",
      "Epoch 15/50\n",
      "235/235 [==============================] - 82s 349ms/step - loss: 0.1411 - val_loss: 0.1410\n",
      "Epoch 16/50\n",
      "235/235 [==============================] - 70s 296ms/step - loss: 0.1403 - val_loss: 0.1406\n",
      "Epoch 17/50\n",
      "235/235 [==============================] - 72s 308ms/step - loss: 0.1397 - val_loss: 0.1401\n",
      "Epoch 18/50\n",
      "235/235 [==============================] - 68s 287ms/step - loss: 0.1392 - val_loss: 0.1394\n",
      "Epoch 19/50\n",
      "235/235 [==============================] - 65s 275ms/step - loss: 0.1385 - val_loss: 0.1388\n",
      "Epoch 20/50\n",
      "235/235 [==============================] - 60s 255ms/step - loss: 0.1380 - val_loss: 0.1385\n",
      "Epoch 21/50\n",
      "235/235 [==============================] - 54s 231ms/step - loss: 0.1376 - val_loss: 0.1386\n",
      "Epoch 22/50\n",
      "235/235 [==============================] - 61s 261ms/step - loss: 0.1370 - val_loss: 0.1385\n",
      "Epoch 23/50\n",
      "235/235 [==============================] - 65s 277ms/step - loss: 0.1367 - val_loss: 0.1373\n",
      "Epoch 24/50\n",
      "235/235 [==============================] - 64s 272ms/step - loss: 0.1361 - val_loss: 0.1371\n",
      "Epoch 25/50\n",
      "235/235 [==============================] - 58s 246ms/step - loss: 0.1358 - val_loss: 0.1369\n",
      "Epoch 26/50\n",
      "235/235 [==============================] - 73s 309ms/step - loss: 0.1354 - val_loss: 0.1372\n",
      "Epoch 27/50\n",
      "235/235 [==============================] - 87s 368ms/step - loss: 0.1351 - val_loss: 0.1366\n",
      "Epoch 28/50\n",
      "235/235 [==============================] - 54s 229ms/step - loss: 0.1338 - val_loss: 0.1350\n",
      "Epoch 32/50\n",
      "235/235 [==============================] - 54s 230ms/step - loss: 0.1334 - val_loss: 0.1349\n",
      "Epoch 33/50\n",
      "235/235 [==============================] - 52s 223ms/step - loss: 0.1331 - val_loss: 0.1345\n",
      "Epoch 34/50\n",
      "235/235 [==============================] - 62s 265ms/step - loss: 0.1328 - val_loss: 0.1343\n",
      "Epoch 35/50\n",
      "235/235 [==============================] - 53s 227ms/step - loss: 0.1325 - val_loss: 0.1347\n",
      "Epoch 36/50\n",
      "235/235 [==============================] - 56s 240ms/step - loss: 0.1324 - val_loss: 0.1342\n",
      "Epoch 37/50\n",
      "235/235 [==============================] - 53s 227ms/step - loss: 0.1321 - val_loss: 0.1340\n",
      "Epoch 38/50\n",
      "235/235 [==============================] - 56s 240ms/step - loss: 0.1317 - val_loss: 0.1335\n",
      "Epoch 39/50\n",
      "235/235 [==============================] - 48s 204ms/step - loss: 0.1316 - val_loss: 0.1334\n",
      "Epoch 40/50\n",
      "235/235 [==============================] - 54s 230ms/step - loss: 0.1314 - val_loss: 0.1328\n",
      "Epoch 41/50\n",
      "235/235 [==============================] - 55s 234ms/step - loss: 0.1311 - val_loss: 0.1333\n",
      "Epoch 42/50\n",
      "235/235 [==============================] - 54s 230ms/step - loss: 0.1308 - val_loss: 0.1328\n",
      "Epoch 43/50\n",
      "235/235 [==============================] - 51s 215ms/step - loss: 0.1305 - val_loss: 0.1335\n",
      "Epoch 44/50\n",
      "235/235 [==============================] - 41s 173ms/step - loss: 0.1304 - val_loss: 0.1326\n",
      "Epoch 45/50\n",
      "235/235 [==============================] - 40s 171ms/step - loss: 0.1300 - val_loss: 0.1324\n",
      "Epoch 46/50\n",
      "235/235 [==============================] - 40s 172ms/step - loss: 0.1298 - val_loss: 0.1316\n",
      "Epoch 47/50\n",
      "235/235 [==============================] - 45s 192ms/step - loss: 0.1295 - val_loss: 0.1316\n",
      "Epoch 48/50\n",
      "235/235 [==============================] - 43s 183ms/step - loss: 0.1292 - val_loss: 0.1310\n",
      "Epoch 49/50\n",
      "235/235 [==============================] - 38s 162ms/step - loss: 0.1288 - val_loss: 0.1310\n",
      "Epoch 50/50\n",
      "235/235 [==============================] - 29s 123ms/step - loss: 0.1285 - val_loss: 0.1304\n",
      "1595257933\n",
      "Wall time: 53min 38s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import os\n",
    "import time\n",
    "\n",
    "CURRENT_TIME = int(time.time())\n",
    "print(CURRENT_TIME)\n",
    "\n",
    "try:\n",
    "    os.makedirs(f'autoencoder_mnist/logs-{CURRENT_TIME}')\n",
    "except Exception:\n",
    "    pass\n",
    "\n",
    "# Run this in a terminal\n",
    "#tensorboard --logdir=autoencoder_mnist/logs\n",
    "\n",
    "CALLBACKS=[\n",
    "    TensorBoard(log_dir=f'autoencoder_mnist/logs-{CURRENT_TIME}')\n",
    "]\n",
    "\n",
    "# Fit data\n",
    "autoencoder.fit(x_train, x_train, epochs=50, batch_size=256, shuffle=True, validation_data=(x_test, x_test), callbacks=CALLBACKS)\n",
    "\n",
    "print(CURRENT_TIME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded_imgs = encoder.predict(x_test)\n",
    "decoded_imgs = decoder.predict(encoded_imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABG0AAABwCAYAAACkaY2RAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAbSklEQVR4nO3daZRUxRnG8RoBFURHGRmQsENQZJVdAwrKEWUTFJQwMQYEMWLEhcUoUQTUc0BRRATJCQqIiLIoCBKVAIpIPBCWsB8wgCO74OgoyMB0Pnh8favobnpm+nbfuf3/fXqKqumupOd291zrrUoLhUIGAAAAAAAA/nJOsicAAAAAAACAM3HTBgAAAAAAwIe4aQMAAAAAAOBD3LQBAAAAAADwIW7aAAAAAAAA+BA3bQAAAAAAAHyoZEEGp6WlcT54koRCobR4PA6vYVIdCYVC5ePxQLyOycO1GAhciwHAtRgIXIsBwLUYCFyLAcC1GAhhr0VW2gCJsyfZEwBgjOFaBPyCaxHwB65FwB/CXovctAEAAAAAAPAhbtoAAAAAAAD4EDdtAAAAAAAAfIibNgAAAAAAAD7ETRsAAAAAAAAf4qYNAAAAAACAD3HTBgAAAAAAwIdKJnsCSE2DBw+WXLp0aauvYcOGknv06BHxMSZNmiT5888/t/pmzJhR1CkCAAAAAJBUrLQBAAAAAADwIW7aAAAAAAAA+BA3bQAAAAAAAHyIPW2QMLNnz5Ycba8aLT8/P2LfgAEDJLdv397qW7FiheS9e/fGOkUkWZ06daz2tm3bJA8aNEjyhAkTEjanVHbBBRdIHjt2rGR97RljzNq1ayX37NnT6tuzZ49HswMAAEiOSy65RHLVqlVj+hn3O9FDDz0kedOmTZJ37NhhjduwYUNhpogAYaUNAAAAAACAD3HTBgAAAAAAwIcoj4JndDmUMbGXROmSmH/+85+Sa9asaY3r0qWL5Fq1all9WVlZkp999tmYnhfJd9VVV1ltXR6XnZ2d6OmkvMsuu0xy//79Jbtli02bNpXcuXNnq2/ixIkezQ5akyZNJM+bN8/qq169umfPe+ONN1rtrVu3Sv7qq688e16cnf6MNMaYBQsWSL7//vslT5482Rp3+vRpbycWQJmZmZLffvttyatWrbLGTZkyRfLu3bs9n9cv0tPTrfa1114recmSJZLz8vISNiegOOjUqZPkrl27Wn1t27aVXLt27Zgezy17qlatmuTzzjsv4s+VKFEipsdHcLHSBgAAAAAAwIe4aQMAAAAAAOBDlEchrpo1aya5e/fuEcdt3rxZsrvc8MiRI5Jzc3Mln3vuuda41atXS27UqJHVl5GREeOM4SeNGze22j/88IPk+fPnJ3o6Kad8+fJWe9q0aUmaCQqqQ4cOkqMtsY43twSnb9++knv16pWweeBn+rPvlVdeiTju5Zdfljx16lSr7/jx4/GfWMDoU2OMsb/T6FKkgwcPWuOSVRKlT/gzxn6v1+WtO3fu9H5ixcxFF11ktXXJff369SW7p5hSauZveluFgQMHStal4MYYU7p0aclpaWlFfl73lFQgVqy0AQAAAAAA8CFu2gAAAAAAAPgQN20AAAAAAAB8KKl72rhHQOs6wn379ll9J06ckDxz5kzJBw4csMZRj5tc+ohgt/ZT13zr/Rf2798f02M/8sgjVvvKK6+MOHbRokUxPSaST9eE62NojTFmxowZiZ5OynnggQckd+vWzepr0aJFgR9PHyVrjDHnnPPrfxvYsGGD5E8++aTAjw1byZK/foR37NgxKXNw98p4+OGHJV9wwQVWn96jCt7Q11/lypUjjps1a5Zk/f0KkV166aWSZ8+ebfWVK1dOst5L6C9/+Yv3E4tg+PDhkmvUqGH1DRgwQDLfm8+UlZUl+emnn7b6qlSpEvZn3L1vvvnmm/hPDHGj3x8HDRrk6XNt27ZNsv5bCPGjj1zX79XG2Hus6mPajTEmPz9f8uTJkyV/9tln1jg/vE+y0gYAAAAAAMCHuGkDAAAAAADgQ0ktjxozZozVrl69ekw/p5d1fv/991ZfIpedZWdnS3b/t6xZsyZh8/CThQsXStZL1YyxX6ujR48W+LHd42NLlSpV4MeA/1xxxRWS3XIKdwk64u+FF16QrJeJFtatt94asb1nzx7Jd9xxhzXOLbPB2bVr107y1VdfLdn9PPKSe/SxLlstU6aM1Ud5VPy5x7s//vjjMf2cLj0NhUJxnVNQNWnSRLK7xF4bOXJkAmZzpnr16lltXVI+f/58q4/P1jPpcpkXX3xRckZGhjUu0vUyYcIEq63LvQvznRexcUthdKmTLnFZsmSJNe6nn36SnJOTI9n9nNLfSz/88EOrb9OmTZL//e9/S163bp017vjx4xEfH7HT2ykYY19j+rum+zsRq5YtW0o+deqU1bd9+3bJK1eutPr079zJkycL9dyxYKUNAAAAAACAD3HTBgAAAAAAwIe4aQMAAAAAAOBDSd3TRh/xbYwxDRs2lLx161arr27dupKj1RW3atVK8ldffSU50hF94eg6tsOHD0vWx1m79u7da7VTdU8bTe9fUVhDhgyRXKdOnYjjdC1puDb8a+jQoZLd3xmuI28sXrxYsj6Su7D00aa5ublWX7Vq1STrY2e/+OILa1yJEiWKPI+gc+u59bHNu3btkvzMM88kbE633HJLwp4LZ2rQoIHVbtq0acSx+rvNBx984NmcgiIzM9Nq33bbbRHH3n333ZL190av6X1sPv7444jj3D1t3P0gYczgwYMl6yPcY+Xu03bTTTdJdo8N1/vfeLkHRlBF22emUaNGkvVRz67Vq1dL1n9X7t692xpXtWpVyXovU2Pisw8gzqTvBwwcOFCye41ddNFFYX/+66+/ttqffvqp5P/9739Wn/4bRO+t2KJFC2ucfk/o2LGj1bdhwwbJ+tjweGOlDQAAAAAAgA9x0wYAAAAAAMCHkloetXTp0qhtzT2q7RfucaONGzeWrJc5NW/ePOZ5nThxQvKOHTskuyVbeqmUXpqOouncubNkfXTmueeea407dOiQ5L/+9a9W348//ujR7FBU1atXt9rNmjWTrK83YzgaMV6uu+46q3355ZdL1st7Y13q6y7/1MuT9dGZxhhz/fXXS452HPGf//xnyZMmTYppHqlm+PDhVlsvEddL8d0StXjTn33u7xbLxRMrWsmOyy0jQHTPP/+81f7DH/4gWX+/NMaYd955JyFzcrVp00ZyhQoVrL7XX39d8htvvJGoKRUbunTXGGP69OkTdtzGjRut9sGDByW3b98+4uOnp6dL1qVXxhgzc+ZMyQcOHDj7ZFOc+/3/zTfflKzLoYyxy4OjlQxqbkmU5m5/gfh79dVXrbYua4t2fLe+b/Df//5X8mOPPWaN03/Xu6655hrJ+nvo1KlTrXH6/oJ+DzDGmIkTJ0qeO3eu5HiXyrLSBgAAAAAAwIe4aQMAAAAAAOBDSS2Piodjx45Z7WXLloUdF630Khq99NgtxdJLsWbPnl2ox8eZdLmMuyRS0/+fr1ixwtM5IX7ccgotkaduBJ0uQ3vrrbesvmjLTTV9mpde8vnUU09Z46KVI+rHuOeeeySXL1/eGjdmzBjJ559/vtX38ssvS87LyzvbtAOlR48ekt0TC3bu3Ck5kSet6TI3txxq+fLlkr/99ttETSllXXvttRH73FNpopUn4kyhUMhq69/1ffv2WX1engBUunRpq62X/t93332S3fn27dvXszkFgS53MMaYCy+8ULI+bcb9zqI/n37/+99LdksyatWqJblixYpW33vvvSf55ptvlnz06NGY5p4KypYtK9ndAkFvo3DkyBGr77nnnpPMVgn+4X6v06c29evXz+pLS0uTrP8ucEvnx44dK7mw2ylkZGRI1qeYjhgxwhqnt2lxSysThZU2AAAAAAAAPsRNGwAAAAAAAB/ipg0AAAAAAIAPFfs9bbyQmZkp+ZVXXpF8zjn2PS59HDV1qIX37rvvWu0bb7wx7Ljp06dbbff4WxQPDRo0iNin9zVB0ZQs+evbe6x72Lh7Q/Xq1UuyWzceK72nzbPPPit53Lhx1rgyZcpIdn8PFixYIHnXrl2Fmkdx1bNnT8n6/yNj7M8nr+k9krKysiSfPn3aGjd69GjJqbb/UKLoI0p1drk1/uvXr/dsTqmmU6dOVlsfp673cnL3YIiV3kelbdu2Vl+rVq3C/sycOXMK9Vyp6rzzzrPaek+gF154IeLP6eODX3vtNcn6vdoYY2rWrBnxMfReK17uh1ScdevWTfKjjz5q9eljuPWx98YYk5OT4+3EUCju+9iQIUMk6z1sjDHm66+/lqz3lv3iiy8K9dx6r5oqVapYffpvy8WLF0t297HV3PnOmDFDspd7+bHSBgAAAAAAwIe4aQMAAAAAAOBDlEeFMXDgQMn6WFr3ePHt27cnbE5Bc9lll0l2l3frJau6JEMvuzfGmNzcXI9mh3jTy7n79Olj9a1bt07yRx99lLA54Wf6qGj3iNjClkRFosucdImNMcY0b948rs9VXKWnp1vtSKUQxhS+9KIw9HHtutxu69at1rhly5YlbE6pKtZrJZG/H0E0fvx4q92uXTvJlSpVsvr00et66XzXrl0L9dz6MdyjvLUvv/xSsnvkNKLTx3W7dPmbW8IfSbNmzWJ+7tWrV0vmu2x40Uo/9ffG7OzsREwHRaRLlIw5s7RaO3XqlOSWLVtK7tGjhzXuiiuuCPvzx48ft9p169YNm42xv+dWqFAh4py0gwcPWu1ElYWz0gYAAAAAAMCHuGkDAAAAAADgQ5RHGWN+97vfWW13l/Jf6J3MjTFm06ZNns0p6ObOnSs5IyMj4rg33nhDcqqdGhMk7du3l1yuXDmrb8mSJZL1qQyIH/fkO00vPfWaXvLvzinaHEeMGCH5zjvvjPu8/MQ90eQ3v/mN5FmzZiV6OqJWrVph/53PwcSLVoYRj5OL8LO1a9da7YYNG0pu3Lix1XfTTTdJ1qeiHD582Bo3bdq0mJ5bn0ayYcOGiONWrVolme9IBeO+n+pSNl2C6JZg6BMwu3fvLtk9bUZfi25f//79JevXesuWLTHNPRW4pTCavt6efPJJq++9996TzIl5/vGvf/3LautSav03gjHGVK1aVfJLL70kOVqpqC63ckuxoolUEpWfn2+158+fL/mBBx6w+vbv3x/z8xUFK20AAAAAAAB8iJs2AAAAAAAAPsRNGwAAAAAAAB9iTxtjTMeOHa12qVKlJC9dulTy559/nrA5BZGuF27SpEnEccuXL5fs1qqieGrUqJFktyZ1zpw5iZ5OSrj33nslu7W5ydKlSxfJV111ldWn5+jOV+9pE3Tff/+91dY1+XpPDWPs/aGOHj0a13lkZmZa7Uj7C6xcuTKuz4vwWrduLbl3794Rx+Xk5EjmKNz4OnbsmGT3aHvdHjZsWJGfq2bNmpL1XmDG2O8JgwcPLvJzpaqPP/7YautrR+9b4+4zE2lfDffxBg4cKPn999+3+n77299K1vtj6M/tVFe+fHnJ7ncCvffbE088YfUNHz5c8uTJkyXrY9aNsfdN2blzp+TNmzdHnFO9evWstv67kPfb6NxjuPV+UBdffLHVp/eW1fvOfvPNN9a4vXv3Sta/E/pvDmOMadGiRYHnO2XKFKv92GOPSdb7VSUSK20AAAAAAAB8iJs2AAAAAAAAPpSy5VGlS5eWrI+OM8aYkydPStblOXl5ed5PLEDco7z10jJdgubSS39zc3PjPzEkRMWKFSW3adNG8vbt261x+hg9xI8uRUokvaTZGGOuvPJKyfo9IBr3mNxUeu91lxDrY3xvu+02q2/RokWSx40bV+Dnql+/vtXWJRnVq1e3+iKVBPil9C7o9OfpOedE/u9tH330USKmA4/pkg/32tPlV+57JWLnlpTefvvtknXZdnp6esTHmDBhgmS3LO7EiROS582bZ/Xp8o8OHTpIrlWrljUulY9xf+655yQ//PDDMf+cfn+87777wuZ40def3tqhV69ecX+uIHPLjfT1URjTp0+32tHKo3RJuv49e/31161x+kjxZGGlDQAAAAAAgA9x0wYAAAAAAMCHuGkDAAAAAADgQym7p82QIUMku0fPLlmyRPKqVasSNqegeeSRR6x28+bNw4579913rTbHfAfDn/70J8n6+OAPPvggCbNBojz++ONWWx97Gs3u3bsl33XXXVafPtYx1ej3Q/fo306dOkmeNWtWgR/7yJEjVlvvnXHppZfG9Bhu3Te8EenIdXcvgFdffTUR00Gc9ezZ02r/8Y9/lKz3XDDmzGNvER/6yG59vfXu3dsap685vfeQ3sPGNWrUKKtdt25dyV27dg37eMac+VmYSvS+JrNnz7b63nzzTcklS9p/ylapUkVytP2/4kHv4ad/Z/Sx48YYM3r0aE/nAWOGDh0quSB7Ct17772SC/M9KpFYaQMAAAAAAOBD3LQBAAAAAADwoZQpj9LLyI0x5m9/+5vk7777zuobOXJkQuYUdLEe0Xf//fdbbY75DoZq1aqF/fdjx44leCbw2uLFiyVffvnlhXqMLVu2SF65cmWR5xQU27Ztk6yPpDXGmMaNG0uuXbt2gR9bH2vrmjZtmtXOysoKO849ohzxUblyZavtlmj8Ijs722qvWbPGsznBOzfffHPEvvfff99q/+c///F6OilPl0rpXFju+6Qu99HlUe3atbPGlStXTrJ7RHnQ6SOW3fe1OnXqRPy5G264QXKpUqUkjxgxwhoXacuGwtLly02bNo3rYyO8fv36SdYlaW7JnLZ582arPW/evPhPzCOstAEAAAAAAPAhbtoAAAAAAAD4UKDLozIyMiS/9NJLVl+JEiUk66X9xhizevVqbycGi17+aYwxeXl5BX6MnJyciI+hl0emp6dHfIyLL77Yasda3qWXcA4bNszq+/HHH2N6jCDq3Llz2H9fuHBhgmeSmvRS3WgnKERblj9lyhTJlSpVijhOP35+fn6sU7R06dKlUD+XytavXx82x8OXX34Z07j69etb7U2bNsV1HqnqmmuusdqRrmH39EUUT+778A8//CD5+eefT/R04LG3335bsi6PuuOOO6xxevsAtm6IzdKlS8P+uy4nNsYujzp16pTk1157zRr397//XfKDDz5o9UUqW4U3WrRoYbX1e2PZsmUj/pzedkOfFmWMMT/99FOcZuc9VtoAAAAAAAD4EDdtAAAAAAAAfIibNgAAAAAAAD4UuD1t9F41S5YskVyjRg1r3K5duyTr47+ReBs3bizyY7zzzjtWe//+/ZIrVKgg2a0XjrcDBw5Y7aefftrT5/OT1q1bW+2KFSsmaSYwxphJkyZJHjNmTMRx+jjZaPvRxLpXTazjJk+eHNM4JIfeEylc+xfsYeMNvSef68iRI5LHjx+fiOnAA3pvBf09xRhjDh06JJkjvoNHf07qz+dbbrnFGvfkk09Kfuutt6y+HTt2eDS7YPrwww+ttv5+ro+I7t+/vzWudu3aktu2bRvTc2VnZxdihjgbd+/DCy+8MOw4vSeYMfa+UZ999ln8J5YgrLQBAAAAAADwIW7aAAAAAAAA+FDgyqNq1aoluWnTphHH6eOcdakU4sc9St1d9hlPPXv2LNTP6WP+opV1LFiwQPKaNWsijvv0008LNY8g6N69u9XWpYrr1q2T/MknnyRsTqls3rx5kocMGWL1lS9f3rPnPXz4sNXeunWr5HvuuUeyLmGE/4RCoahteKtDhw4R+/bu3Ss5JycnEdOBB3R5lHt9LVq0KOLP6ZKASy65RLL+vUDxsX79eslPPPGE1Td27FjJzzzzjNV35513Sj5+/LhHswsO/V3EGPvY9dtvvz3iz7Vr1y5i3+nTpyXra/bRRx8tzBQRhn6/Gzp0aEw/M3PmTKu9fPnyeE4paVhpAwAAAAAA4EPctAEAAAAAAPAhbtoAAAAAAAD4ULHf06ZatWpW2z3S7Rfung76mFt449Zbb7XauhaxVKlSMT1GvXr1JBfkuO6pU6dK3r17d8Rxc+fOlbxt27aYHx8/K1OmjOSOHTtGHDdnzhzJugYY3tmzZ4/kXr16WX3dunWTPGjQoLg+r3vM/cSJE+P6+EiM888/P2If+yd4Q38u6v35XCdOnJCcl5fn6ZyQHPpzMisry+p76KGHJG/evFnyXXfd5f3E4Knp06db7QEDBkh2v1OPHDlS8saNG72dWAC4n1sPPvig5LJly0pu1qyZNS4zM1Oy+/fEjBkzJI8YMSIOs4Qx9uuxZcsWydH+dtTXgH5tg4SVNgAAAAAAAD7ETRsAAAAAAAAfKvblUfoIWWOMqVq1athxK1assNocX5p4Y8aMKdLP9+7dO04zQbzopfnHjh2z+vQx6ePHj0/YnHAm95h13dYlpe77aZcuXSTr13PKlCnWuLS0NMl6KSuKrz59+ljtb7/9VvKoUaMSPZ2UkJ+fL3nNmjVWX/369SXv3LkzYXNCcvTr10/y3XffbfX94x//kMy1GCyHDx+22u3bt5fsluYMGzZMsltCh7M7ePCgZP1dRx+lbowxrVq1kvzUU09ZfYcOHfJodqnt+uuvl1y5cmXJ0f5212WjuoQ4SFhpAwAAAAAA4EPctAEAAAAAAPChtIKUCaWlpfmipqh169aSFy9ebPXpHae1Fi1aWG136bHfhUKhtLOPOju/vIYpam0oFGp29mFnx+uYPFyLgcC1eBYLFy602uPGjZO8bNmyRE8nrCBfi5UqVbLao0ePlrx27VrJATidLWWvRf1dVp8EZIxdwjpp0iSrT5cinzx50qPZFUyQr0W/cE/HvfrqqyW3bNlSchFKlFP2WgySIFyLGzZskNygQYOI48aOHStZlwsGQNhrkZU2AAAAAAAAPsRNGwAAAAAAAB/ipg0AAAAAAIAPFcsjv9u0aSM50h42xhiza9cuybm5uZ7OCQCAoNBHoCLx9u3bZ7X79u2bpJnAKytXrpSsj7gFwunRo4fV1vt+1K5dW3IR9rQBfKFcuXKS09J+3aLHPWL9xRdfTNic/ICVNgAAAAAAAD7ETRsAAAAAAAAfKpblUdHo5YI33HCD5KNHjyZjOgAAAABQaN99953VrlGjRpJmAnhr3LhxYfOoUaOscfv370/YnPyAlTYAAAAAAAA+xE0bAAAAAAAAH+KmDQAAAAAAgA+lhUKh2AenpcU+GHEVCoXSzj7q7HgNk2ptKBRqFo8H4nVMHq7FQOBaDACuxUDgWgwArsVA4FoMAK7FQAh7LbLSBgAAAAAAwIe4aQMAAAAAAOBDBT3y+4gxZo8XE0FU1eL4WLyGycPrWPzxGgYDr2Pxx2sYDLyOxR+vYTDwOhZ/vIbBEPZ1LNCeNgAAAAAAAEgMyqMAAAAAAAB8iJs2AAAAAAAAPsRNGwAAAAAAAB/ipg0AAAAAAIAPcdMGAAAAAADAh7hpAwAAAAAA4EPctAEAAAAAAPAhbtoAAAAAAAD4EDdtAAAAAAAAfOj/KDYux1kFDUwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x288 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABG0AAADPCAYAAABP9SwxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAJ1klEQVR4nO3bz4vO7QLH8WuOn5kZUVhYUEqRksLCxkpsZCEW1izNDkWyYIFs5B8wsWQjG4osFJJjZUfpIfl1yJjxs/iehZ7VcYrvfOe+Prfn9VrfXX3q6pq5vRsDTdMUAAAAALL8q/YAAAAAAP6XaAMAAAAQSLQBAAAACCTaAAAAAAQSbQAAAAACiTYAAAAAgab/zoeHh4ebBQsWTNWWKTVt2rTaE1p79epVef/+/UAXZ82aNasZGhrq4qiemzlzZu0Jk/LixYv/NE2zsIuzZs2a1QwODnZxVM/168+QUkp5+fJlGRsb6+Qtzp49uxkeHu7iqJ77/v177QmT8vbt287e4uDgYDN//vwujuq5Dx8+1J7Q2sePH8uXL186e4v9+ntx7ty5tSdMyuPHj73FUsqXL19qT2htfHy8fP78uZO3OHfu3GbRokVdHNVzX79+rT1hUp4+fdrZW+zn7zffvn2rPaG1Dx8+dPZ7cc6cOc28efO6OKrnJiYmak+YlPHx8Z++xd+KNgsWLChHjx7tblUP9esPj1JK2bdvX2dnDQ0NlS1btnR2Xi8tWbKk9oRJOXny5F9dnTU4OFg2bdrU1XE9tXv37toTWhsZGensrOHh4bJ9+/bOzuulT58+1Z4wKefPn+/sLc6fP7/s3bu3q+N66u7du7UntHbjxo3OzhoaGirbtm3r7Lxe2rx5c+0Jk7Jr165O32KXP6N76eHDh7UntHbp0qXOzlq0aFE5depUZ+f10rNnz2pPmJSRkZHO3mI/f795+/Zt7QmtXb9+vbOz5s2bV/bs2dPZeb10+/bt2hMm5dq1az99i/57FAAAAEAg0QYAAAAgkGgDAAAAEEi0AQAAAAgk2gAAAAAEEm0AAAAAAok2AAAAAIFEGwAAAIBAog0AAABAINEGAAAAIJBoAwAAABBItAEAAAAIJNoAAAAABBJtAAAAAAKJNgAAAACBRBsAAACAQKINAAAAQCDRBgAAACCQaAMAAAAQSLQBAAAACCTaAAAAAAQSbQAAAAACiTYAAAAAgUQbAAAAgECiDQAAAEAg0QYAAAAgkGgDAAAAEEi0AQAAAAgk2gAAAAAEEm0AAAAAAok2AAAAAIFEGwAAAIBAog0AAABAINEGAAAAIJBoAwAAABBItAEAAAAIJNoAAAAABBJtAAAAAAKJNgAAAACBRBsAAACAQKINAAAAQCDRBgAAACCQaAMAAAAQSLQBAAAACDTQNM2vf3hg4Nc/HGZ8fLz2hNY2btxY7t+/P9DFWf18hxcvXqw9YVJ27Njx76Zp1nVxVj/f4+/8zEmzbt26cu/evX/8W+znOyyllIGBAW+x9Pc9eos/9PMdluIt/q2f79Fb/KGf77AUb/Fv/XyP3uIPr1+/rj1hUhYuXPjTt+gvbQAAAAACiTYAAAAAgUQbAAAAgECiDQAAAEAg0QYAAAAgkGgDAAAAEEi0AQAAAAgk2gAAAAAEEm0AAAAAAok2AAAAAIFEGwAAAIBAog0AAABAINEGAAAAIJBoAwAAABBItAEAAAAIJNoAAAAABBJtAAAAAAKJNgAAAACBRBsAAACAQKINAAAAQCDRBgAAACCQaAMAAAAQSLQBAAAACCTaAAAAAAQSbQAAAAACiTYAAAAAgUQbAAAAgECiDQAAAEAg0QYAAAAgkGgDAAAAEEi0AQAAAAgk2gAAAAAEEm0AAAAAAok2AAAAAIFEGwAAAIBAog0AAABAINEGAAAAIJBoAwAAABBItAEAAAAIJNoAAAAABBJtAAAAAAKJNgAAAACBRBsAAACAQKINAAAAQCDRBgAAACCQaAMAAAAQSLQBAAAACDT9dz68YsWKcu7cuanaMqVGRkZqT2jtyZMnnZ21du3acu/evc7O66UNGzbUnhBj1apV5cKFC7VntHLs2LHaE1p7/vx5Z2ctX768nDlzprPzeun06dO1J8RYtmxZOXnyZO0Zrdy8ebP2hNYmJiY6O2vNmjXlxo0bnZ3XSydOnKg9IcbKlSv79jvq5cuXa09obWxsrLOzli5dWg4fPtzZeb109uzZ2hNirF69uly9erX2jFZGR0drT2jtzZs3nZ3Vz/9ePH78eO0JU8Jf2gAAAAAEEm0AAAAAAok2AAAAAIFEGwAAAIBAog0AAABAINEGAAAAIJBoAwAAABBItAEAAAAIJNoAAAAABBJtAAAAAAKJNgAAAACBRBsAAACAQKINAAAAQCDRBgAAACCQaAMAAAAQSLQBAAAACCTaAAAAAAQSbQAAAAACiTYAAAAAgUQbAAAAgECiDQAAAEAg0QYAAAAgkGgDAAAAEEi0AQAAAAgk2gAAAAAEEm0AAAAAAok2AAAAAIFEGwAAAIBAog0AAABAINEGAAAAIJBoAwAAABBItAEAAAAIJNoAAAAABBJtAAAAAAKJNgAAAACBRBsAAACAQKINAAAAQCDRBgAAACCQaAMAAAAQSLQBAAAACCTaAAAAAAQSbQAAAAACiTYAAAAAgUQbAAAAgECiDQAAAEAg0QYAAAAg0PTf+fDg4GBZv379VG2ZUqOjo7UnRHj06FHZunVr7Rmt3Llzp/aEGBMTE+XWrVu1Z7Ry5MiR2hMiPH36tOzfv7/2jFYePHhQe0KMb9++lXfv3tWe0crOnTtrT4gwNjZWrly5UntGKwcPHqw9IcaMGTPK4sWLa89opV+/W3dtYmKi3Lx5s/aMVs6dO1d7Qow3b96U8+fP157RyoEDB2pPiPDp06e+/a536NCh2hOmhL+0AQAAAAgk2gAAAAAEEm0AAAAAAok2AAAAAIFEGwAAAIBAog0AAABAINEGAAAAIJBoAwAAABBItAEAAAAIJNoAAAAABBJtAAAAAAKJNgAAAACBRBsAAACAQKINAAAAQCDRBgAAACCQaAMAAAAQSLQBAAAACCTaAAAAAAQSbQAAAAACiTYAAAAAgUQbAAAAgECiDQAAAEAg0QYAAAAgkGgDAAAAEEi0AQAAAAgk2gAAAAAEEm0AAAAAAok2AAAAAIFEGwAAAIBAog0AAABAINEGAAAAIJBoAwAAABBItAEAAAAIJNoAAAAABBJtAAAAAAKJNgAAAACBRBsAAACAQKINAAAAQCDRBgAAACCQaAMAAAAQSLQBAAAACCTaAAAAAAQSbQAAAAACiTYAAAAAgUQbAAAAgECiDQAAAEAg0QYAAAAg0EDTNL/+4YGB16WUv6ZuDv/H0qZpFnZxkDusyj32P3f4Z3CP/c8d/hncY/9zh38G99j/3OGf4af3+FvRBgAAAIDe8N+jAAAAAAKJNgAAAACBRBsAAACAQKINAAAAQCDRBgAAACCQaAMAAAAQSLQBAAAACCTaAAAAAAQSbQAAAAAC/Rc4/4spBVDTYQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x576 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABG0AAABwCAYAAACkaY2RAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de7BVdfnH8YcsE1NI5SIIcr9ZgIJxU8QbIIroTNY0mWUWUzPVNE3JjNNYUzOVkzb+UflHiaPZVJaKllYWF0ETBT3cBbnI/SIiIKBmZvz+cH6Pn+fh7O0Bz2Xttd+vv57N+nLOPuu7vt+19p7v833aHT582AAAAAAAAFAsH2jrNwAAAAAAAIAj8aUNAAAAAABAAfGlDQAAAAAAQAHxpQ0AAAAAAEAB8aUNAAAAAABAAfGlDQAAAAAAQAF98Ggat2vXjvrgbeTw4cPtmuPn0Idtas/hw4c7N8cPoh/bDmOxFBiLJcBYLAXGYgkwFkuBsVgCjMVSaHQsstIGaD2b2/oNADAzxiJQFIxFoBgYi0AxNDoW+dIGAAAAAACggPjSBgAAAAAAoID40gYAAAAAAKCA+NIGAAAAAACggI6qehTQXNq1e3dz8w98IH53+MEPvntZ/u9///P48OHKG5n/97//bcZ3BwAAAABA22OlDQAAAAAAQAHxpQ0AAAAAAEABkR6F96V9+/bh9Yc//GGPO3XqFI597nOf83jEiBEed+7cObTT9KjXX3/d4+XLl4d2s2fP9viZZ54Jx3bv3u2xplgBaLrjjz/eYx3rb731VsX/8/bbb4fX//nPfzyuluIIAABQVLq1g5nZcccd57F+dtFnp9xOn4nMzN58802P2eoB1bDSBgAAAAAAoID40gYAAAAAAKCA+NIGAAAAAACggNjTBkdNczM/9KEPhWMnnniix5MnTw7Hxo0b5/HHPvYxj0899dTQTkuAa+7n6aefHtrpMd37xszsySef9PiNN95o5K9AUWh/n3nmmeHYpEmTPG5oaPD4ueeeC+3YK6V5nHDCCeH1xz/+cY+nTZvm8cCBA0O7Q4cOebxz585w7L777vN448aNHucxSx++t5xPr3uIdejQweM8p/773//2+KWXXmr034+G5u5rbBbvAfnn61zMXmMtL/eNXhd6LekecGaMRdS2PE9WOpbHh+7HpvNTc4yH/J4YY61Dny81zrQ/cjv9zNOxY8dwTPfu1Osn72mj+9ZobBbvi/oslfcO1NdcP/WJlTYAAAAAAAAFxJc2AAAAAAAABUR6FI6aLgHMywi7dOnicU510fSmj3zkIx5XK3Gnywbz8tKzzz7b4127doVjq1atavRnoO3lftTr4vbbbw/HevTo4fFtt93m8bPPPttC767+6DLeqVOnhmPXXnutx+eee67HWv7bLI6xnGrRvXt3j2fPnu3xQw89VPFnoHGaDmUW+2T8+PEen3LKKaHd6tWrPV68eLHHmzdvDu2q9YGmEugS8d69e4d2OmZXrlwZjml6HEu9j53ed3XpvllMgfrOd74Tjp1//vkeL1iwwOObb745tMslaetVvlfp6zwHDh8+3GN9ptmyZUtop+kP1dIFmzomqqX7nHTSSR7rc5uZ2WuvvVbxWK3T8ZFT+PV+p6mcZjHFVOM8T+7bt8/jaudO+yNfL2U+/21N+79aKq9+DjnttNMajc3Mevbs6XFOIT/55JM91mefPJ41JWrv3r3h2I4dOxr5K8xeffXV8Lpe75M6x+U5uVLKWz7/1VImmzsVsiWx0gYAAAAAAKCA+NIGAAAAAACggFolParasiSly5yq7bReS0uZyi4vpz948KDHuiTfzKxbt24eL1y40GNNZTKLSwI1tWLUqFGhXZ8+fTyeOHFiOPbPf/7T40pLD1EMugw5p1roktJ169a11lsqPU2puOCCCzyeMWNGaDdkyBCP87JgpUu/cwqPpkxqNaoXX3wxtFu0aJHHzOvvqpb+oCmiI0eO9DhX5tq+fbvH1ao26XnPfaD/T9OvRo8eHdppykFOj+Le3fxylRK9JrTim1lMndJ7dU4hqef0qGrjTdMp8jOHvt6wYYPHOq+ZmW3dutVjreSWK8o0dazofKupiWZmffv29Xjt2rXhmKYnapW3Wh2XlVJi8rWt6dg6Z5qZXXzxxR7rOdFqpGZm69ev91irJVariJjTOPQ16VHvT/68qH2u6Uv6rGlmNmjQII+1om3e2kGvhVzRaf/+/R7r82ueO/RziM69+f1WqmBmVv2zdK2O20p0fOizp36ONIvVu7RvOnfuHNr179/f41xdU++hOg/nz6aa3p2fXzXdsSWfc1hpAwAAAAAAUEB8aQMAAAAAAFBAfGkDAAAAAABQQC2yp03Ou/voRz/qseam5byy8847z+Oc46k59FpuT3NLzWKOsO6N0tR9dcxifpvmpuWcY81h09isfPmFTaW58GvWrAnHdF8F3Z8klwjWc6f9nsszarnbnMs9ePBgj5cuXdqk947WkcdGLlmr9Dp5/vnnW+w91ZszzjjD469//esea163WZyvdU7O852WRM37YWgOspbO/MIXvhDavfDCCx5rnni903tXLuU9YcIEj4cOHerxihUrQjs9tzoP5z0Yqu2toPdCLSU8bNiwiu3yfhL1el9sbnqOtby0WdzTRveTMovn/5lnnvG4nvewMYtjTPdSyM8cuq/eZZddVvGYPo/oM6lZ3BfhWPc+0HGle7SMGTMmtNN5fs+ePeHYrl27mvz7aoE+R2h/5j2fxo4d6/Hll18ejunY0X7L40P3EdLPMS+//HJop2Mz39PyPRTHLu8fo/dJ3UdP97Axi88jGud9a5R+/jSLfaxjuFr/5vuuflbVa62e9rDJf6uOMd2ba/z48aGdznm6j03+fkG/h8jPJToPHzp0yGP9jGkWP9POnz8/HHvuuec8rtSfZu+/31hpAwAAAAAAUEB8aQMAAAAAAFBAzZYepUubtMSamVnXrl091rJbF154YWh36aWXepyXNmlJWS2/prGZ2d69ez3WZU6vvPJKaKelG/P71d+lS0rzcigtA6hLo8yOLIVdJtrXuSSdnnMtbWkWlwtqu2pL8nVJaV4Grn2YywxXK2uLYtE5QdMuzOK4yumJaLq8HPSTn/ykx5qWmseRLuXUJZ+zZs0K7bS0cy6r+fnPf95jLc94ySWXhHbnnHOOx3murefxrOkaOlbMYmlS7eM892r/6Dx8NKVm9X3oknNdumwWS2HmObtsS7qLIKcGaJpOPqbPM3/4wx88rpYOUG/0+aZjx47hmKaPDh8+PBw77bTTPF64cKHHudS2PtPoec9jsdpY0feo6VEam8Wl+Tklo9bldAo9X3q/0OdEs9hvkyZNCsf0uVSf4Q8cOBDa6TFNQcylovX1vHnzwjG9n+p1wBzZNHo/yn2sKXDaxwMHDgzt9FlFx4emjprF+2ku163PTHq/y2mR27Zt81jLxOffrc+5ZbsW8pjV1/le1atXL4+vueYaj6+++urQTsefXhP5M7iOYU1RNYvjT9MpdXsAszj/6+/Kv6+hocHj5k49ZqUNAAAAAABAAfGlDQAAAAAAQAHxpQ0AAAAAAEABNdueNpp7l3PJdJ8ZLeE7YMCA0E7Ld+cSzpq3lvdnULrHjf6fPn36hHaaq5xLAmpuneYo9uvXL7TT8nEbNmyo+D7Klpeo8t+m5yvnUOteCtVKXWqeo5bc1PK2ZnH/k7znRe4PFEcu8X3dddd5rPsCmJk9++yzHpd5HLU0LYVoZvaZz3zGYx1HeV8F3QPjz3/+s8czZ84M7XSOz3n9ml+u+5vl/RemT5/ucc4pL9t+DEdD73eDBw8OxzQnX/dtW7RoUWin/XM0+9go3ett3LhxHnfv3j200/LieS8INL+8h5Tu25CfbbTMc87rxzv0+S9f2yNGjPA47+Wk53bJkiUe6x5PZpX382vqHjZmsV91jxbdB8Is7mWV94TT36f7Mxzr/NDWKpVH1vLPZmZDhw71OO9ZofuQLF261GPdoyK30+tl9OjRod2UKVM81vnTLD6j6rMxzzmNy/2r+9jk/fG+9rWveaz7r+V9SHRvmWXLlnm8YsWK0E73o8nXjJaS1rGt/8cszg+5HHilva3Kfi3oZ4E81+peiJ/61Kc81n0RzeKeMbr30OrVq0O7xYsXe7xly5ZwTMfmGWec4bGWEzeL99a8F64+Y+mcn/v6/c6vrLQBAAAAAAAoIL60AQAAAAAAKKBmS49SuXykLrHXcof33ntvaKdlZHPp30olunNJUaXLynN6lJYUz8uyNO3p7LPP9jgvy+rWrZvHedldmemSvbx8T5fgVluOq3Fe9qhL5s466yyPtdyaWVyWmpfh5+VvKI68nF+XtmpaoZnZqlWrWuU9lZGOj4kTJ4ZjvXv39ljHn87PZma//vWvPdb5Wpf6msXxnOdkXSasc0K+DrRUcU6dyikG9URTRDU9I3v55Zc91jRks7iEuKlLrvO8rPddXU6cf9727ds9zkuD0Ty0b3JKjI7t3DeabprT2PEOHW+a2mlmduGFF3qcy4HPmTPH4wULFnisaYtmx5b+kJ8vdQn/Oeec43HeOkDnzbwsX39mGdIw9O+rVg5a/9ZcjlfvVXqP0znNzGz37t0ea7pMPo86Z+bUVk0Fzz8fR9LS2mZmF110kcc33nhjOFYpBU5Td83MHn30UY+1JHu+f6r8OVDpWM99qqXC83YOlVIV8/Wkr2sxjTE/U+i51HQos1jaWz+v51T5p59+2mN9Rs0pbvr/8rWk14jOCVou3iyO2Zx6rNux6LFKaZvHqn6+aQAAAAAAAKghfGkDAAAAAABQQC2SHpXpUjBdepSXje7bt6/R/2NWeflmXnqkrzU9YOHChVX/n9Ld+LVKSq7AsmPHDo/zMvAyLDdtimr9dKxLf3WZ2Re/+EWPdZd2s9iHy5cvD8d0F3cUy7Rp08Jr7dfHHnssHHv11Vdb5T2VkS7NvuKKK8IxTU3SMTx//vzQ7re//a3HunQ8L83VsajpsGZmc+fO9VhT4XJ6VKUl/2ax0k3Z59Z8b9I03FxVUc+FViPRe5PZsS2lzvOyLlHWOKfZ6H09p0qXve9aiz7bfPaznw3HNLU8z58PPfSQx/TFu/Ra19TMXOFUz21+xtBqQ5oakcfesTzL5lSs888/3+P+/ft7nNPCNd04j8X87FZrqqWOqJzKq+dk06ZN4Zim9ur9KVcM0nZ6TeRnVE2JydVmdF7XNPBa75fmpGNAK/eYxXkvV/bV9JfNmzd7nLfkePDBBz3W9OI8VnS+zfdF/eynzz7a92bVt/JQ+jc3d2pNW8vpm5dddlmjsVn8HKhjNlfG/OlPf+qxpoPm861jOB/T/tXUurwlh1aiy/2rc77ed5t7PLPSBgAAAAAAoID40gYAAAAAAKCA+NIGAAAAAACggFplT5tKcg7qseTdV8tr1f1zcmk/lXMUNS9VS3xlmltXr6VNm5pXnGmuZi7JqPtejB492mMtBW4W8xIfeeSRcCznpKJtaS7rjBkzKra76667wmvyu49d165dPR4zZkw4pjm8mn87a9as0G7nzp0e65iqNu5zvrDuJ7Z3716PzzzzzIo/44ILLgjH9H2VfS+OfD/SfTU0p9os5tA/9dRTHjfH/SjPt7qPje6to9eSWdwbIP8tOu+XvR9bkvbF5ZdfXrGdjl+zI/cDwDv0utQxlkvD6j42Bw4cCMeef/75Rn9e3sdBx0T79u091lLjZnFcDRkyJBybMGFCo+3yfmI6Z+c9VXQvKh2Lb775ZmhXi+NU33M+JzpP5s8Fes/UvYKeeeaZ0E5LFev+a3lvFe3fvC+R3uMaGho81nLiZrV5/puLjp28v0jPnj09zvcgvf/961//8vgf//hHaKd7v+XrXulzaP5soeOoWllvHfd5Tqi0F2h+lqrFa0HnwlwuXceA7vFkFs/rSy+95HHed1HnaB1/+dxpu3z+x48f7/H06dM91vnALM75uXy8zhF6HTR3n7HSBgAAAAAAoID40gYAAAAAAKCA2jQ9qqh0eZSW/Vu/fn1ot2DBAo9Jxzk6upyxb9++4djYsWM91rJveZmZLkfWUqZmx5Zqh5aj5Zxzf+tSwrx8FU2XU1F02XZelqpt9fwvWbIktNNSi01NVcvjVNNl9Gfk96tyOXBtW/aUuZwuquV9c8qSljPV0ra5D5p6/nQpcy5zq6UwNT1Kly6bxRK4xx9/fDj2+uuvN/q7anHZd2vT8zVu3DiP8xJuvffdc8894VguU4p36HJ5TafZunVraKclovfv3x+O9e7d22NNWevcuXNop2kemp6j86RZnHt79eoVjvXp08dj7dOcFqlpdLn8u84lmjqVn2Vr8Vmq0jYJZmZz5871OKdQXHXVVR7rPKx9axbnOJ2fcxqblo/Pv2vSpEkea6pFTlHWlLxa7Iv3Q7eqGDRoUDim8161Z45169Z5nMes/r9qpbarpdZoWpUey/c+PZbfb6XtO/I1o/fxWrkWdJ7J6d067+RzrnOZzkkjRowI7XQ+1TlTnzWy7t27h9cXX3xxo8fyXKglxe++++5wbOnSpR5rf5IeBQAAAAAAUAf40gYAAAAAAKCASI+ymLphFneS1qVNc+bMCe3Wrl3bsm+sZCotvdelvmZxSaQuT9N0KDOzH//4xx7rsmUzltsXgfa3LjvOFTmeeOIJj/MSbjRdrqCgS/TzUl21b98+j/MS/eZYgqvLY3Vs5/Qo/V3azuzIpbNlo32nKaFmsapC7kdd7q1zpVYtMYvnVufGvNRb/19OCZgyZYrHmh6Qx6z+LTmdS19rmlatLPVuS9pX06ZN8zj3taZTzJw5s+XfWAlUuhZXrlwZ2umclaud6BJ7TYs566yzQjvtr1deeaXR2CymPubKQ5oSoClcWhnJLN5rdcyaxXGraQVle3bKqSdbtmzxeNu2beGYXgd6zkeOHBna6bkcNmyYx1rRyCzO17k6kVZP/N73vudxTmF8/PHHKx7T9JmypA3rvV7HW35uzK+VPsdonNNz9NrQ/smV3LSKcE5f1vRgvWbyvU9/V+4rHX/V2unYrJUtOfS+lc+JnrucZl0pZWzUqFHhtVaZ0vOl90Gz+FySn210LtdzvH379tDul7/8pcc6Ls1iOlZLzqGstAEAAAAAACggvrQBAAAAAAAoIL60AQAAAAAAKKC63dNGc+s0n9Qs5j3qngG33npraJfLsSHK+1BoTqHuWZH34tBcRC2F+PDDD4d2lFwvNs3d/9KXvlSx3Z133ulx2fLpW1Meb7rvQd4/Rs+zllbMecDN8T7OO+88j7VMZ87X1vl09erV4VhZ8vWVnicdK7ofhlks25vPrZbenjp1qserVq0K7SqVoNSfbVZ5rwYzs9GjR3usezXka0Z/V77uarFkaVFoKdIJEyZ4nPcJqFbiFo3T5wfdW2HJkiWhne6HkvdC1H0SdO7NZbh17Oi+NTt27AjtdCxOnjy54jF97/r+8s+vNk71WNnm2vz36N5B69evD8d27tzp8TnnnONxnid1PyDdnzH/rkp7jpnFZ2DdQ2X48OGhnb7HXIJe97gpW7+Zxf1Q8t5pupdJvpfoHKjjbcCAAaFdv379PNY+GDp0aGjXoUMHj/PeRMuWLWv0WB7Pe/fu9bjataDt8t+lfZzvrUXtf/0b9uzZE44tXrzYY+1Ps1gOXPf5y+e/0s/P+zPqXjjan5nOhffff384pp9B8/XYWueflTYAAAAAAAAFxJc2AAAAAAAABVS36VGDBg3y+Nprrw3HdJnTHXfc4XEu/4Xq8rJtXe6mS4t1KbFZXIqqy0FzyW9dntYSaTVNLTOsf2dezljP6T5a6rR///4e57KVc+bMabX3VGY5zTCXrVSV0qPyz8hLUSvRMaApO2Zm119/vce6VDmPDS2ZuGHDhorvtyx0ftEl9jntQpdB57K9AwcO9FjLkuaymNqPGuffpaVHc9l1/d2aypZLdWoJ+bwMXO+tZezT5pTvP1rmW8dYXpb94IMPekwKd9PoOdT5MD/z6fL7nFrzxBNPeKxjNs+pmvKhfZzH9tVXX+1xfq7Q1Isnn3zS44aGhorvN6dH6Vivp+tE5yRNvzczu/vuuz3WNCotz20W5zxNQcv3XH2df4amp2kZ6V69eoV2lZ6Hzco5h+qY0Os+z4ebNm3yWD9bmMXUGL3Oc9qTvtZUKU1FNYtjNqcg6vvSdLj8fjUFWlOgMr0/53GvP79W+l7nlpyuq3Om9mf+f9q/+X6n40ifL8eMGRPa6fNRLhevqVlz5871WEt8m8XPnG2VjsZKGwAAAAAAgALiSxsAAAAAAIAC4ksbAAAAAACAAqqbPW00n9DMbNasWRWPad7oL37xC48pUfredG8LzdM1i3ucTJkyxeOxY8eGdppTvnLlSo81d9gs5jzm/NGm5nvq/8t78Gjeo+al53aVclrN6uuaqbYHg+b4a6l2syNL5+HY5Bzbajm3Oj40zte29pu209xhM7OOHTt6rHsxmJlddNFFjf6//P52797tcS61W9Ryls2l2h4xmuudy3zqHKW5+9XKqeuxvGfHiy++6LGWMDYz69u3b6O/a/ny5aGdllvN5Y71fdRKTn5b0X0UzGJJdz2W9yrRZxvO8dHT8ZH39NL9E7JjeebQ54pOnTqFdrqvRt4bSvf3e+SRRzzO+23oOM3vnf2ljpyf9Nlk7dq1Hue9UPTc6Zyme/eZxf4dMmRIOKbzac+ePT3u1q1baKd7i1XbI6wsKj2bLF26NLTT83LKKaeEY/osoT8j71Wj/aXnXZ9nzKrv/6TvIz//K91TJfejlprX35Wvz0rnpsj0GtV9C83i3ly5HLj+ffoz8mcqvRfqZ8xJkyaFdtXmU31mufXWWz3Oe0gV4fMcK20AAAAAAAAKiC9tAAAAAAAACqjU6VG6lHz69OnhmJZ3yyWIb775Zo/zUnVE1cra5dK/N9xwg8da5junWuiyOF16mJeN6tJBLfFnFpfTacpHXnKupXa1tGL+3bpcMi9LXLNmjcd5qXo9LUHOqTWTJ0/2WP/23/3ud6FdPZUbbUl5yWdebqoqlZrNy4c1fUZTcfLY1r7+yle+Eo7puNJrJL/fp556yuPVq1dXfO9loXODlpzUZflmZnfccYfHefm99p2W2s50+bimI27cuDG007F4xRVXhGMjRoxo9P3Onz8/tNMlxU0tGY8j5bKkw4YNa7RdXsKdS6fi2OV7dnPcwyv9jA4dOoTX1Z5RFy1a5LGWrc7PH7qcv4ypNO9XPieavqHjaNeuXaGdpj3pc2nuJ+1rTT01i9sCaDnwnJaqaXP5/ZbxmVL/Jn3Gz6l/zz77rMc5tVD7QT8HauqRmdmBAwc81uf9fF7Xr1/v8bZt28IxTfHR95hTm/SzUf68ouXl9f/lZ+Nav5/m81otfVOfUavNXXpev/rVr3p89tlnh3b67Jm32vjZz37mcUNDg8dFSIfKWGkDAAAAAABQQHxpAwAAAAAAUEClTo/SZYY//OEPwzFdbqW775uZ/f73v2+0HY6Uq5noEt+8nFt39tbloHlZv6YiabtrrrkmtNOl/Tk1R5ev9urVy+OckqHpBflv0aV7ulQ9L63TJX95GW09ydXCBg8e7LGmUyxcuLDV3lM9yUs5Z8+e7fHo0aPDMb3WtarFt7/97dDur3/9q8cDBw70+NOf/nRop5UwTj755Iq/S+fTDRs2hHY/+clPPM7LmMtOl0HnChSazqTLtM3iPFep2kI+pkusczsdw3m5sr7Wahd52X+99V1L6dq1a3itKQDanzk9rdaX0JedLvvX1PBLL700tNNnn5zGuGLFCo/1mSOnU5Qxfaa1VKt6o32o5zxXwtS5MKc9aSqy/j+tMmRmNmbMGI/vu+++iu+jjH2t51YrQpkdmT6q9Llen0fydg7ar3v37vU4P5voNhnaziymp2pKerV0n2ppT/qsnOfysn0e1Ws2P7/mvvp/eYuLSy65xOMrr7zSY936wiymoN11113h2AMPPOCxnv8iYqUNAAAAAABAAfGlDQAAAAAAQAHxpQ0AAAAAAEABlW5PG83x/9WvfuVx3mdh2bJlHs+YMSMcy7mIqCznHWoeoe5hYxZL6qmco3jiiSd6rCVu+/TpE9qNHz++4s/WnFbd7yb3reYc53KN2nbx4sUVf0al39VY2zIbNWpUeK05x1qWtJ73/WlJOaf9nnvu8fi6664Lx3SfJ91DKu9Vo2WftV3OJ9d5IO8NpXnYWkI875+jJVbLmJ9fTVPL3lfLaddzdqznT3PotUy4WexXLZWa9+Cpt75rKV/+8pfDa7236N5sM2fODO04/7VD9zWZMmVKOKb9mJ9NtNyx7sFA37cOPc+6L0re+0bnTL33mcU9NvQ5NH9W0b2NdA+keqDnJT9XaNnsfM70mVz3VsyfISrtVblkyZLQTp/j8zO+vi99H7q/jVn18tZ6/6+251w90TGmnxHHjh0b2t16660en3rqqR7n/UsfffRRj++8885wLI/bImOlDQAAAAAAQAHxpQ0AAAAAAEAB1Xx6VE7P0bLQmq6Rl5d+97vf9TgvW0TT5eV7umz+6aefDsc0vUL7Ji/Dr1TGNpdwa9++vcd52aguN9SliLmEnr7fXOZPX8+bN8/jXHZXS5bXW7lVXRr6jW98IxzTvtNrIS9bRMvQee2WW24Jx2677TaPNbWwQ4cOoV1+3RR5TtBxdNNNN3m8YMGC0K6pKUJl15bnQZeL9+jRo+IxvS7yvFypxDvemy4Dv+qqqyq207Kz69ata9H3hOalfazpGvk5SJ9tdWm/WUy9ICWquHT+y+XA9f6s7XLJbz3WtWvXcEyfPaup1dLg1cqu67Gcrq1pSpq+qylK+WfoZ4F8T+vSpYvHuX80zU1TvPVzjFns/3ys3j43NIX26cSJEz3++c9/Htpp+qDKn+duv/12j3PZ9lrCShsAAAAAAIAC4ksbAAAAAACAAuJLGwAAAAAAgAKq+T1tNNfQzOyGG27wWPM4Z82aFdrNnTvX41rK8SyafO40bzPvabNq1SqP+/Xr53Hfvn1DO92fRnM/c1k/zQ3Pe2/ovilvv/22x5q3ahbLT+e9AbZu3erxa1zI6KMAAAXgSURBVK+95nHOP9Vc23rbw0HLs+c84N27d3v8l7/8pdXeE96he6P88Y9/DMcGDBjgse5FpPtEmR25Z9j/y+Nex4eWozUzu/HGGz1evny5x7nsJdqezrc5717nNp1787jXayZfP9xrq9P7mJ5js3jfmT9/fqP/juLJY6BTp04eT5061eNBgwaFdmvWrPFY500z5s5alPfy27hxo8e670q+XnROOPfccyv+DL0m8r5oZZh387O17k+jz/Fm8blU96DJe1/q/mv62UU/M5iZnX766R7nvXV0fyn9+bm/de+bvKeKzuFl6Ktjkfvm4osv9lj3scl72Oh40XP8ox/9KLRbuXKlx7V8jllpAwAAAAAAUEB8aQMAAAAAAFBANZkeddxxx3ms6VBmZiNHjvRYl+zn9CiWlzaPvMxMlwTqUjWzuCRwy5YtHj/++OOhnS5Z1Fj73SwuYWxqWlJeelqtPK2+1jj/zbW81O790nLRL730Ujimy/s19ayez1dbOXjwYHj9gx/8wOP777/f4yuvvDK0Gz58uMcdO3b0OKcS/u1vf/N48eLF4djOnTs9zsuOUSxvvPGGx/PmzQvHevfu7bGmuubxXCmlDo3T86WpM/kZRVN79Z7ZliXi8d7yeNBUi0984hMV/98LL7zg8Y4dO8Ix7qG1J/eZPg///e9/93j8+PGhnaYs9+jRIxzr1q2bx3qN1MOcoM/k+lnPLJZ71nSmnLqtfaKfBTp37hzarV271uNTTjklHNu8ebPHhw4d8jiXY9ftAqqVL68nOjfqvGhmdtNNN3ncvXt3j7WfzOI5v+WWWzx+4IEHQruynGNW2gAAAAAAABQQX9oAAAAAAAAUUM2kR+kyqiFDhnh83XXXhXZa8WL27NkeL1iwILRjeWnr03Ne7fyTQlE7dMnnN7/5zXBMl4CyO36xaBqMLhluaGgI7SpVAsp9WC19ELVDU3JyepT2q1a2yZUwdGk+18LR0XtfXsqv6cb6PMP9snh0rsxVwHR8aD/qmDKLKae56iXjqvbpM5FeB1oRyszs5Zdf9ljT0c3idZarFdWTPB702fPFF19s0s/Qc5nTv3UM5/QcHc/V5uJq/dPUz0Zlo1tejBgxIhzTanp6/nOa00MPPeTxb37zG4+1uliZsNIGAAAAAACggPjSBgAAAAAAoID40gYAAAAAAKCAamZPmxNOOMFjLUvbpUuX0E7zAXNuKIDmpXtg5L0tUHvKUhYRx0Zz8rWUqZnZvffe67Hm/+c8/nrKyW8Oer7Wr1/v8fXXXx/aMTZrh+57ofs2mMUStY899pjH+/fvD+22bt3qse5/gnLQcb9p0yaPv/Wtb4V2e/bs8bhfv37hmD5z1UOZ76Z6v3vE5HuaPufqvS+/1t/FfP3edJ7M51z3XdT9aXbt2hXa/elPf/L44MGDzf0WC4eVNgAAAAAAAAXElzYAAAAAAAAFVDPpUVryS5egLVy4MLRbunSpxw8//LDHWgIOAABUlpeVU1q6dbG8vnbp2MmlZzXtadu2bR7n9Bb6v37o55Onn346HNP0Ok2jyv8PLadeS3K3NH2maGhoCMe+//3ve9y+fXuPFy1aFNqtXLnSY01jKytW2gAAAAAAABQQX9oAAAAAAAAUEF/aAAAAAAAAFFC7o8nPa9euXZsl8+meNieddJLHBw4cCO00/1Pz5Wo9H//w4cPt3rvVe2vLPoQ9d/jw4XOb4wfRj22HsVgKjMUSYCyWAmOxBBiLpcBYLAHGYik0OhZZaQMAAAAAAFBAfGkDAAAAAABQQEdb8nuPmW1uiTfyXt566y2P9+3bV7FdradBVdCrGX9Wm/Uh6McSoA/LgX6sffRhOdCPtY8+LAf6sfbRh+XQaD8e1Z42AAAAAAAAaB2kRwEAAAAAABQQX9oAAAAAAAAUEF/aAAAAAAAAFBBf2gAAAAAAABQQX9oAAAAAAAAUEF/aAAAAAAAAFBBf2gAAAAAAABQQX9oAAAAAAAAUEF/aAAAAAAAAFND/AXnqmSVBA93aAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1440x288 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 3.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "n = 10\n",
    "plt.figure(figsize=(20, 4))\n",
    "for i in range(n):\n",
    "    ax = plt.subplot(1, n, i+1)\n",
    "    plt.imshow(x_test[i].reshape(28, 28))\n",
    "    plt.gray()\n",
    "    ax.get_xaxis().set_visible(False)\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "\n",
    "plt.show()\n",
    "\n",
    "n = 10\n",
    "plt.figure(figsize=(20, 8))\n",
    "for i in range(n):\n",
    "    # What does encoded images looks like?\n",
    "    ax = plt.subplot(1, n, i+1)\n",
    "    plt.imshow(encoded_imgs[i].reshape(4*1,2)) # encoded 64 # (8,8) # (4, 4*4)\n",
    "    #plt.imshow(decoded_imgs[i].reshape(4, 14*14).T) # input or decoded 784 # (28,28) # (4, 14*14)\n",
    "    plt.gray()\n",
    "    ax.get_xaxis().set_visible(False)\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "\n",
    "plt.show()\n",
    "\n",
    "n = 10\n",
    "plt.figure(figsize=(20, 4))\n",
    "for i in range(n):\n",
    "    ax = plt.subplot(1, n, i+1)\n",
    "    plt.imshow(decoded_imgs[i].reshape(28,28))\n",
    "    plt.gray()\n",
    "    ax.get_xaxis().set_visible(False)\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE = 0.0066, PSNR (dB) = 21.83\n",
      "MSE = 0.0375, PSNR (dB) = 14.26\n",
      "MSE = 0.0018, PSNR (dB) = 27.34\n",
      "MSE = 0.0256, PSNR (dB) = 15.91\n",
      "MSE = 0.0159, PSNR (dB) = 17.98\n",
      "MSE = 0.0018, PSNR (dB) = 27.50\n",
      "MSE = 0.0297, PSNR (dB) = 15.27\n",
      "MSE = 0.0384, PSNR (dB) = 14.15\n",
      "MSE = 0.0567, PSNR (dB) = 12.46\n",
      "MSE = 0.0181, PSNR (dB) = 17.43\n",
      "Wall time: 2 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# https://dsp.stackexchange.com/questions/38065/peak-signal-to-noise-ratio-psnr-in-python-for-an-image\n",
    "import math\n",
    "import numpy as np\n",
    "\n",
    "def mse(img1, img2):\n",
    "    return np.mean( (img1 - img2) ** 2 )\n",
    "\n",
    "def psnr(img1, img2):\n",
    "    mserr = mse(img1, img2)\n",
    "    PIXEL_MAX = 1.0\n",
    "    try:\n",
    "        return 20 * math.log10(PIXEL_MAX / math.sqrt(mserr))\n",
    "    except ZeroDivisionError:\n",
    "        return 'Same image'\n",
    "\n",
    "n = 10\n",
    "for i in range(n):\n",
    "    original = x_test[i]\n",
    "    contrast = decoded_imgs[i]\n",
    "    a = mse(original, contrast)\n",
    "    b = psnr(original, contrast) # can also use cv2.PSNR rather than defined function psnr\n",
    "    if b != 'Same image':\n",
    "        print(f'MSE = {a:.4f}, PSNR (dB) = {b:.2f}')\n",
    "    else:\n",
    "        print(f'MSE = {a:.4f}, PSNR too high')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9055\n",
      "0.5552\n",
      "0.9734\n",
      "0.7618\n",
      "0.7242\n",
      "0.9788\n",
      "0.5394\n",
      "0.5446\n",
      "0.3731\n",
      "0.7573\n",
      "Wall time: 235 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# https://github.com/imamun93/Image-Similarities-using-SSIM/blob/master/blog5_ssim.ipynb\n",
    "import numpy as np\n",
    "from tensorflow.image import ssim\n",
    "from tensorflow import convert_to_tensor\n",
    "\n",
    "n = 10\n",
    "for i in range(n):\n",
    "    original = x_test[i].reshape(1, 28, 28, 1)\n",
    "    contrast = decoded_imgs[i].reshape(1, 28, 28, 1)\n",
    "    a = np.asarray(ssim(convert_to_tensor(original), convert_to_tensor(contrast), max_val=1)) # <class 'tensorflow.python.framework.ops.EagerTensor'>\n",
    "    print(f'{a[0]:.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9815\n",
      "0.7511\n",
      "0.9957\n",
      "0.8855\n",
      "0.8858\n",
      "0.9944\n",
      "0.8264\n",
      "0.8070\n",
      "0.6949\n",
      "0.9315\n",
      "Wall time: 850 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# https://github.com/tensorflow/models/blob/master/research/compression/image_encoder/msssim.py\n",
    "import numpy as np\n",
    "from scipy import signal\n",
    "from scipy.ndimage.filters import convolve\n",
    "\n",
    "def _FSpecialGauss(size, sigma):\n",
    "  \"\"\"Function to mimic the 'fspecial' gaussian MATLAB function.\"\"\"\n",
    "  radius = size // 2\n",
    "  offset = 0.0\n",
    "  start, stop = -radius, radius + 1\n",
    "  if size % 2 == 0:\n",
    "    offset = 0.5\n",
    "    stop -= 1\n",
    "  x, y = np.mgrid[offset + start:stop, offset + start:stop]\n",
    "  assert len(x) == size\n",
    "  g = np.exp(-((x**2 + y**2)/(2.0 * sigma**2)))\n",
    "  return g / g.sum()\n",
    "\n",
    "\n",
    "def _SSIMForMultiScale(img1, img2, max_val=255, filter_size=11,\n",
    "                       filter_sigma=1.5, k1=0.01, k2=0.03):\n",
    "  \"\"\"Return the Structural Similarity Map between `img1` and `img2`.\n",
    "  This function attempts to match the functionality of ssim_index_new.m by\n",
    "  Zhou Wang: http://www.cns.nyu.edu/~lcv/ssim/msssim.zip\n",
    "  Arguments:\n",
    "    img1: Numpy array holding the first RGB image batch.\n",
    "    img2: Numpy array holding the second RGB image batch.\n",
    "    max_val: the dynamic range of the images (i.e., the difference between the\n",
    "      maximum the and minimum allowed values).\n",
    "    filter_size: Size of blur kernel to use (will be reduced for small images).\n",
    "    filter_sigma: Standard deviation for Gaussian blur kernel (will be reduced\n",
    "      for small images).\n",
    "    k1: Constant used to maintain stability in the SSIM calculation (0.01 in\n",
    "      the original paper).\n",
    "    k2: Constant used to maintain stability in the SSIM calculation (0.03 in\n",
    "      the original paper).\n",
    "  Returns:\n",
    "    Pair containing the mean SSIM and contrast sensitivity between `img1` and\n",
    "    `img2`.\n",
    "  Raises:\n",
    "    RuntimeError: If input images don't have the same shape or don't have four\n",
    "      dimensions: [batch_size, height, width, depth].\n",
    "  \"\"\"\n",
    "  if img1.shape != img2.shape:\n",
    "    raise RuntimeError('Input images must have the same shape (%s vs. %s).',\n",
    "                       img1.shape, img2.shape)\n",
    "  if img1.ndim != 4:\n",
    "    raise RuntimeError('Input images must have four dimensions, not %d',\n",
    "                       img1.ndim)\n",
    "\n",
    "  img1 = img1.astype(np.float64)\n",
    "  img2 = img2.astype(np.float64)\n",
    "  _, height, width, _ = img1.shape\n",
    "\n",
    "  # Filter size can't be larger than height or width of images.\n",
    "  size = min(filter_size, height, width)\n",
    "\n",
    "  # Scale down sigma if a smaller filter size is used.\n",
    "  sigma = size * filter_sigma / filter_size if filter_size else 0\n",
    "\n",
    "  if filter_size:\n",
    "    window = np.reshape(_FSpecialGauss(size, sigma), (1, size, size, 1))\n",
    "    mu1 = signal.fftconvolve(img1, window, mode='valid')\n",
    "    mu2 = signal.fftconvolve(img2, window, mode='valid')\n",
    "    sigma11 = signal.fftconvolve(img1 * img1, window, mode='valid')\n",
    "    sigma22 = signal.fftconvolve(img2 * img2, window, mode='valid')\n",
    "    sigma12 = signal.fftconvolve(img1 * img2, window, mode='valid')\n",
    "  else:\n",
    "    # Empty blur kernel so no need to convolve.\n",
    "    mu1, mu2 = img1, img2\n",
    "    sigma11 = img1 * img1\n",
    "    sigma22 = img2 * img2\n",
    "    sigma12 = img1 * img2\n",
    "\n",
    "  mu11 = mu1 * mu1\n",
    "  mu22 = mu2 * mu2\n",
    "  mu12 = mu1 * mu2\n",
    "  sigma11 -= mu11\n",
    "  sigma22 -= mu22\n",
    "  sigma12 -= mu12\n",
    "\n",
    "  # Calculate intermediate values used by both ssim and cs_map.\n",
    "  c1 = (k1 * max_val) ** 2\n",
    "  c2 = (k2 * max_val) ** 2\n",
    "  v1 = 2.0 * sigma12 + c2\n",
    "  v2 = sigma11 + sigma22 + c2\n",
    "  ssim = np.mean((((2.0 * mu12 + c1) * v1) / ((mu11 + mu22 + c1) * v2)))\n",
    "  cs = np.mean(v1 / v2)\n",
    "  return ssim, cs\n",
    "\n",
    "\n",
    "def MultiScaleSSIM(img1, img2, max_val=255, filter_size=11, filter_sigma=1.5,\n",
    "                   k1=0.01, k2=0.03, weights=None):\n",
    "  \"\"\"Return the MS-SSIM score between `img1` and `img2`.\n",
    "  This function implements Multi-Scale Structural Similarity (MS-SSIM) Image\n",
    "  Quality Assessment according to Zhou Wang's paper, \"Multi-scale structural\n",
    "  similarity for image quality assessment\" (2003).\n",
    "  Link: https://ece.uwaterloo.ca/~z70wang/publications/msssim.pdf\n",
    "  Author's MATLAB implementation:\n",
    "  http://www.cns.nyu.edu/~lcv/ssim/msssim.zip\n",
    "  Arguments:\n",
    "    img1: Numpy array holding the first RGB image batch.\n",
    "    img2: Numpy array holding the second RGB image batch.\n",
    "    max_val: the dynamic range of the images (i.e., the difference between the\n",
    "      maximum the and minimum allowed values).\n",
    "    filter_size: Size of blur kernel to use (will be reduced for small images).\n",
    "    filter_sigma: Standard deviation for Gaussian blur kernel (will be reduced\n",
    "      for small images).\n",
    "    k1: Constant used to maintain stability in the SSIM calculation (0.01 in\n",
    "      the original paper).\n",
    "    k2: Constant used to maintain stability in the SSIM calculation (0.03 in\n",
    "      the original paper).\n",
    "    weights: List of weights for each level; if none, use five levels and the\n",
    "      weights from the original paper.\n",
    "  Returns:\n",
    "    MS-SSIM score between `img1` and `img2`.\n",
    "  Raises:\n",
    "    RuntimeError: If input images don't have the same shape or don't have four\n",
    "      dimensions: [batch_size, height, width, depth].\n",
    "  \"\"\"\n",
    "  if img1.shape != img2.shape:\n",
    "    raise RuntimeError('Input images must have the same shape (%s vs. %s).',\n",
    "                       img1.shape, img2.shape)\n",
    "  if img1.ndim != 4:\n",
    "    raise RuntimeError('Input images must have four dimensions, not %d',\n",
    "                       img1.ndim)\n",
    "\n",
    "  # Note: default weights don't sum to 1.0 but do match the paper / matlab code.\n",
    "  weights = np.array(weights if weights else\n",
    "                     [0.0448, 0.2856, 0.3001, 0.2363, 0.1333])\n",
    "  levels = weights.size\n",
    "  downsample_filter = np.ones((1, 2, 2, 1)) / 4.0\n",
    "  im1, im2 = [x.astype(np.float64) for x in [img1, img2]]\n",
    "  mssim = np.array([])\n",
    "  mcs = np.array([])\n",
    "  for _ in range(levels):\n",
    "    ssim, cs = _SSIMForMultiScale(\n",
    "        im1, im2, max_val=max_val, filter_size=filter_size,\n",
    "        filter_sigma=filter_sigma, k1=k1, k2=k2)\n",
    "    mssim = np.append(mssim, ssim)\n",
    "    mcs = np.append(mcs, cs)\n",
    "    filtered = [convolve(im, downsample_filter, mode='reflect')\n",
    "                for im in [im1, im2]]\n",
    "    im1, im2 = [x[:, ::2, ::2, :] for x in filtered]\n",
    "  return (np.prod(mcs[0:levels-1] ** weights[0:levels-1]) *\n",
    "          (mssim[levels-1] ** weights[levels-1]))\n",
    "\n",
    "n = 10\n",
    "for i in range(n):\n",
    "    original = x_test[i].reshape(1, 28, 28, 1)\n",
    "    contrast = decoded_imgs[i].reshape(1, 28, 28, 1)\n",
    "    a = MultiScaleSSIM(original, contrast, max_val=1.0)\n",
    "    print(f'{a:.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "autoencoder.save(\"./Deep Autoencoders in Keras - 784-256-128-64-32-16-8-16-32-64-128-256-784-tf1.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}