Deep Autoencoders in Keras - 784-256-128-64-128-256-784-tf2-v2-make.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using plaidml.keras.backend backend.\n"
     ]
    }
   ],
   "source": [
    "# https://blog.keras.io/building-autoencoders-in-keras.html\n",
    "\n",
    "import os\n",
    "os.environ['KERAS_BACKEND'] = 'plaidml.keras.backend' # https://github.com/plaidml/plaidml/blob/master/docs/index.md\n",
    "\n",
    "from keras.models import Model, save_model, load_model\n",
    "from keras.layers import Input, Dense\n",
    "from keras.datasets import mnist\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "\n",
    "from tensorflow.keras.callbacks import TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_VERSION = 'v2'\n",
    "INPUT_SIZE = (784,)\n",
    "BOTTLENECK_DIM = 64\n",
    "BOTTLENECK_SIZE = (BOTTLENECK_DIM,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:plaidml:Opening device \"opencl_intel_hd_graphics_4000.0\"\n"
     ]
    }
   ],
   "source": [
    "# Autoencoder Architecture\n",
    "input_img = Input(shape=INPUT_SIZE)\n",
    "x = Dense(256, activation='relu')(input_img)\n",
    "x = Dense(128, activation='relu')(x)\n",
    "encoded = Dense(BOTTLENECK_DIM, activation='relu')(x)\n",
    "\n",
    "x = Dense(128, activation='relu')(encoded)\n",
    "x = Dense(256, activation='relu')(x)\n",
    "decoded = Dense(784, activation='sigmoid')(x)\n",
    "\n",
    "# Autoencoder Model\n",
    "autoencoder = Model(input_img, decoded, name='autoencoder')\n",
    "autoencoder.compile(optimizer='adam', loss='binary_crossentropy') # use binary_crossentropy, mean_squared_error doesnt work\n",
    "autoencoder.run_eagerly = True # plaidml / keras not running eager mode by default\n",
    "\n",
    "# Encoder Architecture\n",
    "Elayer_input = Input(shape=INPUT_SIZE) # layer start from 0\n",
    "x = autoencoder.layers[1](Elayer_input)\n",
    "x = autoencoder.layers[2](x)\n",
    "Elayer_output = autoencoder.layers[3](x)\n",
    "\n",
    "# Encoder Model\n",
    "encoder = Model(Elayer_input, Elayer_output, name='encoder')\n",
    "\n",
    "# Decoder Architecture\n",
    "Dlayer_input = Input(shape=BOTTLENECK_SIZE) # use model.summary() to check input_shape\n",
    "x = autoencoder.layers[-3](Dlayer_input)\n",
    "x = autoencoder.layers[-2](x)\n",
    "Dlayer_output = autoencoder.layers[-1](x) # extract last layer, start from -1\n",
    "\n",
    "# Decoder Model\n",
    "decoder = Model(Dlayer_input, Dlayer_output, name='decoder')\n",
    "\n",
    "# Remove unused variable to reduce RAM usage\n",
    "del x, input_img, encoded, decoded, Elayer_input, Elayer_output, Dlayer_input, Dlayer_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               200960    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               8320      \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 256)               33024     \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 784)               201488    \n",
      "=================================================================\n",
      "Total params: 484,944\n",
      "Trainable params: 484,944\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               200960    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 64)                8256      \n",
      "=================================================================\n",
      "Total params: 242,112\n",
      "Trainable params: 242,112\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               8320      \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 256)               33024     \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 784)               201488    \n",
      "=================================================================\n",
      "Total params: 242,832\n",
      "Trainable params: 242,832\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# use model.summary() or data.shape() to find shape\n",
    "autoencoder.summary()\n",
    "encoder.summary()\n",
    "decoder.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape = (60000, 784)\n",
      "x_test shape = (10000, 784)\n",
      "Wall time: 496 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import numpy as np\n",
    "\n",
    "# Load mnist dataset\n",
    "(x_train, _),(x_test, _) = mnist.load_data()\n",
    "\n",
    "# Preprocess\n",
    "x_train = x_train.astype('float32')/255 # normalise to 0 and 1\n",
    "x_test  = x_test.astype('float32')/255\n",
    "x_train = x_train.reshape(len(x_train), (np.prod(x_train.shape[1:]))) # 28x28 flatten into 1x784\n",
    "x_test  = x_test.reshape(len(x_test), (np.prod(x_test.shape[1:])))\n",
    "\n",
    "print(f'x_train shape = {x_train.shape}') # 60000 images 784 vectors = 60000 row 784 col\n",
    "print(f'x_test shape = {x_test.shape}') # 10000 images 784 vectors = 10000 row 784 col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1595007482\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "60000/60000 [==============================] - 74s 1ms/step - loss: 0.1292 - val_loss: 0.0971\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.09707, saving model to autoencoderv2/checkpoint-1595007482/model.001-0.0971.h5\n",
      "Epoch 2/100\n",
      "60000/60000 [==============================] - 68s 1ms/step - loss: 0.0924 - val_loss: 0.0873\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.09707 to 0.08729, saving model to autoencoderv2/checkpoint-1595007482/model.002-0.0873.h5\n",
      "Epoch 3/100\n",
      "60000/60000 [==============================] - 67s 1ms/step - loss: 0.0860 - val_loss: 0.0839\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.08729 to 0.08386, saving model to autoencoderv2/checkpoint-1595007482/model.003-0.0839.h5\n",
      "Epoch 4/100\n",
      "60000/60000 [==============================] - 69s 1ms/step - loss: 0.0826 - val_loss: 0.0814\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.08386 to 0.08145, saving model to autoencoderv2/checkpoint-1595007482/model.004-0.0814.h5\n",
      "Epoch 5/100\n",
      "60000/60000 [==============================] - 69s 1ms/step - loss: 0.0805 - val_loss: 0.0796\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.08145 to 0.07963, saving model to autoencoderv2/checkpoint-1595007482/model.005-0.0796.h5\n",
      "Epoch 6/100\n",
      "60000/60000 [==============================] - 72s 1ms/step - loss: 0.0790 - val_loss: 0.0780\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.07963 to 0.07798, saving model to autoencoderv2/checkpoint-1595007482/model.006-0.0780.h5\n",
      "Epoch 7/100\n",
      "60000/60000 [==============================] - 70s 1ms/step - loss: 0.0779 - val_loss: 0.0771\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.07798 to 0.07706, saving model to autoencoderv2/checkpoint-1595007482/model.007-0.0771.h5\n",
      "Epoch 8/100\n",
      "60000/60000 [==============================] - 71s 1ms/step - loss: 0.0770 - val_loss: 0.0762\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.07706 to 0.07617, saving model to autoencoderv2/checkpoint-1595007482/model.008-0.0762.h5\n",
      "Epoch 9/100\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.0762 - val_loss: 0.0754\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.07617 to 0.07537, saving model to autoencoderv2/checkpoint-1595007482/model.009-0.0754.h5\n",
      "Epoch 10/100\n",
      "60000/60000 [==============================] - 71s 1ms/step - loss: 0.0756 - val_loss: 0.0748\n",
      "\n",
      "Epoch 00010: val_loss improved from 0.07537 to 0.07479, saving model to autoencoderv2/checkpoint-1595007482/model.010-0.0748.h5\n",
      "Epoch 11/100\n",
      "60000/60000 [==============================] - 71s 1ms/step - loss: 0.0750 - val_loss: 0.0747\n",
      "\n",
      "Epoch 00011: val_loss improved from 0.07479 to 0.07472, saving model to autoencoderv2/checkpoint-1595007482/model.011-0.0747.h5\n",
      "Epoch 12/100\n",
      "60000/60000 [==============================] - 76s 1ms/step - loss: 0.0746 - val_loss: 0.0740\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.07472 to 0.07396, saving model to autoencoderv2/checkpoint-1595007482/model.012-0.0740.h5\n",
      "Epoch 13/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0742 - val_loss: 0.0737\n",
      "\n",
      "Epoch 00013: val_loss improved from 0.07396 to 0.07366, saving model to autoencoderv2/checkpoint-1595007482/model.013-0.0737.h5\n",
      "Epoch 14/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0738 - val_loss: 0.0737\n",
      "\n",
      "Epoch 00014: val_loss improved from 0.07366 to 0.07365, saving model to autoencoderv2/checkpoint-1595007482/model.014-0.0737.h5\n",
      "Epoch 15/100\n",
      "60000/60000 [==============================] - 70s 1ms/step - loss: 0.0736 - val_loss: 0.0733\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.07365 to 0.07330, saving model to autoencoderv2/checkpoint-1595007482/model.015-0.0733.h5\n",
      "Epoch 16/100\n",
      "60000/60000 [==============================] - 70s 1ms/step - loss: 0.0733 - val_loss: 0.0730\n",
      "\n",
      "Epoch 00016: val_loss improved from 0.07330 to 0.07304, saving model to autoencoderv2/checkpoint-1595007482/model.016-0.0730.h5\n",
      "Epoch 17/100\n",
      "60000/60000 [==============================] - 72s 1ms/step - loss: 0.0730 - val_loss: 0.0727\n",
      "\n",
      "Epoch 00017: val_loss improved from 0.07304 to 0.07275, saving model to autoencoderv2/checkpoint-1595007482/model.017-0.0727.h5\n",
      "Epoch 18/100\n",
      "60000/60000 [==============================] - 74s 1ms/step - loss: 0.0728 - val_loss: 0.0723\n",
      "\n",
      "Epoch 00018: val_loss improved from 0.07275 to 0.07234, saving model to autoencoderv2/checkpoint-1595007482/model.018-0.0723.h5\n",
      "Epoch 19/100\n",
      "60000/60000 [==============================] - 72s 1ms/step - loss: 0.0726 - val_loss: 0.0725\n",
      "\n",
      "Epoch 00019: val_loss did not improve from 0.07234\n",
      "Epoch 20/100\n",
      "60000/60000 [==============================] - 74s 1ms/step - loss: 0.0724 - val_loss: 0.0722\n",
      "\n",
      "Epoch 00020: val_loss improved from 0.07234 to 0.07217, saving model to autoencoderv2/checkpoint-1595007482/model.020-0.0722.h5\n",
      "Epoch 21/100\n",
      "60000/60000 [==============================] - 68s 1ms/step - loss: 0.0722 - val_loss: 0.0722\n",
      "\n",
      "Epoch 00021: val_loss did not improve from 0.07217\n",
      "Epoch 22/100\n",
      "60000/60000 [==============================] - 68s 1ms/step - loss: 0.0720 - val_loss: 0.0720\n",
      "\n",
      "Epoch 00022: val_loss improved from 0.07217 to 0.07199, saving model to autoencoderv2/checkpoint-1595007482/model.022-0.0720.h5\n",
      "Epoch 23/100\n",
      "60000/60000 [==============================] - 71s 1ms/step - loss: 0.0719 - val_loss: 0.0718\n",
      "\n",
      "Epoch 00023: val_loss improved from 0.07199 to 0.07181, saving model to autoencoderv2/checkpoint-1595007482/model.023-0.0718.h5\n",
      "Epoch 24/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0717 - val_loss: 0.0714\n",
      "\n",
      "Epoch 00024: val_loss improved from 0.07181 to 0.07138, saving model to autoencoderv2/checkpoint-1595007482/model.024-0.0714.h5\n",
      "Epoch 25/100\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.0715 - val_loss: 0.0714\n",
      "\n",
      "Epoch 00025: val_loss did not improve from 0.07138\n",
      "Epoch 26/100\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.0714 - val_loss: 0.0712\n",
      "\n",
      "Epoch 00026: val_loss improved from 0.07138 to 0.07117, saving model to autoencoderv2/checkpoint-1595007482/model.026-0.0712.h5\n",
      "Epoch 27/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0712 - val_loss: 0.0719\n",
      "\n",
      "Epoch 00027: val_loss did not improve from 0.07117\n",
      "Epoch 28/100\n",
      "60000/60000 [==============================] - 76s 1ms/step - loss: 0.0711 - val_loss: 0.0715\n",
      "\n",
      "Epoch 00028: val_loss did not improve from 0.07117\n",
      "Epoch 29/100\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.0710 - val_loss: 0.0708\n",
      "\n",
      "Epoch 00029: val_loss improved from 0.07117 to 0.07075, saving model to autoencoderv2/checkpoint-1595007482/model.029-0.0708.h5\n",
      "Epoch 30/100\n",
      "60000/60000 [==============================] - 76s 1ms/step - loss: 0.0710 - val_loss: 0.0710\n",
      "\n",
      "Epoch 00030: val_loss did not improve from 0.07075\n",
      "Epoch 31/100\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.0709 - val_loss: 0.0711\n",
      "\n",
      "Epoch 00031: val_loss did not improve from 0.07075\n",
      "Epoch 32/100\n",
      "60000/60000 [==============================] - 74s 1ms/step - loss: 0.0708 - val_loss: 0.0706\n",
      "\n",
      "Epoch 00032: val_loss improved from 0.07075 to 0.07065, saving model to autoencoderv2/checkpoint-1595007482/model.032-0.0706.h5\n",
      "Epoch 33/100\n",
      "60000/60000 [==============================] - 72s 1ms/step - loss: 0.0708 - val_loss: 0.0709\n",
      "\n",
      "Epoch 00033: val_loss did not improve from 0.07065\n",
      "Epoch 34/100\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.0706 - val_loss: 0.0704\n",
      "\n",
      "Epoch 00034: val_loss improved from 0.07065 to 0.07039, saving model to autoencoderv2/checkpoint-1595007482/model.034-0.0704.h5\n",
      "Epoch 35/100\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.0706 - val_loss: 0.0708\n",
      "\n",
      "Epoch 00035: val_loss did not improve from 0.07039\n",
      "Epoch 36/100\n",
      "60000/60000 [==============================] - 76s 1ms/step - loss: 0.0706 - val_loss: 0.0707\n",
      "\n",
      "Epoch 00036: val_loss did not improve from 0.07039\n",
      "Epoch 37/100\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.0705 - val_loss: 0.0706\n",
      "\n",
      "Epoch 00037: val_loss did not improve from 0.07039\n",
      "Epoch 38/100\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.0704 - val_loss: 0.0707\n",
      "\n",
      "Epoch 00038: val_loss did not improve from 0.07039\n",
      "Epoch 39/100\n",
      "60000/60000 [==============================] - 72s 1ms/step - loss: 0.0704 - val_loss: 0.0700\n",
      "\n",
      "Epoch 00039: val_loss improved from 0.07039 to 0.07002, saving model to autoencoderv2/checkpoint-1595007482/model.039-0.0700.h5\n",
      "Epoch 40/100\n",
      "60000/60000 [==============================] - 69s 1ms/step - loss: 0.0703 - val_loss: 0.0701\n",
      "\n",
      "Epoch 00040: val_loss did not improve from 0.07002\n",
      "Epoch 41/100\n",
      "60000/60000 [==============================] - 72s 1ms/step - loss: 0.0703 - val_loss: 0.0703\n",
      "\n",
      "Epoch 00041: val_loss did not improve from 0.07002\n",
      "Epoch 42/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0702 - val_loss: 0.0709\n",
      "\n",
      "Epoch 00042: val_loss did not improve from 0.07002\n",
      "Epoch 43/100\n",
      "60000/60000 [==============================] - 72s 1ms/step - loss: 0.0702 - val_loss: 0.0702\n",
      "\n",
      "Epoch 00043: val_loss did not improve from 0.07002\n",
      "Epoch 44/100\n",
      "60000/60000 [==============================] - 77s 1ms/step - loss: 0.0701 - val_loss: 0.0703\n",
      "\n",
      "Epoch 00044: val_loss did not improve from 0.07002\n",
      "Epoch 45/100\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.0701 - val_loss: 0.0702\n",
      "\n",
      "Epoch 00045: val_loss did not improve from 0.07002\n",
      "Epoch 46/100\n",
      "60000/60000 [==============================] - 76s 1ms/step - loss: 0.0700 - val_loss: 0.0704\n",
      "\n",
      "Epoch 00046: val_loss did not improve from 0.07002\n",
      "Epoch 47/100\n",
      "60000/60000 [==============================] - 76s 1ms/step - loss: 0.0700 - val_loss: 0.0701\n",
      "\n",
      "Epoch 00047: val_loss did not improve from 0.07002\n",
      "Epoch 48/100\n",
      "60000/60000 [==============================] - 76s 1ms/step - loss: 0.0701 - val_loss: 0.0700\n",
      "\n",
      "Epoch 00048: val_loss improved from 0.07002 to 0.07000, saving model to autoencoderv2/checkpoint-1595007482/model.048-0.0700.h5\n",
      "Epoch 49/100\n",
      "60000/60000 [==============================] - 77s 1ms/step - loss: 0.0700 - val_loss: 0.0702\n",
      "\n",
      "Epoch 00049: val_loss did not improve from 0.07000\n",
      "Epoch 50/100\n",
      "60000/60000 [==============================] - 78s 1ms/step - loss: 0.0699 - val_loss: 0.0701\n",
      "\n",
      "Epoch 00050: val_loss did not improve from 0.07000\n",
      "Epoch 51/100\n",
      "60000/60000 [==============================] - 77s 1ms/step - loss: 0.0699 - val_loss: 0.0703\n",
      "\n",
      "Epoch 00051: val_loss did not improve from 0.07000\n",
      "Epoch 52/100\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.0699 - val_loss: 0.0699\n",
      "\n",
      "Epoch 00052: val_loss improved from 0.07000 to 0.06986, saving model to autoencoderv2/checkpoint-1595007482/model.052-0.0699.h5\n",
      "Epoch 53/100\n",
      "60000/60000 [==============================] - 81s 1ms/step - loss: 0.0699 - val_loss: 0.0697\n",
      "\n",
      "Epoch 00053: val_loss improved from 0.06986 to 0.06974, saving model to autoencoderv2/checkpoint-1595007482/model.053-0.0697.h5\n",
      "Epoch 54/100\n",
      "60000/60000 [==============================] - 74s 1ms/step - loss: 0.0698 - val_loss: 0.0697\n",
      "\n",
      "Epoch 00054: val_loss improved from 0.06974 to 0.06972, saving model to autoencoderv2/checkpoint-1595007482/model.054-0.0697.h5\n",
      "Epoch 55/100\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.0698 - val_loss: 0.0697\n",
      "\n",
      "Epoch 00055: val_loss did not improve from 0.06972\n",
      "Epoch 56/100\n",
      "60000/60000 [==============================] - 74s 1ms/step - loss: 0.0698 - val_loss: 0.0700\n",
      "\n",
      "Epoch 00056: val_loss did not improve from 0.06972\n",
      "Epoch 57/100\n",
      "60000/60000 [==============================] - 75s 1ms/step - loss: 0.0698 - val_loss: 0.0696\n",
      "\n",
      "Epoch 00057: val_loss improved from 0.06972 to 0.06965, saving model to autoencoderv2/checkpoint-1595007482/model.057-0.0696.h5\n",
      "Epoch 58/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0698 - val_loss: 0.0704\n",
      "\n",
      "Epoch 00058: val_loss did not improve from 0.06965\n",
      "Epoch 59/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0698 - val_loss: 0.0698\n",
      "\n",
      "Epoch 00059: val_loss did not improve from 0.06965\n",
      "Epoch 60/100\n",
      "60000/60000 [==============================] - 74s 1ms/step - loss: 0.0697 - val_loss: 0.0700\n",
      "\n",
      "Epoch 00060: val_loss did not improve from 0.06965\n",
      "Epoch 61/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0697 - val_loss: 0.0699\n",
      "\n",
      "Epoch 00061: val_loss did not improve from 0.06965\n",
      "Epoch 62/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0697 - val_loss: 0.0695\n",
      "\n",
      "Epoch 00062: val_loss improved from 0.06965 to 0.06952, saving model to autoencoderv2/checkpoint-1595007482/model.062-0.0695.h5\n",
      "Epoch 63/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0697 - val_loss: 0.0700\n",
      "\n",
      "Epoch 00063: val_loss did not improve from 0.06952\n",
      "Epoch 64/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0696 - val_loss: 0.0699\n",
      "\n",
      "Epoch 00064: val_loss did not improve from 0.06952\n",
      "Epoch 65/100\n",
      "60000/60000 [==============================] - 74s 1ms/step - loss: 0.0696 - val_loss: 0.0692\n",
      "\n",
      "Epoch 00065: val_loss improved from 0.06952 to 0.06925, saving model to autoencoderv2/checkpoint-1595007482/model.065-0.0692.h5\n",
      "Epoch 66/100\n",
      "60000/60000 [==============================] - 74s 1ms/step - loss: 0.0696 - val_loss: 0.0697\n",
      "\n",
      "Epoch 00066: val_loss did not improve from 0.06925\n",
      "Epoch 67/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0696 - val_loss: 0.0695\n",
      "\n",
      "Epoch 00067: val_loss did not improve from 0.06925\n",
      "Epoch 68/100\n",
      "60000/60000 [==============================] - 72s 1ms/step - loss: 0.0696 - val_loss: 0.0699\n",
      "\n",
      "Epoch 00068: val_loss did not improve from 0.06925\n",
      "Epoch 69/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0695 - val_loss: 0.0697\n",
      "\n",
      "Epoch 00069: val_loss did not improve from 0.06925\n",
      "Epoch 70/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0695 - val_loss: 0.0697\n",
      "\n",
      "Epoch 00070: val_loss did not improve from 0.06925\n",
      "Epoch 71/100\n",
      "60000/60000 [==============================] - 72s 1ms/step - loss: 0.0695 - val_loss: 0.0699\n",
      "\n",
      "Epoch 00071: val_loss did not improve from 0.06925\n",
      "Epoch 72/100\n",
      "60000/60000 [==============================] - 73s 1ms/step - loss: 0.0695 - val_loss: 0.0694\n",
      "\n",
      "Epoch 00072: val_loss did not improve from 0.06925\n",
      "Epoch 73/100\n",
      "60000/60000 [==============================] - 72s 1ms/step - loss: 0.0695 - val_loss: 0.0704\n",
      "\n",
      "Epoch 00073: val_loss did not improve from 0.06925\n",
      "Epoch 74/100\n",
      "60000/60000 [==============================] - 72s 1ms/step - loss: 0.0695 - val_loss: 0.0699\n",
      "\n",
      "Epoch 00074: val_loss did not improve from 0.06925\n",
      "Epoch 75/100\n",
      "60000/60000 [==============================] - 72s 1ms/step - loss: 0.0695 - val_loss: 0.0695\n",
      "\n",
      "Epoch 00075: val_loss did not improve from 0.06925\n",
      "Epoch 00075: early stopping\n",
      "Wall time: 1h 33min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import os\n",
    "import time\n",
    "\n",
    "CURRENT_TIME = int(time.time())\n",
    "print(CURRENT_TIME)\n",
    "\n",
    "try:\n",
    "    os.makedirs(f'autoencoder{MODEL_VERSION}/logs-{CURRENT_TIME}')\n",
    "except Exception:\n",
    "    pass\n",
    "\n",
    "try:\n",
    "    os.makedirs(f'autoencoder{MODEL_VERSION}/checkpoint-{CURRENT_TIME}')\n",
    "except Exception:\n",
    "    pass\n",
    "\n",
    "# Run this in a terminal\n",
    "#tensorboard --logdir=autoencoder/logs\n",
    "\n",
    "CALLBACKS=[\n",
    "    TensorBoard(log_dir=f'autoencoder{MODEL_VERSION}/logs-{CURRENT_TIME}'),\n",
    "    ModelCheckpoint(filepath=f'autoencoder{MODEL_VERSION}/checkpoint-{CURRENT_TIME}'+'/'\n",
    "                             +'model.{epoch:003d}-{val_loss:.4f}.h5',\n",
    "                    verbose=1, save_best_only=True, mode='min'),\n",
    "    EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)\n",
    "]\n",
    "\n",
    "# Fit data\n",
    "training = autoencoder.fit(x_train, x_train,\n",
    "                           validation_data=(x_test, x_test),\n",
    "                           epochs=100, batch_size=32, shuffle=True,\n",
    "                           callbacks=CALLBACKS) # 50 epochs, 256 bs; 100 epochs; 32 bs\n",
    "\n",
    "print(CURRENT_TIME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 2.15 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import os\n",
    "\n",
    "try:\n",
    "    os.makedirs(f'autoencoder{MODEL_VERSION}/model')\n",
    "except Exception:\n",
    "    pass\n",
    "\n",
    "save_model(autoencoder, f'autoencoder{MODEL_VERSION}/model/autoencoder{MODEL_VERSION}-last_train.h5')\n",
    "save_model(encoder, f'autoencoder{MODEL_VERSION}/model/encoder{MODEL_VERSION}-last_train.h5')\n",
    "save_model(decoder, f'autoencoder{MODEL_VERSION}/model/decoder{MODEL_VERSION}-last_train.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5Sc9X3v8fd3ys5IWyXtqq1WBUkIFSOKkOnN4AiMITbxDfiQGDs25gaCSUJsnNzr2Dm5qdjH5poLxgZ3QwImtiCKAdNCRw2BCqqorNquykraPuV7/3hmpWG9kkbSSjN69vM6Z87O02a+o/KZ3/6e3/P8zN0REZHwihS7ABEROb4U9CIiIaegFxEJOQW9iEjIKehFREIuVuwC+lJbW+vjx48vdhkiIieNhQsX7nD3ur62lWTQjx8/ngULFhS7DBGRk4aZbTjYNnXdiIiEnIJeRCTkFPQiIiFXUNCb2RwzW2lma8zs7j62n2Zmr5tZl5ndlbc+aWZvmdkSM1tmZt/oz+JFROTwDnsy1syiwH3AlUAjMN/M5rr78rzddgF3AL/f6/Au4HJ3bzWzOPCKmf2Xu7/RP+WLiMjhFNKinw2scfd17t4NPApcl7+Duze5+3wg1Wu9u3trbjGee+guaiIiJ1AhQV8PbMpbbsytK4iZRc3sbaAJeNbd3zzIfreY2QIzW9Dc3Fzoy4uIyGEUEvTWx7qCW+XunnH3M4AxwGwzm3GQ/R5091nuPquurs8x/4d173OreWmVviRERPIVEvSNQEPe8hhgy5G+kbu3AC8Cc4702EJ976W1vKygFxH5gEKCfj4w2cwmmFkZcAMwt5AXN7M6M6vJPR8EXAG8d7TFHk4iHqUrnT1eLy8iclI67Kgbd0+b2e3A00AUeNjdl5nZrbntD5jZSGABUAVkzexOYBowCvhxbuROBPh3d3/qOH0WkrEInanM8Xp5EZGTUkH3unH3ecC8XuseyHu+jaBLp7d3gDOPpcAjkYxH6VSLXkTkA0J1ZWxZLEKXWvQiIh8QqqBXi15E5HeFLOjVRy8i0luogj4Ri6rrRkSkl1AFfTIe0fBKEZFeQhb0UXXdiIj0EqqgT8QidKbUohcRyReqoE/Go3Sl1aIXEckXuqBXi15E5INCFfSJWITOdAZ33fJeRKRHqII+GY/iDqmMgl5EpEeogj4RCz5Op/rpRUT2C1fQx6MAGmIpIpInVEGfzLXou3RCVkRkv3AFfa5FryGWIiIHhCro9/fRq0UvIrJfqIJeLXoRkd8VyqBXi15E5IBQBf2Brhu16EVEeoQq6A903ahFLyLSI2RBrxa9iEhvoQr6REx99CIivYUq6Hta9Bp1IyJyQMiCXi16EZHeCgp6M5tjZivNbI2Z3d3H9tPM7HUz6zKzu/LWN5jZC2a2wsyWmdmX+rP43sqi6qMXEektdrgdzCwK3AdcCTQC881srrsvz9ttF3AH8Pu9Dk8Df+nui8ysElhoZs/2OrbfRCJGWUwThIuI5CukRT8bWOPu69y9G3gUuC5/B3dvcvf5QKrX+q3uvij3fB+wAqjvl8oPIhmLqEUvIpKnkKCvBzblLTdyFGFtZuOBM4E3D7L9FjNbYGYLmpubj/Tl90to3lgRkQ8oJOitj3VHNIWTmVUAvwTudPe9fe3j7g+6+yx3n1VXV3ckL/8ByXhEtykWEclTSNA3Ag15y2OALYW+gZnFCUL+5+7+xJGVd+SSsahmmBIRyVNI0M8HJpvZBDMrA24A5hby4mZmwEPACnf/1tGXWbhEPKLhlSIieQ476sbd02Z2O/A0EAUedvdlZnZrbvsDZjYSWABUAVkzuxOYBpwO/BHwrpm9nXvJv3b3ecfhswBBi1599CIiBxw26AFywTyv17oH8p5vI+jS6e0V+u7jP26S8SgdGnUjIrJfqK6MheBWxRpeKSJyQOiCPhmP6oIpEZE8oQv64GSsWvQiIj3CF/SxqEbdiIjkCV3QJ+MRjboREckTwqCP6spYEZE8oQv6RCxCdyZLJntEd2kQEQmt0AV9z+Qj3Rp5IyIChDHoY5p8REQkX+iCPtEznaBOyIqIACEM+v0ThOuErIgIEMagj6lFLyKSL3RBn4j39NGrRS8iAiEM+p4WfZdOxoqIACEM+gMnY9WiFxGBMAa9hleKiHxA6IK+54IpBb2ISCCEQZ8bXqmuGxERIJRBr5OxIiL5Qhf0B/ro1aIXEYEQBv3+Fr0umBIRAUIY9PFohGjE1KIXEckJXdBD0H2jUTciIoGCgt7M5pjZSjNbY2Z397H9NDN73cy6zOyuXtseNrMmM1vaX0UfTjIe1agbEZGcwwa9mUWB+4CrgGnAjWY2rdduu4A7gHv6eIkfAXOOrcwjk1SLXkRkv0Ja9LOBNe6+zt27gUeB6/J3cPcmd58PpHof7O7/TfBFcMIk4lHdAkFEJKeQoK8HNuUtN+bWlaxELKJx9CIiOYUEvfWxrt9n3jazW8xsgZktaG5uPqbXSqpFLyKyXyFB3wg05C2PAbb0dyHu/qC7z3L3WXV1dcf0Whp1IyJyQCFBPx+YbGYTzKwMuAGYe3zLOjYadSMicsBhg97d08DtwNPACuDf3X2Zmd1qZrcCmNlIM2sE/gL4X2bWaGZVuW2PAK8DU3Lr/+R4fZgeybj66EVEesQK2cnd5wHzeq17IO/5NoIunb6OvfFYCjwaiVhUXTciIjmhvDI2GY+o60ZEJCekQa8WvYhIj1AGfTDqRi16EREIadAHo24yuPf7cH8RkZNOaIM+65DKKOhFREIZ9PtnmdLkIyIiIQ36/fPGqp9eRCSUQZ/cP2+sWvQiIqEM+oTmjRUR2S+UQX+gRa+uGxGRcAa9WvQiIvuFMugTatGLiOwXyqBXi15E5IBQB71a9CIiIQ36hIZXiojsF8qgP9B1oxa9iEhIg14tehGRHqEM+kRMffQiIj1CGvTBx9KoGxGRkAZ9JGKUafIREREgpEEPPbNMqUUvIhLaoA9mmVKLXkQkxEEfoUstehGR8AZ9IhbVDFMiIhQY9GY2x8xWmtkaM7u7j+2nmdnrZtZlZncdybHHS9CiV9eNiMhhg97MosB9wFXANOBGM5vWa7ddwB3APUdx7HGRVIteRAQorEU/G1jj7uvcvRt4FLgufwd3b3L3+UDqSI89XhJxDa8UEYHCgr4e2JS33JhbV4hjOfaYJGNRDa8UEaGwoLc+1nmBr1/wsWZ2i5ktMLMFzc3NBb78wWl4pYhIoJCgbwQa8pbHAFsKfP2Cj3X3B919lrvPqqurK/DlD04XTImIBAoJ+vnAZDObYGZlwA3A3AJf/1iOPSaJeFR99CIiQOxwO7h72sxuB54GosDD7r7MzG7NbX/AzEYCC4AqIGtmdwLT3H1vX8cerw+TLxmP6KZmIiIUEPQA7j4PmNdr3QN5z7cRdMsUdOyJkIhFNY5eRIQQXxmbjEfozmTJZAs9bywiEk4hDvpg8pFujbwRkQEutEGvCcJFRALhCfpsBl7+Fqx9HjjQotdtEERkoAtP0Eei8Oq34b3gvG/PBOE6ISsiA114gh6gZiy0bADyJghXi15EBriQBf042B0EfU+LXhdNichAF66gHzIeWjaCO8lci16zTInIQBeuoK8ZC+kOaGsm0dOi1/BKERngQhb044Kfuzcc6KNXi15EBrhwBf2QXNC3bNg/vFK3KhaRgS5cQV+duyNyywZdMCUikhOuoE9UwOBa2J3XolfQi8gAF66gh6D7pmXDgQum1HUjIgNc+IK+Zhy0bNTJWBGRnBAG/Vho2UTcskRMF0yJiIQv6IeMg2wKa92WmyBcLXoRGdjCF/Q9Y+lbNuYmCFeLXkQGtvAGfW7kjfroRWSgC2HQNwAGLRsYVBalrTtd7IpERIoqfEEfS0DlKGjZyCm1Faza3lrsikREiip8QQ/ByJvdG5hRX8W65lba1aoXkQEsnEGfu2hq+uhqsg4rtu4rdkUiIkUTzqCvGQd7NzNj5CAAlm3ZU+SCRESKp6CgN7M5ZrbSzNaY2d19bDczuze3/R0zOytv25fMbKmZLTOzO/uz+IMaMg48y0h2Mqy8jKWbFfQiMnAdNujNLArcB1wFTANuNLNpvXa7Cpice9wC3J87dgbwBWA2MBO4xswm91v1B1MzNqi9ZQPTRlexbMve4/6WIiKlqpAW/Wxgjbuvc/du4FHgul77XAf8xANvADVmNgqYCrzh7u3ungZeAj7Rj/X3LW8s/Yz6alZt36crZEVkwCok6OuBTXnLjbl1heyzFLjYzIaZ2WDgaqDh6MstUFU9WBRaNjJjdDWpjLNawyxFZIAqJOitj3VeyD7uvgL4Z+BZ4DfAEqDPsY5mdouZLTCzBc3NzQWUdQjRGFTXQ0swxBJQP72IDFiFBH0jH2yFjwG2FLqPuz/k7me5+8XALmB1X2/i7g+6+yx3n1VXV1do/QdXMw52b6BhyGAqEzH104vIgFVI0M8HJpvZBDMrA24A5vbaZy7wx7nRN+cCe9x9K4CZDc/9HAt8Enik36o/lCHBfekjEWPa6CqWaoiliAxQscPt4O5pM7sdeBqIAg+7+zIzuzW3/QFgHkH/+xqgHfhs3kv80syGASngNnff3c+foW8146B1G6Q6mFFfzc/f3EA6kyUWDeelAyIiB3PYoAdw93kEYZ6/7oG85w7cdpBjLzqWAo/a/tsVb2L66Co6U1nW7Wjj1BGVRSlHRKRYwtu8HXLgvvQz6qsBXSErIgNTeIN+f4t+PafUlpOMR1i6WSdkRWTgCW/QV4yAaAJ2byAWjTB1VJWGWIrIgBTeoI9EYOgE2PYuANNHV7F8y16y2d6XAIiIhFt4gx7gtI/B+y/Bvu3MGF3Nvq40m3a3F7sqEZETKtxBf/oN4FlY+vj+E7LqpxeRgSbcQV93Kow+E5Y8yuQRFcSjpgunRGTACXfQQ9Cq3/YOiV2rmDy8kiWbWopdkYjICRX+oJ9xfXAnyyWPcumUOt58fxdNezuLXZWIyAkT/qCvqINJV8C7j3H9WaPJZJ1fvb252FWJiJww4Q96gJl/CHs3M7F1MWeNreHxhY0Ed20QEQm/gRH0U66GRBUseZTrzx7Dqu2tvKuLp0RkgBgYQR8fBNOuhRVzuWZqDWWxCL9c2FjsqkREToiBEfQQjL7pbqV6/TP83vSR/HrJFs0jKyIDwsAJ+nEXQHUDLP4pf3D2GFraUzy/oqnYVYmIHHcDJ+gjEZj9BXj/JS4sW82IqgSPq/tGRAaAgRP0AOd8ASpGEH3h7/nkmfW8uKqZpn0aUy8i4Tawgr5sMFx0F2x4lZvq1pHJOr9e3HuecxGRcBlYQQ9w9meguoH6Rf/KGWOqeWT+RtKZbLGrEhE5bgZe0McScMlXYMti/vfk9axrbuORtzYWuyoRkeNm4AU9wMwbYehEzlp7HxecMoR7nlnF7rbuYlclInJcDMygj8bgsr/Gmpbzr1PX0tqV5pvPrix2VSIix8XADHqA6Z+E4dMZvegePnfOcH7x5kaWb9GkJCISPgM36CMRuPpfYPcG7or8nOpBcb7+5DLd7ExEQqegoDezOWa20szWmNndfWw3M7s3t/0dMzsrb9ufm9kyM1tqZo+YWbI/P8AxGX8hnHcbicUP862zdvLW+7t46p2txa5KRKRfHTbozSwK3AdcBUwDbjSzab12uwqYnHvcAtyfO7YeuAOY5e4zgChwQ79V3x8u/99QdxqXrvwGHx4Z4e//czk7WruKXZWISL8ppEU/G1jj7uvcvRt4FLiu1z7XAT/xwBtAjZmNym2LAYPMLAYMBkrrCqV4Ej7xANbWzPeGPcqejhRf/OlC3fBMREKjkKCvBzblLTfm1h12H3ffDNwDbAS2Anvc/Zm+3sTMbjGzBWa2oLm5udD6+8foM+GSr1Cz9lf84rytLNywm68+8a7660UkFAoJeutjXe8E7HMfMxtC0NqfAIwGys3spr7exN0fdPdZ7j6rrq6ugLL62YV/AaPP4qx3/46vXVTFE4s288BL6058HSIi/ayQoG8EGvKWx/C73S8H2+cK4H13b3b3FPAEcP7Rl3scRWNw/Q8gk+KzW/+O604fzr88/R5PL9tW7MpERI5JIUE/H5hsZhPMrIzgZOrcXvvMBf44N/rmXIIumq0EXTbnmtlgMzPgI8CKfqy/fw2bCNfeizW+xTeH/orTx9Rw56Nv8/LqE9yVJCLSjw4b9O6eBm4HniYI6X9392VmdquZ3ZrbbR6wDlgDfB/409yxbwKPA4uAd3Pv92B/f4h+NeN6mPUnxN74Lj+5YCfjhg3mcz+az9wlpXUOWUSkUFaKJxxnzZrlCxYsKF4BqU546Epo2cjem1/g87/ezvz1u/jba6Zx8wUTileXiMhBmNlCd5/V17aBe2XsocST8KkfQTZD1ZOf5yefnsKVU0fw9SeXc8/TKzUaR0ROKgr6gxk2ET5xP2xdQvIHl/D/Lslw4+wGvvvCGj7/4wXs0t0uReQkoaA/lKkfh889DQaxH13FP9Q9x9evOY2XV+/gqu/8N6+v3VnsCkVEDktBfzhjZsEXX4apH8ee+zo3v38XT352EuWJGJ/+wRt865mVpDRDlYiUMAV9IQbVBH32H/8ObHiNKU/MYd6cdq4/awz3Pr+Gj//fV5i/flexqxQR6ZOCvlBmcPbN8MX/hspRJB+7kXsG/5Tvf3o6+zrTfOqB17nrsSXs1A3RRKTEKOiPVN0U+MJzcO5tMP/7XPnyDfz2xir+56UT+dXizVz+zZf43kvBrFUiIqVAQX80YgmY8w9w0y+hYzeDfnQlX7Gf8ps/PZuZDTX843+9x4X//Dz3PreaPR2pYlcrIgOcLpg6Vh0t8Nuvw8IfQvVYuOZbvJ08h+8+v4bfrthOZSLGTeeN47MXjGd4ZenMuSIi4XKoC6YU9P1lw+vw5B2wYxWcchlceCfLE2dy30trmffuVuLRCJ86ewy3XHwK44aVF7taEQkZBf2Jku6CN+6H1++DtqbgPvcXfIkN1efwvTd38PiiraSzWS6bMpyrPzSKK6aNoHpQvNhVi0gIKOhPtFQnLHkEXrsXdh24p302Xk6rlfNyejp/2/4/2BOt4YJJtXzsQ6OYM2MklUmFvogcHQV9sWQzsPrZIOw790DXXmhtwpf/mmxsEM+N+gL/p+k8NuzuJhGLcMW0EXzijHouPrWOspjOk4tI4RT0pWbHaph3F6x7ER81k7Uz7uTfto/hl8v2sKutm4pEjNkThnL+xGGcN3EYU0dWEYn0NYmXiEhAQV+K3GHZE/D038C+rRCJkR11Bpsqz+CttpGsamqjaV8XjpEtqyTTcC7TJ4zh7HFDmdlQzeCyWLE/gYiUEAV9Ketuh42vwYbcY/NCyPzunTHTRJmfmcKL2Zm8yunERk7nzPG1zBo3lFnjhzCiSkM3RQYyBf3JJNUBe3vNZrV3C6z5LZnVzxJtWgZApyVYlh3PkswElmRPYWPF6YxsmMzMsUOYOaaGaaOqqB6sk7siA4WCPkz2boX1L8PmRWQ3L4KtS4hkOgFosmG8kT6V+dkpPJ85k0xVA1NGVjJlZCWT6iqYUFfOhNpyhpWXEUzhKyJhoaAPs0wamlfAxjdg4+tk179GpHUrABuTU/itncej+06nI2OMsR3U2w4mxHdD1Wg66y+kfsIUpo+uZtLwCgaVRYv8YUTkaCnoBxL3YDjniidhxdygz/8QNmbreD07nW0MpbYsxYhEmmFlKcoGV2KjTqdqwtmMnHw2sWTFCfoAInI0FPQDWcsmWP1McCO26gaoaYCqeti1juy6l+hY9SLxTa9RltpDV2QQ7STZl01S6fsYYq0AZNzYEhnF9rIG9gweR3vleDK1Uykb/2HGDqugYejgI7vCt7sN3v4FDBoCEy+HwUOP04cXGTgU9HJo2dwMWZEDF2m1dqbY+P4q9qxdQHbrEspbVjG0axMj0ltIEIwK2uJDeSpzHk9mzmNj4lQmDq9g0vAKJg9LMGloGSPqahlZnWTI4HhwTiCbhXcfC24Ct6/nhLMFt4qY9BE449Mw9JQT+9lFQkJBL/0nm4W9m+lY9yrptx+jfNNLRDzFvtgwMtkMyWw7ydwXwaZsHYt8Mu8wmY5Bo7nZ/4NTU++xdfBpLJz6ZarKBzG+5Q1qt7/CoKbFEIljl/wVnP8liJUV+YOKnFwU9HL8tO8KzgdseA3igyBRSWeknF0daWzbO1TtWEx5VxMAuyNDuD92Ez9uP4/e87IMZzdfi/+Ea6Jv8n5kLN+vvoNt1WcwKtbKKb6JsZkN1GWaqPRWyrP7GJTZi5WV0z1xDjb1Y1QMG0UilncyuW1HcCHa8Okf+E2lqLJZaN8BFcOLXYmE0DEHvZnNAb4DRIEfuPs/9dpuue1XA+3Aze6+yMymAP+Wt+spwNfc/duHej8FfcjsaYSm92DshyFRibvT0p5iR2sXO9u62dXWzc7c87otL3DVxm8yNL2d3VbDEG/Z/zJdHmc3FezxclqoYDi7mRDZTsaNt7JTeZdJnBrdzFTWM4KdAGyPjWb+0I+ztv46kjUjKY9ladizkHFNv2VE02t4JEZmUC3ZwbVQXodV1hGvHEFZ9QiilcOhax/seh92rw8etZPhor88svMK6W5Y+ji8+h1ofg+mXA1X/l3wWiL95JiC3syiwCrgSqARmA/c6O7L8/a5GvgzgqD/MPAdd/9wH6+zGfiwu2841Hsq6Ae4rlZ49dvBhWLDp8HwqfjwqXQk6tjTmWZPR4o97Sn2dqSwpmXUNT5Nw9ZnqWlfz47kWBoTk1kfn0RLNsnsvc8yI72Ubo8yP3saMyLvU23ttHqSl7MfIkWMYexlmO2l1vYwlH1E7Hf/T7QxiOboCMZmNtIRKec3dZ9j8YhPMDiZpCIRCx7JGOVlMcpiEcropqpzM7XbXmHk8oeIt24hXTsNn3gZscU/hnQnqTM/S9eFf0V5VS2R9mbYuxn2bYdkNVTXQ+Wo4CR6qhN2rITty6BpRbB97HlQf1bwW9SxaN8FrU1Qe+qJ+81n21LYvjT4wktWnZj3LAVrnw/+bU/+KMT7/0r2Yw3684Cvu/vv5Za/CuDu/5i3z/eAF939kdzySuBSd9+at89Hgb919wsOV7CCXo5KJg3RPu4B1LwKX/hDfPWzdI84k7aJH2Pv6Avp8DI60xk6uzN0pDK0d2fo6OoitW8ntDURaWtmTzbBZkayPV1OWypDbfsabt77IGeml7CWMTyWvYx4poMqa6eaNuqshQm2jQZrIpr7wngzexr3p6/lxexMwBjGHv489jg3Rp8nRYwoWeKW6fMj7YtWMzjTSpRge8biRD2Vex5jZ/V0Wson0Rarpi1aTWu0hog5dentDE1vo7JzK4n0PrKJquALIllDNBoj1rKW2K41wRcM4BUjyZ72cXzadUTHnYf19edYiJaNwTUdkRhMuuKDQb5vGzz/97D4Z4BDWQXMvBFm3wJ1pwZDg1u3B19mezYFX3Q146Bm7KGDMZuFTW9CNg3VY4JRZaV0jifdBb/5Kix4KFhOVMO0a+H0P4RxF/TbF+yxBv0fAHPc/fO55T8iaJXfnrfPU8A/ufsrueXngK+4+4K8fR4GFrn7dw/yPrcAtwCMHTv27A0bDtnoFyked1g5L7gh3e73g1XxwWTKqkkPGkZn1QTaKyfQVjGelqpTaR48ibauNO3dwZeJGUQMhratY+rmx2jzJM2RWrb5MLZkqoh276Wqu4nqVDM1mR3syFayNNPAO931rMuOoJJ2zo6s4pzIKmZFVjLWmhjCvg98WWTdaKKGRq9jj5dTae1U0U61tREjzXofyZpsPWt9NHso5/LIYi6LvE3SUuzwKtb7SHZSww6GsMtqSEWSQXhHY0SicWKWpSySJUGaMkszLrORKV1LGZZp2l9D2uKsq5zFezWXUJHexYXbfkrEU7xRez3Lqy7inN1P8aHdzxHzFE2DJ1HV3UQyvbfPP/KO5Ah21c5ix4gL2DHiAlKDR5Bs38qY9U9Qv/4JBrVvPvDXg5EpH0Fq7EW0nnMHmWGnAkGeJqJREvEIZek2Ih07gyHH+V9qnXuD4cgrnoQti6B8ePDbVdUYqBoFiSpIVARfUrFEMHx555rgsXsDjJ4JMz8d/MYViQTrHvsMbFkM598BEy+Ddx4LrnHpbg2+zKZcDVOvgfEXQfTob1tyrEH/KeD3egX9bHf/s7x9/hP4x15B/2V3X5hbLgO2ANPdffvhClaLXk4K2UwwZ3Cy6pj+gxbK3elKZ+nOZPEsZNzJZJ1oxIhHIJ5uo6xrFxl39pWNYG8qwp6OFK1dabrSGbpSWTpzPzPupDNOOutkslkyWYim22jY8Qpjd71CRdd2yrt3Ut69k0GZvsM3367IEJZGp/N2ZBoL/TTKMu1ckn2Ty/wN6gl+a3iO2dxrN9EYGY0ZZLJOVbaFT2af5RyWsz47nFU+hlU+hkavo44WGqyZBmtiYmQL50eWM9yCczYbssNpsGYi5rycmcFjmUvZQRX1uau/x1oTcyLzSdLNvOxsvpv+BJu9lo9EFnF19E0uibxDwlKkPMpGH856H4nhXBhZSpml2UENSyLTqaaV4exgeHYHSbr6/OxpYmyLjWZXtJbJ3SsY5B00x0bxTvn5nLvvGYwsvxh1N8uqLiZiRixiDLIuZux7lWktLzBp7xuUZTtpj1SwuuZCZt72s6P691T0rhszuw64zd0/WkjBCnqREpLuhnRn0DWSTUMmBZEoRMtyrfw4xJLQ1/2T3GHbu+BZGH3GId8mk3U6Uxk6U0FXmnuwLuvBw7NOfMcKBm96kUFb36Kjdga7T/0U3RUNZHNfgm3daTq6M7R2pYl17GTK+p9x6oZfEE+3kbEYUU/TWjac1bWXsz05kZquzQzt3EhN5yZimS7W1JzP0upLWZuYSlfGSGeypDJOKp0hlm6lLNNGIttOWaadWKaTndE6mqIjyFoUHKKZds7peJVLO5/jjPQS1kYm8I3kl2m0kWTcyWYhnc3u/5LNulMRSXG+vctl/iajo3s4629eOKq/pmMN+hjBydiPEJxMnQ982t2X5e3zMeB2DpyMvdfdZ+dtfxR42t1/WEjBCnoR6Tcdu2H+Q8Esb1OvhfqzT1ntdfQAAASYSURBVMyJ5849QRdP5MTcQ+pQQX/YMy7unjaz24GnCYZXPuzuy8zs1tz2B4B5BCG/hmB45Wfz3nwwwYidLx7rBxEROWKDhsDFd534901Wn/j3PIiCTq27+zyCMM9f90DecwduO8ix7cCwY6hRRESOQYlcMigiIseLgl5EJOQU9CIiIaegFxEJOQW9iEjIKehFREJOQS8iEnIlOfGImTUDR3tXs1pgRz+Wczyoxv6hGvvHyVAjnBx1FrPGce5e19eGkgz6Y2FmCw52GXCpUI39QzX2j5OhRjg56izVGtV1IyIScgp6EZGQC2PQP1jsAgqgGvuHauwfJ0ONcHLUWZI1hq6PXkREPiiMLXoREcmjoBcRCbnQBL2ZzTGzlWa2xszuLnY9PczsYTNrMrOleeuGmtmzZrY693NIEetrMLMXzGyFmS0zsy+VWo25epJm9paZLcnV+Y0SrTNqZovN7KlSrC9X03oze9fM3jazBaVYp5nVmNnjZvZe7t/meaVUo5lNyf359Tz2mtmdpVRjvlAEvZlFgfuAq4BpwI1mNq24Ve33I2BOr3V3A8+5+2TgudxysaSBv3T3qcC5wG25P7tSqhGgC7jc3WcCZwBzzOxcSq/OLwEr8pZLrb4el7n7GXljvkutzu8Av3H304CZBH+mJVOju6/M/fmdAZxNMLPef5RSjR/g7if9AziPYE7anuWvAl8tdl159YwHluYtrwRG5Z6PAlYWu8a82n5NMPVjKdc4GFhEMD9xydQJjCH4z3058FSp/l0D64HaXutKpk6gCnif3GCRUqyxV10fBV4t5RpD0aIH6oFNecuNuXWlaoS7bwXI/Rxe5HoAMLPxwJnAm5RgjblukbeBJuBZdy+1Or8NfBnI5q0rpfp6OPCMmS00s1ty60qpzlOAZuCHuW6wH5hZeYnVmO8G4JHc85KsMSxBb32s07jRI2BmFcAvgTvdfW+x6+mLu2c8+FV5DDDbzGYUu6YeZnYN0OTuC4tdSwEucPezCLo6bzOzi4tdUC8x4Czgfnc/E2ijVLpAejGzMuBa4LFi13IoYQn6RqAhb3kMsKVItRRiu5mNAsj9bCpmMWYWJwj5n7v7E7nVJVVjPndvAV4kOPdRKnVeAFxrZuuBR4HLzexnJVTffu6+JfeziaBfeTalVWcj0Jj7jQ3gcYLgL6Uae1wFLHL37bnlUqwxNEE/H5hsZhNy37A3AHOLXNOhzAU+k3v+GYJ+8aIwMwMeAla4+7fyNpVMjQBmVmdmNbnng4ArgPcokTrd/avuPsbdxxP8+3ve3W8qlfp6mFm5mVX2PCfoX15KCdXp7tuATWY2JbfqI8BySqjGPDdyoNsGSrPGcJyMzZ34uBpYBawF/qbY9eTV9QiwFUgRtFT+BBhGcNJude7n0CLWdyFBN9c7wNu5x9WlVGOuztOBxbk6lwJfy60vqTpzNV3KgZOxJVUfQf/3ktxjWc//lRKs8wxgQe7v+1fAkBKscTCwE6jOW1dSNfY8dAsEEZGQC0vXjYiIHISCXkQk5BT0IiIhp6AXEQk5Bb2ISMgp6EVEQk5BLyIScv8fHzXtFr215ygAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# need to put training = autoencoder.fit(...) first\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(training.history['loss'], label='train')\n",
    "plt.plot(training.history['val_loss'], label='test')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 4.22 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "encoded_imgs = encoder.predict(x_test)\n",
    "decoded_imgs = decoder.predict(encoded_imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABG0AAABwCAYAAACkaY2RAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAbSklEQVR4nO3daZRUxRnG8RoBFURHGRmQsENQZJVdAwrKEWUTFJQwMQYEMWLEhcUoUQTUc0BRRATJCQqIiLIoCBKVAIpIPBCWsB8wgCO74OgoyMB0Pnh8favobnpm+nbfuf3/fXqKqumupOd291zrrUoLhUIGAAAAAAAA/nJOsicAAAAAAACAM3HTBgAAAAAAwIe4aQMAAAAAAOBD3LQBAAAAAADwIW7aAAAAAAAA+BA3bQAAAAAAAHyoZEEGp6WlcT54koRCobR4PA6vYVIdCYVC5ePxQLyOycO1GAhciwHAtRgIXIsBwLUYCFyLAcC1GAhhr0VW2gCJsyfZEwBgjOFaBPyCaxHwB65FwB/CXovctAEAAAAAAPAhbtoAAAAAAAD4EDdtAAAAAAAAfIibNgAAAAAAAD7ETRsAAAAAAAAf4qYNAAAAAACAD3HTBgAAAAAAwIdKJnsCSE2DBw+WXLp0aauvYcOGknv06BHxMSZNmiT5888/t/pmzJhR1CkCAAAAAJBUrLQBAAAAAADwIW7aAAAAAAAA+BA3bQAAAAAAAHyIPW2QMLNnz5Ycba8aLT8/P2LfgAEDJLdv397qW7FiheS9e/fGOkUkWZ06daz2tm3bJA8aNEjyhAkTEjanVHbBBRdIHjt2rGR97RljzNq1ayX37NnT6tuzZ49HswMAAEiOSy65RHLVqlVj+hn3O9FDDz0kedOmTZJ37NhhjduwYUNhpogAYaUNAAAAAACAD3HTBgAAAAAAwIcoj4JndDmUMbGXROmSmH/+85+Sa9asaY3r0qWL5Fq1all9WVlZkp999tmYnhfJd9VVV1ltXR6XnZ2d6OmkvMsuu0xy//79Jbtli02bNpXcuXNnq2/ixIkezQ5akyZNJM+bN8/qq169umfPe+ONN1rtrVu3Sv7qq688e16cnf6MNMaYBQsWSL7//vslT5482Rp3+vRpbycWQJmZmZLffvttyatWrbLGTZkyRfLu3bs9n9cv0tPTrfa1114recmSJZLz8vISNiegOOjUqZPkrl27Wn1t27aVXLt27Zgezy17qlatmuTzzjsv4s+VKFEipsdHcLHSBgAAAAAAwIe4aQMAAAAAAOBDlEchrpo1aya5e/fuEcdt3rxZsrvc8MiRI5Jzc3Mln3vuuda41atXS27UqJHVl5GREeOM4SeNGze22j/88IPk+fPnJ3o6Kad8+fJWe9q0aUmaCQqqQ4cOkqMtsY43twSnb9++knv16pWweeBn+rPvlVdeiTju5Zdfljx16lSr7/jx4/GfWMDoU2OMsb/T6FKkgwcPWuOSVRKlT/gzxn6v1+WtO3fu9H5ixcxFF11ktXXJff369SW7p5hSauZveluFgQMHStal4MYYU7p0aclpaWlFfl73lFQgVqy0AQAAAAAA8CFu2gAAAAAAAPgQN20AAAAAAAB8KKl72rhHQOs6wn379ll9J06ckDxz5kzJBw4csMZRj5tc+ohgt/ZT13zr/Rf2798f02M/8sgjVvvKK6+MOHbRokUxPSaST9eE62NojTFmxowZiZ5OynnggQckd+vWzepr0aJFgR9PHyVrjDHnnPPrfxvYsGGD5E8++aTAjw1byZK/foR37NgxKXNw98p4+OGHJV9wwQVWn96jCt7Q11/lypUjjps1a5Zk/f0KkV166aWSZ8+ebfWVK1dOst5L6C9/+Yv3E4tg+PDhkmvUqGH1DRgwQDLfm8+UlZUl+emnn7b6qlSpEvZn3L1vvvnmm/hPDHGj3x8HDRrk6XNt27ZNsv5bCPGjj1zX79XG2Hus6mPajTEmPz9f8uTJkyV/9tln1jg/vE+y0gYAAAAAAMCHuGkDAAAAAADgQ0ktjxozZozVrl69ekw/p5d1fv/991ZfIpedZWdnS3b/t6xZsyZh8/CThQsXStZL1YyxX6ujR48W+LHd42NLlSpV4MeA/1xxxRWS3XIKdwk64u+FF16QrJeJFtatt94asb1nzx7Jd9xxhzXOLbPB2bVr107y1VdfLdn9PPKSe/SxLlstU6aM1Ud5VPy5x7s//vjjMf2cLj0NhUJxnVNQNWnSRLK7xF4bOXJkAmZzpnr16lltXVI+f/58q4/P1jPpcpkXX3xRckZGhjUu0vUyYcIEq63LvQvznRexcUthdKmTLnFZsmSJNe6nn36SnJOTI9n9nNLfSz/88EOrb9OmTZL//e9/S163bp017vjx4xEfH7HT2ykYY19j+rum+zsRq5YtW0o+deqU1bd9+3bJK1eutPr079zJkycL9dyxYKUNAAAAAACAD3HTBgAAAAAAwIe4aQMAAAAAAOBDSd3TRh/xbYwxDRs2lLx161arr27dupKj1RW3atVK8ldffSU50hF94eg6tsOHD0vWx1m79u7da7VTdU8bTe9fUVhDhgyRXKdOnYjjdC1puDb8a+jQoZLd3xmuI28sXrxYsj6Su7D00aa5ublWX7Vq1STrY2e/+OILa1yJEiWKPI+gc+u59bHNu3btkvzMM88kbE633HJLwp4LZ2rQoIHVbtq0acSx+rvNBx984NmcgiIzM9Nq33bbbRHH3n333ZL190av6X1sPv7444jj3D1t3P0gYczgwYMl6yPcY+Xu03bTTTdJdo8N1/vfeLkHRlBF22emUaNGkvVRz67Vq1dL1n9X7t692xpXtWpVyXovU2Pisw8gzqTvBwwcOFCye41ddNFFYX/+66+/ttqffvqp5P/9739Wn/4bRO+t2KJFC2ucfk/o2LGj1bdhwwbJ+tjweGOlDQAAAAAAgA9x0wYAAAAAAMCHkloetXTp0qhtzT2q7RfucaONGzeWrJc5NW/ePOZ5nThxQvKOHTskuyVbeqmUXpqOouncubNkfXTmueeea407dOiQ5L/+9a9W348//ujR7FBU1atXt9rNmjWTrK83YzgaMV6uu+46q3355ZdL1st7Y13q6y7/1MuT9dGZxhhz/fXXS452HPGf//xnyZMmTYppHqlm+PDhVlsvEddL8d0StXjTn33u7xbLxRMrWsmOyy0jQHTPP/+81f7DH/4gWX+/NMaYd955JyFzcrVp00ZyhQoVrL7XX39d8htvvJGoKRUbunTXGGP69OkTdtzGjRut9sGDByW3b98+4uOnp6dL1qVXxhgzc+ZMyQcOHDj7ZFOc+/3/zTfflKzLoYyxy4OjlQxqbkmU5m5/gfh79dVXrbYua4t2fLe+b/Df//5X8mOPPWaN03/Xu6655hrJ+nvo1KlTrXH6/oJ+DzDGmIkTJ0qeO3eu5HiXyrLSBgAAAAAAwIe4aQMAAAAAAOBDSS2Piodjx45Z7WXLloUdF630Khq99NgtxdJLsWbPnl2ox8eZdLmMuyRS0/+fr1ixwtM5IX7ccgotkaduBJ0uQ3vrrbesvmjLTTV9mpde8vnUU09Z46KVI+rHuOeeeySXL1/eGjdmzBjJ559/vtX38ssvS87LyzvbtAOlR48ekt0TC3bu3Ck5kSet6TI3txxq+fLlkr/99ttETSllXXvttRH73FNpopUn4kyhUMhq69/1ffv2WX1engBUunRpq62X/t93332S3fn27dvXszkFgS53MMaYCy+8ULI+bcb9zqI/n37/+99LdksyatWqJblixYpW33vvvSf55ptvlnz06NGY5p4KypYtK9ndAkFvo3DkyBGr77nnnpPMVgn+4X6v06c29evXz+pLS0uTrP8ucEvnx44dK7mw2ylkZGRI1qeYjhgxwhqnt2lxSysThZU2AAAAAAAAPsRNGwAAAAAAAB/ipg0AAAAAAIAPFfs9bbyQmZkp+ZVXXpF8zjn2PS59HDV1qIX37rvvWu0bb7wx7Ljp06dbbff4WxQPDRo0iNin9zVB0ZQs+evbe6x72Lh7Q/Xq1UuyWzceK72nzbPPPit53Lhx1rgyZcpIdn8PFixYIHnXrl2Fmkdx1bNnT8n6/yNj7M8nr+k9krKysiSfPn3aGjd69GjJqbb/UKLoI0p1drk1/uvXr/dsTqmmU6dOVlsfp673cnL3YIiV3kelbdu2Vl+rVq3C/sycOXMK9Vyp6rzzzrPaek+gF154IeLP6eODX3vtNcn6vdoYY2rWrBnxMfReK17uh1ScdevWTfKjjz5q9eljuPWx98YYk5OT4+3EUCju+9iQIUMk6z1sjDHm66+/lqz3lv3iiy8K9dx6r5oqVapYffpvy8WLF0t297HV3PnOmDFDspd7+bHSBgAAAAAAwIe4aQMAAAAAAOBDlEeFMXDgQMn6WFr3ePHt27cnbE5Bc9lll0l2l3frJau6JEMvuzfGmNzcXI9mh3jTy7n79Olj9a1bt07yRx99lLA54Wf6qGj3iNjClkRFosucdImNMcY0b948rs9VXKWnp1vtSKUQxhS+9KIw9HHtutxu69at1rhly5YlbE6pKtZrJZG/H0E0fvx4q92uXTvJlSpVsvr00et66XzXrl0L9dz6MdyjvLUvv/xSsnvkNKLTx3W7dPmbW8IfSbNmzWJ+7tWrV0vmu2x40Uo/9ffG7OzsREwHRaRLlIw5s7RaO3XqlOSWLVtK7tGjhzXuiiuuCPvzx48ft9p169YNm42xv+dWqFAh4py0gwcPWu1ElYWz0gYAAAAAAMCHuGkDAAAAAADgQ5RHGWN+97vfWW13l/Jf6J3MjTFm06ZNns0p6ObOnSs5IyMj4rg33nhDcqqdGhMk7du3l1yuXDmrb8mSJZL1qQyIH/fkO00vPfWaXvLvzinaHEeMGCH5zjvvjPu8/MQ90eQ3v/mN5FmzZiV6OqJWrVph/53PwcSLVoYRj5OL8LO1a9da7YYNG0pu3Lix1XfTTTdJ1qeiHD582Bo3bdq0mJ5bn0ayYcOGiONWrVolme9IBeO+n+pSNl2C6JZg6BMwu3fvLtk9bUZfi25f//79JevXesuWLTHNPRW4pTCavt6efPJJq++9996TzIl5/vGvf/3LautSav03gjHGVK1aVfJLL70kOVqpqC63ckuxoolUEpWfn2+158+fL/mBBx6w+vbv3x/z8xUFK20AAAAAAAB8iJs2AAAAAAAAPsRNGwAAAAAAAB9iTxtjTMeOHa12qVKlJC9dulTy559/nrA5BZGuF27SpEnEccuXL5fs1qqieGrUqJFktyZ1zpw5iZ5OSrj33nslu7W5ydKlSxfJV111ldWn5+jOV+9pE3Tff/+91dY1+XpPDWPs/aGOHj0a13lkZmZa7Uj7C6xcuTKuz4vwWrduLbl3794Rx+Xk5EjmKNz4OnbsmGT3aHvdHjZsWJGfq2bNmpL1XmDG2O8JgwcPLvJzpaqPP/7YautrR+9b4+4zE2lfDffxBg4cKPn999+3+n77299K1vtj6M/tVFe+fHnJ7ncCvffbE088YfUNHz5c8uTJkyXrY9aNsfdN2blzp+TNmzdHnFO9evWstv67kPfb6NxjuPV+UBdffLHVp/eW1fvOfvPNN9a4vXv3Sta/E/pvDmOMadGiRYHnO2XKFKv92GOPSdb7VSUSK20AAAAAAAB8iJs2AAAAAAAAPpSy5VGlS5eWrI+OM8aYkydPStblOXl5ed5PLEDco7z10jJdgubSS39zc3PjPzEkRMWKFSW3adNG8vbt261x+hg9xI8uRUokvaTZGGOuvPJKyfo9IBr3mNxUeu91lxDrY3xvu+02q2/RokWSx40bV+Dnql+/vtXWJRnVq1e3+iKVBPil9C7o9OfpOedE/u9tH330USKmA4/pkg/32tPlV+57JWLnlpTefvvtknXZdnp6esTHmDBhgmS3LO7EiROS582bZ/Xp8o8OHTpIrlWrljUulY9xf+655yQ//PDDMf+cfn+87777wuZ40def3tqhV69ecX+uIHPLjfT1URjTp0+32tHKo3RJuv49e/31161x+kjxZGGlDQAAAAAAgA9x0wYAAAAAAMCHuGkDAAAAAADgQym7p82QIUMku0fPLlmyRPKqVasSNqegeeSRR6x28+bNw4579913rTbHfAfDn/70J8n6+OAPPvggCbNBojz++ONWWx97Gs3u3bsl33XXXVafPtYx1ej3Q/fo306dOkmeNWtWgR/7yJEjVlvvnXHppZfG9Bhu3Te8EenIdXcvgFdffTUR00Gc9ezZ02r/8Y9/lKz3XDDmzGNvER/6yG59vfXu3dsap685vfeQ3sPGNWrUKKtdt25dyV27dg37eMac+VmYSvS+JrNnz7b63nzzTcklS9p/ylapUkVytP2/4kHv4ad/Z/Sx48YYM3r0aE/nAWOGDh0quSB7Ct17772SC/M9KpFYaQMAAAAAAOBD3LQBAAAAAADwoZQpj9LLyI0x5m9/+5vk7777zuobOXJkQuYUdLEe0Xf//fdbbY75DoZq1aqF/fdjx44leCbw2uLFiyVffvnlhXqMLVu2SF65cmWR5xQU27Ztk6yPpDXGmMaNG0uuXbt2gR9bH2vrmjZtmtXOysoKO849ohzxUblyZavtlmj8Ijs722qvWbPGsznBOzfffHPEvvfff99q/+c///F6OilPl0rpXFju+6Qu99HlUe3atbPGlStXTrJ7RHnQ6SOW3fe1OnXqRPy5G264QXKpUqUkjxgxwhoXacuGwtLly02bNo3rYyO8fv36SdYlaW7JnLZ582arPW/evPhPzCOstAEAAAAAAPAhbtoAAAAAAAD4UKDLozIyMiS/9NJLVl+JEiUk66X9xhizevVqbycGi17+aYwxeXl5BX6MnJyciI+hl0emp6dHfIyLL77Yasda3qWXcA4bNszq+/HHH2N6jCDq3Llz2H9fuHBhgmeSmvRS3WgnKERblj9lyhTJlSpVijhOP35+fn6sU7R06dKlUD+XytavXx82x8OXX34Z07j69etb7U2bNsV1HqnqmmuusdqRrmH39EUUT+778A8//CD5+eefT/R04LG3335bsi6PuuOOO6xxevsAtm6IzdKlS8P+uy4nNsYujzp16pTk1157zRr397//XfKDDz5o9UUqW4U3WrRoYbX1e2PZsmUj/pzedkOfFmWMMT/99FOcZuc9VtoAAAAAAAD4EDdtAAAAAAAAfIibNgAAAAAAAD4UuD1t9F41S5YskVyjRg1r3K5duyTr47+ReBs3bizyY7zzzjtWe//+/ZIrVKgg2a0XjrcDBw5Y7aefftrT5/OT1q1bW+2KFSsmaSYwxphJkyZJHjNmTMRx+jjZaPvRxLpXTazjJk+eHNM4JIfeEylc+xfsYeMNvSef68iRI5LHjx+fiOnAA3pvBf09xRhjDh06JJkjvoNHf07qz+dbbrnFGvfkk09Kfuutt6y+HTt2eDS7YPrwww+ttv5+ro+I7t+/vzWudu3aktu2bRvTc2VnZxdihjgbd+/DCy+8MOw4vSeYMfa+UZ999ln8J5YgrLQBAAAAAADwIW7aAAAAAAAA+FDgyqNq1aoluWnTphHH6eOcdakU4sc9St1d9hlPPXv2LNTP6WP+opV1LFiwQPKaNWsijvv0008LNY8g6N69u9XWpYrr1q2T/MknnyRsTqls3rx5kocMGWL1lS9f3rPnPXz4sNXeunWr5HvuuUeyLmGE/4RCoahteKtDhw4R+/bu3Ss5JycnEdOBB3R5lHt9LVq0KOLP6ZKASy65RLL+vUDxsX79eslPPPGE1Td27FjJzzzzjNV35513Sj5+/LhHswsO/V3EGPvY9dtvvz3iz7Vr1y5i3+nTpyXra/bRRx8tzBQRhn6/Gzp0aEw/M3PmTKu9fPnyeE4paVhpAwAAAAAA4EPctAEAAAAAAPAhbtoAAAAAAAD4ULHf06ZatWpW2z3S7Rfung76mFt449Zbb7XauhaxVKlSMT1GvXr1JBfkuO6pU6dK3r17d8Rxc+fOlbxt27aYHx8/K1OmjOSOHTtGHDdnzhzJugYY3tmzZ4/kXr16WX3dunWTPGjQoLg+r3vM/cSJE+P6+EiM888/P2If+yd4Q38u6v35XCdOnJCcl5fn6ZyQHPpzMisry+p76KGHJG/evFnyXXfd5f3E4Knp06db7QEDBkh2v1OPHDlS8saNG72dWAC4n1sPPvig5LJly0pu1qyZNS4zM1Oy+/fEjBkzJI8YMSIOs4Qx9uuxZcsWydH+dtTXgH5tg4SVNgAAAAAAAD7ETRsAAAAAAAAfKvblUfoIWWOMqVq1athxK1assNocX5p4Y8aMKdLP9+7dO04zQbzopfnHjh2z+vQx6ePHj0/YnHAm95h13dYlpe77aZcuXSTr13PKlCnWuLS0NMl6KSuKrz59+ljtb7/9VvKoUaMSPZ2UkJ+fL3nNmjVWX/369SXv3LkzYXNCcvTr10/y3XffbfX94x//kMy1GCyHDx+22u3bt5fsluYMGzZMsltCh7M7ePCgZP1dRx+lbowxrVq1kvzUU09ZfYcOHfJodqnt+uuvl1y5cmXJ0f5212WjuoQ4SFhpAwAAAAAA4EPctAEAAAAAAPChtIKUCaWlpfmipqh169aSFy9ebPXpHae1Fi1aWG136bHfhUKhtLOPOju/vIYpam0oFGp29mFnx+uYPFyLgcC1eBYLFy602uPGjZO8bNmyRE8nrCBfi5UqVbLao0ePlrx27VrJATidLWWvRf1dVp8EZIxdwjpp0iSrT5cinzx50qPZFUyQr0W/cE/HvfrqqyW3bNlSchFKlFP2WgySIFyLGzZskNygQYOI48aOHStZlwsGQNhrkZU2AAAAAAAAPsRNGwAAAAAAAB/ipg0AAAAAAIAPFcsjv9u0aSM50h42xhiza9cuybm5uZ7OCQCAoNBHoCLx9u3bZ7X79u2bpJnAKytXrpSsj7gFwunRo4fV1vt+1K5dW3IR9rQBfKFcuXKS09J+3aLHPWL9xRdfTNic/ICVNgAAAAAAAD7ETRsAAAAAAAAfKpblUdHo5YI33HCD5KNHjyZjOgAAAABQaN99953VrlGjRpJmAnhr3LhxYfOoUaOscfv370/YnPyAlTYAAAAAAAA+xE0bAAAAAAAAH+KmDQAAAAAAgA+lhUKh2AenpcU+GHEVCoXSzj7q7HgNk2ptKBRqFo8H4nVMHq7FQOBaDACuxUDgWgwArsVA4FoMAK7FQAh7LbLSBgAAAAAAwIe4aQMAAAAAAOBDBT3y+4gxZo8XE0FU1eL4WLyGycPrWPzxGgYDr2Pxx2sYDLyOxR+vYTDwOhZ/vIbBEPZ1LNCeNgAAAAAAAEgMyqMAAAAAAAB8iJs2AAAAAAAAPsRNGwAAAAAAAB/ipg0AAAAAAIAPcdMGAAAAAADAh7hpAwAAAAAA4EPctAEAAAAAAPAhbtoAAAAAAAD4EDdtAAAAAAAAfOj/KDYux1kFDUwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x288 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABG0AAABwCAYAAACkaY2RAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAATgklEQVR4nO3b+1OU9d/H8Q+6iHIKOaoFtwrlAWvUtNFOZo4w0njKLEfTLE8dNdMcOllRjWlWmmlFOf5ANTSMlifQxgk1NSvSMtM0T4SBCiSgiwjC3n/Anfv9vL6ze/uxeT5+bJ6+ueS6rt1r324hPp/PAAAAAAAAwC2trvQBAAAAAAAA4P9iaQMAAAAAAOAgljYAAAAAAAAOYmkDAAAAAADgIJY2AAAAAAAADmJpAwAAAAAA4CCPEkdFRfni4+Ot+9OnT0sH07FjR6mvqamxbv/++29pdrt27aQ+IiJC6hsaGqS2qakpRPoBlxEdHe1LSEiw7svLy6X5Fy9elPq4uDjrtqqqSpodGhoq9cq1bYwxFy5ckPqampoqn89n/8v3w+Px+JS/X3NzszRfvf6V+V6vV5odHh4u9eq9WFtba91eunTJNDc3B+RejI+P93Xu3Nm6P3XqVCB+7GU1NjZat5WVldJs9ZzExMRI/blz56S+rq4uYPdibGysLzk52bovLS2V5qempkr98ePHrduzZ89KsxMTE6W+devWUt+pUyfr9sSJE6aqqiog92KrVq18Ho/941CbNm2k+T6fL2i9+j4UFhYm9VFRUVKvvJ4aY0xTU1PA7sWwsDBfZGSkda++t6ivMwr196a85hijX4PKs1lZWZmprq4OyL0YHh7uU17/1XOo/h7atm1r3R44cECa3b59e6lX/66tWmn/Ll9WVhawe/Gaa67xJSUlWffqZwf1fmlpabFu1fu8R48eUq+8RxujnXev12saGhoCci9GRkb6YmNjrXv1HNbX10u9cj3X1dVJs5X73Bj9GUB1uWdUaWkTHx9vcnJyrPvFixcr483zzz8v9Rs2bLBuP/30U2l2t27dpP7mm2+W+qNHj1q3JSUl0mx/EhISzKJFi6z7V155RZp/6NAhqR8zZox1+9FHH0mzleWUMcZMmzZN6vft2yf1X375pfZpzY/Q0FCjfOA/f/68NL9Xr15Sr7xA7tq1K6jHot6LhYWF1m1FRYU025/OnTtL9/abb74ZsJ/9T5RlwocffijNVs/hqFGjpH779u1SX1RUFLB7MTk52WzatMm6V19nVq9eLfWTJ0+2bvPz86XZ48aNk3p1+fbqq69at/369ZNm++PxeKSFlPLaa4z2jzTGaEvwn3/+WZqtftgfNGiQ1BcVFUl9eXl5wO7FyMhIk5mZad2r7xXbtm2TemVBoDzPGmPMvHnzpP7SpUtSP378eOs2IyNDmu1PTEyM9BrZu3dvaX5TU5PUK58F1GMZMmSI1KvXq7rkmTVrVsDuxaSkJPP+++9b9+oiY+PGjVKvPAMXFxdLs/Py8qReeY82xpg+ffpYt+rvxZ/Y2Fgzd+5c6/7IkSPS/L1790q98o9/mzdvlmanpaVJ/XXXXSf1ytLQGGO+/vrrf7wX+d+jAAAAAAAAHMTSBgAAAAAAwEEsbQAAAAAAABzE0gYAAAAAAMBBLG0AAAAAAAAcxNIGAAAAAADAQSxtAAAAAAAAHMTSBgAAAAAAwEEsbQAAAAAAABzkUeKamhqzZs0a637fvn3SwVRWVkr9xYsXrduPP/5Ymj1t2jSp7969u9S3b9/euvV4pNPkV2Njozlx4oR1v3//fml+r169pP7kyZPW7eDBg6XZxcXFUh8dHS31p0+flvpAiomJMaNHj7buFyxYIM1Xf9edO3e2bsPDw6XZW7ZskfqamhqpnzhxonW7cuVKabY/paWlZsaMGdZ9bm6uNP+bb76R+tdff926HTVqlDT7q6++kvqEhASpV36PxhhTVFQk9f4cO3bMjBs3zrrfvn27NH/ZsmVSr7y+T5kyRZr93nvvSf2tt94q9Zs3b7Zuf//9d2m2P5GRkdKxFhQUSPMHDRok9X369LFur7nmGmn2tm3bpD45OVnqlfclY4xZvny51PsTFRUlvXdNnz5dml9XVyf1ynksLCyUZmdlZUn97Nmzpb53797WbVVVlTTbH5/PJz3b33vvvdL8e+65R+pHjBhh3a5du1aaPXLkSKnv1q2b1GdmZkp9IJWXl5ucnBzrfufOnUE8GmPS09OtW/X31q9fP6l/9tlnpX7dunXWrdfrlWb7c+bMGbNixQrr/tChQ9L8WbNmSb0y/4MPPpBmP/bYY1I/cOBAqQ/U53i+aQMAAAAAAOAgljYAAAAAAAAOYmkDAAAAAADgIJY2AAAAAAAADmJpAwAAAAAA4CCWNgAAAAAAAA5iaQMAAAAAAOAgljYAAAAAAAAOYmkDAAAAAADgIJY2AAAAAAAADmJpAwAAAAAA4CCP+gdCQkKCcRzGGGO2bt0q9atXr7Zuw8PDxaPRHDlyROpLSkqCdCT+eb1e88MPPwRtflZWltQvWrTIum3Xrp16OJIePXpI/a5du4J0JP9ZVFSUGTRokHW/YMECaX5sbKzUL1261Lr1+XzSbPU1Jzs7W+ofeeQRqQ+UkJAQ06ZNm6DNnz59utR7vV7rtqysTD0cyejRo6V+1KhRQTqSK+/8+fNSv3v3but2zJgx6uFI1OtbfQYIlLZt25r09HTrvqCgQJofFRUl9UuWLLFug/16+swzz0j98OHDpT6QamtrTWFhYdDmFxcXS/3Ro0et2/Xr16uHI2lsbJT6ioqKIB2Jfx6PxyQmJgZt/tChQ6V+xowZ1q3ymeS/UVpaKvV9+/YN0pH8Z/X19UH9rBERESH1v/32m3Ub7M8aaWlpUn/o0KEgHYl/HTt2NC+88IJ1P2nSJGl+XFyc1G/atMm6VT8HqFJSUqT+pZdeCsjP5Zs2AAAAAAAADmJpAwAAAAAA4CCWNgAAAAAAAA5iaQMAAAAAAOAgljYAAAAAAAAOYmkDAAAAAADgIJY2AAAAAAAADmJpAwAAAAAA4CCWNgAAAAAAAA5iaQMAAAAAAOAgljYAAAAAAAAOCvH5fPZxSIh9/F8YPHhw0GYfOXJE6keMGCH1y5cvl/qnn37auv3888/N6dOnQ6QfcBnBPocvvfSS1NfX11u3JSUl0uyRI0dK/TPPPCP177zzjjr/J5/P10/6Q5cR7PMYFxcn9W+99ZZ16/V6pdl33XWX1N94441SX1ZWZt1mZWWZffv2XRX34scffyz1S5cutW5nzZolzc7Ozpb66upqqZ89e7bUv/vuu1fNvfjQQw9JfXh4uHXbt29faXZmZqbUp6SkSP2ECROs26KiIlNdXX1V3Ivdu3eX+pkzZ1q3u3fvlmbffffdUj958mSp//DDD6X+0UcfvWruxYULF0p9p06drFv1+WbJkiVSHxKi3SrDhg2zbnfu3Glqa2uvinvxk08+kfoLFy5Yt1VVVdLsV155RerVc1hYWCj1WVlZV829eNNNN0n9Cy+8YN2qz6i//PKL1CvPWsYYM2PGDOt29erVprKy8qq4F+fNmyf1+fn51q16fSi7EGOM2bhxo9TPnz9f6nNycv7xXuSbNgAAAAAAAA5iaQMAAAAAAOAgljYAAAAAAAAOYmkDAAAAAADgIJY2AAAAAAAADmJpAwAAAAAA4CCWNgAAAAAAAA5iaQMAAAAAAOAgljYAAAAAAAAOYmkDAAAAAADgIE8wh48ZM0bq33jjDal/7rnnrNva2lppdkFBgdQPGzZM6k+ePGndNjU1SbMDqbCwUOp37twp9QcOHLBup06dKs3esWOH1D/22GNSn5ubK/VX0n333Sf11dXVUl9VVWXdnjlzRpr9xBNPSP3cuXOlPjs727r966+/pNmBdPDgQalfvHix1Le0tFi3K1eulGa//fbbUh8aGir1+/fvl/orafDgwVIfExMTtH7hwoXS7F9//VXqBwwYIPWtW7e2bkNCQqTZgaT+vc6fPy/1zc3N1u3Fixel2UuWLJH6nJwcqd+8ebPUX0nq61J6errUx8fHW7fqc94nn3wi9V27dpX6u+++27pVXxcC6emnn5Z65ZwYY0xeXp51O2nSJGn27NmzpV69XqdMmSL1V9Ltt98e1Pl79uyxbg8fPizNHj16tNQfP35c6pV73eMJ6sd6vyZOnCj1rVpp3xtJSUmxbi9cuCDNPnv2rNQvWrRI6rds2SL1l8M3bQAAAAAAABzE0gYAAAAAAMBBLG0AAAAAAAAcxNIGAAAAAADAQSxtAAAAAAAAHMTSBgAAAAAAwEEsbQAAAAAAABzE0gYAAAAAAMBBLG0AAAAAAAAcxNIGAAAAAADAQSxtAAAAAAAAHORR4oSEBHP//fdb97m5udLB1NbWSn1aWpp1m52dLc1OTU2V+vz8fKnv1q2bdbtjxw5ptj8ej8fExcVZ93l5edL8jIwMqZ8+fbp1e+rUKWn2ypUrpT4lJUXqb7nlFqn//fffpd6fdu3aSddQSUmJNH/KlClS/+eff1q3TU1N0uyRI0dK/bFjx6R+y5Yt1m1dXZ00258uXbqYN954w7pftWqVNL+4uFjqlXtRPYfq9efxSG9NZvTo0VK/YMECqfcnKSnJTJo0ybpXj/W1116T+qioKOu2Z8+e0uyEhASpj4iIkPqjR49atw0NDdJsfyIiIkyvXr2se/X6nzBhgtQrr0mhoaHS7KFDh0r93r17pb6iokLqA6lDhw7m4Ycftu6vu+46af4NN9wg9RMnTrRuBwwYIM1OTEyU+uuvv17q77zzTutWfdbyp23bttLzd0FBgTRffW/p0qWLdVtfXy/N7t27t9SPGzdO6lVz5swJ2Kzw8HDpNfXbb7+V5s+cOVPqO3XqZN126NBBmq1eg/Pnz5f6a6+91ro9d+6cNNufG264waxYscK6/+ijj6T50dHRUt+jRw/r9rvvvpNmP/DAA1KvfuZfu3at1CcnJ//jf+ebNgAAAAAAAA5iaQMAAAAAAOAgljYAAAAAAAAOYmkDAAAAAADgIJY2AAAAAAAADmJpAwAAAAAA4CCWNgAAAAAAAA5iaQMAAAAAAOAgljYAAAAAAAAOYmkDAAAAAADgIJY2AAAAAAAADvIocUpKinn//fet+6amJulgWlpapP7OO++0br/77jtp9t69e6V+9uzZUv/4449btzU1NdJsf8LCwkxqaqp1v2zZMmn+jBkzpP6RRx6xbj/77DNpdnFxsdRnZGRIfWJiotQHUlpamlm7dq11/9dff0nz16xZI/UlJSXW7bXXXivN7t69u9TX1dVJ/a233mrdqteUP16v13z//ffWfVRUlDS/TZs2Up+SkmLd7tmzR5rdrVs3qf/pp5+kXn1tCCSv12t+/PFH6z4iIkKaHxsbK/Xt2rWzbpOSkoI22xhjOnbsKPWRkZHW7eHDh6XZ/iQnJ5t3333Xuv/111+l+du3b5d65fofOnSoNNvjkR77TN++faV+/fr1Uh9IkZGRZuDAgda91+uV5kdHR0v9iy++aN22b99emq0+a+3atUvq1depQAkLCzNdu3a17s+ePSvNb2xslPrS0lLrNjc3V5q9YcMGqX/vvfekfuzYsVI/Z84cqfcnKirK3HHHHdZ9dna2NL9///5S/8UXX1i3w4cPl2a3bt1a6seNGyf1ynXS3NwszQ4k9ZmzvLxc6pVn1MrKSmm28gxnjP6MquxO/OGbNgAAAAAAAA5iaQMAAAAAAOAgljYAAAAAAAAOYmkDAAAAAADgIJY2AAAAAAAADmJpAwAAAAAA4CCWNgAAAAAAAA5iaQMAAAAAAOAgljYAAAAAAAAOYmkDAAAAAADgIJY2AAAAAAAADvIo8f79+01aWpp1n56eLh1MU1OT1B86dMi6XbZsmTRb+XsaY8y3334r9f3797duKyoqpNn+tGrVykRERFj3kydPluavX79e6p988knrdvz48dLs+fPnS/3ChQulPi4uTuo/++wzqffn8OHDJiMjw7qfM2eONF+9nsvKyqzb3bt3S7NPnDgh9f369ZP6X375xbq9ePGiNNuflpYW09DQYN0fPHhQmv/UU09JfW5urnW7detWafasWbOkfs+ePVJ/1113SX0gRUdHmyFDhlj3LS0t0nzl+jTGmAULFli3w4cPl2bn5eVJfc+ePaW+ubnZug0NDZVm+3Ps2DHz4IMPWvdTp06V5qvn8NKlS9btqlWrpNmZmZlSr77mKe8FgdbQ0GD++OMP675Lly7S/M2bN0t9fHy8dTtgwABp9rp166R+x44dUn/hwgXr9rbbbpNm+5OammrWrFlj3S9dulSa37t3b6kPCwuzbteuXSvNVp9VcnJypH7s2LFSH0gej0d6Rq6vr5fmv/POO1KvPIM8/PDD0uyXX35Z6s+cOSP1gwcPDtpsf/7++2+Tn59v3aufG1JSUqReeV9UP2e89tprUq9ef3369JH6y+GbNgAAAAAAAA5iaQMAAAAAAOAgljYAAAAAAAAOYmkDAAAAAADgIJY2AAAAAAAADmJpAwAAAAAA4CCWNgAAAAAAAA5iaQMAAAAAAOAgljYAAAAAAAAOYmkDAAAAAADgIJY2AAAAAAAADgrx+Xz2cUhIpTGmNHiHg8v4H5/PlxCIQZzDK4rzePXjHP47cB6vfpzDfwfO49WPc/jvwHm8+nEO/x3+8TxKSxsAAAAAAAD8/+B/jwIAAAAAAHAQSxsAAAAAAAAHsbQBAAAAAABwEEsbAAAAAAAAB7G0AQAAAAAAcBBLGwAAAAAAAAextAEAAAAAAHAQSxsAAAAAAAAHsbQBAAAAAABw0P8CK18jo42jvxgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x576 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABG0AAABwCAYAAACkaY2RAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZQU1RXH8TeJihJQNoEoAooiKKAYUZbIIoKyKCIBNS6oGNGcuMWwKdGoKMclkSMq4oZKjhLQsMRdFmVzBUTRgKAwirLLHkFNJn94cvN7l+lhBrpnqnu+n79u+R41ZVe/quo6776bV1BQEAAAAAAAAJAsPynrAwAAAAAAAMCueGkDAAAAAACQQLy0AQAAAAAASCBe2gAAAAAAACQQL20AAAAAAAASiJc2AAAAAAAACbRPSTrn5eVRH7yMFBQU5KVjP5zDMrW+oKDg4HTsiPNYdhiLOYGxmAMYizmBsZgDGIs5gbGYAxiLOaHQschMG6D05Jf1AQAIITAWgaRgLALJwFgEkqHQschLGwAAAAAAgATipQ0AAAAAAEAC8dIGAAAAAAAggXhpAwAAAAAAkEC8tAEAAAAAAEggXtoAAAAAAAAkEC9tAAAAAAAAEmifsj4AlE+VKlWyuEePHlHbwIEDLa5Zs6bF++23X9Rvzpw5Fp9//vlR2/bt29NynAAAAAAAlBVm2gAAAAAAACQQL20AAAAAAAASiJc2AAAAAAAACcSaNsiYn/70p9H2sccea/FTTz1lccOGDaN+++yzT6FxXl5e1K9Lly4Wv/nmm1FbmzZtLN65c2dJDhtl6KCDDoq2//rXv1r86quvWjxq1Kio37///e/MHlg5petIDRkyxOILL7ww6jd16lSLr7nmmqjt+++/z9DRAQAAlA39XaLx/vvvH/WrWLGixT179oza6tata/Gnn35qsT5XhRDCqlWr9u5gkfWYaQMAAAAAAJBAvLQBAAAAAABIINKjkFZVq1a1eOLEiVFb8+bNLT7ggANS7kPTmXQ6oE+dOfDAAy1u1qxZ1NauXTuLX3vttd0dNsqQTint2rVr1KbndfTo0RaTDpUZPgWxc+fOFg8ePNhiP/W3Tp06Fo8bNy5q86mLyIz69etb3LZt26hNz0F+fn5a/66/LuvY3L59e9RWUFCQ1r+NXekYvuCCC6K2e++91+IxY8ZYPHTo0Kgf19fd89fKWrVqWdyyZUuL/fjQNN/Vq1dn6Oh25Y/jhhtusPiuu+6y2I9ZIFfpGNalGH72s59F/Y4++miLq1WrFrXpMg2nnXaaxXo9CCFONa9QoULK41i4cKHFH3/8cdSP9Cgw0wYAAAAAACCBeGkDAAAAAACQQGWaHuWnlyqmUWenU045xeLjjz8+avvJT/7/jnD27NkWDxgwIOq3ePFii7XyjK6wHkIIM2bMsLhmzZpRm1akIj0q2XSqaL9+/aK2bdu2WUyaTebVqFEj2n7iiScs9ilRat9997V44MCBUds777xj8Y4dO/b2ECG0Qt+1115rcevWraN+7777rsV6393T+6yeb18tbOXKlRb7VLlvv/12j/4eiq9KlSoW33///VGbpi/36dPH4ptvvjnqR3rU7h188MHR9siRIy3Wa+Vnn30W9dP7mD4T/ec//0n3IYZ69epZPH78+Kjtm2++sfi7775L+9/OdnpuDjvssKjtpptuKrTfH//4x6gf6SzJ4n9zVq9e3WIdv926dYv6FbWcQ6rqUV5R91r9d1pld9KkSVE/TcXyqVMoWlHnKRPX3kxhpg0AAAAAAEAC8dIGAAAAAAAggXhpAwAAAAAAkEClvqaN5ltrmcEQQmjQoIHFfm2FWbNmWfzAAw9YrHm5IcRrJhSVQ6hrpfzwww9R257kt5XXNXh8bqCW654zZ07UtnTpUos193fr1q3F+lu+n66r4HOy169fX6x9ouw1adLEYr0GhBDC9OnTLdb1bZA+us7FM888k7JN+eudXgdOPfXUqO2hhx6yWNfO0LVPsGd0XY0TTzzRYl+2V9dWSMe9StehOuGEE6K2vn37WvzKK69Ebaxpk37+Hqzrgh144IEp/92GDRssZg2b4tGywPocGkIILVq0sPjBBx+0eOzYsVG/devWWZzutRR8WW/920ceeWTUptdifR4uz/T89ujRw+K//OUvUb+f//znFutnp2MqhBCGDBlicTatm5Ft9LyFEP/O3Lx5c8p+p59+usU9e/a02JfkVn6s6G8PvY7659X8/HyLp0yZErXpvbtp06YW+9LjX3/9dcrjKq/0/qfXvyuuuCLqp88l/je/Ph+99dZbFvs1+VasWGGx/81ZWu8AmGkDAAAAAACQQLy0AQAAAAAASKBSSY/S6UuVK1e22KdC6BSxihUrRm3a9+yzz7ZY03FCiKfzazlUPy1bp675qXC6T51aV1S/K6+8MmqbP3++xbk8LdJPCXv//fct9tPT9Bz46ftKvy9akk9LB4cQpwb478Enn3xS1GGjDPnp/Dot1ZdW1JK15TUFMR38Z37EEUdYPGrUKIvbtGkT9UtVkrao0pb77bdftH3RRRdZfNppp1ms5z2E+JrJuS6cno8Q4tKkdevWtfiyyy6L+m3ZsiWtx6HXW3++a9asabHe70OgBG4m+OeS/v37W+y/Lzq1/5ZbbrE4l59R0ql3794Wt2/fPmqbPXu2xaNHj7Y43WMvhPj6q+e4bdu2Ub9mzZpZvHz58qjtueeeS/txZRs/PjTVU0tA+/Lu+vnr74zWrVtH/SpVqmSxT+/nHrd3dHkEfa4IIYT69etbPGPGDIt9etGSJUss/uyzzyw+6qijon5r1qyx2KfKzZs3z2K9F/7rX/+K+i1evNhif03Q79P+++8fUinuUhK5RsfY4YcfHrX97ne/s/j888+3WH87ev5+17hxY4v1uzR06NCo38aNGy326bF6zdfvS7ox0wYAAAAAACCBeGkDAAAAAACQQLy0AQAAAAAASKC8kuRV5uXl7XUSpubuadm8EEK49NJLLa5Xr17UpqXUDjnkEIvr1KkT9dPcU80n9TlsmgeuuZEhxKW8NHdfy8iFEOek+ty3e+65p9B+e6qgoCD1IhIlkI5zuJv9FxqHEH8ORX0mmtM5YcIEi7t27Zryby1atChqO+WUUyzWdYnK2LyCgoITd99t9zJ9HjPJ55HrOkiaixxCfB1ISj5vtoxF5dcWmTZtmsXNmze3WHOHPb2G+vGr10xfElOvwzpmfd6vln7/5ptvUh5HmmTlWPQlfSdPnmyxfs6//OUvo347duxI63Fo6dRly5ZFbVqmVNdOCiH9Yzgbx2K6+Weljz76yGL9ToQQwurVqy1u2LChxb48bSlL7FjUNRJDiNet8eXUtUS0rs+VaXocM2fOjNr0mbVXr15Rm67FkQ7ZOBb92pm6/omusemfZbVksP428dauXWvxsGHDorYnnniiWPsoZYkdi/4cdOzY0WJd+zCEuPS6rnPi17TR5xgd6/obM4QQvvjiC4v9tbK4v+9Kcw2jbByLvhx7o0aNLNZ1hFq1ahX109/y+tvCl+TWNYvWrVsXtR133HEW6/OLPyZ9Bvbrsr7wwgsWX3fddRbrd7GECh2LzLQBAAAAAABIIF7aAAAAAAAAJFCplPxWOkXMT1W78847i7UPnSbnpy/p9PFatWpZ7FMydEqxnyr15ZdfWtyyZUuLH3300aiflieeM2dO1FZey/kVNwWqKLVr17ZYSyj6KaQ61V7Ll4awa4oGksOXz9S0GD/lkFK06XHxxRdH2zod1F9DlX7+mirqp9ZrOqifPnzXXXdZrKlY/nug/a644oqorbxeT0OI73dXXnll1KZjR1N0050O5WlZ70MPPTRq0+uyH89ID/1O9O3bN2rT6d3eO++8Y7EvSYtdXXvttdH2YYcdZrGmmoUQwooVK0rjkEIIcWnhO+64w+IGDRpE/V577TWLFy5cmPkDyzJ+iYamTZtarPccLfUbQpzyoKWFfTpd3bp1LfYlgpcuXWrxG2+8UYKjLp98avCtt95qsY7LEEJ49tlnLdZxWtTz5KZNmwqNd/fvsOd0KQx9hgwhhIsuushivadpamII8XuEsWPHWuzHm6bc62/3EELo0qWLxe3atbO4TZs2UT991vH7aNGihcU1atQo9O+GsPfPssy0AQAAAAAASCBe2gAAAAAAACRQqadHFWVPVuH2K0RrqpPGfuVxrbDg/672/eqrryz2U851ZfjSrBaQa3zFmt69e1uslRH8FEVd6X/q1KlR27fffpvOQ0Qa6fTDEOL0HF3hPQTO497QVfWvueaaqE2n1ys/9fTzzz+3+E9/+pPFU6ZMifppqoW/1mpq1j/+8Q+LfdWbPn36WKxV40KIp/mXN40bN7b4xhtvjNr0nvTMM8+U2jFp5ROfepyfn28x08ozQ6dma9XNEOLz4dOENZWGc1M4vX7pNSmEuNqQT63RyofpqH6nz0X+ej1y5EiL9frqq7MNGTLEYn9tR5x+H0L8ma9atcriM888M+qnzylaqW/ixIlRv6Kq1Pbr18/iWbNmWZygSlKJcvXVV0fbmmrtUz1HjRplMd/75NBUwhBCePrppy3u3Llz1KbXYU1P9Cni+ixa3GUx/L3v7bfftljfKfhnm3PPPddiv6yA/lbV1P8lS5YU65iKi5k2AAAAAAAACcRLGwAAAAAAgATipQ0AAAAAAEACJWpNm0wqSZktzRcfM2aMxVWqVIn6jRs3zuJMl1jNNZqvqKXdQohL+WlOoS8l/Mgjj1jsS8uW5xLBSaT5n9ddd13UpjnHw4cPj9pYd2HPaV5tnTp1UvbTz/iDDz6I2rp27Wrx+vXrLS5qfPk2XRfnqaeesnjQoEFRv0qVKlmsJTtDCOGoo46yOB3rRSSZz6PWdRJ8OWctk+nXs0i3WrVqWdy2bVuL/Rj1pTaRfrrOkS+5rtasWRNtL168OGPHlIt8We9GjRpZ7Mfi3//+d4uffPJJi/141tLFWq5Wy9+GEK+Zc8YZZ0RtWkpa+bW/9NqLH+mz5w033BC16bnSNcI+/PDDqJ9e87Rct66DE0K8bpu/L/7iF7+wWNcsYh2//6tdu7bF/lzpM6VfY29vnxF47syMjh07ptz2a8ToOkV//vOfLX799dejfqnWLPJrK+rven/9PP300y3u27evxQ0aNIj6Va5c2WL/Hfniiy8sXrRoUaHHlA7MtAEAAAAAAEggXtoAAAAAAAAkULlJjyqKn5alU+10Wr6WHQshhGHDhllMOk7J6LSz+++/P2rTMolafm3gwIFRv+nTp1vMdMZk0/ScJk2aRG2a9qbnFHvn+OOPt9hf49SKFSssbt++fdTm0w73hJZhfO655yzu379/1E/TDXwqardu3SweO3bsXh9TkmkaUgjxFHFNUQshnjac7nuQT+vo1KmTxVoad9OmTVG/SZMmpfU48COd7q3lUfVchBB/D/xUcl8aF7vSz++yyy6L2ubOnWuxpp+GEE+5v+WWW1LuX0s6ayrMzp07o36aMqOpoyHE3wUdfwMGDIj6Ue54V/rZ+c9Vr3k+NSLVPqpWrZpyf6n+TWHb2NXMmTMt1pLKIcTfbU27DmHP7oX8hsi8mjVrRtt67/LnbN9997W4V69eFh9yyCFRv7Vr11qsY7Zly5ZRvxo1aljsU1H1b2lc1Bj199IRI0ZY7J+J0omZNgAAAAAAAAnESxsAAAAAAIAEIj0qxGkEIYTQunVri3XK1iWXXBL1y/UqJulWsWJFi59//nmLfRUGnaao05G1IkMITP1NOp1aqBWjdAX2EEKYP3++xVu2bMn8gZUTxx57rMU+PUrHjk49TUc6VFE0FU6ntYZQdBUcP602l/lp+VqZcM6cOVGbT6lIJ01TDSFOUdMUj/fffz/qt3nz5owdU3mm6TI6Zn0am6YjPv7441Ebadwlo6mjIcRjU6uMhBDCr3/9a4t1Cr+vBqQVnebNm2fxhg0bon79+vWzuGnTpimP8frrr7d45cqVKfvhR0VVS6xfv77FqSrKhJC6GqZWBvN/y9PqYJpqp1VoyiO9nlWvXt3iolJV/PjQZ0pNYynu9a8kqVJ6XFxfi/a3v/0t2u7evbvFJ510UtSm1Z50SYVmzZpF/fz973982nBx0xH1HOpzTggh5OfnW3zhhRdGbe+++26x9r+3mGkDAAAAAACQQLy0AQAAAAAASCBe2gAAAAAAACRQuV3TRvPgzjvvvJRtM2bMsPjll1/O/IHlEJ9r2Lt3b4u1lLrPG9Sytloi2PdDsmlO6TnnnGOxz/u99957LeYcp88xxxxjsc/n1TzvZcuWZfQ49G8fffTRFmu+uue/B1OnTk3/gSXU8uXLo21d68Kv7aPrgW3durXEf8uvdaTrjrVq1Spq69ixo8VaFnPcuHFRP8ZwZmjp96LWONH7p66ZEgJrLuwtXZ/m4Ycfjtr8dkn5NRjq1KljcePGjaO2jz76yOLx48fv1d8tzy6//PJo++STT7ZYr7X+3Oq50vVP/Hj78ssvLe7QoUPUpiWs77jjDov92pnl7Xqq9yR9TtHS6iHE58CXutfz+PHHH1u8cePGqJ+uoahloPV3Xwjx2nH16tVLuY/PPvus0H+DH61bty7aPvPMMy3297SzzjrLYl13tlq1alE/vd/p2lC6PlUI8f3TP/foGNb96XpVIYQwefJki33J79LCTBsAAAAAAIAE4qUNAAAAAABAApXb9CidVq5pOyHEJTN1elRJysAhhFq1akXb9913n8VavvTrr7+O+ulUR02P4vPPLlr+UlNh/BTJ6dOnl9oxlScnnHCCxT49Sqf5p3tc+bRIPY4HHnjAYl/iW49jyZIlUdunn36azkNMtG+++Sba1lKSWnY7hLhkrZYDf+mll6J+mjqlJTP9uWrRooXFPiWjRo0aFuuUfVJwMsOP2VNPPdViTU/zn/e0adMsZop+9vBpMOeee67F/ho9aNAgi8tqmn4u2LRpU7StqUmadla5cuWU+5g7d67FXbp0idr0vA0ePDhqu+mmmywuKk3ElyXPdfr7S9ONfDp1hQoVLD7kkEOiNk3H79Gjh8Xfffddyn0on0alqXI+tUbH7fDhwy2+7bbbon78ftmVnmst0x5CCAsWLLC4qLLq2qa/OR588MGoX58+fSz250LPt6ZMvvjii1G/JJxDZtoAAAAAAAAkEC9tAAAAAAAAEoiXNgAAAAAAAAlUbta08fnhWmbYr63w/vvvW7x48eLMHliO0c/5hhtuiNo03/CHH36w+Pbbb4/6aS4x6yNkr7Zt2xb63/Pz86NtLbGHPeevcT7PW+maGP7f7Qktlzls2LCorV+/fhbr2gD+727bts3i3//+91Hbjh079voYs4WuNxRCnCfv1zs44ogjLO7Zs6fFvXr1ivppLr9+7v5z1fKlVapUSXmMev324xnp4cfHxRdfXGg/vxbKyJEjLeb+mT20rG0Icclav8bG22+/XRqHVO7MnDnTYl3P8le/+lXUb9y4cRZPmDDBYr0ueo899li0rWtn6Bqb/tq9aNGiYu0/V+g1q3v37oXGIYTQt29fi/3aNPrso7/vKlWqFPXTa6yu7+Z/E/q131K16XOLrnUUQgivv/66xVyXd08/o+J+XnoN1eehEOIS8f6eOWbMGIt1PcAkrGHjMdMGAAAAAAAggXhpAwAAAAAAkEB5JZmmlZeXl7Vzutq1axdta5lhPwWqefPmFuvUxLJUUFCw9zkMIfPnUKcbvvfee1Gblv5dtWqVxX7Kvy95W5y/lenphlqi3P8tnWq3m+l08woKCk5Mx/EkcSz66fzPPvusxTrl9+GHH476XX311Zk9sDRL6lj0n7+mnVWrVi1q03QZLQFdVGltnV7apEmTqO3xxx8vdH/+3+kx+imq+n3p379/1JaBsrZZORb1OhRCCHXr1rVYp8771CZ/Tv5n3bp10fby5cst9mluZ599tsWawlW7du2o3/bt2wv9W5mQ1LGYDpoyEUIIK1assFjL365cuTLqd+SRR1rsS9wmVFaOxXTQ6+HSpUujtgYNGlj87rvvRm1t2rSxOCkpM7k2FotKidmTtAm9D4YQwpAhQyzW+92GDRuifp06dbLYX68zIGvGoo4dTfcOIU4tHDhwoMUdOnSI+h144IEWF5W6rc8q/rlFU8OV//3Tvn17izN9Xc61saj8uWnVqpXFmoJWsWLFlPvQUvIhxL9BfXp6GSp0LDLTBgAAAAAAIIF4aQMAAAAAAJBAOV09SlcQf/XVV6M2nfq4cOHCqO2TTz7J7IGVE36avE7j1emlfur+ggULLNZUJD9dXKcp+ioomk6h+/DTKDXd4JhjjonaLrjgAot79Ohh8QEHHBD1+8Mf/mDxnDlzorZly5aF8sJP/9XqUTqlUVdnR+asWbPGYp8epde/Rx991OJnnnkm6qdVFLRaQ61ataJ+++zz/1uJn1au40/HrE/ruOWWWyzOQDpUTvDTqot7ffnggw9K/LdmzJgRbZ9zzjkW6zn14x7pcdxxx0XbWn1R75+33XZb1C9LUqIQ4ucRf03V8/jII49EbT5FA+mX7soxfn96r9UUKP89uOqqqyy+++67o7byVFXR03uQv+Zpmvf1119v8YgRI6J+Z5xxRqH73rlzZ7Stvwm///77qE1/N/jfKIr7ZHr4dOwnn3zS4qJSorQ6qaZ6h5ColKjdYqYNAAAAAABAAvHSBgAAAAAAIIF4aQMAAAAAAJBAObemjeYNTpkyxWJfKlXLSvu8xnTnspYnmmf68ccfR21amq1GjRoWP//881E/PVcVKlSw2Jd60/Pkc1q1VLuuQVO1atWon7b5tWq0lF+qssUhhDBy5EiLtfRxCHE+ba7zawLVrFnTYv1e/POf/yy1YypPfCn6Sy+91OI333wzatMxpuOydevWUT9dn8Z/71Px10/ND9drwrXXXhv1y8/PL9b+UTq0HGoIqcu1s75GZlx33XXRtq4bpWPqlVdeKbVjQnqddNJJFvu1wLQs7bRp04q1v6Ku0f7+gKLpZ5mOz87vY/ny5RbrujUPPfRQ1E/XVpw5c2bU9sYbb+z1ceW6rVu3WjxmzJio7ayzzrJY15fyY7FRo0YW+/VP9Lqs69289dZbUT+/Tg6KT59X/bpOhx9+eKH/xv8m1Pup/j7MNsy0AQAAAAAASCBe2gAAAAAAACRQ1qdH+emgmuqk6RpbtmyJ+nXo0MHitWvXZujoyrcNGzZE2zo9VNOeNA6h+GkYypfybtGiRaF/10/l1zadRhlCCF9//bXFlStXttiX9XvwwQctXrVqVUkOO6fcc8890bamlGl5Z1/qGZmhZZ5ffPHFqE2nBev03uLyU71/+OEHizdt2hS1LViwwOKbb77Z4nnz5kX9SLPJTkz7zgxf8lvvi3qvWrduXakdE9Jr+PDhFvuUDH3+8M+v2levxX4fmqpKelTJZPrz0v1rierJkydH/a655hqLe/fuHbXNmjXLYu6fu+efOTQFUdOBfXnuSpUqWazLJoQQjzFN/R81alTKfiiZrl27Wty9e/eoTc+VjgGfRvXEE09k6OhKFzNtAAAAAAAAEoiXNgAAAAAAAAnESxsAAAAAAIAEyvo1bapXrx5t+zzC/5k6dWq0/cknn2TsmPCjRx99NNr+/PPPLdYSh7Vr1476HXTQQYXuz+fsakk3n9evZfl0jY358+dH/V599VWLly5dGrXpv9N1G7SsX3mnpfh8yW/N4R0xYoTF5F6XDv3ODho0KGrT8vYdO3a02K8NlarsqV/HRNdymjRpUtT29NNPW6wlv3UdHCRPp06dUrZpeXbGc/roeNuxY0fKfrouGGslZBe9xh577LEW+7X9mjRpYrFfy+SFF16wWJ+DfJnbbdu27d3BolTovVDvlyGEMGDAAItPP/30qO2oo46yeMmSJRazflHh/DX1yiuvtPixxx6zuGnTplE/XSvKl/zW8tGXXnqpxbpeDkpOr4e33367xbr2kPfhhx9afOedd0ZtuTImmGkDAAAAAACQQLy0AQAAAAAASKCsTI/S6aU+HerQQw+1WKcc3nfffVE/pnRn3hdffBFtjxkzptAY2UvH4ldffRW1aclSnXqaK9MUs8myZcui7csvv9xiTY/S6b0hhHD00UdbrGWGx48fH/V76aWXLNYUqBBC2L59u8WkciSbls88+OCDozZNC3355ZdL7ZjKE02PWr16ddTWuHFji3VM+fK0pO8mW8WKFS3eZ5/Uj+BVqlSx+IILLoja3nvvPYs3bNhgsU+x0u8JssPGjRuj7blz51p80kknRW2a9vzb3/7WYp/Cgx/5Z09Np7nkkkssvvHGG6N+Oo6efPLJqG3BggUW6+fOc+7eGTp0qMWNGjWyWFPVQog/cz1vuToGmGkDAAAAAACQQLy0AQAAAAAASKC8kkzhysvLS8R8r86dO1vsp2nr1CmtKNS2bduo3+LFizN0dJlRUFCQt/teu5eUc1hOzSsoKDgxHTviPJYdxmJOYCwWQtMdx40bF7Vplb/zzjvP4i+//DLzB5ZCLo/Fnj17Rtt33323xYMHD7Z44sSJUb8sTEEsV2NRU+BatWplsU/hX79+vcWjR4+O2l577TWLi0r11yUCMp2ukctjsSxVq1bNYl/1Vqs2aspc//79o34lOPflaizmqmwciz5VVL/rWiXNf5e1MnT37t0t9pX0slChY5GZNgAAAAAAAAnESxsAAAAAAIAE4qUNAAAAAABAAmVlye8ePXpYrPnB3ooVKyxeu3ZtJg8JAICspuWie/XqVYZHAr9Wjd9GdtI1GbSc88knn1wWh4OE27x5s8X62yeEEBo2bGjxhAkTLKbcNLJNlSpVUrbpul26Vm0IIVx//fUW58A6NrvFTBsAAAAAAIAE4qUNAAAAAABAAmVletTs2bMt/s1vfhO1bdu2zeJu3bpZvGnTpswfGAAAAADsJU0Neeedd6K2rVu3Wrxjx45SOyYg3TQNMIS4bP1VV11l8a233hr109Lg5QEzbQAAAAAAABKIlzYAAAAAAAAJxEsbAAAAAACABMorSWm4vLy8xNWRK6rkdy6VvSsoKEj9P1oCSTyH5ci8goKCE9OxI85j2WEs5gTGYg5gLOYExjH8B5YAAADGSURBVGIOYCzmBMZiDmAs5oRCxyIzbQAAAAAAABKIlzYAAAAAAAAJVNKS3+tDCPmZOJA9lUspUEWol8Z9Je4cliOcx+zHOcwNnMfsxznMDZzH7Mc5zA2cx+zHOcwNhZ7HEq1pAwAAAAAAgNJBehQAAAAAAEAC8dIGAAAAAAAggXhpAwAAAAAAkEC8tAEAAAAAAEggXtoAAAAAAAAkEC9tAAAAAAAAEoiXNgAAAAAAAAnESxsAAAAAAIAE4qUNAAAAAABAAv0X9Dff6/bupMMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x288 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1.72 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "n = 10\n",
    "plt.figure(figsize=(20, 4))\n",
    "for i in range(n):\n",
    "    ax = plt.subplot(1, n, i+1)\n",
    "    plt.imshow(x_test[i].reshape(28, 28))\n",
    "    plt.gray()\n",
    "    ax.get_xaxis().set_visible(False)\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "    \n",
    "plt.show()\n",
    "\n",
    "n = 10\n",
    "plt.figure(figsize=(20, 8))\n",
    "for i in range(n):\n",
    "    # What does encoded images looks like?\n",
    "    ax = plt.subplot(3, n, i+1)\n",
    "    plt.imshow(encoded_imgs[i].reshape(8*1,8)) # encoded 64 # (8,8) # (4, 4*4)\n",
    "    #plt.imshow(decoded_imgs[i].reshape(4, 14*14).T) # input or decoded 784 # (28,28) # (4, 14*14)\n",
    "    plt.gray()\n",
    "    ax.get_xaxis().set_visible(False)\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "    \n",
    "plt.show()\n",
    "\n",
    "n = 10\n",
    "plt.figure(figsize=(20, 4))\n",
    "for i in range(n):\n",
    "    ax = plt.subplot(1, n, i+1)\n",
    "    plt.imshow(decoded_imgs[i].reshape(28,28))\n",
    "    plt.gray()\n",
    "    ax.get_xaxis().set_visible(False)\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "    \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE = 0.0014, PSNR (dB) = 28.68\n",
      "MSE = 0.0049, PSNR (dB) = 23.07\n",
      "MSE = 0.0007, PSNR (dB) = 31.69\n",
      "MSE = 0.0030, PSNR (dB) = 25.25\n",
      "MSE = 0.0026, PSNR (dB) = 25.92\n",
      "MSE = 0.0006, PSNR (dB) = 31.88\n",
      "MSE = 0.0048, PSNR (dB) = 23.17\n",
      "MSE = 0.0036, PSNR (dB) = 24.49\n",
      "MSE = 0.0048, PSNR (dB) = 23.21\n",
      "MSE = 0.0035, PSNR (dB) = 24.56\n",
      "Wall time: 7.99 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# https://dsp.stackexchange.com/questions/38065/peak-signal-to-noise-ratio-psnr-in-python-for-an-image\n",
    "def mse(img1, img2):\n",
    "    return np.mean( (img1 - img2) ** 2 )\n",
    "\n",
    "def psnr(img1, img2):\n",
    "    mserr = mse(img1, img2)\n",
    "    PIXEL_MAX = 1.0\n",
    "    try:\n",
    "        return 20 * math.log10(PIXEL_MAX / math.sqrt(mserr))\n",
    "    except ZeroDivisionError:\n",
    "        return 'Same image'\n",
    "\n",
    "n = 10\n",
    "for i in range(n):\n",
    "    original = x_test[i]\n",
    "    contrast = decoded_imgs[i]\n",
    "    a = mse(original, contrast)\n",
    "    b = psnr(original, contrast) # can also use cv2.PSNR rather than defined function psnr\n",
    "    if b != 'Same image':\n",
    "        print(f'MSE = {a:.4f}, PSNR (dB) = {b:.2f}')\n",
    "    else:\n",
    "        print(f'MSE = {a:.4f}, PSNR too high')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9834\n",
      "0.9632\n",
      "0.9903\n",
      "0.9734\n",
      "0.9687\n",
      "0.9924\n",
      "0.9357\n",
      "0.9684\n",
      "0.9584\n",
      "0.9698\n",
      "Wall time: 229 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# https://github.com/imamun93/Image-Similarities-using-SSIM/blob/master/blog5_ssim.ipynb\n",
    "import numpy as np\n",
    "from tensorflow.image import ssim, rgb_to_yuv\n",
    "from tensorflow import convert_to_tensor\n",
    "\n",
    "n = 10\n",
    "for i in range(n):\n",
    "    original = x_test[i].reshape(1, 28, 28, 1)\n",
    "    contrast = decoded_imgs[i].reshape(1, 28, 28, 1)\n",
    "    a = np.asarray(ssim(convert_to_tensor(original), convert_to_tensor(contrast), max_val=1)) # <class 'tensorflow.python.framework.ops.EagerTensor'>\n",
    "    #a = np.asarray(ssim(rgb_to_yuv(convert_to_tensor(original)), rgb_to_yuv(convert_to_tensor(contrast)), max_val=1))\n",
    "    print(f'{a[0]:.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9983\n",
      "0.9889\n",
      "0.9967\n",
      "0.9963\n",
      "0.9944\n",
      "0.9988\n",
      "0.9845\n",
      "0.9922\n",
      "0.9918\n",
      "0.9916\n",
      "Wall time: 113 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# https://github.com/tensorflow/models/blob/master/research/compression/image_encoder/msssim.py\n",
    "import numpy as np\n",
    "from scipy import signal\n",
    "from scipy.ndimage.filters import convolve\n",
    "\n",
    "def _FSpecialGauss(size, sigma):\n",
    "  \"\"\"Function to mimic the 'fspecial' gaussian MATLAB function.\"\"\"\n",
    "  radius = size // 2\n",
    "  offset = 0.0\n",
    "  start, stop = -radius, radius + 1\n",
    "  if size % 2 == 0:\n",
    "    offset = 0.5\n",
    "    stop -= 1\n",
    "  x, y = np.mgrid[offset + start:stop, offset + start:stop]\n",
    "  assert len(x) == size\n",
    "  g = np.exp(-((x**2 + y**2)/(2.0 * sigma**2)))\n",
    "  return g / g.sum()\n",
    "\n",
    "\n",
    "def _SSIMForMultiScale(img1, img2, max_val=255, filter_size=11,\n",
    "                       filter_sigma=1.5, k1=0.01, k2=0.03):\n",
    "  \"\"\"Return the Structural Similarity Map between `img1` and `img2`.\n",
    "  This function attempts to match the functionality of ssim_index_new.m by\n",
    "  Zhou Wang: http://www.cns.nyu.edu/~lcv/ssim/msssim.zip\n",
    "  Arguments:\n",
    "    img1: Numpy array holding the first RGB image batch.\n",
    "    img2: Numpy array holding the second RGB image batch.\n",
    "    max_val: the dynamic range of the images (i.e., the difference between the\n",
    "      maximum the and minimum allowed values).\n",
    "    filter_size: Size of blur kernel to use (will be reduced for small images).\n",
    "    filter_sigma: Standard deviation for Gaussian blur kernel (will be reduced\n",
    "      for small images).\n",
    "    k1: Constant used to maintain stability in the SSIM calculation (0.01 in\n",
    "      the original paper).\n",
    "    k2: Constant used to maintain stability in the SSIM calculation (0.03 in\n",
    "      the original paper).\n",
    "  Returns:\n",
    "    Pair containing the mean SSIM and contrast sensitivity between `img1` and\n",
    "    `img2`.\n",
    "  Raises:\n",
    "    RuntimeError: If input images don't have the same shape or don't have four\n",
    "      dimensions: [batch_size, height, width, depth].\n",
    "  \"\"\"\n",
    "  if img1.shape != img2.shape:\n",
    "    raise RuntimeError('Input images must have the same shape (%s vs. %s).',\n",
    "                       img1.shape, img2.shape)\n",
    "  if img1.ndim != 4:\n",
    "    raise RuntimeError('Input images must have four dimensions, not %d',\n",
    "                       img1.ndim)\n",
    "\n",
    "  img1 = img1.astype(np.float64)\n",
    "  img2 = img2.astype(np.float64)\n",
    "  _, height, width, _ = img1.shape\n",
    "\n",
    "  # Filter size can't be larger than height or width of images.\n",
    "  size = min(filter_size, height, width)\n",
    "\n",
    "  # Scale down sigma if a smaller filter size is used.\n",
    "  sigma = size * filter_sigma / filter_size if filter_size else 0\n",
    "\n",
    "  if filter_size:\n",
    "    window = np.reshape(_FSpecialGauss(size, sigma), (1, size, size, 1))\n",
    "    mu1 = signal.fftconvolve(img1, window, mode='valid')\n",
    "    mu2 = signal.fftconvolve(img2, window, mode='valid')\n",
    "    sigma11 = signal.fftconvolve(img1 * img1, window, mode='valid')\n",
    "    sigma22 = signal.fftconvolve(img2 * img2, window, mode='valid')\n",
    "    sigma12 = signal.fftconvolve(img1 * img2, window, mode='valid')\n",
    "  else:\n",
    "    # Empty blur kernel so no need to convolve.\n",
    "    mu1, mu2 = img1, img2\n",
    "    sigma11 = img1 * img1\n",
    "    sigma22 = img2 * img2\n",
    "    sigma12 = img1 * img2\n",
    "\n",
    "  mu11 = mu1 * mu1\n",
    "  mu22 = mu2 * mu2\n",
    "  mu12 = mu1 * mu2\n",
    "  sigma11 -= mu11\n",
    "  sigma22 -= mu22\n",
    "  sigma12 -= mu12\n",
    "\n",
    "  # Calculate intermediate values used by both ssim and cs_map.\n",
    "  c1 = (k1 * max_val) ** 2\n",
    "  c2 = (k2 * max_val) ** 2\n",
    "  v1 = 2.0 * sigma12 + c2\n",
    "  v2 = sigma11 + sigma22 + c2\n",
    "  ssim = np.mean((((2.0 * mu12 + c1) * v1) / ((mu11 + mu22 + c1) * v2)))\n",
    "  cs = np.mean(v1 / v2)\n",
    "  return ssim, cs\n",
    "\n",
    "\n",
    "def MultiScaleSSIM(img1, img2, max_val=255, filter_size=11, filter_sigma=1.5,\n",
    "                   k1=0.01, k2=0.03, weights=None):\n",
    "  \"\"\"Return the MS-SSIM score between `img1` and `img2`.\n",
    "  This function implements Multi-Scale Structural Similarity (MS-SSIM) Image\n",
    "  Quality Assessment according to Zhou Wang's paper, \"Multi-scale structural\n",
    "  similarity for image quality assessment\" (2003).\n",
    "  Link: https://ece.uwaterloo.ca/~z70wang/publications/msssim.pdf\n",
    "  Author's MATLAB implementation:\n",
    "  http://www.cns.nyu.edu/~lcv/ssim/msssim.zip\n",
    "  Arguments:\n",
    "    img1: Numpy array holding the first RGB image batch.\n",
    "    img2: Numpy array holding the second RGB image batch.\n",
    "    max_val: the dynamic range of the images (i.e., the difference between the\n",
    "      maximum the and minimum allowed values).\n",
    "    filter_size: Size of blur kernel to use (will be reduced for small images).\n",
    "    filter_sigma: Standard deviation for Gaussian blur kernel (will be reduced\n",
    "      for small images).\n",
    "    k1: Constant used to maintain stability in the SSIM calculation (0.01 in\n",
    "      the original paper).\n",
    "    k2: Constant used to maintain stability in the SSIM calculation (0.03 in\n",
    "      the original paper).\n",
    "    weights: List of weights for each level; if none, use five levels and the\n",
    "      weights from the original paper.\n",
    "  Returns:\n",
    "    MS-SSIM score between `img1` and `img2`.\n",
    "  Raises:\n",
    "    RuntimeError: If input images don't have the same shape or don't have four\n",
    "      dimensions: [batch_size, height, width, depth].\n",
    "  \"\"\"\n",
    "  if img1.shape != img2.shape:\n",
    "    raise RuntimeError('Input images must have the same shape (%s vs. %s).',\n",
    "                       img1.shape, img2.shape)\n",
    "  if img1.ndim != 4:\n",
    "    raise RuntimeError('Input images must have four dimensions, not %d',\n",
    "                       img1.ndim)\n",
    "\n",
    "  # Note: default weights don't sum to 1.0 but do match the paper / matlab code.\n",
    "  weights = np.array(weights if weights else\n",
    "                     [0.0448, 0.2856, 0.3001, 0.2363, 0.1333])\n",
    "  levels = weights.size\n",
    "  downsample_filter = np.ones((1, 2, 2, 1)) / 4.0\n",
    "  im1, im2 = [x.astype(np.float64) for x in [img1, img2]]\n",
    "  mssim = np.array([])\n",
    "  mcs = np.array([])\n",
    "  for _ in range(levels):\n",
    "    ssim, cs = _SSIMForMultiScale(\n",
    "        im1, im2, max_val=max_val, filter_size=filter_size,\n",
    "        filter_sigma=filter_sigma, k1=k1, k2=k2)\n",
    "    mssim = np.append(mssim, ssim)\n",
    "    mcs = np.append(mcs, cs)\n",
    "    filtered = [convolve(im, downsample_filter, mode='reflect')\n",
    "                for im in [im1, im2]]\n",
    "    im1, im2 = [x[:, ::2, ::2, :] for x in filtered]\n",
    "  return (np.prod(mcs[0:levels-1] ** weights[0:levels-1]) *\n",
    "          (mssim[levels-1] ** weights[levels-1]))\n",
    "\n",
    "n = 10\n",
    "for i in range(n):\n",
    "    original = x_test[i].reshape(1, 28, 28, 1)\n",
    "    contrast = decoded_imgs[i].reshape(1, 28, 28, 1)\n",
    "    a = MultiScaleSSIM(original, contrast, max_val=1.0)\n",
    "    print(f'{a:.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 400 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import os\n",
    "\n",
    "try:\n",
    "    os.makedirs(f'autoencoder{MODEL_VERSION}/model')\n",
    "except Exception:\n",
    "    pass\n",
    "\n",
    "save_model(autoencoder, f'autoencoder{MODEL_VERSION}/model/autoencoder{MODEL_VERSION}-784-256-128-64-128-256-784-best.h5')\n",
    "save_model(encoder, f'autoencoder{MODEL_VERSION}/model/encoder{MODEL_VERSION}-784-256-128-64-best.h5')\n",
    "save_model(decoder, f'autoencoder{MODEL_VERSION}/model/decoder{MODEL_VERSION}-64-128-256-784-best.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}