ch15/ch15_part1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Python Machine Learning 3rd Edition* by [Sebastian Raschka](https://sebastianraschka.com) & [Vahid Mirjalili](http://vahidmirjalili.com), Packt Publishing Ltd. 2019\n",
    "\n",
    "Code Repository: https://github.com/rasbt/python-machine-learning-book-3rd-edition\n",
    "\n",
    "Code License: [MIT License](https://github.com/rasbt/python-machine-learning-book-3rd-edition/blob/master/LICENSE.txt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 15: Classifying Images with Deep Convolutional Neural Networks (Part 1/2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that the optional watermark extension is a small IPython notebook plugin that I developed to make the code reproducible. You can just skip the following line(s)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sebastian Raschka & Vahid Mirjalili \n",
      "last updated: 2019-11-01 \n",
      "\n",
      "numpy 1.17.2\n",
      "scipy 1.2.1\n",
      "matplotlib 3.1.0\n",
      "tensorflow 2.0.0\n",
      "tensorflow_datasets 1.3.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -a \"Sebastian Raschka & Vahid Mirjalili\" -u -d -p numpy,scipy,matplotlib,tensorflow,tensorflow_datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  The building blocks of convolutional neural networks\n",
    "\n",
    "### Understanding CNNs and feature hierarchies\n",
    "\n",
    "### Performing discrete convolutions\n",
    "\n",
    "### Discrete convolutions in one dimension\n",
    "\n",
    "### Padding inputs to control the size of the output feature maps\n",
    "\n",
    "### Determining the size of the convolution output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow version: 2.0.0\n",
      "NumPy version:  1.17.2\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "print('TensorFlow version:', tf.__version__)\n",
    "print('NumPy version: ', np.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Conv1d Implementation: [ 5. 14. 16. 26. 24. 34. 19. 22.]\n",
      "Numpy Results: [ 5 14 16 26 24 34 19 22]\n"
     ]
    }
   ],
   "source": [
    "def conv1d(x, w, p=0, s=1):\n",
    "    w_rot = np.array(w[::-1])\n",
    "    x_padded = np.array(x)\n",
    "    if p > 0:\n",
    "        zero_pad = np.zeros(shape=p)\n",
    "        x_padded = np.concatenate(\n",
    "            [zero_pad, x_padded, zero_pad])\n",
    "    res = []\n",
    "    for i in range(0, int(len(x)/s),s):\n",
    "        res.append(np.sum(\n",
    "            x_padded[i:i+w_rot.shape[0]] * w_rot))\n",
    "    return np.array(res)\n",
    "\n",
    "\n",
    "## Testing:\n",
    "x = [1, 3, 2, 4, 5, 6, 1, 3]\n",
    "w = [1, 0, 3, 1, 2]\n",
    "\n",
    "print('Conv1d Implementation:',\n",
    "      conv1d(x, w, p=2, s=1))\n",
    "\n",
    "print('Numpy Results:',\n",
    "      np.convolve(x, w, mode='same')) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Performing a discrete convolution in 2D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Conv2d Implementation:\n",
      " [[11. 25. 32. 13.]\n",
      " [19. 25. 24. 13.]\n",
      " [13. 28. 25. 17.]\n",
      " [11. 17. 14.  9.]]\n",
      "SciPy Results:\n",
      " [[11 25 32 13]\n",
      " [19 25 24 13]\n",
      " [13 28 25 17]\n",
      " [11 17 14  9]]\n"
     ]
    }
   ],
   "source": [
    "import scipy.signal\n",
    "\n",
    "\n",
    "def conv2d(X, W, p=(0, 0), s=(1, 1)):\n",
    "    W_rot = np.array(W)[::-1,::-1]\n",
    "    X_orig = np.array(X)\n",
    "    n1 = X_orig.shape[0] + 2*p[0]\n",
    "    n2 = X_orig.shape[1] + 2*p[1]\n",
    "    X_padded = np.zeros(shape=(n1, n2))\n",
    "    X_padded[p[0]:p[0]+X_orig.shape[0],\n",
    "    p[1]:p[1]+X_orig.shape[1]] = X_orig\n",
    "\n",
    "    res = []\n",
    "    for i in range(0, int((X_padded.shape[0] - \n",
    "                           W_rot.shape[0])/s[0])+1, s[0]):\n",
    "        res.append([])\n",
    "        for j in range(0, int((X_padded.shape[1] - \n",
    "                               W_rot.shape[1])/s[1])+1, s[1]):\n",
    "            X_sub = X_padded[i:i+W_rot.shape[0],\n",
    "                             j:j+W_rot.shape[1]]\n",
    "            res[-1].append(np.sum(X_sub * W_rot))\n",
    "    return(np.array(res))\n",
    "\n",
    "X = [[1, 3, 2, 4], [5, 6, 1, 3], [1, 2, 0, 2], [3, 4, 3, 2]]\n",
    "W = [[1, 0, 3], [1, 2, 1], [0, 1, 1]]\n",
    "\n",
    "print('Conv2d Implementation:\\n',\n",
    "    conv2d(X, W, p=(1, 1), s=(1, 1)))\n",
    "\n",
    "\n",
    "print('SciPy Results:\\n',\n",
    "    scipy.signal.convolve2d(X, W, mode='same'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Subsampling layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Putting everything together – implementing a CNN\n",
    "\n",
    "### Working with multiple input or color channels\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**TIP: Reading an image file**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image shape: (252, 221, 3)\n",
      "Number of channels: 3\n",
      "Image data type: <dtype: 'uint8'>\n",
      "tf.Tensor(\n",
      "[[[179 134 110]\n",
      "  [182 136 112]]\n",
      "\n",
      " [[180 135 111]\n",
      "  [182 137 113]]], shape=(2, 2, 3), dtype=uint8)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "img_raw = tf.io.read_file('example-image.png')\n",
    "img = tf.image.decode_image(img_raw)\n",
    "print('Image shape:', img.shape)\n",
    "print('Number of channels:', img.shape[2])\n",
    "print('Image data type:', img.dtype)\n",
    "print(img[100:102, 100:102, :])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image shape: (252, 221, 3)\n",
      "Number of channels: 3\n",
      "Image data type: uint8\n",
      "[[[179 134 110]\n",
      "  [182 136 112]]\n",
      "\n",
      " [[180 135 111]\n",
      "  [182 137 113]]]\n"
     ]
    }
   ],
   "source": [
    "import imageio\n",
    "\n",
    "\n",
    "img = imageio.imread('example-image.png')\n",
    "print('Image shape:', img.shape)\n",
    "print('Number of channels:', img.shape[2])\n",
    "print('Image data type:', img.dtype)\n",
    "print(img[100:102, 100:102, :])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**INFO-BOX: The rank of a grayscale image for input to a CNN**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rank: 3\n",
      "Shape: TensorShape([252, 221, 1])\n"
     ]
    }
   ],
   "source": [
    "img_raw = tf.io.read_file('example-image-gray.png')\n",
    "img = tf.image.decode_image(img_raw)\n",
    "tf.print('Rank:', tf.rank(img))\n",
    "tf.print('Shape:', img.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rank: 2\n",
      "Shape: (252, 221)\n",
      "New Shape: TensorShape([252, 221, 1])\n"
     ]
    }
   ],
   "source": [
    "img = imageio.imread('example-image-gray.png')\n",
    "tf.print('Rank:', tf.rank(img))\n",
    "tf.print('Shape:', img.shape)\n",
    "\n",
    "img_reshaped = tf.reshape(img, (img.shape[0], img.shape[1], 1))\n",
    "tf.print('New Shape:', img_reshaped.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regularizing a neural network with dropout\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "\n",
    "\n",
    "conv_layer = keras.layers.Conv2D(\n",
    "    filters=16, kernel_size=(3, 3),\n",
    "    kernel_regularizer=keras.regularizers.l2(0.001))\n",
    "\n",
    "fc_layer = keras.layers.Dense(\n",
    "    units=16, kernel_regularizer=keras.regularizers.l2(0.001))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loss Functions for Classification\n",
    "\n",
    " * **`BinaryCrossentropy()`**\n",
    "   * `from_logits=False` \n",
    "   * `from_logits=True`\n",
    "\n",
    " * **`CategoricalCrossentropy()`**\n",
    "   * `from_logits=False`\n",
    "   * `from_logits=True`\n",
    "   \n",
    " * **`SparseCategoricalCrossentropy()`**\n",
    "   * `from_logits=False`\n",
    "   * `from_logits=True`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BCE (w Probas): 0.3711 (w Logits): 0.3711\n",
      "CCE (w Probas): 0.5996 (w Logits): 0.5996\n",
      "Sparse CCE (w Probas): 0.5996 (w Logits): 0.5996\n"
     ]
    }
   ],
   "source": [
    "####### Binary Crossentropy\n",
    "bce_probas = tf.keras.losses.BinaryCrossentropy(from_logits=False)\n",
    "bce_logits = tf.keras.losses.BinaryCrossentropy(from_logits=True)\n",
    "\n",
    "logits = tf.constant([0.8])\n",
    "probas = tf.keras.activations.sigmoid(logits)\n",
    "\n",
    "tf.print(\n",
    "    'BCE (w Probas): {:.4f}'.format(\n",
    "    bce_probas(y_true=[1], y_pred=probas)),\n",
    "    '(w Logits): {:.4f}'.format(\n",
    "    bce_logits(y_true=[1], y_pred=logits)))\n",
    "\n",
    "\n",
    "####### Categorical Crossentropy\n",
    "cce_probas = tf.keras.losses.CategoricalCrossentropy(\n",
    "    from_logits=False)\n",
    "cce_logits = tf.keras.losses.CategoricalCrossentropy(\n",
    "    from_logits=True)\n",
    "\n",
    "logits = tf.constant([[1.5, 0.8, 2.1]])\n",
    "probas = tf.keras.activations.softmax(logits)\n",
    "\n",
    "tf.print(\n",
    "    'CCE (w Probas): {:.4f}'.format(\n",
    "    cce_probas(y_true=[0, 0, 1], y_pred=probas)),\n",
    "    '(w Logits): {:.4f}'.format(\n",
    "    cce_logits(y_true=[0, 0, 1], y_pred=logits)))\n",
    "\n",
    "####### Sparse Categorical Crossentropy\n",
    "sp_cce_probas = tf.keras.losses.SparseCategoricalCrossentropy(\n",
    "    from_logits=False)\n",
    "sp_cce_logits = tf.keras.losses.SparseCategoricalCrossentropy(\n",
    "    from_logits=True)\n",
    "\n",
    "tf.print(\n",
    "    'Sparse CCE (w Probas): {:.4f}'.format(\n",
    "    sp_cce_probas(y_true=[2], y_pred=probas)),\n",
    "    '(w Logits): {:.4f}'.format(\n",
    "    sp_cce_logits(y_true=[2], y_pred=logits)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implementing a deep convolutional neural network using TensorFlow\n",
    "\n",
    "### The multilayer CNN architecture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading and preprocessing the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['test', 'train'])\n"
     ]
    }
   ],
   "source": [
    "## MNIST dataset\n",
    "\n",
    "mnist_bldr = tfds.builder('mnist')\n",
    "mnist_bldr.download_and_prepare()\n",
    "datasets = mnist_bldr.as_dataset(shuffle_files=False)\n",
    "print(datasets.keys())\n",
    "mnist_train_orig, mnist_test_orig = datasets['train'], datasets['test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "BUFFER_SIZE = 10000\n",
    "BATCH_SIZE = 64\n",
    "NUM_EPOCHS = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist_train = mnist_train_orig.map(\n",
    "    lambda item: (tf.cast(item['image'], tf.float32)/255.0, \n",
    "                  tf.cast(item['label'], tf.int32)))\n",
    "\n",
    "mnist_test = mnist_test_orig.map(\n",
    "    lambda item: (tf.cast(item['image'], tf.float32)/255.0, \n",
    "                  tf.cast(item['label'], tf.int32)))\n",
    "\n",
    "tf.random.set_seed(1)\n",
    "\n",
    "mnist_train = mnist_train.shuffle(buffer_size=BUFFER_SIZE,\n",
    "                                  reshuffle_each_iteration=False)\n",
    "\n",
    "mnist_valid = mnist_train.take(10000).batch(BATCH_SIZE)\n",
    "mnist_train = mnist_train.skip(10000).batch(BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implementing a CNN using the TensorFlow Keras API\n",
    "\n",
    "#### Configuring CNN layers in Keras\n",
    "\n",
    " * **Conv2D:** `tf.keras.layers.Conv2D`\n",
    "   * `filters`\n",
    "   * `kernel_size`\n",
    "   * `strides`\n",
    "   * `padding`\n",
    "   \n",
    "   \n",
    " * **MaxPool2D:** `tf.keras.layers.MaxPool2D`\n",
    "   * `pool_size`\n",
    "   * `strides`\n",
    "   * `padding`\n",
    "   \n",
    "   \n",
    " * **Dropout** `tf.keras.layers.Dropout2D`\n",
    "   * `rate`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Constructing a CNN in Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.Sequential()\n",
    "\n",
    "model.add(tf.keras.layers.Conv2D(\n",
    "    filters=32, kernel_size=(5, 5),\n",
    "    strides=(1, 1), padding='same',\n",
    "    data_format='channels_last',\n",
    "    name='conv_1', activation='relu'))\n",
    "\n",
    "model.add(tf.keras.layers.MaxPool2D(\n",
    "    pool_size=(2, 2), name='pool_1'))\n",
    "    \n",
    "model.add(tf.keras.layers.Conv2D(\n",
    "    filters=64, kernel_size=(5, 5),\n",
    "    strides=(1, 1), padding='same',\n",
    "    name='conv_2', activation='relu'))\n",
    "\n",
    "model.add(tf.keras.layers.MaxPool2D(pool_size=(2, 2), name='pool_2'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([16, 7, 7, 64])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compute_output_shape(input_shape=(16, 28, 28, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([16, 3136])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "    model.add(tf.keras.layers.Flatten())\n",
    "    \n",
    "model.compute_output_shape(input_shape=(16, 28, 28, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add(tf.keras.layers.Dense(\n",
    "    units=1024, name='fc_1', \n",
    "    activation='relu'))\n",
    "\n",
    "model.add(tf.keras.layers.Dropout(\n",
    "    rate=0.5))\n",
    "    \n",
    "model.add(tf.keras.layers.Dense(\n",
    "    units=10, name='fc_2',\n",
    "    activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([16, 10])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.random.set_seed(1)\n",
    "model.build(input_shape=(None, 28, 28, 1))\n",
    "\n",
    "model.compute_output_shape(input_shape=(16, 28, 28, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv_1 (Conv2D)              multiple                  832       \n",
      "_________________________________________________________________\n",
      "pool_1 (MaxPooling2D)        multiple                  0         \n",
      "_________________________________________________________________\n",
      "conv_2 (Conv2D)              multiple                  51264     \n",
      "_________________________________________________________________\n",
      "pool_2 (MaxPooling2D)        multiple                  0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            multiple                  0         \n",
      "_________________________________________________________________\n",
      "fc_1 (Dense)                 multiple                  3212288   \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            multiple                  0         \n",
      "_________________________________________________________________\n",
      "fc_2 (Dense)                 multiple                  10250     \n",
      "=================================================================\n",
      "Total params: 3,274,634\n",
      "Trainable params: 3,274,634\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "782/782 [==============================] - 29s 38ms/step - loss: 0.1450 - accuracy: 0.9549 - val_loss: 0.0000e+00 - val_accuracy: 0.0000e+00\n",
      "Epoch 2/20\n",
      "782/782 [==============================] - 28s 35ms/step - loss: 0.0491 - accuracy: 0.9858 - val_loss: 0.0401 - val_accuracy: 0.9874\n",
      "Epoch 3/20\n",
      "782/782 [==============================] - 27s 35ms/step - loss: 0.0311 - accuracy: 0.9903 - val_loss: 0.0493 - val_accuracy: 0.9846\n",
      "Epoch 4/20\n",
      "782/782 [==============================] - 28s 35ms/step - loss: 0.0241 - accuracy: 0.9920 - val_loss: 0.0489 - val_accuracy: 0.9855\n",
      "Epoch 5/20\n",
      "782/782 [==============================] - 27s 34ms/step - loss: 0.0197 - accuracy: 0.9942 - val_loss: 0.0453 - val_accuracy: 0.9874\n",
      "Epoch 6/20\n",
      "782/782 [==============================] - 27s 35ms/step - loss: 0.0161 - accuracy: 0.9953 - val_loss: 0.0361 - val_accuracy: 0.9897\n",
      "Epoch 7/20\n",
      "782/782 [==============================] - 28s 35ms/step - loss: 0.0125 - accuracy: 0.9961 - val_loss: 0.0401 - val_accuracy: 0.9903\n",
      "Epoch 8/20\n",
      "782/782 [==============================] - 27s 34ms/step - loss: 0.0117 - accuracy: 0.9963 - val_loss: 0.0371 - val_accuracy: 0.9918\n",
      "Epoch 9/20\n",
      "782/782 [==============================] - 28s 35ms/step - loss: 0.0113 - accuracy: 0.9966 - val_loss: 0.0367 - val_accuracy: 0.9904\n",
      "Epoch 10/20\n",
      "782/782 [==============================] - 27s 35ms/step - loss: 0.0097 - accuracy: 0.9967 - val_loss: 0.0426 - val_accuracy: 0.9912\n",
      "Epoch 11/20\n",
      "782/782 [==============================] - 28s 35ms/step - loss: 0.0098 - accuracy: 0.9974 - val_loss: 0.0367 - val_accuracy: 0.9925\n",
      "Epoch 12/20\n",
      "782/782 [==============================] - 27s 35ms/step - loss: 0.0093 - accuracy: 0.9974 - val_loss: 0.0395 - val_accuracy: 0.9912\n",
      "Epoch 13/20\n",
      "782/782 [==============================] - 27s 35ms/step - loss: 0.0074 - accuracy: 0.9975 - val_loss: 0.0509 - val_accuracy: 0.9912\n",
      "Epoch 14/20\n",
      "782/782 [==============================] - 27s 35ms/step - loss: 0.0082 - accuracy: 0.9975 - val_loss: 0.0595 - val_accuracy: 0.9885\n",
      "Epoch 15/20\n",
      "782/782 [==============================] - 27s 35ms/step - loss: 0.0056 - accuracy: 0.9982 - val_loss: 0.0601 - val_accuracy: 0.9895\n",
      "Epoch 16/20\n",
      "782/782 [==============================] - 27s 35ms/step - loss: 0.0086 - accuracy: 0.9976 - val_loss: 0.0431 - val_accuracy: 0.9917\n",
      "Epoch 17/20\n",
      "782/782 [==============================] - 27s 34ms/step - loss: 0.0058 - accuracy: 0.9982 - val_loss: 0.0538 - val_accuracy: 0.9912\n",
      "Epoch 18/20\n",
      "782/782 [==============================] - 28s 35ms/step - loss: 0.0080 - accuracy: 0.9979 - val_loss: 0.0500 - val_accuracy: 0.9896\n",
      "Epoch 19/20\n",
      "782/782 [==============================] - 29s 37ms/step - loss: 0.0064 - accuracy: 0.9980 - val_loss: 0.0424 - val_accuracy: 0.9923\n",
      "Epoch 20/20\n",
      "782/782 [==============================] - 29s 37ms/step - loss: 0.0041 - accuracy: 0.9988 - val_loss: 0.0469 - val_accuracy: 0.9915\n"
     ]
    }
   ],
   "source": [
    "model.compile(optimizer=tf.keras.optimizers.Adam(),\n",
    "              loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
    "              metrics=['accuracy']) # same as `tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy')`\n",
    "\n",
    "history = model.fit(mnist_train, epochs=NUM_EPOCHS, \n",
    "                    validation_data=mnist_valid, \n",
    "                    shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtwAAAELCAYAAADqelFjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd3hU1dbA4d9KL5CEkgKhCiogUiOKgAqKoFIsIHAtiAULXOWqKCiXpthQQSzwoSLXijQBFUFUsCu9CIiiICQhoQSSAOnZ3x9nJqRMyqTMTMh6n2ceZk7dk4Qza/ZZe20xxqCUUkoppZSqGl7uboBSSimllFJnMw24lVJKKaWUqkIacCullFJKKVWFNOBWSimllFKqCmnArZRSSimlVBXycXcDqlL9+vVNs2bN3N0MpZQql02bNh01xoS7ux2upNdtpVR1VdI1+6wOuJs1a8bGjRvd3QyllCoXEfnH3W1wNb1uK6Wqq5Ku2ZpSopRSqlgiMk9EDovIb8WsFxGZJSJ7RWS7iHTKt264iPxpewx3XauVUsqzaMCtlFKqJPOBviWsvwY41/YYCcwGEJG6wCTgYqALMElE6lRpS5VSykNpwK2UUqpYxpjvgKQSNhkIvGssvwBhItIA6AOsMcYkGWOOA2soOXBXSqmzlgbcSimlKiIaOJjvdaxtWXHLixCRkSKyUUQ2HjlypMoaqpRS7qIBt1JKqYoQB8tMCcuLLjRmrjEmxhgTEx5eo4qyKKVqiLO6Somzlm2JY/rqPcSfSKNhWCBj+5zP9R0ddsgopZSyxAKN871uBMTbll9RaPk6l7VKKaWcVJVxoAbcNsu2xDF+6Q7SsnIAiDuRxvilOwA06D7LpKSkcPjwYbKystzdFFVD+fr6EhERQUhIiLubUhlWAKNFZAHWAMlkY8whEVkNPJNvoOTVwHh3NVIppUpS1XGgBtw201fvyfsh26Vl5TB99R4NuM8iKSkpJCYmEh0dTWBgICKO7norVXWMMaSlpREXFwfg8UG3iHyE1VNdX0RisSqP+AIYY+YAK4Frgb3AaWCEbV2SiDwFbLAdaqoxpqTBl0op5TbTV/9epXGgBtw28SfSnFquqqfDhw8THR1NUFCQu5uiaigRISgoiOjoaOLj4z0+4DbGDCtlvQFGFbNuHjCvKtqllHKOu9JmPTVdNyfX8HtCChv3H2fD/iTiTqQ73K6y4kANuG0ahgUS5+CH2jAs0A2tUVUlKyuLwED9nSr3CwwM1LQmpZRLuCtttiLnrUig7mjfvm2j2HrwBBv3J7F+/3G2/HOc1IxsABqEBhDo612khxsqLw7UgNtmbJ/zC/xRAAT6ejO2z/lubJWqCppGojyB/h0qpVwhJ9fwzMrdDtMlnv58Fz1bRRAa6Ftp50vPymFPQio74pKLPe+EZTs4diqTBqEBtkcg4bX98fayrosVCdQ/2RzL+E92kJ6Vm7fvfxZu5eGFkGurk3R+ZG0GdGjIRc3qclHzukSHBRY5J1RuHKgBt439F/jsF7tJTMkgLNCXyQMu8IjbHkoppZSqGpXdk1qWfavynMdPZbLl4HE2/3OCzQeOs+3gCU5lFu25BTh6MpP2U74kOiyQ1g1CaNOgNm0ahtC6QQiN6wThlS8AdnTOtMwcdiek8FtcMjtik/ktPoU/E1PJznVYATTPyYwcnvpsV4Fl3l5CZG1/okID2H0ohTRbwGxnBeq/sfGfJE5l5HAyI5tTGdmcysyx/rU9UtKzi5zPGAj292HWsA50alKHsCC/ItvYf4ZVlf4iVvqd64hIX+AVwBt4yxjzXKH1lwEzgXbAUGPM4kLrQ4DdwCfGmNElnSsmJsZs3LjRqfZlZOdw/oRVPNz7PB688lyn9lWeb/fu3bRu3drdzVAKKP3vUUQ2GWNiXNgktyvPdVup8iquV/PZGy8sU8pDefat7HP6+3gxoENDcnNhy4Hj/H30FGAFsK2iatOpSR0+2x7P8dNFU9jqBftxV4/m7D6Uyq74ZPYdPZXXC1zL34dWUbUJ8PXi131JZOWciRe9RQiv7ceRk5nk2HaoG+xH2+hQ2jYM4cLoUNpGhzJ07s8Oc6OjwwL4/MEexJ9IJyElzfo3OZ345DQSktP56a9jxf4M6gb7EezvTbCfD8H+1qOWvzdBfj7U8vdh/k/7He4nwL7nrivx51tRJV2zXdrDLSLewOtAb6warRtEZIUxJv/XnAPAHcCjxRzmKeDbqmqjv483dYP9SEhxnDyvlDuVJQ1h7dq1XHHFFRU6T1RUFHfffTdPP/10hY6zatUqrrnmGv78809atmxZoWMppVRlK65C2VOf7aJ2QMkh0lOf7XK47+QVO0lJzyIzO5fMnFyysg1ZOdbzzOxcFm486HC/8Ut38N0fRxARRMBLQBC8vKxrv2AF3IX3zcjOZdHGWOoG+9GpSRg3dW5EpyZ1aN84lCA/6z10blrHYZD/335tCgT5aZk57ElMZfehFHYfSmFXfAo/7j1WZMaqHGM4fjqLB65oQdvoUC6MDqVBaECRz6ixfVoVk6bRirAgP8KC/GjTsOjA8W7PfeNwXF10WCA/jutV9JeRz5pdiR45Js/VKSVdgL3GmL8BbHVbBwJ5AbcxZr9tXW7hnUWkMxAJrAKqrNcnMiSAxGQNuJXn+fnnn/Oep6Wl0atXLyZMmMB115351t6mTZsKn2flypVERERU+DhKKeVpTmdms/XgCTbtP+4wMAM4diqTu/5XvjstJ9KymLh8Z4Fl3l6Cr7fg6+3F6WLSO9KyctjwTxK5uVb5UAPkGoMx9txjU2xqiACbJlxVbKdMWdMlAv286dA4jA6Nw/KWNR/3ucNjZmbn8sjVJec3lzdNoyLj6jx1TJ6rA+5o4GC+17FYEyWUSkS8gJeA24ArS9huJDASoEmTJuVqZFSIv/ZwK490ySWX5D0/efIkAC1atCiwvDjp6ekEBASU6TydOnUqXwOVUqoCqiK3+VByGhv3H2fTP9Zj16GUvDQIHy9xmG8cXtuft4eX3K931/82ciQ1o8jyqBB/Pn+wB74+Xvh5e+Hr7ZU3GBBK7r39/rGSe2+L27dhWOnzSlzfMbpc+cgVreJWnvNWJJ+6qnOxy8vLxedz9NdQ1iTyB4CVxpiDJW1kjJlrjIkxxsSEh4c73UCAqNAAEjXgVqVYtiWObs99Q/Nxn9PtuW9YtiXO3U3KM2fOHESEzZs306NHDwIDA3n11VcxxvDII4/Qtm1bgoODady4McOHD+fIkSMF9o+KimLChAl5r4cOHUr37t1ZuXIlF1xwAbVq1eLyyy9nz549Trft5MmTPPDAA0RERBAYGMjFF1/M2rVrC2yzbt06Lr30UmrXrk1oaCidOnVi+fLleeuXLFlCx44dCQoKom7dunTt2pWffvrJ6bYopSpfea+N9vzkuBNpGM5UpijL/o72fWThNjpMWU3XZ7/h3x9t4eMNB6nl78P9l7fgnREXsW3i1bw4uD2Bvt4FjhXo682T17amXaOwEh9PXtva4b7jrmlNvVr+hAT4EuDrXSDYBqsH1tF+Ze29Le++5eWOc4IVOP84rhf7nruOH8f1cipgrsi+VcXVPdyxQON8rxsB8WXctyvQQ0QeAGoBfiJy0hgzrpLbSGRIAEdPZpKZnYufj6u/k6jqwF01TZ01ZMgQRo0axdSpU6lbty65ubkkJSUxYcIEGjRoQGJiItOnT+fqq69m8+bNJfaQ7N27lwkTJjB58mR8fX15+OGHGTZsGJs3b3aqTcOHD+err77i2WefpVmzZsyePZs+ffrwww8/0KVLF44dO0b//v0ZMmQIU6dOJScnh+3bt3P8+HEAdu3axdChQxk7diwvv/wyp0+fZuPGjXnrlVKVozy9zWW9Nmbn5HIiLYukU5kcO5nJ8dOZTFqx02Fu87gl21m6JY4se060LRfanhedlW04nJpO4Y7qHGPIyM5lUv82xDStS6sGtfH1LviZ7o6e1OrWe+upPcbVjasD7g3AuSLSHIgDhgL/KsuOxphb7M9F5A4gpiqCbYCoEOu2++HUdBrV0RkJz3ZTPt3JrvgUp/bZcuAEmTlFSxY9tng7H60/UObjtGkYwqT+Fzh1bmc8+uij3HvvvQWWvfPOO3nPc3Jy6Ny5My1btmTDhg106dKl2GMlJSXx66+/0rRpU8BKURk2bBj79++nWbNmZWrP1q1bWbp0KQsWLGDIkCEA9OnTh1atWjFt2jSWL1/O7t27OXXqFK+//jr+/v5529ht3ryZiIgInnnmmbxl+XPYlVIV50ynQm6uITkti+OnM3n6c8d1l8ct2c77v/xD0qlMkk5nkpyWRVmLpKVn55KcloW/txcBvl7UDvCxUjVsKRt+3l58vNHxze/0rFxGdGte4vHLm2pRkX3dcc6KcMc5zzYuDbiNMdkiMhpYjVUWcJ4xZqeITAU2GmNWiMhFwCdAHaC/iEwxxlRdROJAZKgVcCemaMCtHCscbJe23F0cBaIrVqzgmWeeYffu3aSknPmi8ccff5QYcJ933nl5wTacGZwZGxtb5oB7/fr1eHt7c+ONN+Yt8/b2ZtCgQcydOzfvPAEBAQwdOpQ777yTyy67jNDQ0Lzt27Vrx6FDh7j77rsZOnQol156KUFB+v9UqcpUXPWO8Ut3sGJbPMdPZ3LitBVklyV4Ts/Oxdfbi9YNQ6gb5Efd4KKPEe9scDh+KjoskOWjupV4/B/2HvXIyhRK2bl84htjzEpgZaFlE/M934CValLSMeYD86ugecCZHu6E5KKDIdTZpzw9zCUNevn43q6V0axKERkZWeD1jz/+yA033MDQoUN58sknCQ8PJysri8suu4z09JLHLYSFhRV47ednTRxQ2n75HTp0iDp16uDrW3BWs8jIyLyUkIiICFavXs3UqVO56aabAOjbty+vvvoqTZs2pV27dixdupTp06fTp08f/P39GTRoEDNnzqRu3bplbotSqqBjJzPYsP846/clFVu9Iy0rh4TkdOoE+9IwLJA6Qb7UCfKzHsG+PP3Zbo6dyiyyX3RYIB+NLHlw97hriishV30rUyhlpzNNOpAXcOvASVWM6nJxL5yTvWTJEpo0acIHH3yQt6w8Ax/Lq0GDBhw/fpysrKwCQXdiYiJ16tTJe92jRw/WrFnDqVOnWLNmDf/5z38YPnw469atA+D666/n+uuv58SJE3z66aeMGTMGLy8v5s+f77L3olR1UFIedvyJNNbvS2L9/iTW70ti72Gr8pG/jxd+Pl5kZhe9YxcdFsjKh3oUez5Byn1trG65zUo5QwNuB8KCfPHz8dJKJapY1fXinpaWltczbZc/+K5qXbp0IScnh08++YSbb74ZsPLIlyxZQvfu3YtsHxwczPXXX8+WLVuYPXt2kfVhYWHcdtttfPXVV+zatavIeqVqMkd52GMXb+P9X/aTkJJB7HGrF7u2vw8xzepwY6doLm5el7bRoXyxI6FcgXNFr43VLbdZqbLSgNsBESEqJIAEnfxGlaA6Xtx79+7NnDlzGDt2LH379uW7775jwYIFLjt/hw4duPHGGxk5ciRJSUk0bdqU2bNns3///rzA3z6ocuDAgTRq1IiDBw8yb948evWy6tPOmjWL7du307t3bxo0aMDvv//OsmXLuP/++132PpSqDl5Y/XuRPOysHMPmAyfoc0EUd3VvzkXN6tK6QUiR0nUV7W2ubtdGpaqaBtzFiAoJ0JQSdda58cYbeeqpp3jjjTd444036NGjB8uWLeOCC1w3Lvl///sfY8eO5b///S+pqam0b9+eVatWcdFFFwHWoMns7Gwef/xxjhw5QkREBAMGDMirStKhQwe++OILxowZw/Hjx2nYsCGjR49m8uTJLnsPSnkqYwy7DqWwfGs88Sccf4YZA7Nv7VzqsTRwVqryiClrXZ5qKCYmxmzcWL6pWf/90Ra2x57g27E9K7lVyp12795N69at3d0MpYDS/x5FZJMxpuTp7s4yFblu12T/HDvFiq3xLN8Wz97DJ/HxEny8hPRi8rB/HFfyjIZKKeeVdM3WHu5iRIX482VyOsaYUqdLVUoppaqSo8GP3VrW5/Pt8SzbGs/WgycA6NK8LtNuaMu1bRvw7R9HqsXgbqVqAg24ixEZEkCGrdh+WJBf6TsopZRSVcDR4MeHF27Nm1mxdYMQxl3Tiv7tGxKdr+50dR3crdTZSAPuYkSFnikNqAG3Ukopd3E0CU2ugVr+Pix94FLOi6xd7L6ah62UZ/BydwM81ZnJb3TgpFJKKfcwxhQ7Cc2pjOwSg22llOfQgLsYkSFnpndXSimlXO1wajr3vrep2PU6bblS1YcG3MWI1OndlVJKuYExhuVb47h6xnes++MIA9o1INC34Me1Dn5UqnrRHO5i+Pl4US/YT2txK6WUcpkjqRlMWLaD1TsT6dA4jBcHt6dlRK0Sp2hXSnk+DbhLEBkSoCklSimlqpwxhhXb4pm0YienM3MYf00r7u5xTt4MkDr4UanqTQPuEkSF6vTuSimlqtaR1Az+u+w3Vu1MsPVqt6NlhA6GdIvUBPj2BYhdD/f94O7WqMrgIb9TzeEugfZwK0/Tr18/LrzwwmLXjx49mjp16pCRUbaxB3v37kVEWLVqVd6yRo0aMW7cuBL327p1KyLCDz84d/GaM2cOK1asKLK8LOesLNnZ2YgIc+bMccn5lCqOvVf76hnf8s2ew4y7phVL7r9Ug213SE2Azx6GV9rDlvcgYYe7W6QqysN+p9rDXYKokACOncokIzsHfx9vdzdHKYYNG8att97Kzp07ueCCCwqsy8nJYfHixdx44434+/uX+xyffvop9evXr2hTHZozZw4xMTEMGDDAZedUylPkz8OODAkgorYf2+NSaN84jJe0V9s9/vkZVj4Kh3eDlzfkZLq7Raqi7D3aW94HciEny90tArSHu0RRoVbQcjhFK5UozzBw4ECCgoJYsGBBkXVr164lMTGRYcOGVegcHTt2pHHjxhU6RnU4p1KuZJ8tMu5EGgZrUrXtcSn0b9eAJfd1PXuDbXsv45zu7m7JGdkZsGMxzO8H7/SFxN/A5BQNtn+YASePuKeNnswTf6f5LRoBG9+GnAyPCbZBA+4SaS1uVWYuugDVqlWLfv368fHHHxdZt2DBAiIjI+nZsycAcXFxjBgxgubNmxMYGMh5553HpEmTyMoq+QLkKL3j1VdfpXHjxgQHBzNw4EASEhKK7Dd9+nRiYmIICQkhMjKSgQMH8tdff+Wt7969O9u2bePtt99GRBAR3n///WLPuWDBAtq2bYu/vz9NmjRh4sSJ5OScmW3vrbfeQkTYuXMnV111FcHBwbRu3Zrly5eX8lN0bNasWbRs2RJ/f3/OPfdcZs2aVWD9gQMHGDRoEOHh4QQGBtKyZUsmT56ct37Hjh306dOHOnXqUKtWLdq0aaNpKyqPo9kiATYfOIGP91n4Uexht/ML+GEmLLkLThyA7o9A+1vAJwC8C80q/dVkeLk1LLoDjv3l6Eg1iyf/Tu2MgcA61nPxth757VoBr18Mnz8Cvy2Fk4eLP1Ylf65rSkkJ8k/vrpRD9ltXWz8Ak+uS25HDhg1j4cKFbNq0ic6dOwOQlZXFJ598wi233IK3t3WBOXLkCPXr12fmzJmEhYXx+++/M2XKFI4ePcrrr79e5vMtWbKEBx98kFGjRtG/f3/Wrl3LPffcU2S72NhYHnzwQZo0aUJycjKzZ8+me/fu/PHHH9SuXZu5c+dy/fXX07p1a8aPHw9Ay5YtHZ5z5cqVDBs2jBEjRvDiiy+ydetWJk6cSFJSEq+99lqRn8fIkSN57LHHmDlzJkOGDGHfvn00aNCgzO9x9uzZjBkzhkceeYTevXvz9ddfM2bMGDIzM3n00UcBuPXWW8nJyeGtt94iJCSEv//+mz///BOwcnH79etH+/bt+fDDD/Hz8+P3338nJSWlzG1QZ689CanFzhYZX8zyais1AdZMgl3LXHZNLLYd374AB3+FHo/Apnfgkgfg/Gug023QKAbO6Qleti87V02Cb58veC0ftQE2zYftC8Db19ru2F8QEArBDlLgyjs4z0MG9RXL3r7N71o/G1P0i6PTx6qq9yoCrftB8x5wwQ1FP5/9a0FoI9i2ADa8Ze1T/zy4czUE1bV6xE8fq5LPdZcH3CLSF3gF8AbeMsY8V2j9ZcBMoB0w1Biz2La8AzAbCAFygGnGmKLdfJVIp3evQd65ruiyC66HLvdA5mn4YHDBddmZ4OsPsRtstyLz9Rrbj3XRndD2JkiOhaX3Fj3+paOti7+TrrnmGsLCwliwYEFewL169WqSkpIKpJN06NCBDh065L3u1q0bgYGB3Hfffbzyyiv4+JTtv/+0adPo169fXqDbp08fEhMTmT9/foHtXnnllbznOTk59O7dm/DwcD799FP+9a9/0aZNG4KCgggPD+eSSy4p8ZwTJ07kqquuYt68eQD07duX3NxcJk6cyJNPPlkgmH700Ue5/fbb895zVFQUn3/+OXfffXeZ3l92djZTpkzhrrvuYvr06QBcffXVHD9+nGnTpvHggw/i5+fH+vXr+eSTT7jmGut3Zr+TAJCYmMiBAwdYtWoVrVu3BuDKK68s0/nV2Wvf0VPM/OoPVmyLRwDjYJuzZrZIY+CfH2HBLZB+wvE2u5bD+deBdxWGHvnzd3OzrIBp8QgIbQLZts/ykIbWI7/akdDvZbj8cSvwjl0P4edB32eg95QzAfeqcfD3OmjdHzqPgGbd4WRi+QI0N3TYlMviEXDgF6uNhf3fZXDvd9bzfd9ZPcr1z4XgcCv4tavq9xq7EVLioc0A6PCvM8sL/05b9LIeOdlwaBv88wMk/Gb1iqcmWJ/fSX9Z+fy52ZXaRJcG3CLiDbwO9AZigQ0issIYsyvfZgeAO4BHC+1+GrjdGPOniDQENonIamNMMf+zKy400Bd/Hy9NKVFFHf0dMlJx/BFatfz9/bnhhhtYuHAhL7zwAiLCxx9/TNOmTQsEsrm5ucyYMYO33nqL/fv3k55+5u84NjaWZs2alXquzMxMtm3bxgMPPFBg+Y033lgk4P7pp5+YOHEiW7ZsISkpKW/5H3/84dT7y8rKYuvWrbzxxhsFlg8ZMoQnn3ySX375hRtuuCFv+dVXX533PCIigvr16xMbG1vm8x04cIDExEQGDy74pWrIkCG8+eab7Ny5k44dO9KhQwcef/xxDh8+TK9evQrknIeHhxMdHc29997L6NGjueKKK4iIiHDqfauzR9yJNGZ99SeLN8fi5+3FfZe3oHGdQJ76bHeBtJKzYrbI3Bz4/XP4cSbEbYLAutCwMxzeWTSwWng7hDaGa56HVg46OSoiNQHWPgPbPy563luWQIueVhBVGnvgnZ892AboPdXq9d72Efy2BPxDICvNCi7zn3PPF2ee1z8P6rWAzFNWUJp23Epn2LfO+gjJ9Zw84wKMgR2L4OIHILy142C5Rb6OhTWTIH6z9TwgFOqdC40usravyi8V2xfB8lFQ9xw4/9qiX+gc/k59oFFn62G3eIQVbGMqPdgG1/dwdwH2GmP+BhCRBcBAIC/gNsbst60r8FXKGPNHvufxInIYCAeqLOAWEasWtw6aPPuN+Lz4dX5BRdenJha9/VjcsUIblXz8chg2bBjvvPMOP//8M506dWL58uWMGjUKydej8NJLLzF+/HieeOIJevToQVhYGL/88gsPPvhggeC7JIcPHyY3N7dI8Fj49b59++jTpw+XXnopc+fOpUGDBvj5+dGnT58ynyv/OXNycoiMjCyw3P46fzAPEBYWVuC1n5+fU+c8dOhQgeMXd77FixfzxBNP8NBDD5GcnEzHjh156aWX6NmzJ97e3nz55ZdMmDCBESNGkJ6eTrdu3Xj11Vdp3759mduiqrfDqem8sfYvPvz1AAC3XdKUB3q2IKK2dbc0yM/n7Jst8mSirQe5EVz3EnS4BXwDHV8jh3wAv86xglSA5Dir4yKiVfnOfTrJ6lXf/4OV7pB12vF2515VvuM7EtHa+sJw1WSY0wOO/el4u4+Gnnl+1RToPsb6UpB/uSc7dRQ+GwO7P4W2g2DQ22d6ivP/Tq+adGafm9+Fo3vg6J+2xx+wcymcOuK4d7yicnNh7TT4/kVo2g1ufq9id08GzS/+c70SuDrgjgYO5nsdC1zs7EFEpAvgB1T5KIbIkAASNaVEFVb49qOLbwn26tWLyMhIFixYwKFDh0hNTS1SnWTRokUMHTqUqVOn5i3bvn27U+eJiIjAy8uLw4cLDiwp/PqLL74gIyODZcuWERho3SLPzMzkxAnnvw9HRETg7e1d5ByJiYkA1K1b1+ljlsSenlLa+Ro1asS7775LTk4O69evZ+LEiQwYMICDBw8SFhZGmzZtWLp0KZmZmXz//fc89thj9OvXjwMHDhT4IqSqv8LTrI/q2YIDSWn876f9ZObkMrhzI/595blEF0oXqdazRdpTAg78DO1uhsRdcNObVmrGnauhQYeCwY6jFI3W/ayH3U+zrAD8nJ5wyf3QsveZnOr857Tn+548AgEh4OMPv8yBVY9b2/kEQoP2Vmqfo571quAbCHd8Xvz1f+S6M89r29JXQqKt5aePWV8Q9qyyelIrkhNdFXZ/Cp+OgYwUq0e/62hruaPfaX5hja1Hy3xfcPK+eL1vpWLmvyuclQ6+AeVrY042LL7DamvH2+C6l8HHr9TdSlTFn+uuDrgdfeo4dU9eRBoA7wHDjSn6lUlERgIjAZo0aVKeNhYQFRLA1oNV1omuqrvSLkBVxNvbm8GDB7No0SLi4uJo3bo17dq1K7BNWlpakXrcH3zwgVPn8fPzo127dixfvrxATvTSpUuLnMvb27tAXviCBQvIzS34X7Qsvc++vr507NiRRYsWFRicuXDhQry9vUvN/3ZW06ZNiYyMZNGiRfTu3bvA+erUqVOk3rm3tzddu3Zl4sSJXHbZZRw4cKBAL7ufnx9XXnklY8aM4fbbbyclJYXQ0NBKbbNyH3t5P3tqSNyJNJ745DcABnZoyJirzqN5/WB3NrFyOcqJ/mqyFSRnpVmBZ6OY4vd3dDvf7rKxVq7vhrfgw5utlIBL/22lBdjPabKttJXXL4Yjv3aaFB4AACAASURBVMOtS6HlldC0K/T6r5VD3bDTmWCrpLuPla2kAK1hx6Lb+wacWd7yKsdtTYkvml/uSn9+BR/fan2BueEzq0e/sJJ+p8Vta/8ZbXnPNubJWNVO7vgc6jsePF8ibx+o0wz6PGMNhq3MTo0q+lx3dcAdC+QvttsIiC/rziISAnwOTDDG/OJoG2PMXGAuQExMTIUTbKNCA0jYmY4xRnupVPGcuQBVkmHDhvHaa6/xySefFOjFtuvduzezZ88mJiaGc845h3fffZf9+/c7fZ4nnniCm2++mdGjRzNgwADWrl3LV199VWCbK6+8kscee4wRI0YwYsQIduzYwYwZMwgJCSmwXatWrVi7di1ffvkldevW5ZxzznHYYz1lyhSuu+467r77bgYPHsy2bduYPHky9913n1PVR8rC29ubSZMmMWrUKOrUqcOVV17J2rVrefPNN3nhhRfw8/Pj2LFj9O/fn9tuu43zzjuPtLQ0XnzxRRo2bMj555/P5s2bGT9+PEOGDKF58+YkJSUxffp0OnfuXO2D7TIMdG8KzMNK8UsCbjXGxNrWPQ/Yk3WfquqB7q5QXHm/iNr+vDLUQZBVXeUf5JabXTCndeS30LBD8fuWVXB9uOxR6PaQNaDyp1fh5zdg1XgryM6f2xzaCNoNgXq24KxBe+tRmDs6Qcp7zvz7rRxr9dbO7QnDPoTozqXvX5lOHbV+Hy16wYBXof2wgrnrFVX4Z/TXN9CkK9Rtbq2P3WSlFvmV8mU1dpN1F6RhR7j66cprX0ltriSuDrg3AOeKSHMgDhgK/KvkXSwi4gd8ArxrjFlUdU0sKDIkgMzsXI6fzqJucAVvVyhVibp27UqzZs3Yv38/Q4cWzQucMmUKx44d44knnkBEGDRoEDNmzOD666936jyDBw9m5syZvPDCC8ybN49evXrx5ptv5lXrAKs6yNtvv83UqVNZsmQJHTt2ZMmSJUXONXHiROLi4hg8eDApKSm899573HrrrUXOee211/Lhhx8ybdo03n33XSIiInjssccK1L2uTPfffz+ZmZnMmjWLGTNm0KRJE2bMmMFDDz0EQFBQEG3atGHmzJkcPHiQ4OBgunbtypdffom/vz8NGzYkPDycp59+mvj4eOrUqUOvXr14/vnnq6S9rlLGge4vYl2X/ycivYBngdtE5DqgE9AB8Ae+FZEvjDHVulZicWX8jqRWg7E+JZVkO3XUqqucsMOaCGb3p1YPtqOb0JURbOfn7QsXDrImK0nY7jjf99Ylzh3TDZ0g5T5n7UgY8q6VpvPREHjnWrh+NrS9sfLbWFhGKnw5wfrC88AvUDsKOt1ededz9DPKSoMPB4N4WXc4Yu6yyvcVtmOxNTiyYUcY8UXl9mq7gBjj2ioLInItVtk/b2CeMWaaiEwFNhpjVojIRViBdR0gHUgwxlwgIrcC7wA78x3uDmPM1uLOFRMTYzZu3Fih9n6+/RCjPtzMygd70KZhSOk7KI+2e/fuvLJtSrlbaX+PIrLJGFPC/fqqJSJdgcnGmD621+MBjDHP5ttmJ9DHGBMr1m3AZGNMiIiMBfyNMU/btnsbWG2MWVjSOSvjul2Vukz7isMOguvosEB+HNfLDS0qA0cl2QbNs3KfW11r5cM+G32mbF7thlZpN/GycrYLp2ZMTq6idpaQDlJV5/Q0p45aKR0Hfrbyki+6q2rOk5oAn/0H/lxj3b249N/Q88ny51RX1IFfzvR8B9Wz8sa73AP+ta3BkV9OgF9eB98gGPMbBNdzTztLUdI12+V1uI0xK4GVhZZNzPd8A1aqSeH93gfer/IGFmKf3j0xJV0DbqVUTVOWge7bgJuw0k5uAGqLSD3b8kki8jIQBPQkX0Wq/Cp77E1VyczOxc/BrJAeW94vf6Cdk1mw53jxndDkUivg9vaBfjMhpAFEXlgwmPGUnOiaIrg+3L4c1ky0ShlWttQEWPc8bP7fmcGad66GJpU7NsZpTS6B2z6Bg+ut3/3XU6DxxdZA0/cGwvH9gJdVicZDg+3S6EyTpbBP766zTSqlaqCyDHR/FHhNRO4AvsNKF8w2xnxpu2P5E3AE+BlwWNy2ssfeVJVXvv6D2BNpjOjWjC93Jnp+eb+SJiy59zsIz1eOr8OwottA9cqJPlv4+FulB8Gqhb3uOeg8vGKDKQvPAJq/Moq7g+38Gnex0of2fW/VOd/6AWTb7yhVQWlBF9KAuxT2+qk626RSqgYqdaC7MSYeuBFARGoBNxljkm3rpgHTbOs+BIopWuz51u9L4o11fzEkpjGT+l/ApP4XlL6TO508bKWMnH8t7P2qaE+xowGHJalOOdFnk2N74efXrB7poR9CdKey72sMHN4Fv6+EH2dYE+9UF+ueKf7LYjVV9N6YKsDPx4v6tfx0tkmlVE2UN9DdNnB9KLAi/wYiUl9E7J8l47EqliAi3rbUEkSkHdAO+NJlLa9EKelZ/OfjrTSpG8TE/m3c3ZzS/fElzL7UmhCmdX94aLtVq9gnALx18H+1Uv9cuOtL8PK1BlP+trT0fQAO74ZZHay/g7VPQ92WEB1j9Z5Xh7+BQfOh84iz6m9We7jLIDIkQFNKziJa4lF5AlcPWC8PY0y2iIwGVnNmoPvO/APdgSuAZ0XEYKWUjLLt7gt8b/u/loJVLrDy50t2gYnLfiMhJZ3F93Ul2N+DPzaz0qzc3/VzIbItDP/0TB3lmpyiUd1FXgD3fGMNplw8ApJjoduD1rrUBGu2xb/XQdPu1p2LS+6DsCYQ0Qa6jYHzr7Gqj4Brc/Ir4izM5/fgK4fniAoJIK6YUlCqevH19SUtLY2goCB3N0XVcGlpafj6VmKd2ypShoHui4HFDvZLB6pBd3DJlm+NY9nWeP5z1Xl0bFLH3c0p2baPrGD7klFw5cSiFSc0RaP6qhUOw1fA549YQXVqAiy5B/754UzaRfpKqNPUeu4XDMM+Knqc6pYfX93aWwINuMsgMjSAzQeOu7sZqhJEREQQFxdHdHQ0gYGB2tOtXM4YQ1paGnFxcURGRrq7OaoEscdPM2HZb3RuWodRPVu4uzmO5ebC8X1QrwV0Gm71bDfu4u5Wqarg4w+9JpypPJOdQYExzGP3ln2ymur25au6tdcBDbjLICokgOOns0jPyiHA19vdzVEVYJ/5MD4+nqysrFK2Vqpq+Pr6EhkZWWQmTuU5cnINDy/chjEw4+YO+DgoB+h2KYdg2f3WZDGjN0JQXQ22z3YlVZ6pzJkhVaXTgLsMomylAQ+nZNCknqYiVHchISEa6CilSjT3u79Zvy+JFwe398zr/u7PYMW/rclq+j4LgR6e7qIqx6D5Z01Oc03jgV/ZPU9kqNbiVkqps1ZqAnz2MMzpDsBvccm8vGYP113YgJs6eVh97ZwsWHoPfHyLNQnIvd9B5zuq3TTXqpzsqRVaeaba0R7uMojSyW+UUursk/Q3fPcS/LY4r7cwLTOHBxdsoV6wP9NuaOtZ4zxSE6zezR2LQbyt3u3657q7VcodzqLBhDWFBtxlYA+4E3XyG6WUqv5SE2DlWNi9osiqaSt38feRU3xw98WEBXlIz+GRPbDoDjj2F2DOqslAVAWdBYMJawoNuMsgJNCHAF8v7eFWSqmzwUfDIH6zw1Xv/3KAe3o0p1vL+i5ulAO5ObD5Xfj8UaieJcyVUjaaw10GIkKUTn6jlFLVm32yoWELoGk3W/5rwcoOrRuE8Gif893QuEIO/Apv9oTPxkB0R7jgBs3XVaoa04C7jCJDAjSlRCmlqqtjf8GbvSB+i3UbfsRK28Cz2zE+AWTZbvjOucoXf3Lc3Fhgz0o4dRQGzYO71sDg+TpQTqlqTAPuMooK1R5upZSqlnZ/CnOvsCaISU85s9yW/7q4x0o+yr6C48EtaLp8EHwwCNJOuLaN2Rnww0z4a631+vLHYPQGaHvTmQokhStURF3o2jYqpcpNc7jLKCokgMMpGRhjPGvUulJKKcdysuHrKfDTLGjYCW7+H4Q1AWDZljimr95D/Ik0DNAq6iFue6iHNT36in/DvL5wy8K87avUH1/CqnGQ9Jc1LXuLntbU3MXRgXJKVTvaw11GkSEBZObkknRKi8wrpVS1sHm+FWxfdDfcuapAsD1+6Q7ibME2wP6jp1i+NR46/AtuXQop8fDWVRDneHBlhaUmwOK74dlG8OFgEC+4ZQn0faZqzqeUcivt4S6jBvkmv6lXy9/NrVFKKVWsrDTwDYROd0BYUzi3d4HV01fvIS2rYJ52enYu01fv4fqO0XDO5XDXl/DBYGtGv+hOlde21AT49gXruDlZYHKg91Nw8X3go3nZSp2tNOAuI/tsk4kp6VzQMNTNrVFKKVWEMfDza/Dr/8E930CtiCLBNkD8iTSHuxdYHtEKRq6FANv1Pu14xaZPT46HVY9bgyExVsk/u24Plv+4SqlqweUpJSLSV0T2iMheERnnYP1lIrJZRLJFZFChdcNF5E/bY7jrWp1vtsnkDFeeVimlVFmkJ1s90l9OsJ77BBS7acOwwLItD65vlQ08nQT/dzmsGl8wUC6Lnctgyd3wyoXWRDu52c4fQylV7bk04BYRb+B14BqgDTBMRNoU2uwAcAfwYaF96wKTgIuBLsAkEalAd4Nzwmv7I6LTuyullMf5+1uY0Rb2rrGmPM9IgYCQYjcf2+d8An29CywL9PVmbHH1twNCodV18Msb8PFtkHnqzLrUBPjsYZjT3ao08ve3VrURu98Ww9/r4PzroPkVWtJPqRrK1SklXYC9xpi/AURkATAQ2GXfwBiz37au8Ny1fYA1xpgk2/o1QF/go6pvNvh6e1G/lr/W4lZKKU9hz4fe9M6Z6c5N6b3H13eMJiMrh8eX7gAgOiyQsX3Ot/K3HfHyhr7PQp1mVjWR+ddB/1et82553+q1NjnwfHPIOgVevtDxVquHfMBr4B8CXrb+rdRE+PZ5K4fb5EKODsRXqiZwdcAdDRzM9zoWq8e6vPsWc3WsGjrbpFJKeZDFI+DAL2eCbSfENK8LwIwh7bmhY6Oy7XTxveBfG5aPhrmXgZdPwYC5wzBoeRU0625tBxAYVvAY9pJ+lz9uBd6x651uu1Kq+nF1wO2ogLVxsKzc+4rISGAkQJMmlVs/NTIkgNjjpyv1mEoppcpp0Pxy9xYn2jpPIkOKz/V2aMt7tgDfFD3fdS+V/ThaS1upGsXVgyZjgcb5XjcC4itzX2PMXGNMjDEmJjw8vNwNdSQq1F97uJVSylMUnnnRifxoe8Ad5WzAPWg+xNypudhKKae4OuDeAJwrIs1FxA8YCqwo476rgatFpI5tsOTVtmUuExUSwInTWaRn6QhzpZTyGOWY8txecSoq1MmAuwJBvlKq5nJpwG2MyQZGYwXKu4GFxpidIjJVRAYAiMhFIhILDAb+T0R22vZNAp7CCto3AFPtAyhdxX7rMVF7uZVSHk5E+olIzZpN2B4M3/dDqZsmpqRTO8CHIL9yZlaWI8hXStVcLp/4xhizElhZaNnEfM83YKWLONp3HjCvShtYAntPSEJyOk3rBburGUopVRbLgcMi8i4w3xiz290N8iQJyenO5287ornYSqkyqFm9HxWUN/mN9nArpTxfC2AucDPwm4j8LCL3iEjxBaprkISUdOfzt5VSqpw04HZC/undlVLKkxlj9htjJhljmgO9gb3ADOCQiLwnIj3d20L3OpxSST3cSilVBhpwO6G2vw9Bft46vbtSqloxxnxjjLkNOA/YBNwCfCUi+0TkPyLi8vRCd8rNNRxOzSAq1N/dTVFK1RAacDtBRIgKCdAebqVUtSIil4vIfGAP0BZ4HavS0yJgCvCu+1rnekdPZZCdazSlRCnlMjWqV6MyROpsk0qpakBEmgLDbY9mwDqsScGWGmPst+m+FpGfgffd0UZ3SbTdpYzQgFsp5SIacDspKjSA9ftcWo1QKaXK42+sycHmA/OMMfuK2W4nUKPmF08o76Q3SilVThpwOykyJIDDqenk5hq8vBzNNq+UUh6hP7DKGJNb0kbGmD+AGjWAMm+WSWcnvVFKqXLSHG4nRYX4k5VjSDqd6e6mKKVUSb4HIh2tEJEGIlLLxe3xGIkp6Xh7CfVr6aBJpZRraMDtpPyT3yillAd7G5hazLrJwFuua4pnSUhOJ7yWP956l1Ip5SIacDtJp3dXSlUTlwGfF7NupW19jZSQkp43r4JSSrmCBtxOyuvh1oBbKeXZQoHTxaxLB+q4sC0eJTElncjamk6ilHIdDbidFF7LHy+BRE0pUUp5tj+B64pZdy3wlwvb4lESktN1wKRSyqW0SomTfLy9qF/LX3u4lVKe7lVgjohkYpUGPAQ0wKrLPQq4331Nc5+0zBxS0rN1WnellEtpD3c5RIUGkJCi07srpTyXMeZNYBLwALAdOGL7dxQwwba+VCLSV0T2iMheERnnYH1TEflaRLaLyDoRaZRv3QsislNEdovILBFx+yjFRK3BrZRyAw24yyEyJEBTSpRSHs8Y8zTQECu15Hbbvw2NMc+VZX8R8caaBv4aoA0wTETaFNrsReBdY0w7rKooz9r2vRToBrTDmk7+IuDyir6nikrQGtxKKTfQlJJyiArR2SaVUtWDMSYZWFXO3bsAe40xfwOIyAJgILAr3zZtgP/Ynq8FltlPDQQAfoAAvkBiOdtRaew93JpSopRyJQ24yyEqNIDktCzSs3II8PV2d3OUUsohWwpHN+A8rOC3AGPMG6UcIho4mO91LHBxoW22ATcBrwA3ALVFpJ4x5mcRWYuVOy7Aa8aY3cW0cyQwEqBJkyalva0Ksc+hEBmiVUqUUq6jAXc52HtGEpLTaVY/2M2tUUqpokQkEvgaqwfaYAW92J7blRZwO8q5NoVePwq8JiJ3AN8BcUC2iLQEWgP2nO41InKZMea7Igc0Zi4wFyAmJqbw8StVQko6wX7e1A7wrcrTKKVUAU7lcItIhIg0z/daRGSkiMwUkf6V3zzPZB9so5VKlFIe7CUgGWiMFThfDDQD/otVMvC8Mhwj1ra/XSMgPv8Gxph4Y8yNxpiOwJO2ZclYvd2/GGNOGmNOAl8Al1TkDVWGwykZOumNUsrlnB00OZ8zuXoAU7B6SPoCn9h6OEpUhhHv/iLysW39ryLSzLbcV0T+JyI7bCPexzvZ9koTFWrditTZJpVSHuxyrKD7kO21GGMOGGOeAd6n9N5tgA3AuSLSXET8gKHAivwbiEh9EbF/lowH5tmeHwAuFxEfEfG1tcdhSokrJaSka4USpZTLORtwdwK+AbBdYO8HnjDGtAKmAWNK2rmMI97vAo4bY1oCM4DnbcsHA/7GmAuBzsC99mDc1fKnlCillIcKA44YY3KBFCAi37qfgEtLO4AxJhsYDazGCpYXGmN2ishUERlg2+wKYI+I/AFEYn0WACzGmlxnB1ae9zZjzKcVflcVlJCsAbdSyvWczeEOBY7ZnncG6gIf2F5/AzxSyv5lGfE+EJhse74YKzdQsPIGg0XEBwgEMrE+RFyudoAvwX7emlKilPJk+7AmugHYCdwCfGZ73R8oU6klY8xKYGWhZRPzPV+Mda0uvF8OcK/Tra5CubmGw6npmlKilHI5Z3u4Y7F6psGq5/q7MSbO9joUKC0CdTTiPbq4bWy9K8lAPawL+ims26MHgBeNMW6rzRcZGqApJUopT7YSuNr2/GngJhGJFZF9wINYM1HWKEmnM8nKMUTW1golSinXcraHex7wgohchRVw58+jvoTS8/PKMuK9uG26ADlYkzjUAb4Xka/sveV5O7uovFRUSICmlCilPJYxZly+51/YJqK5AesO4RpjzBdua5yb2K/ZOumNUsrVnAq4jTHPikgc1oxh/+bM4Biw0kveKuUQpY54z7dNrC19JBTr1ue/gFXGmCzgsIj8CMQABQJuV5WXigoJ4Fed/EYp5YFExB+rXN9nxphtAMaYjcBGtzbMzQ6n6qQ3Sin3cHpqd2PMu8aYfxtj3jbGmHzL7zPG/K+U3Usd8W57Pdz2fBDwje08B4BetlKEwVg96r872/7KYk8pyc2t0pKxSinlNGNMBlaJvjB3t8WTJCRnANrDrZRyPWfrcLcWkUvyvQ4SkWdEZJmI/Lu0/cs44v1toJ6I7AUeBuy3RV8HagG/YQXu7xhjtjvT/soUFRJAdq7h2KlMdzVBKaVK8ivW4HZlk5CSjgiE19IcbqWUazmbw/0GVjmpX2yvpwN3AN8Dz4tIgDFmekkHKMOI93SsEoCF9zvpaLm72G9JJqakE64DcJRSnucx4EMRycS65iZSaMyMMea0OxrmLonJ6dSv5Y+Pt9M3d5VSqkKcveq0BX4GayIa4FZgjDGmL/AEcGflNs9z2W9J6sBJpZSH+hVoAczCmlkyBUgt9KhRdNIbpZS7ONvDHcyZ2teX2F4vtb3eDDStpHZ5PJ3eXSnl4e6kaBWoGi0xJZ1GdYLc3QylVA3kbMD9N1ag/R1Weaktxhj7RDj1qUE9JvVr+eElOr27UsozGWPmu7sNniYxJZ2YZnXc3QylVA3kbMA9A5gtIoOBjsCIfOuuANw2iNHVfLy9CK/tryklSilVDaRn5XD8dJamlCil3MLZOtxvi8ifWHW4xxljvs63OgmYWZmN83RRIQGaUqKU8kgicoRSUkqMMREuao7bHU6xSgJqDW6llDs428ONMeY7rJSSwssnV0aDqpPIkAD2Hzvl7mYopZQjr1M04K4L9AJCsEqw1hj2zhENuJVS7uB0wC0iYcC9QHesi3cSVlnAucaYE5XbPM8WFRrAL38fK31DpZRyseI6QUREgIVAtksb5Gb2gFsnvVFKuYOzE9+0wJp4ZipWhZIDtn+nAttt62uMyJAAUtKzScvMcXdTlFKqTGwz976FNQlZjZGYrD3cSin3cbYO9wzgOHCOMaaXMWaYMaYXVq3XE8DLld1AT6alAZVS1dQ5gJ+7G+FKiSnpBPp6ExLg9I1dpZSqMGevPFcAw40xcfkXGmPiRGQK8E5lNaw6yD/5TfP6wW5ujVJKnSEiDzhY7Ae0Bm4BFrm2Re6VkJJOVGgAVkaNUkq5lrMBtwG8i1nnRQ2bZCH/9O5KKeVhXnOwLAOIBd4Apri2Oe6VmJJOZIi/u5uhlKqhnA241wJPicgGY8w/9oUi0hQrj/vrYvc8C+X1cGvArZTyMMYYZ1MGz2oJKel0aqKT3iil3MPZgHsM8A3wp4hsBhKBCKAzcBB4uHKb59lq+ftQy99HJ79RSikPZowhMSVDJ71RSrmNUz0gxpj9QCvgQWAn4Avswhrt3hVoUsnt83iRIf6aUqKU8jgiMk1E/q+YdXNE5ClXt8ldjp/OIjM7VyuUKKXcpjwT32QCc2yPPCJyE1Zt1+JyvM9KUaE626RSyiMNAyYWs+57rDTA/7quOe6TqDW4lVJupjl+FRQZEpBX31UppTxIQyCumHXxtvU1gs4yqZRyNw24KygqJIDDqRnk5taoAi1KKc+XAHQqZl0n4IgL2+JWZya90SolSin30IC7gqJCA8jONRw9leHupiilVH4LgYkicl3+hSJyLVYqyQK3tMoN7D3cEbW1h1sp5R465VYF5dXiTs7Qi7lSypNMBDoAn4rIMeAQ0ACoC3xJDcnfBiuHu34tP/x8tI9JKeUepQbcInKEsk1oUyPv1eWf3v1CQt3cGqWUshhj0oGrRaQP0BOoBxwDvjbGrHFr41wsITld87eVUm5Vlh7u16nEGSRFpC/wClY1k7eMMc8VWu8PvItV2/sYMMRWjhARaQf8HxAC5AIX2T5U3EYnv1FnldQE+PYFiF0P9/1Q9fupKmeMWQ2sdnc73CkxJYMGWqFEKeVGpQbcxpjJlXUyEfHGCuB7Y00vvEFEVhhjduXb7C7guDGmpYgMBZ4HhoiID/A+cJsxZpuI1AOyKqtt5VW/lj/eXqKVSlT1Zg+Yt34AJhdyMkvePicLMlLh+H7Y8n7Z91MuY7t+NjbGTHew7lHggDFmoetb5nqJKel0aBLm7mYopWowV+dwdwH2GmP+BhCRBcBArMlz7AYCk23PFwOviYgAVwPbjTHbAIwxx1zV6JJ4ewnhtfy1h1tVT/kD7dxs62G3+C7IPAmD5oFfMHz/Evz8hrUsO9/fu7efBtqeaRzwdjHrTgPjsQZWntUysnM4diqTSB1jo5RyI1ePIInGmgLeLta2zOE2xphsIBkr9/A8wIjIahHZLCKPOTqBiIwUkY0isvHIEddUvYoMDdDZJlX1tPB22DTPCqDzB9sA8Zsh9RBk2yrw1DsX2gyALiMhtAkg1nINtj3VucBvxazbbVt/1jucYv39RoXWyGFGSikP4eoebnGwrHB+eHHb+ADdgYuweme+FpFNxpivC2xozFxgLkBMTIxLimNHhfjz95FTrjiVUpUnaR+cToK6LSA5tmhKyINbCm7fZoD1AOg6Gr59XlNJPNtpoFEx6xoDNaKWaaJOeqOU8gCu7uGOxbrQ2zXCmvHM4Ta2vO1QIMm2/FtjzFFjzGlgJcVP6uBSUSE6vbuqZvZ9B2/2hFNH4LqX4aHt0PE28AmwUkRKUzsS+tn2az+06turyuMr4L8iEpF/oYiEA09ilQY86yXotO5KKQ/g6oB7A3CuiDQXET9gKLCi0DYrgOG254OAb4wxBmuUfTsRCbIF4pdTMPfbbSJDA0hNz+Z0ZnbpGyvlbhvegvdugOAIuOcbOOfyggF0x9sg6sKyHat2JAx4FfrPsr1uUHXtVs56HKgF/CUii0RklogsAv4CggCHaXlnm0R7Son2cCul3MilAbctJ3s0VvC8G1hojNkpIlNFxHavmreBeiKyF3gYa+APxpjjwMtYQftWYLMx5nNXtr84ebW4tVKJ8nQnj8DXU6FFL7h7DdRrUXC9PfB2trRfp9uhZW+rcklK4ZtWyh2MMQeA9sBrWHcNr7H9+yrWhDgJ7mud6ySmpOPv40Vo2utExQAAIABJREFUoK+7m6KUqsFcPtOkMWYlVjpI/mUT8z1PBwYXs+/7WKUBPUr+yW/OCa/l5tYo5UB6CvjXhlrhcNdXVqDt5V15xxeBa6fDG5fAmklw05uVd+ziaO3vUhljjmBVIwFARLyAK4DngBuxBqSf1eyT3ljFrpRSyj10avdKEGnLDdRKJcojHf4dPhoCMXdCt4cg/LyqOU/d5nDDHGjYsWqOb+dszXCFiFwMDANuBiKxxsUscGujXCQhJV3TSZRSbqcBdyU4k1JSIwb9q+pkzypYcjf4BkKTS6v+fBfcYP1rjBUMV2YvugbaThGRtlhB9lCgGZAJ+GGl6r1uS/Ery3FKmx24KTAPCMcK5G81xsSKSE9gRr5NWwFDjTHLKvK+nJWYkk67RjrpjVLKvVw9aPKsFOzvQ21/H+3hVp7DGPhhJnw0FOqdAyPXQeOLXHPu7Ez4YDCse670bZ2xeARstNUMLxxsG5dUAPV4InKOiDwhIjuAbcCjWONlbsequy3AFieCbfvswNcAbYBhItKm0GYvAu8aY9oBU4FnAYwxa40xHYwxHYBeWGUKXVoZxRhDYko6USFag1sp5V4acFeSyNAAHTSpPENqAiy6A76aBBdcDyNWQWjh+aWqkI8fBIbBDzPg6J+Vd9wBr0FwfRCvoqULX+0M6563aovXbHuBp4BU4F4gyhjTzxjzgW2Zs/JmBzbGZGKloQwstE0bwD4fwloH68GqOPWFraSry6SkZZOelas1uJVSbqcBdyXRWtzK7VITYMVD8Ep72GMblzzoHfALcn1brp4GvkHw+cOV0/uceQqWj7Ym6uk/q2jN8JCGsO4Z+OTeM/tk1cj/j/9g9WK3xRoceamtjGp5lWV24G3ATbbnNwC1RaTwYMyhwEfFnaSqZgjWGtxKKU+hAXcliQzR6d2Vm6QmwGcPw8wLYfP8gikX7qrMUDsSrvyvNcHOjsUVO1bmafhwCBz8BW56CzrdVrRm+B2fwZjf4JoXrH1OJ8GL51pT1/++0kpzyc/+M5vT/f/bu/OwKMv1gePfm0UERVEETNTUTHNNjeOx1CzLrSy3TK1OZVanc7LtlB2zfma2W53qtJm5lOlJc83K0krbyzIVyzUzS0DAFVABYXh+f7wzOAMz7LMA9+e65mLmXea952Xm4eaZ+32eysUWYIwxrYHewFvAJcD7QJqIvGF/XN7/fsoyO/B9QD8R2Yw1P0IyUFiyIiJnAF2whoP1FPcsY0yCMSYhJiamnCF6lqqzTCqlAoReNFlFmjYMIz0rF1uBITiogklOZYY50yHSaq+l4+GP7yh/LuVlCTdZFzh++yJ0uariyf+qO2Df1zByFnQeeXq5Y8xwh6gW1g3AlgfdroWfl8D29yC8MXQeBd3GweaFNfrCS2PMd8B3InIXVpI9DqsHegLWm+QWETlpjNlYhqcrdXZgY0wK1hCDiEh9YJQxJsNpk6uBFcaYvAq+pApLs5f56SglSil/04Tbk3ImsE0b1MVWYDh8PJfY8jbulRl9QUduUAOfgDcvg7yTEBQKBT7Pa9wLCoar5kJEdOV62vv+C9oNgq5Xl32fyDgY8hQMfBR+Wwcb51kXXG6aD5ha8TkxxhQAnwCfiMhtwGVYpR0jgGtEZLcxpkMpT1M4OzBWz/VY4BrnDUSkCXDEfrwHsEYscTYOp7HAfcnRwx2rF00qpfxME+6iKpjAxjlNflPmhDsrFb54Gjbb5/JxPtZ/OlpDuYVGQGwHq3cPrJEfMpKs+wcSId0+u32BTitfK+VkwvJbrET7+lVWb24g/fPVuI3105YHJw5BgzJO/W7Ls15L51EQ18m6VURwqJWsf/MCUAC22jl0p/2Cx5XAShGpBwzHSp5L2y9fRByzAwcDcx2zAwMbjTGrsGrFnxQRA3wJ3O7YX0RaYfWQf1GlL6iM0jJzaFyvDmEhVTg8pVJKVYAm3A4uibbN+oNfDjtTMwG48uVviI8KZ9Kg9gzvXsrIEEtuhD+/c7/urIut2tW8bGuGQIcDiZCyGY6nW3G6k58LIdqjUyuERVplFm0ugjMvgDb9oN+/rX/kkn7wd3Sn/e9qyD4KN39W+tjctnxYNsFKuKNaQouelT/+VW9a58Td5zv/lDWyircFSNmXMeYEsNB+K8v2pc0OvBRwW6hvjNlH8YssfSYtM0frt5VSAUETboel4+HP762ewXJauTmZV9f/Vvg4+Vg2Dyz/GcBz0p173OqVBAgKsYY6c+6RHPaK+/3G2S/0z0rznEA8czZ0GAqdRloJWHBouV+TCnD5p+B4qpWQXjzFdV3R2uZA0O1aK4neOBd63uJ5uwIbrLzNSrYHPVE1yTacPieOf0a2LLQ+b6YA5g6EUXOs6e69Qcu+/CY1M4c4LSdRSgUAHaXE4ao34bzx1lBjUr6vH59Zs4ucfNdEPTvPxjNrdrnf4Xg6vHk5/PE1DH0B7tlefJiz0jgSiLu2QvfrXfftMBR2fAALR8Gz7awa1pLU0BEbaqyCAlj5D3ijvzUaR3XQeRS07gefTbf+WXSnoMAa+u/nJXDJw3D+7e63qwznz8154yHqTDiyFxaMtHrWq5Ljc/V8Z9j8tvsJe5RXpWbk6gWTSqmAoAm3g8sf4htdE19HjbUHKceyy7WcfV9bE4KMfQcSxhdJnu3DnFUkbse+w1+FSfbnP6s/NGlnbbvjA1g9yRrRoqDgdELw4rlWQpD6c9mPq/zDGFgzBX5ZCr3+CRGN/R1R2YjA5f+xks61D7rfJnkjbF0EF02xLpT0Jsfn5u6tcNs3MHwmBIdYPeynTlTuuQtssPNDWHwd/DTPuohVE22fy7MVcPhErpaUKKUCgpaUFOX81fNnj8D2lfDDG9B1jMfSjGZR4SS7Sa6bRYW7LsjJhLoNrJrbM3tbx3J37MrE7RASBudcZt0cDv9qjdLwwywIrWclP0HBmgxUJ1//Bza8ZiXbfe7xdzTl06Qt9L7bmpQn9ziE1Xdd36KnVdscW3TmcC9zHk7w6/9A4iJrdJUzzi3f8+RkWmUjG2bC0X1w4SRo2lVLSfwkPSsXY3TSG6VUYNAebk8i46ye4sn74brlVrKdfQyO/lFs00mD2hMe6lqGEhYSxKRB7U8v2PmhNTHJnxtOP7+v9bkHJu2B6LMh74S99luTgGpjxwdWSUaXq62ZHP01qU1lXHgf3Pr56WTbGOtblgVXWSVNcZ38+7pa9LJ6uGdfCt+/VrZZMo2BNQ9aIwt9PBnqN4XRb0G/ya7fPpWnZExVmmMiMi0pUUoFAk24SxMUDPXssxR/9G+Y1Q9+/dRlk+Hd43lyZBfio8IRrHyhWcO6XHluM2uDH2dbXy9Hn+W9C7PKKiwSbvwQEiZYf/yD9EuOUgVKjftZF8PFD1oX1AZV049uSNjpf163LoHX+8LGOfDbZ4FR0tS6r1ViclZ/K3l+ZxycOHx6vfN7Ic0+JKcIHPsT2g+BW9bBhDXQabhVogKVKxlTFeaY9EZLSpRSgUCzrfK46N+Qtg0WXmWVnPT7d2HiM7x7fOGIJO9u3M/9S7ey+Mc/GZc1D75+HtoNtr6mrlPPn6/A4nbEhjzPwwzWVoEyukTKFms867oNoN/9/omhKmWlwltXwiGni4orMDqQ19SLhnGLYMPrVlnZkb1WHbbjveD4rLx2Ady5yfrdjH6r9H+CAnH0mBrs9LTuOkqJUsr/qmk3mZ80bgMT1sK54+CLp+B/o92OEjH6vOb8tXVjEj+abSXb542HMQsDI9l25kgA/rkBwhpYyxIX+zemQFB4MWlX/48ukfqLlZy+f5d/ju8NS8fDod3+jqJkItDrNrhlPSS+Y11Y/NOb1nvB8Y/pFS9ApH0in+r6jUMNlpqZQ53gIBrX0zIepZT/6V+J8qoTYdV2D30eDu6yJpkpQkR4YmQXVuT1YnazR6xtgwP4y4TGreDuRGjVF1bcCl89V7ba1ZrqnbFWmUN+rn9r3I/ugwWjrH/UBkz3XxxV7ao3IeGm6lHT/OE91kgjzom2w3k3WrPBqoCUlpFDbIMwpDpe66CUqnF8nnCLyGAR2SUie0Rkspv1YSKy2L5+g31qYOf1LUXkuIjc56uYixGxEoaJG62pqgtssOtjK0nNPABvXsFZX/6Lr6Km8djes/l890G/hVpmdRtaF4d2GW1dmPdxsV9N1QqUumhnJw5bwyamJFqTEgWFFE8IN71t/b69LXUbzOwLJ9Lhb8tPj6JRExStaQ7kxNt5fP5AjVG5lZqZoxdMKqUChk+7XUUkGHgFGAAkAT+KyCpjzHanzSYAR40xbUVkLPA0MMZp/fPAR76KuUSh9sY88R1473Y4ZyjsXW+NchAUQmxBPmfF1OOhlb+w9p4LiagTwL3cYE1vPWIWNIiH2A7eOUag1EUXlfoLzLsMTmVZY6NfPMVKrB017gX51m3VRGtYxcFPQisv/LPgOD8/zTtd1+yt34W/Fb2WIJCmondwd71DIL1vlUfpmbl0aNbA32EopRTg+x7unsAeY8xeY8wpYBEwrMg2w4C37PeXApeI/TtBERkO7AW2+SjesmlzMZzRDXZ+cHrSjAJr1ronRnQh6Wg2L376qx8DLIegIBjwCJw71nq851M4cahyz3nqBPy69vQkOxvnBsase8bAsf3W/ZhzoMso+Me3VoJVr4lrT2yPG6zRJUbNser2P7zPmjyoqmSlwgf3nK4bD6SLCL3NcZ5v+9rfkXimI41UK8YY7eFWSgUUX3e5xgP7nR4nAX/1tI0xJl9EMoBoEckG/o3VO+6xnEREbgVuBWjZsmXVRV6S5TdD6la3q/7aJpoxCS2Y/fXvDOsWT8fq1OOSkwFLb4KIaLh2afmGNDQG9m+wZuncthLyTgLGfSL5ej84fyJ0HV1loZcqZbM1dvKh3XDHJmsEkKHPu9+26OgS51wOmSnWPyc5GfDty3DBHdZzlFdOJvz+BXx4Lxz3MOW5Chw60ki1kJWbz8lTNh2hRCkVMHzdw+3u6pWiV+d52uYR4HljzPGSDmCMmWWMSTDGJMTExFQwzHIqpc7zgcvOISo8lAdW/IytoBpdjFi3oZVoZx+DOQMh6aey7bd7Lbx0HswdBL8sh45XWsOmeTpHoeHWRDxgJbKLr7NGhMhIKv7clan9zkqF5X+Hp1rBrIvh4E64aDKERpTveULDT//zsecz+HIGvNTDirlofXfReJ0vRv1kKsxobb3e3CxoeKZVO661wkpVio7BrZQKNL7u4U4CnK/+ag6keNgmSURCgIbAEaye8KtEZAYQBRSISI4x5mXvh12KUuo8oyLq8H9DO3L34i0s+P4PbriglX/jLY8WPWHCJ7BgJLx5OYyeZ03wAafrjfdvgL7/gthOEHuOlajXj7OWdRx+elbBjle4P0c3fXz6eEf/gORNsON963FMB2h7iXUx56b5FauhdcS5eQHY7KPK9L4L+t5rxVoZnUdCo1awZoo1dN8Pb8CgJyCmvVOtus0au3nZzbD3c/jHd1A/xipDuuBO6/U172nV0Gelaa2wUpWUqrNMKqUCjBgfDv9mT6B3A5cAycCPwDXGmG1O29wOdDHG3Ga/aHKkMebqIs8zDThujHm2pOMlJCSYjRs3VvGrKANH0pT0Q2FdqjGG6+f+wOY/j/Hpv/rRtGE1+0NwPB3+d7V1oeD5E63Xt3mBVavuKBPpfbdV/10Wbs5RIWOs3uc9n8LOD+HPDdbshBjXBLRxGwitZw3VeP17Vs/z1iWw7ytrKD1TAEk/2st9xHXfaRmVORvFGQPbV8LHD1qj2Jw8dPpCS4d6MdYMhhdPsZL0kpR0flStISI/GWMS/B2HL1VFu71k434mLd3KF5Mu4szoAJv/QJVLZmYm6enp5OXl+TsUVcuFhoYSGxtLgwaey0dLarN92sNtr8meCKwBgoG5xphtIjId2GiMWQXMAd4WkT1YPdtjfRljlXBT5ykiPDa8MwOf/5KHV/3C63+rZn9D68fCVfPgm/9aFz7m5+JSDfS3ldD6wrI/X0m1sCLWyByxHWDXahBzumfaWbMeVm34qROnyzAO7YLdH8Opk9aII74iAp1GwIaZ1j8IxSqlgHt3l32CFK0VVqrC0rOs9kJLSqq3zMxM0tLSiI+PJzw8XMdUV35jjCE7O5vk5GSAEpNuT3w+Tp0xZjWwusiyqU73c4ASr54zxkzzSnBedmZ0Pe669GxmfLyLtdtSGdipqb9DKp/3/gl/fu/+wsezLvbOMa9603OJxVVzim/f/yHrBlYpyedPQuIi35VnjJ7vOV6djVApn0jNyKFheCh1Q4P9HYqqhPT0dOLj44mIKOd1NkpVMREhIiKC+Ph4UlJSKpRwawbgY7f0bUP7uEgeXrWN47n5pe8QSPwxCUhlJkmJbApXvOjbCVaq06QuStVQOiRgzZCXl0d4uM7mqgJHeHh4hcubNOH2sdDgIJ4Y2YXUzByeW7vL3+GUjz+TycqMg+yPMZR13Gal/CYtM4e46nadjHJLy0hUIKnM+zHApz6smc47sxHX/rUlb327jxHd4+naPMrfIZWPP2cIrExtsz/qorUWWymfS83I4Zymkf4OQymlCmkPt5/cP/gcmtQPY/Kyn8m3VdNZBavDDIFKqVol31bAoeO5WlKilAoomnD7SYO6oUy7shPbD2TS49FPaD35Q3o/tY6Vm5P9HZpSSlVbh46fosCgJSXK70Sk1Nvnn39e6eM0bdqUhx56qPIBK6/SkhI/ys2zESSQmWNdPJl8LJsHlv8MwPDu8f4MTSmlqiXHpDdxkZpwK//67rvvCu9nZ2fTv39/HnroIS6//PLC5R07dqz0cVavXk1sbGyln0d5lybcfvTs2t0Unek9O8/GM2t2acKtlFIVkGqf1r3aTS6mapxevXoV3j9+/DgAZ511lstyT3Jycqhbt2zv4R49elQsQOVTWlLiRynHssu1XCmlVMnSHD3cWsOtnKzcnEzvp9YFZPnmzJkzERE2bdpE3759CQ8P56WXXsIYw7333kvnzp2pV68eLVq04IYbbuDgwYMu+xctKRk7dix9+vRh9erVdOrUifr169OvXz927Sp5ZLTMzEz+8Y9/0K5dOyIiImjTpg133XVX4T8LDvn5+Tz66KO0bduWsLAwWrRowa233uqyzZIlS0hISCA8PJwmTZowdOjQwkljaitNuP2oWZT78UXrh4WQk2fzcTRKKVX9pWbmEBosRNfT8e+VZeXmZB5Y/jPJx7IxnC7fDKSkG2DMmDGMGjWK1atXM3DgQAoKCjhy5AgPPfQQq1ev5rnnnmP79u0MHDgQY9zMZuxkz549PPTQQ0ybNo0FCxawf/9+xo0bV+I+WVlZBAcH8+STT/LRRx/x8MMPs3r1aq699lqX7W688UYee+wxrrvuOj788ENmzJhBVtbpmZ1nz57N1VdfTceOHVmyZAlz5syhdevWHD58uOInpwbQkhI/mjSoPQ8s/5lsp+Q6WISs3HyGvPgVjw/vzAVtm/gxQqWUql7SMnKIjaxLUJCO31wTPfL+NranZJZrn81/HuNUkdHAsvNs3L90K+/88GeZn6djswY8fEWnch27PO677z7+/ve/uyybN29e4X2bzcZ5551H27Zt+fHHH+nZs6fH5zpy5AgbNmzgzDPPBKwSlXHjxrFv3z5atWrldp/4+Hhefvnlwse9e/emefPmDBgwgLS0NOLi4khMTGThwoW8/vrrLr3ajmQ+Ly+PKVOmMG7cOObPn1+4ftiwYWU/ETWU9nD70fDu8Tw5sgvxUeEIEB8VznNXn8vbE3pSYAzXzN7Ave8mcuSED6YkV0qpGiAtK4e4BmH+DkMFkKLJdmnL/cX5YkqHVatW0atXLxo2bEhISAht27YFYPfu3SU+V7t27QqTbTh9cWZSUlKJ+82dO5dzzz2XevXqERoayqWXXooxhl9//RWAdevWERQUxPXXX+92/19++YWDBw8yfvz4Eo9TG2kPt58N7x7v9gLJNXdfyEvrfuX1L/aybmcaD17ekVE94nXWLaWUKkFqRg7t4nTSm5qqIj3MvZ9aR7Kba6Pio8JZ/PfzqyKsKhEXF+fy+JtvvmHEiBGMHTuWBx98kJiYGPLy8rjwwgvJyckp8bmiolwn1KtTxyqxKmm/d955hwkTJnDHHXfw1FNPER0dze+//87YsWML9zt8+DCNGjXyeEGno2zkjDPOKPnF1kLawx2g6oYGM2nQOay+qy9tYupz35JErnljA3sPHi99Z6WUqqXSMnP1gknlYtKg9oSHBrssCw8NZtKg9n6KyL2iHWrLli2jZcuWLFy4kCuuuIJevXp5dfi/JUuW0K9fP/773/8yZMgQevbsWSxxj46O5ujRox4T9+joaAAOHDjgtTirK024A1y7uEiW/P18Hh/RmV9SMhj84lf897Nfyc3XiyqVUsrZ8dx8jufm65CAyoW78s0nR3YJ+OF3s7OzC3umHRYuXOjV44WFuZZjFT3eJZdcQkFBAQsWLHD7HF26dCE2Npa33nrLa3FWV1pSUg0EBQnX/vVMBnSMY/r72/nPJ7tZlZjCkM5NWb4pmZRj2TSLCmfSoPYB34AopZS3FI7BrT3cqghP5ZuBbMCAAcycOZNJkyYxePBgvvzySxYtWuTV402aNIkZM2bQvXt3Vq1axddff+2yTdeuXbn++uuZOHEiKSkp9O7dm8OHD7Nq1SoWLFhASEgITz31FDfddBMhISGMHj0aYwyffvopN910E127dmXt2rVcdtllfPPNN/z1r3/12usJNJpwVyOxkXV5+ZoejDovnXsWbeGldXsK1+kslUqp2k7H4FY1yciRI3n00Ud59dVXefXVV+nbty8rV66kUyfvjJRyxx138Mcff/Dss8+Sk5PDkCFDmD9/Pn379nXZbs6cObRp04Z58+bx2GOPERcX53LB5/jx44mIiOCpp57inXfeITIykgsuuIAmTaxR1woKCrDZbKUObVjTSE1+wQkJCWbjxo3+DsMrzn/yMw5kFK+hio8K55vJ/f0QkVKqqonIT8aYBH/H4UuVabeXb0riX+8msu7efrSJqV/FkSlf27FjBx06dPB3GEq5KOl9WVKbrTXc1VSqm2QbrJ7ujJN5Po5GKaX8LzVTp3VXSgUmTbirKU+zVAL0eXod/1m7i2MndfxupVTliMhgEdklIntEZLKb9WeKyGcislVEPheR5k7rWorIWhHZISLbRaSVN2NNy8ghsm4IEXW0WlIpFVh8nnCXofEOE5HF9vUbHA20iAwQkZ9E5Gf7z1pdN1HSMEd92zXhv+v20Ofp9TynibdSqoJEJBh4BRgCdATGiUjHIps9C8w3xnQFpgNPOq2bDzxjjOkA9ATSvRlvamaOXjCplApIPu0GcGq8BwBJwI8issoYs91pswnAUWNMWxEZCzwNjAEOAVcYY1JEpDOwBqi1Vwc6Lox8Zs0ut6OU7EzN5KXP9vDSuj3M+2YfN17Qigl9WtOoXp2SnlYppZz1BPYYY/YCiMgiYBjg3GZ3BO6x318PrLRv2xEIMcZ8AmCM8fokAqmZuVpOopQKSL7+3q0sjfcwYJr9/lLgZRERY8xmp222AXVFJMwYk+v9sANTScMcndO0Aa9c24M7U7P477pfeeXzPcz75ndu7N2Km/u04YvdBz0m60opZRcP7Hd6nAQUHccrERgFvAiMACJFJBpoBxwTkeVAa+BTYLIxptgkAiJyK3ArQMuWLSscbFpGDmfHNqnw/kop5S2+TrjL0ngXbmOMyReRDCAaq4fbYRSw2V2yXVUNd03Rvmkkr1zTg132xPvVz3/jjS/3UmAgv8AaoUaHFFRKeSBulhUd2uo+rI6RG4EvgWQgH+vvS1+gO/AnsBi4EZhT7AmNmQXMAmuUkooEaiswHDyeS1yDsNI3VkopH/N1DXdZGu8StxGRTlhlJn93dwBjzCxjTIIxJiEmJqbCgdY0jsR7zd0XEhwkhcm2Q3aejWfW7PJTdEqpAJUEtHB63BxIcd7AGJNijBlpjOkOPGhflmHfd7MxZq8xJh+r1KSHtwI9fDwXW4HRGm6lVEDydcJdauPtvI2IhAANgSP2x82BFcD1xpjfvB5tDdQuLpKcvAK365KPZTP7q738cfiEj6NSSgWoH4GzRaS1iNQBxgKrnDcQkSYi4vhb8gAw12nfRiLi6Pnoj2v5YJVK1UlvlFIBzNclJYWNN9bXjmOBa4psswq4AfgOuApYZ4wxIhIFfAg8YIz5xocx1zjNosJJPpZdbHlIkPDYhzt47MMdnB1bn0s7xjGgYxzdmkcRFGR98bByc7LWfitVS9jL+iZiXaQeDMw1xmwTkenARmPMKuAi4EkRMVglJbfb97WJyH3AZyIiwE/AG96KtXBad71oUikVgHzaw23/WtHReO8A3nU03iJypX2zOUC0iOwB/gU4hg6cCLQF/k9Etthvsb6Mv6bwNKTgs6PP5ctJFzN1aEdiIsOY9eVeRr76LT2f+JR/L93Kox9sZ/LyrSQfy8ZwuvZ75eZk/7wQpZTXGWNWG2PaGWPOMsY8bl821Z5sY4xZaow5277Nzc7X1hhjPjHGdDXGdDHG3GiM8doYpY5p3bWkRAWKoUOH0qVLF4/rJ06cSKNGjcjNLdvYD3v27EFE+PjjjwuXNW/enMmTi42w7GLLli2ICF9//XXZArebOXMmq1atKra8LMdUxfl8dgBjzGpgdZFlU53u5wCj3ez3GPCY1wOsBUobUvCmPq25qU9rMk7m8fnudD7Znsbqnw+QlZtf7Lkctd/ay62U8qe0zFyCg4To+nrRpAoM48aN47rrrmPbtm106tTJZZ3NZmPp0qWMHDmSsLCKv2fff/99mjTxzsg8M2fOJCEhgSuvvNJluTePWZPpdFy1VElDCjo0jAhlWLd4hnWL51R+Ae0e+sjtdsnHsvnpj6N0axFFcJC7a16VUsq7UjNziKkfpm2QChjDhg0jIiKJjxzdAAAaqklEQVSCRYsW8eijj7qsW79+PWlpaYwbN65Sx+jevXul9q8ux6wJdGp3VSZ1QoKIL2E6+VGvfctfHv+UexZvYVViis5uqZTyqbTMHOK0fluVRVYqfPAvmNnHq4epX78+Q4cOZfHixcXWLVq0iLi4OC6++GIAkpOTGT9+PK1btyY8PJx27drx8MMPk5eXV+Ix3JV3vPTSS7Ro0YJ69eoxbNgwUlNTi+33zDPPkJCQQIMGDYiLi2PYsGH89tvpsSj69OlDYmIic+bMQUQQERYsWODxmIsWLaJz586EhYXRsmVLpk6dis12esj92bNnIyJs27aNSy+9lHr16tGhQwfee++9Us5i6bE6LFu2jL/85S+Eh4fTpEkTLr/8cvbvPz0SdWJiIpdffjkNGzYkMjKSXr16sW7dulKPX1W0h1uV2aRB7Xlg+c9k553+EIWHBvN/QztQv24o63em8/mudFZsTiZI4LwzG3HxObH0PyeW9nGRvLclRS+4VEp5RWpGDm1i6vk7DBXIslLhixmwZSGYArB5v2No3LhxvPvuu/z000+cd955AOTl5bFixQquvfZagoOt66kOHjxIkyZNeOGFF4iKimLnzp088sgjHDp0iFdeeaXMx1u2bBl33nknt99+O1dccQXr16/nlltuKbZdUlISd955Jy1btiQjI4PXXnuNPn36sHv3biIjI5k1axbDhw+nQ4cOPPDAAwC0bdvW7TFXr17NuHHjGD9+PM8++yxbtmxh6tSpHDlyhJdffrnY+bj11lu5//77eeGFFxgzZgy///47Z5xxhsfXVFqsAG+++Sbjx4/n2muv5eGHH6agoIDPPvuMQ4cO0aJFC7Zt20bv3r3p2LEjr7/+Oo0bN2bjxo38+eefZT63laUJtyqz0mq/rzy3GbYCQ2LSMdbvTGfdznRmfLyLGR/vIio8hKxcGzaXyXa2ujyvJzoyilKqNKmZOVxwVrS/w1C+MO/y4ss6DYeet8Cpk7CwyGVg+acgNAySfgRjA5tTr7Hjuf5yE3QeBRlJsNzNNB8XTIT2Q8od6pAhQ4iKimLRokWFCfeaNWs4cuSISzlJt27d6NatW+Hj3r17Ex4ezm233caLL75ISEjZ0rXHH3+coUOHFia6gwYNIi0tjTfffNNluxdffLHwvs1mY8CAAcTExPD+++9zzTXX0LFjRyIiIoiJiaFXr14lHnPq1KlceumlzJ1rjQg6ePBgCgoKmDp1Kg8++KBLMn3fffdx/fXXF77mpk2b8uGHH3LzzTd7fP7SYrXZbEyePJnRo0cX9sIDLrXn06ZNo3Hjxnz55ZfUrWt9EzZw4MASX1dV04RblUtptd/BQUKPlo3o0bIR9w5sT1pmDut3pjNt1bbCZNshO6+Af727hTe+2kt0/TCa1K9Dk/phRNerQ3T9MKLr1+GXpAxeXr+H3Hxr7PDyzoqpybpSNd/JU/lk5eRrSYly79BOyM2i+Dx73hcWFsaIESN49913mTFjBiLC4sWLOfPMM10S2YKCAp5//nlmz57Nvn37yMnJKVyXlJREq1atSj3WqVOnSExM5J///KfL8pEjRxZLuL/99lumTp3K5s2bOXLkSOHy3bt3l+v15eXlsWXLFl599VWX5WPGjOHBBx/k+++/Z8SIEYXLnZPc2NhYmjRpQlJSUonHKC3W7du3k5aWxvjx4z0+x7p167j55psLk21/0IRbeVVcg7qM7dmyMEkuqsBY2xw+nstv6cc5dDy3MLn2JDvPxv1Lt/LB1hQa1A2lQXgoDeqGEFk3lAbhIYXLNv1xtMLJuibqSlUfaZnWsGo6JGAtMf5Dz+vqRBRfn5UGXzztvpSk6LYNm5f8/BUwbtw45s2bx3fffUePHj147733uP3227GGp7c899xzPPDAA0yZMoW+ffsSFRXF999/z5133umSfJckPT2dgoICYmNdR0wu+vj3339n0KBBXHDBBcyaNYszzjiDOnXqMGjQoDIfy/mYNpuNuLg4l+WOx84JMkBUVJTL4zp16pR4zLLEevjwYYASy1KOHj1a4npf0IRb+YSnyXbio8KZe+NfCh8bYzh5ysbh46c4eDyXUa996/b5TtkKSDmWw67cLDKz88nMycOUofMiO8/GpKWJLP0piYYRoUSFh9IwPJSoiFCiwuvQMCKUn5OO8cZXv1e4V10p5VuOSW90lknlVmQcDP0P9Pu358Tbi/r3709cXByLFi3iwIEDZGVlFRudZMmSJYwdO5bp06cXLtu6dWu5jhMbG0tQUBDp6ekuy4s+/uijj8jNzWXlypWEh1uDIZw6dYpjx46V63iOYwYHBxc7RlpaGgCNGzcu93OWN9boaKuU7MCBAy5lOc4aNWrEgQMHKhVLZWnCrXzC0wWXkwa1d9lORKgXFkK9sBBaRkcQX0KivvquvoWPCwoMJ07lk5mTT2Z2HpnZeYyZ9b3bWPJshpOn8knJyCbjZB4Z2XnkF5ScrWfn2Xhg+c/8dvA4LRpF0LxxOC0aRXBGw7qEBLsO9lPR3vHa1Ktem16r8r40ndZdlUXRxDvpB58cNjg4mNGjR7NkyRKSk5Pp0KEDXbt2ddkmOzu72HjcCxcuLNdx6tSpQ9euXXnvvfdcaqKXL19e7FjBwcEudeGLFi2ioMD12+XSep8BQkND6d69O0uWLHG5OPPdd98lODi41Prv0pQl1o4dO9K0aVPeeusthgxxX2d/ySWXsGjRIqZPn16pcc8rQxNu5ROlXXDpSVkT9aAgIbJuKJF1QwuHLywpWV/+z96Fj40xnDhl49jJUxw7mcfQl9zPxpWdZ+PVz39zqUUPCRKaRYXTwp6AZ+XksXZ7Gnm20xeH/nvZVlIysrmoXSwGgzFgDBQYg8H6+cWug7z2xW+cculVL9tFpVC5BNbXye/Kzckuv1P9BkFVVmqmTuuuysGRePvQuHHjePnll1mxYoVLL7bDgAEDeO2110hISKBNmzbMnz+fffv2lfs4U6ZM4eqrr2bixIlceeWVrF+/nk8//dRlm0suuYT777+f8ePHM378eH7++Weef/55GjRo4LLdOeecw/r161m7di2NGzemTZs2bnusH3nkES6//HJuvvlmRo8eTWJiItOmTeO2226rdBlHWWINDg7m6aef5oYbbqBOnTqMGTMGgM8++4y//e1vdO/enUceeYSePXvSr18/7rnnHqKjo9m0aRNxcXHccMMN2Gw2wsLCmD59OlOmTKlUzJ5owq18piyT7bjbB8qfqEP5etXrh4VQPyyE5o1KTtQ/n3QRqRk5/HnkJPuPnGT/0ZPsP5LNn0dO8umONA4dL/4VZW5+QeFoLeWRnVfAfUsSWb45mZb2hL5l4wha2G8Nw0OByiWwvk5+jTE8sXqHy+8EfDNjqfaq11ypGTmFn2GlAtH5559Pq1at2LdvH2PHji22/pFHHuHw4cNMmTIFEeGqq67i+eefZ/jw4eU6zujRo3nhhReYMWMGc+fOpX///rzxxhsuPb/dunVjzpw5TJ8+nWXLltG9e3eWLVtW7FhTp04lOTmZ0aNHk5mZydtvv811111X7JiXXXYZ//vf/3j88ceZP38+sbGx3H///UybNq1csbtT1livv/56IiIieOKJJ1i8eDGRkZGcf/75xMTEANChQwe++uorJk+ezIQJEwgKCqJTp0488cQTgPW3yWazFevlr0piylL4Wk0lJCSYjRs3+jsM5UcVSbKKJqFgJepPjuxS6r6tJ3/o8Tr4mdf1sCYQAIJEELF+IjB+3o8en7NLfEP2Hz3JsZOuEyA0DA+lReNw9qQfJyeveCMRFR7Kvwa2I89myLcVkF9gyLMVkG8z5BVYPxf98CcnTtmK7RsfFc43k/uX+FrLKj0rh2/2HOKrXw/x9a+HSM/K9bjtEyO6cFH7GJqVMMlSRazcnMzk5VtdzlN4aBBPjuwa0BfQishPxpgEnxwsQFSk3f7Hgp/YnZbFZ/de5J2glF/s2LGDDh06+DsMpVyU9L4sqc3W7gBVo/m6V72ki0MHd/b81VpJverv32HNhpaRncf+IydJOnrS3sNu9az/kpfp9jmPZecx9b1txZaLQGhQECHBwkk3yTZYPd1/f3sjnZs1pFN8Azo3a0hskfpYT4loTp6NH34/wle/HuSrXw+xMzULgEYRofRu24Svfz3Esezis6cFizBlhdW7fk7TSC5qH8vF7WPocWYjQu118mVNfjNO5rH9QCbbD2Sy40Am721JLizzcXAMSznzi99o4jQsZZPIMGt4yvp1+CU5g1fW7SFHL6ANaGmZOVq/rZQKaJpwK+VGRRJ1KHsZS0X2axgeSsP4hnSOb+iyb++n1rlN1ps2COP9O/oSGiyEBAcREiSEBgcRHCSl7hseGsTutOOs2ZZWuKxJ/TA6NWtA5/gGnMjN550f9ruM5HLfkkReXb+HfUdOciq/gDrBQSS0asT9g9vTt20MnZo1IChIPH6D8MSIznSOb8j6Xems33mQ2V/tZeYXvxFZN4QLz46hQXgIKzYlF0l+t3IwK5f4RuHsOJDJ9hQrwU7JyHGJu2iy7VBgoHmjCA4dz+WPIyc4lHWqWLlLUdl5Nh79YDt9z25CdH3/XHyjXKVl5vLX1pUbDUEppbxJE26lqlBFe8e9Uas+eUgHYiJLTgg97eson8nKyWPHgSy2pWSwLSWTX5Iz+GbPIbejuuQXGPYeOsENF7Si79lN6Nm6MRF1ijcxpb3Ws+MiufXCs8jKyeObPYdYv/Mg63eluy1Fyc4r4PHVOwBr0qWzYurxl9aN6XBGA/stktjIuh7/sYiPCmf2Da7f/p3IzefQ8VwOHT/lcVjKwydOcd5jn9KicTjnNo+iWwvr1jm+IXVDgwu307px71vxUxLJx7JZvjmZDb8f0XOslApIWsOtVA3gy1FKcvJsdPi/j93Wqgvw+1Nupl2uJGMMbR5Y7bE+/v2JfTg7rr5LsuusonX5nhL1JvXrcOuFbdiy/xiJ+zMKtwkOEs5pGkm3FlHYjGHFpmSXiZzKei2Ag9Zwl2zl5mQmL9ta+K0HlP8cq8ClNdwqEGkNt1K1WEVLYCqyb93QYI+16lV9saODiJRYH9+leUM3e51W1cNSPnR5R5d907NySNyfQeL+Y2zZf4xViSlk5eQXez5fjMZSmzyzZpdLsg16jmsaY4zLjIxK+VNlOqk14VZKlVtFa9X9eUxvXkAbG1mXAR3rMqCjNZ1xQYHhrCnue+RT3PzToCrG07nUc1wzhIaGkp2dTUREhL9DUQqwJuIJDQ2t0L6acCulyq0yNefV6ZiO45b3GEFBnnvkvfUtQG2k57hmi42NJTk5mfj4eMLDw7WnW/mNMYbs7GySk5OJi4ur0HNowq2UqpDKlLFUp2NWlD++Baht9BzXbI7ZBFNSUsjLKz6UqFK+FBoaSlxcXLEZOcvK5wm3iAwGXgSCgdnGmKeKrA8D5gPnAYeBMcaYffZ1DwATABtwpzFmjQ9DV0qpMvNXj3xtoue45mvQoEGFExylAolPE24RCQZeAQYAScCPIrLKGLPdabMJwFFjTFsRGQs8DYwRkY7AWKAT0Az4VETaGWNKHjRXKaX8pDr1yFdXeo6VUtVBkI+P1xPYY4zZa4w5BSwChhXZZhjwlv3+UuASsQq3hgGLjDG5xpjfgT3251NKKaWUUipg+Trhjgf2Oz1Osi9zu40xJh/IAKLLuC8icquIbBSRjQcPHqzC0JVSSimllCo/Xyfc7i4xLjpylqdtyrIvxphZxpgEY0xCTExMBUJUSimllFKq6vg64U4CWjg9bg6keNpGREKAhsCRMu6rlFJKKaVUQPF1wv0jcLaItBaROlgXQa4qss0q4Ab7/auAdcaa2mcVMFZEwkSkNXA28IOP4lZKKaWUUqpCpDLTVFbogCKXAS9gDQs41xjzuIhMBzYaY1aJSF3gbaA7Vs/2WGPMXvu+DwI3AfnA3caYj0o51kHgD++9mnJrAhzydxBFBFpMGk/pAi2mQIsHAi+misZzpjGmVtXGBVi7HWjvIwi8mAItHgi8mDSe0gVaTFXeZvs84a7NRGSjMSbB33E4C7SYNJ7SBVpMgRYPBF5MgRaPKptA/L0FWkyBFg8EXkwaT+kCLSZvxOPrkhKllFJKKaVqFU24lVJKKaWU8iJNuH1rlr8DcCPQYtJ4ShdoMQVaPBB4MQVaPKpsAvH3FmgxBVo8EHgxaTylC7SYqjwereFWSimllFLKi7SHWymllFJKKS/ShFsppZRSSikv0oS7iolICxFZLyI7RGSbiNzlZpuLRCRDRLbYb1O9HNM+EfnZfqyNbtaLiPxXRPaIyFYR6eHleNo7vfYtIpIpIncX2car50hE5opIuoj84rSssYh8IiK/2n828rDvDfZtfhWRG9xtU4UxPSMiO+2/lxUiEuVh3xJ/x1UYzzQRSXb6vVzmYd/BIrLL/p6aXBXxlBDTYqd49onIFg/7euMcuf28+/u9pMouENts+zEDpt0OhDbbfoyAare1za5wTLWzzTbG6K0Kb8AZQA/7/UhgN9CxyDYXAR/4MKZ9QJMS1l8GfAQI0AvY4MPYgoFUrMHifXaOgAuBHsAvTstmAJPt9ycDT7vZrzGw1/6zkf1+Iy/GNBAIsd9/2l1MZfkdV2E804D7yvA7/Q1oA9QBEot+BqoypiLrnwOm+vAcuf28+/u9pLfK/w6LbOPTNtt+zIBst/3VZtuPEVDttrbZFYupyPpa02ZrD3cVM8YcMMZsst/PAnYA8f6NqlTDgPnG8j0QJSJn+OjYlwC/GWN8OrOcMeZLrJlMnQ0D3rLffwsY7mbXQcAnxpgjxpijwCfAYG/FZIxZa4zJtz/8HmheFceqaDxl1BPYY4zZa4w5BSzCOrdejUlEBLgaeKcqjlXGeDx93v36XlJlV03bbPBfu+2XNhsCr93WNrtyMdW2NlsTbi8SkVZYU9RvcLP6fBFJFJGPRKSTl0MxwFoR+UlEbnWzPh7Y7/Q4Cd/9wRmL5w+bL88RQJwx5gBYH0og1s02/jxXN2H1aLlT2u+4Kk20f10618PXbv46R32BNGPMrx7We/UcFfm8B/p7SbkRQG02BG67HUhtNgT2Z03b7JLVqjZbE24vEZH6wDLgbmNMZpHVm7C+jjsXeAlY6eVwehtjegBDgNtF5MKi4brZx+vjRYpIHeBKYImb1b4+R2Xlr3P1IJAPLPSwSWm/46ryGnAW0A04gPV1YFF+OUfAOEruKfHaOSrl8+5xNzfLdJxWPwmwNhsCsN2upm02+OdcaZtdulrVZmvC7QUiEor1i1xojFledL0xJtMYc9x+fzUQKiJNvBWPMSbF/jMdWIH19ZGzJKCF0+PmQIq34nEyBNhkjEkrusLX58guzfGVrP1nupttfH6u7BdmDAWuNfZCsqLK8DuuEsaYNGOMzRhTALzh4Tj+OEchwEhgsadtvHWOPHzeA/K9pNwLtDbbfpxAbLcDrc2GAPysaZtdutrYZmvCXcXsNUlzgB3GmP942KapfTtEpCfW7+Gwl+KpJyKRjvtYF3T8UmSzVcD1YukFZDi+WvEyj//d+vIcOVkFOK46vgF4z802a4CBItLI/tXcQPsyrxCRwcC/gSuNMSc9bFOW33FVxeNcIzrCw3F+BM4Wkdb2HrGxWOfWmy4Fdhpjktyt9NY5KuHzHnDvJeVeoLXZ9mMEarsdaG02BNhnTdvsMqt9bbap4JWeevN4BWwfrK8YtgJb7LfLgNuA2+zbTAS2YV0J/D1wgRfjaWM/TqL9mA/alzvHI8ArWFcp/wwk+OA8RWA1xg2dlvnsHGH90TgA5GH91zoBiAY+A361/2xs3zYBmO20703AHvttvJdj2oNVM+Z4L820b9sMWF3S79hL8bxtf49sxWqgzigaj/3xZVhXf/9WVfF4ism+/E3He8dpW1+cI0+fd7++l/RWJb9Dv7TZ9uMFXLuNn9ts+zECqt32EI+22aXEZF/+JrWszdap3ZVSSimllPIiLSlRSimllFLKizThVkoppZRSyos04VZKKaWUUsqLNOFWSimllFLKizThVkoppZRSyos04Va1gohMExHj4XadH+IxIjLR18dVSqnqQNtsVdOE+DsApXwoAxjsZvkeXweilFKqVNpmqxpDE25Vm+QbY773dxBKKaXKRNtsVWNoSYlSgIi0sn9leI2IvC0iWSKSLiIPu9m2v4hsEJEcEUkTkVdFpH6RbaJF5HUROWDfbpeI3F3kqYJF5AkROWg/1isiEubVF6qUUjWAttmqutEeblWriEix97wxJt/p4TPAB8BVwIXAwyJyyBjzin3/jsDHwCfAKKAF8BTWNLSD7duEA58DscAjwE6grf3m7F5gHXAd0BV4EvgDmFH5V6qUUtWfttmqptCp3VWtICLTgGI9H3at7T9/Bz4xxgx02u8N4DKghTGmQEQWAecB5xhjbPZtrgYWAxcYY74Tkb8DrwE9jDFbPMRjgK+MMRc6LVsJNDXG9KrES1VKqWpP22xV02hJiapNMoC/uLmlOG2zosg+y4FmQHP7457ACkfDbbcMyAf62B/3BzZ7aridrC3yeLvTcZRSqrbTNlvVGFpSomqTfGPMRncrRMRxN73IKsfjM4A/7T/TnDcwxthE5DDQ2L4oGjhQhniOFXl8Cqhbhv2UUqo20DZb1Rjaw62Uq1gPjw84/XTZRkSCsRrsI/ZFh7EaeaWUUt6lbbaqFjThVsrViCKPR2I12En2xxuAEfYG23mbEOBr++PPgO4i0tWbgSqllNI2W1UPWlKiapMQEXF3cct+p/udROR1rBq/C4EJwF3GmAL7+seAzcBKEXkNq37vaWCNMeY7+zbzgduBtfYLf3ZhXeTTzhgzuYpfk1JK1VTaZqsaQxNuVZs0BL5zs/z/gAX2+/cDQ7Ea7xzgUeBlx4bGmG0iMgR4AuvinEzgHft+jm1yRKQ/1tBT04EGwD7g1ap9OUopVaNpm61qDB0WUCmsSRSwhpi6whjzgX+jUUopVRJts1V1ozXcSimllFJKeZEm3EoppZRSSnmRlpQopZRSSinlRdrDrZRSSimllBdpwq2UUkoppZQXacKtlFJKKaWUF2nCrZRSSimllBdpwq2UUkoppZQX/T9aLkzxQ/RyZwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "hist = history.history\n",
    "x_arr = np.arange(len(hist['loss'])) + 1\n",
    "\n",
    "fig = plt.figure(figsize=(12, 4))\n",
    "ax = fig.add_subplot(1, 2, 1)\n",
    "ax.plot(x_arr, hist['loss'], '-o', label='Train loss')\n",
    "ax.plot(x_arr, hist['val_loss'], '--<', label='Validation loss')\n",
    "ax.set_xlabel('Epoch', size=15)\n",
    "ax.set_ylabel('Loss', size=15)\n",
    "ax.legend(fontsize=15)\n",
    "ax = fig.add_subplot(1, 2, 2)\n",
    "ax.plot(x_arr, hist['accuracy'], '-o', label='Train acc.')\n",
    "ax.plot(x_arr, hist['val_accuracy'], '--<', label='Validation acc.')\n",
    "ax.legend(fontsize=15)\n",
    "ax.set_xlabel('Epoch', size=15)\n",
    "ax.set_ylabel('Accuracy', size=15)\n",
    "\n",
    "#plt.savefig('figures/15_12.png', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500/500 [==============================] - 2s 5ms/step - loss: 0.0428 - accuracy: 0.9927\n",
      "\n",
      "Test Acc. 99.27%\n"
     ]
    }
   ],
   "source": [
    "test_results = model.evaluate(mnist_test.batch(20))\n",
    "print('\\nTest Acc. {:.2f}%'.format(test_results[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorShape([12, 10])\n",
      "tf.Tensor([6 2 3 7 2 2 3 4 7 6 6 9], shape=(12,), dtype=int64)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAq8AAADoCAYAAADBow1gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de7zNVf748fchd4mcI6Ickmshl5HGtQvVkGiiaca1XAeZCt+QESW6YCgRTg0zk5lEUm6TLm4Rp9I0IhmHJHHcI0Ln98fn12qt1fls+3r2Xvu8no9HD+9lffZnr87H/ux1Pp/3571ScnJyBAAAAHBBgXgPAAAAAAgWk1cAAAA4g8krAAAAnMHkFQAAAM5g8goAAABnMHkFAACAMy4KZePU1NSc9PT0GA0FgWRlZUl2dnZKNPbFcYyvzMzM7JycnLRo7ItjGV8cy+TBsUweHMvkEGjeE9LkNT09XTZv3hydUSEkjRo1itq+OI7xlZKSsjta++JYxhfHMnlwLJMHxzI5BJr3kDYAAAAAZzB5BQAAgDOYvAIAAMAZTF4BAADgDCavAAAAcEZI1QZc9+WXX6p44MCBRl+hQoVUvGTJkjwbEwAAQLhq1apltLdt25brdqNHjzbaw4YNU3GJEiWiP7AY4sorAAAAnMHkFQAAAM5I+rSBU6dOqbhXr14qXrNmjbHdqFGj8mxMAAAA0TBgwACjPXjw4Fy3Gzt2rNF+9tlnVbx48WKj76abborS6GKDK68AAABwBpNXAAAAOIPJKwAAAJyR9DmvQ4YMUbGe51qjRg1juw4dOuTZmBC6zz//XMX9+/dX8fvvv29sV6VKFRW/8sorRl+TJk1iNLrkpf98P/30U6NPPyYzZszw3UdOTo7RTklJUXG/fv1UbJd7qVu3ropbtmwZ5IgBIH+ZOnVqWK87efKkijt16mT0vf322ypu3LhxeAOLIa68AgAAwBlMXgEAAOCMpEsbmD17ttGeNWtWrtvZJSMaNWoUszEhdB988IHR/s1vfqPiI0eO+L5u165dKu7WrZvR984776i4YsWKkQ4xaRw8eNBo9+7dW8WrV69W8bFjx3z3oacCXIi+7cyZM323K1WqlIpbtWpl9L344osqTktLC/q9ASDZ3H///Ub7f//7n4pvueUWFY8bN87YTk8FO378uNG3ceNGFZM2AAAAAESAySsAAACcweQVAAAAzkiKnNcffvhBxU8//bTvdtddd52KKY2VePScG7tsx+nTp1XctWtXFT/00EPGdm+88YaKR48ebfTdfPPNKt60aZPRV7JkyTBGnBy2bt1qtN99910VFy9eXMXVq1c3ths5cqSKA+WdBiqVpZfY2rZtm7GdntusH1cRkX379qm4bNmyRt/SpUt9xwLkJytXrjTabdq0UXFGRobRpy+lrpcjLFiwYIxGh2gZPnx4UNvdfvvtRltfAnbDhg1Gn35+t58R6dixY6hDjDquvAIAAMAZTF4BAADgjKRIG1i0aJGKv/jiC9/txowZo+IiRYrEckgIw4QJE1S8f/9+o+/vf/+7iu+9917ffeirMunlQkRE5s6dq2L7ltngwYNDG2wSsVevmjdvnorT09NVrP9so6Vt27a+fVu2bPHdbvPmzb6vW7BggYrvuusuFYdSzgtINOfPn1fxnj17VHzgwAFjuz/96U8q3rlzp9F31VVXqfijjz4y+vT0O/18a39XvvrqqyqmTJ1b9DQwEZG33npLxfXq1TP69u7dq+LJkycbfaQNAAAAACFg8goAAABnOJk2cOjQIaPds2dP3231W9Ht27eP2ZgQOf2JSb0yhIjIHXfcEdQ+9FvDL730ktGnP80+adIko09fjat06dJBvVeyCvZnHWv6bSw7jUR/anbFihVGX+fOnVWsr7jXq1evaA8xISxevFjFlSpVMvoaNmyY62vWrl1rtPUnzW3VqlVTcdWqVcMZIqJAr8ai3/4Phb2anh99dSWbvtqhncI1ZMiQsMaF+Lj00ktVPGjQIKNP/z7++OOPjT69bX9X5xWuvAIAAMAZTF4BAADgDCavAAAAcIaTOa9Tpkwx2t9//72KixYtavTpea6Uyklsl1xyiYrvvvvuqO///vvvzzUWMfOo83vOayKyc/XsvHc/derUicVw4urqq6822npJuAIFzOsRJUqUyHUf9nb6KmjHjh0z+goVKuT7Op2+AlvNmjWNPj1H2S7Nppdjw89q1apltO2V6n5il7KqXbu2iu3j9eOPPwbVd/ToURXv2rXL2E4vU6eXsxMROXv2rIqHDh2a63iRmC677DLfPj03VkSkTJkysR7OBXHlFQAAAM5g8goAAABnOJM2cOTIERXPnDnTdzv7Vot+CwX5W+XKleM9BFj0VX12795t9K1evVrFL774otEXaIWtpUuXqvjaa6+NdIgJ55///KfR1lfCsW/nlStXTsX6bWI79UC/Zbxu3Trf9165cqXR1ss36X16+S4Rc1Ume5UfvZzZPffc4/ve+Y1e2k/ETHtr1qyZirt3725sd99990X83jt27FDxwoULjb5HHnlExXqagEjwpbiQePTUHhHzHKH/exARef3111Ucr/JoXHkFAACAM5i8AgAAwBlMXgEAAOAMZ3JeT58+reJAeTUZGRlRf2/9/fQcL5Hwl+lD3tuwYYOK7XJYxYoVy+vhOG/r1q0qtnNSdXaJHz13Ty/JM2/ePN/X2WXu9FzOkSNHGn233nproGE7r0GDBgHb4dDzYZs3b+67XaA+3TPPPGO0hw0bpmJ7Kdqvv/46qH3mNzfeeKPR1n+GTZs2VXGpUqWi/t56vuPDDz9s9OmlswJ97uGWtLQ0o12/fn0V2zmven47Oa8AAADABTB5BQAAgDOcSRtYsGCBb59+y8tebSRYesmPvn37Gn36JXI7bWDRokUqvummm8J6b+SNTZs2qbhGjRpG3+WXX57Xw3GCfuu+c+fORt9rr70W8j5Egl/prm7duiru16+f0We3kVjOnz/v22cf/2ikPSSjVatWxXsIImKWsxMhVSBZ6eXyRH5ZBk1XuHDhWA/ngrjyCgAAAGcweQUAAIAzmLwCAADAGQmb83r48GGj/eSTT/pu27BhQxXby8P6sfM5unbtqmJ7+cVA9Nd99tlnKr700kuD3gdiQz8eIiLLli1Tcc+ePfN6OM677bbbjPaKFStUfPLkyaD3E2zOa3p6uoobNWoU9P4Rf4HKX1WvXt1ot27dOtbDAXAB06dPN9r6ErDVqlULuG08cOUVAAAAzmDyCgAAAGckbNrAtm3bjPY333zju629WlIw7BVgAqUK6PvXVwSyx7V9+3YV6yugIO/oZZkmTJhg9FWsWFHFo0ePzrMxuUy/xd+rVy+jT7+V9P333/vuY/DgwUZbL223b98+FX/33XfGdm+88YaKN27caPS1bdtWxfYKW/rqQMg7+spZ//73v323C+d8DeQH9jzn888/V/Hf/vY3o08/H1922WW++9RXAbXnL3pJu3fffdd3H+fOnfN9Xbxw5RUAAADOYPIKAAAAZyRs2kAo7rzzzqC2mzhxoorHjh3ru13NmjWN9vjx41XcqVMn39fpt0MRGfvpdf22sb5S1pIlS4ztrrzyShW/8sorRp9+/PUUAoSnRYsWQW2np9PY9NSAKVOmGH3vvfeeig8cOGD0zZs3L9dYJDFuaeUXp0+fVvHs2bNVHOiY2ykmffr0Cfl9y5cvb7QDnc8RGf1ziOjQ0yL1FLYPPvjA2G7v3r2++3jppZeCeq+OHTuqeMeOHUaf/vn98ssvffdxxRVXBGzHA7MtAAAAOIPJKwAAAJzB5BUAAADOSNicV7ucSrFixVRs50zpOTkDBgww+vbv36/iMWPGqFjP9RARadCggYpXrlxp9D366KO+49RXh2nSpInvdvDopazs0hwLFixQ8dKlS42+3bt3B7X/devW+fbp+T56WR8RkeLFiwe1f0TXHXfckWssIvL++++rePLkyUafnitra9WqlYoXL16s4ksuuSTcYSa9//3vfyp+4oknjL7s7GwV2+dG+zwajJ07dxrt2rVr5zoOEZG6devmug+/v0dgx48fV7G9ymTZsmVVrH++AuUk2ysv9e3bN9IhJiU7D18v77dw4cKYvveiRYsi3of+70bEXL3ymmuuiXj/4eDKKwAAAJzB5BUAAADOSNi0Af1WkohInTp1VLx582ajT78svmLFCqPvL3/5i4oD3eL69a9/reJhw4YZfRkZGb6vC9SHX9LLe9x3331Bv87v1oR++0LELFd2+eWXG316KZ9jx44Zff/617+CHgvyRsuWLVXcuHFjo09PAdLTTUREVq9ereLf//73KrZLy6SlpUVjmEnh4osvVrFdRk5vd+3a1ejTS9OtXbtWxQ8//LDvew0fPtxojxo1KrTBImjffvut0b7nnntUrK9uJyLSu3dvFeu3te30At3MmTONtr6aE35m/5z8UgXs1KYbbrghqP3r5zyRX5aajNSWLVuMtr66l/7/dt1110X1fQPhyisAAACcweQVAAAAzmDyCgAAAGckbM6rbc6cOSquV6+e0Xfu3DkV33rrrWHtf9q0aUFt16NHD6Nt51XC9MUXXxhtfSk8m57Xaucn6iVZ7r77bhXbOa8PPvigiu2SP3q5pUDL7iF3R48eNdp6ubFYfw7sUmZPPfWUinv27Gn06fnxy5YtU/FHH31kbNe2bdtoDtFpev5vuEutFilSJKjt9FxmxNbXX39ttPXyczb7WY9g9OvXz2h36dJFxXp+bbTYz8K4IthzTZkyZYy2Pd/wU7RoUaMdTnmsX/3qV0Zbz7/997//bfTpS7TfeOONKh48eLCx3WOPPRbyOILFlVcAAAA4g8krAAAAnOFM2sDVV1+t4nHjxhl9+q1ofQWncJUoUcJoT5gwQcUDBw6MeP/5iV5+RcS8jXXzzTcbffpqSPaKJPotLb0kj32LUy/DU7hwYd8+BEcvQ/X8888bfSVLllTxkiVL8mxMtlq1asXtvfGzzMzMeA8BFjud5/HHH1fxxIkTjb4TJ06EvH991UJ7/3ocLT/++GPU95kXqlSpYrS7deum4rlz56o4KyvL2E5Pw4gF/Tv4tddeM/r0NKD169cbffqcSF91b8qUKcZ2F1308xQz0Eql4eDKKwAAAJzB5BUAAADOYPIKAAAAZziT81qsWDEV28sJ1qxZU8V2eaTt27er+Pvvvw/qvfR8FBHyXCNhL8mbmpqq4ueee87o0/OZOnfubPQtX75cxXfccYeKo51HA9OaNWtUbJfZ0UtS6XlbIr/8DEWbnh+9Z88eo0/Pe69cubKK9aVMEX32cqN+9u/fH+OR4Cfly5c32iNGjFCxvdSv/pyBXgpp6tSpYb23vUTpJ598ouJGjRqpOFCJtQEDBoT13olGX7ZcROS2225TsX3ujLb+/fur2H7ORC8tapcj1LVu3dpo68vW6uXSXn75ZWM7vVSh/cyJ/UxKqLjyCgAAAGcweQUAAIAznEkbCOS3v/1trrGIyJtvvqnioUOHqnjnzp3Gdv/3f/+nYm5FR0YvuXL48GGj75ZbbvHtGzRokIrtFT06dOig4unTp0dlnLiwlJSUXGMRka1bt6q4b9++Rp+evtOsWTOjT/+MLly4UMWdOnUytnvggQd83/ubb75RsX17Ut9WX7WNklqJwb6Vjfi44oorfPvS09NVrK9oGAr7/D5//nwV6ykLF198cVj7d1mbNm1UrN/Wf+GFF3xfY6diVaxY0XfbIUOGqFhfPc8+j4ZLT/WYNWuWivVUMhHze+Ds2bNGH2kDAAAAyDeYvAIAAMAZSZE2EEi7du1yjRE7+m2gSy+91Oh75ZVXco1FRAoVKqRie2WRp556SsX2qjGIHT0dwH4a9W9/+5uK7VtCX375pYrtVXjsJ1J/kpGRYbT1qgGBbneVK1fOaLdv317FdjoD4qNo0aIq1iuOIHnZ5/5kqRwQDfrPRk+DczElTl9F6+GHHzb67HY0ceUVAAAAzmDyCgAAAGcweQUAAIAzkj7nNVGdOyfyzDMic+aI7NkjkpYmcvfdIpMnx3tk0dW2bVujra+GVKNGDaNvwoQJKr7++utjO7AoefVVkXnzRDIzRY4dE6lRQ+Thh0V+97t4jyw69PJS48ePN/r0fNJjx44ZfatXr/bd5+OPP67igwcPqtguyWOX2PLTokULo123bt2gXmdbsEBk0iSR7dtFTp4UqVxZpGtXkWHDRCKs6pIv6M8U6PnKIiIdO3ZUcV6ULGvVSsRaEE5Zv16kadOYDwFRkOzn1/wk2seSyWuc9OwpsmqVyJ//LFKzpshXX4loZTPhiEmTRKpU8X7pSE0VWbpU5N57RbKzRbSytXDAoUMirVuLDB0qUrq0yIcfiowZI7J/v4i1kjES3PTpIsePm383erTIxx+LNG4cnzEhdJxfk0e0jyWT1zhYvlxk/nyRLVtEateO92gQiSVLvA/iT268UWTfPu+DysnVLXZhgtatvQnQ88+LTJsmEqX63sgD9nn1hx9ENm8W6dJF5CK+9ZzB+TV5RPtY8jGOg4wM78Dlh4nr2LFjA7Zdl1vVn+uuE1m8OO/HktcqV67s2xfo1v3AgQNjMZyYKFvWm/jgwurXr59rnAiWLxc5coTbza7Jz+fXZBPtY8kDW3GwcaNI9eoiAweKlColUry4SKdO3m8hcN/69fnjF5Nkdf68yKlTImvXikydKtK/P1ddXTd/vkjFiiLNm8d7JIgU59fkEcmx5MprHOzfL/LyyyL16nkn1RMnvIdCOnYU2bCBL0qXrVrl/SZp1duHQ0qUEDlzxou7dRN5+un4jgeROXXKu2XZpw/nVtdxfk0ekR5LJq9xkJPj/bd4sXdbUkSkQgWRli1F3nlH5Kab4js+hCcry0tA79BBpEePeI8G4Vq/3pvwfPihyNix3h0SBxe+wf+3ZInId9+RMuA6zq/JIxrHkslrHJQpI1K16s8TVxGRZs28cjxbtzJ5ddHhwyK33SZy5ZUi2qqpcFCDBt6fzZp5eVrdu4s89JDIVVfFd1wIz/z5ItWqiTRqFO+RIFycX5NHtI4lOa9x4FfmMCdHpABHxDmnTom0a+c92PPWW95tZySHnyayu3bFdxwIz7FjIsuWcdXVZZxfk0c0jyVTpTho107k00+9+mY/Wb1a5OxZLw8W7jh3zltcYscO70uyXLl4jwjRtG6d92eVKvEdB8KzaJGXv8zk1U2cX5NHtI8laQNx0KeP9xRz+/YiI0Z4D2wNHy5y883erUq4Y8AAr9jyX/7i3Q7ZsOHnvuuuEylSJH5jQ2huvdX7DNapI1KwoDdxffZZrzYoKQNumj/fuyCQB4t6IQY4vyaPaB9LJq9xUKqU92DW4MEi99zj5bp26JB8S8PmBytXen8+8MAv+3btEklPz9PhIAKNG3tVQLKyvEL2VauKPPmkSL9+8R4ZwpGd7T3RPG5cvEeCcHF+TR7RPpZMXuOkWjXvtxC4LSsr3iNAtIwbx0QnmaSmeqlYcBfn1+QR7WNJzisAAACcweQVAAAAzmDyCgAAAGcweQUAAIAzUnJycoLfOCXloIjsjt1wEEDlnJyctGjsiOMYdxzL5MGxTB4cy+TBsUwOvscxpMkrAAAAEE+kDQAAAMAZTF4BAADgDCavAAAAcAaTVwAAADiDySsAAACcweQVAAAAzmDyCgAAAGcweQUAAIAzmLwCAADAGUxeAQAA4AwmrwAAAHAGk1cAAAA4g8krAAAAnMHkFQAAAM5g8goAAABnMHkFAACAM5i8AgAAwBlMXgEAAOAMJq8AAABwBpNXAAAAOIPJKwAAAJxxUSgbp6am5qSnp8doKAgkKytLsrOzU6KxL45jfGVmZmbn5OSkRWNfHMv44lgmD45l8uBYJodA856QJq/p6emyefPm6IwKIWnUqFHU9sVxjK+UlJTd0doXxzK+OJbJg2OZPDiWySHQvIe0AQAAADiDySsAAACcweQVAAAAzmDyCgAAAGcweQUAAIAzmLwCAADAGUxeAQAA4IyQ6rwCABCJDz74wGh37txZxXYx+CFDhqj4rrvuium4ALiDK68AAABwBpNXAAAAOIPJKwAAAJxBzisAIKZmzJih4kceecToO3bsmIorVKhg9NWrVy+2A0NEDh48aLRbtGih4u3btxt9OTk5Kh41apSKx40bF6PRIZlx5RUAAADOYPIKAAAAZziTNpCSkpJrfCG9e/dWcdOmTX23q1+/fq4xouvUqVMqLly4sNF30UWR/3M8c+aMigcNGmT0zZ49W8VvvPGG0deuXbuI3xuAZ8KECUb7scceU3GRIkWMvhdeeEHFXbt2NfqKFy8eg9EhEnqqwO2332706akCgb6nx48fr+KsrCyjb968eRGOEH52795ttGfNmqXirVu3Gn2vv/66ilNTU42+Tp06qVhP+0hLS4vKOIPBlVcAAAA4g8krAAAAnMHkFQAAAM5I2JxXO2dKz58pWLBg0PvJyMhQsZ7fYe/jyiuvVHHlypV99/frX//aaI8cOVLFRYsWDXpc+ZWed/r2228bfXYeajjeeustFc+ZM8fo0/8N/fGPfzT6yHkFQnfu3DkVP/jggyqeNm2asZ1e8mru3LlGX926dWM0OkTD6tWrjXbfvn1VHKgclv7dKCLy3//+V8V6PuWKFSuM7fbs2aNi/XsZ4Vm4cKGK+/fvb/RlZ2er2M5r1Y/zgQMHjD59LtWgQQMV9+nTJ7LBhoArrwAAAHAGk1cAAAA4I2HTBmrVqpWn76ffqtBj23vvvWe0b7nlFhXrq4vAo9+yEBEZPXq0ik+cOJHXw1H0kl0iInv37lVxpUqV8no4gJO6deum4vnz56u4fPnyxnbLly/37UPi0cth6aleIoHLYempAvZKajq9JNqiRYuMPv1WNmkD/k6ePKli+2e4du3aXPv0W/wiIh07dlRxoFv+mZmZRlvfZ/PmzYMccXRx5RUAAADOYPIKAAAAZyRs2kCHDh2M9q5du4J6nf3048SJE1X8zTffqHjHjh0RjO5n+hPtpA149HQAffUNuy8W9NsiFSpUMPr042+v8lO2bNmYjssl+i2hVatWqXjZsmXGdnrljVDSfFq2bKni999/P6jXtGrVymjr6TsrV640+qZOnaria6+9Nuhx4cLsfwMLFixQccmSJVX8n//8x9jOfpIZiU1/0lyvDCBirnpmr4al34YOpG3btiq20/RIFQiOfp7u3r270ad/3kaMGKHiBx54IKz3sj+/5cqVC2s/0cSVVwAAADiDySsAAACcweQVAAAAzkjYnFdboFWvAm3Xpk0bFb/88ssq7t27d1TGdd9990VlP8lk8eLFKt6yZYvvdnrZrGhJT09X8RVXXGH07du3T8UFCpi/txUrVizqY3HVhg0bVDx9+nTf7YLNQ7cVLlxYxT/88ENQr7FzlM+cOeO77Z133qninTt3hjg62PTjbK/Qc/78eRX/4x//UDE5ru55/PHHVaznudrlsDp16qTiYHNcbXqOvJ1Hzb+d3NkrnT3xxBMq1lc2ExEZNGiQisPNc9XZ86r9+/dHvM9IceUVAAAAzmDyCgAAAGc4kzYQLPtydufOnVW8Zs0aFdu3QoJll6S47LLLwtpPMtNvP9k/56ZNm6p4+PDhMR2H/d56O9zjnx+88cYbMd2/fss/2OMQKE3Advbs2ZDHBH8vvviiiu2yRvfff7+K27Vrl2djQuRGjRpltMePH69i/Ta0XQJy7ty5Eb93vFZlco2eKvDQQw8ZfYcPH1bx5MmTjb5opAp8/vnnKtb/bYiIbNu2TcWbNm2K+L3CwZVXAAAAOIPJKwAAAJzB5BUAAADOcDLn9dy5c0b7ueeeU7Gdj6MvU6jn1xUsWNDYrnTp0iouVaqU0afndT3yyCNhjDh/+eKLL1Rs5zRWqlRJxZSnSgz28o9ZWVlBva5JkyYqtpfiDUTPn9LLcoWSa7t+/XoVHzhwIOjX4cKWL19utKdNm+a7bfny5VWs58Hp51N7O8RPoDxG/Vydlpam4kmTJsV+YMiVvpS2/V06Y8YMFffp0yes/R88eFDF+pLAIub3gl2KKxGeGeHKKwAAAJzB5BUAAADOcDJtYO/evUZ76NChQb2ufv36KtZX4RExL8/rMYIza9aseA8BYTp+/LjR9itL1aFDB6OdkZGh4jJlyoT13jVr1lRxjx49gn7dLbfcouJVq1aF9d7I3W233Wa0A90i1Mvi6bG9Ilq1atVU3LBhQ6Nv9uzZKr7oIie/khLWyZMnjfbIkSNVbN8K1ukrUJYtWzb6A0Ou9LQOkcDlHfWVzoJlr9Kll9/KzMz0fW9b7dq1Q37vaOPKKwAAAJzB5BUAAADOYPIKAAAAZ+SrBKOpU6equFmzZnEcSfI5ceKEigPlUg0ZMiSm41i5cqWK9TJMtkBjzG/mzJkT1HZ2nni4ea7h+OijjwK2EZkVK1b49umfFTsfVl+GUi9fpi8pKyLy2Wef5RqLiKSnp6t4zJgxQY0XwdHLl4mILF68WMWBls/Wy2jpOckiIgMHDlSxvcQsIqMvYS8S+Hsq0M9+5syZKtaPa6CSV4Heq23btkZ73rx5vtvmFa68AgAAwBlMXgEAAOCMfJU20KtXLxXbt0CfeuqpvB5OUvEr6WHfmlq6dKmKr7/++qiPQ78FHqjURyKsEJIoChQwf4fVy1ddffXVKtZv7+a1w4cPG+0jR474bmvf4sKFBSoPqJe9mjhxotF37bXXqlj/uffr18/YTi+lp5fUEjFvUd97771GX/Xq1QMNG7nQyyHZxzXQbWL9c6+nd9llEPX0u5IlSxp9sU4LS3Z2+Sv9c2R/Z+mpOYFSQPSyVjVq1DC2s1dX1N11110qfvXVVwMNOy648goAAABnMHkFAACAM5xMG7BvVVxzzTUq3rJli+/rduzYoeJnnnnG6NPbdt8dd9yhYn2lmPzsiy++MNr79+8P6nUzZsxQ8dtvv230/elPf/J9XaVKlVTctGnTXGMRkZ07dwY1Dvzs3XffjfcQLmjBggVBbxvoyXnkrmjRor59derUUbGeJhBIhQoVjPbo0aNVfPToUaNvypQpKj537lxQ+4c/vcJAoNvJepqAiKeEns8AAAueSURBVMjmzZtVXLx4cd/967ey9RWaREgbiFRqaqrR/vbbb1W8aNGioPfTvHlzFdeqVUvFdlqCnjpiH3M7hSfRcOUVAAAAzmDyCgAAAGcweQUAAIAznMx5tfNC9FwQfYUlEbO0y65du1RcsGBB3/0PHz7caGdkZKj4zTffNPrKly+v4kB5Y8nGLmGj/5yrVq2q4gEDBhjbHTp0KNdYRKRLly4qtnO19HycSy+9VMVfffWVsV2wJbAaNGgQ1HaIHz1/XV8Z6EL0HHUE55NPPvHt08ulRUOpUqWiuj+Y9NJWgVZNqly5stEOlOeqC7TPgwcPqjgtLS2o/cGf/jPs06dPWPvQS9PZpbH070s7H7Zjx45hvV9e4corAAAAnMHkFQAAAM5wMm3Apq/8Y19a19tjx45V8V//+ldjuz179vjuf/v27SrWb4mLiLz33nsqbtGiRVDjTXZ9+/ZVccWKFY0+fYWt2bNnG31nz5713efJkydzjQPdwgpELyWCxKSXVdNLxtjKlStntO3VnXBhgX6+0WavlobI6LfqRUSys7NVHKhU1ogRI8J6v0ArKOopfOHe5kbkli9frmK9TJ39fTl37lwV/+EPf4j9wKKIK68AAABwBpNXAAAAOIPJKwAAAJyRFDmvwdJzP/S8TBGz1NO0adOC3ufQoUNVvHHjxghGl5zatWvn27799tuNPj13a86cOUHt/8cffzTaX375pYrtUly6YEtqIX70fPJAuc16bqyIuVw0gmPnTerCzSvXffrppyq2z696/nnt2rUjfq/8Zvfu3b7tQMcu3Lz/aPx7QGx1795dxfp3nV2+zOXndLjyCgAAAGcweQUAAIAz8lXagO6ll14y2votylDoZUkQGjulQNezZ8+w9qmv0rVgwQLf7aK9ahAit23bNqN99OhRFdtpHo0aNVJxoH9HCE6rVq18+86dOxfy/s6fP2+0n3/+eRXbx7Jz584h7x8/C1QO60LbBsP+XAYqlYW8o5eMfPLJJ42+AwcOqFg/RvY858orr4zN4PIAV14BAADgDCavAAAAcAaTVwAAADjDyZzX7777zmgHym3MyMhQ8Zo1a1Qcbq6OXSZE3z/cQZ5k4pk5c6bRDrRkqZ7zetFFTp7GEkqlSpVUXLp0aaNvyZIlKn7uueeMvv79+6v4xIkTKraX6P3Xv/6l4jvvvNN3HwidnbfYoEEDFW/evDmsfU6ePFnFU6ZMMfr070C79BJLwuadrl27qnjx4sVGn75k9rx581Rcq1at2A8sj3DlFQAAAM5g8goAAABnJNT9tr1796rYvoW4du1aFZ85c8bo27RpU1D711MFChYs6Ltd0aJFjbZ+Cd5OE6hXr15Q7428oa8a8+qrr/puN3bsWKOtr76GvLNz504Vz50713e7G264wWg//fTTMRtTfrdw4UKj3bZtWxUPHjzY6Bs1apSK9fJYdmpXkyZNVPzCCy8YfYHOxbgw+9Z97969VZyZmWn06d+B9nHWbylPmDAh19fYRowYEdpgETY9TUBEZNGiRSq2j5GeOnLFFVfEdmBxwpVXAAAAOIPJKwAAAJyRUGkDLVq0UPFXX31l9Om3pKJxm8l+onbQoEEqTk9PN/q6desW8fshbzRs2FDFgW53LV261GiTNhAfy5YtU/GRI0d8tytevLjRLlGiRMzGlN+1bt3aaK9YsULFdrrG6dOnVayvxGXf4uzRo4eKCxUqFI1hwof+xP+ePXuMvvHjx6v4rrvuMvr086VeUcA+j+ppJL///e8jGywCevzxx1X8+uuvG32BVjrT0zmSqcKAjiuvAAAAcAaTVwAAADiDySsAAACckVA5r8GyS1n96le/Cup1zZo1U7G9EkjFihUjH1iQFiwQmTRJZPt2kZMnRSpXFunaVWTYMJHChfNsGEnpmmuuyTUWEfnss89U3KVLl5i8/9dfi9So4R3XEydESpaMydskja+//jqo7Xr16hXjkYi0aiXy/vu5961fL9K0acyHkJD0HFg7HzZRnTsn8swzInPmiOzZI5KWJnL33SLawlH5gl42S0Tk0KFDKrbLUfo9I6CXQxP5ZZnBWMtPx1LPcRURefTRR1VsH58hQ4aouGPHjkafXjIykbz+usjo0d7c5/LLRQYNEnnwwfD25eTk1XWHDom0bi0ydKhI6dIiH34oMmaMyP79Itbqi3DM0KHehPXkyXiPBKGaPl3k+HHz70aPFvn4Y5HGjeMzJoSnZ0+RVatE/vxnkZo1Rb76SmTr1niPCuHgWCaHdetEOnUS6dXL+2Vk40aR4cNFChQQ0ebhQWPyGgd9+5rt1q29L83nnxeZNk0kwEPySGBr1ogsXy4yYoQ3iYVbatc22z/8ILJ5s0iXLiIXcaZ0xvLlIvPni2zZ8stjCrdwLJPH2LEizZqJzJ7ttdu0ETlyxPv7AQNCv+ucUKdkvQzLSevS1bp161Rsl7J65JFHYjquvFC2rPdlichcfPHFKh5i/Tp3//33x+x9z5/3boGMHu1dTUdwpk+fHtR2V111VYxH8kvLl3sn19/9Ls/fGhHIyBC58UYmOyIilStXNtr66mb2SmeJKL8dywIFzMeQ9FQBu7TZpEmT8mRM0fLJJyJ//KP5d23aeOkfH3wg0rJlaPvjga04On9e5NQpkbVrRaZOFenfn6uurpoxQ+T06V9+OOGu+fNFKlYUSdD0MfjYuFGkenWRgQNFSpUSKV7cu125b1+8R4ZQcSyTx+nTv7y6WqSI9+fnn4e+PyavcVSihPdf8+bebx0s1+6mQ4dEHn3UewiP+uvJ4dQpkSVLvJQBfqF0y/79Ii+/7F3pmT9f5KWXRDIzRTp2FNFq78MBHMvkUa2ayKZN5t99+KH35+HDoe8vodIG8pv1670vyQ8/9PI+Bg70HhqBW0aOFGnSROT22+M9EkTLkiUi331HyoCLcnK8/xYv9tKxREQqVPAuELzzjshNN8V3fAgexzJ59Ovn3V2eNUvkt7/15j3PPuv1hbNoakJNXu2cDl0yLtHaoIH3Z7NmIqmpIt27izz0kEgc0vuSUs+ePQO2o+G///XyslavFjl61Pu7U6e8P48d8z6UxYpF/W2dtmTJEhXry4vaGjVqpOIqVarEdEy2+fO9KwXaEOCIMmVEqlb9ebIj4p1jCxf2nlJnwuOO/HYs9WVdc2u7rFcv78G7/v1F+vTxUkAmTvSeFbnsstD3R9pAgvhpIrtrV3zHgdDs2CFy9qxXA7RMGe+/n/JeK1XyPphwy7FjIsuWcdXVVX5LuefkeGV54A6OZfIoWNArBXrwoMinn4p8+63I9dd7fT/9GYqEuvKan/1UTCGPLzAhQs2aibz7rvl3y5d7v1EuXepdNYBbFi0SOXOGyaur2rXzaoJmZ3t3tES8OyNnz4rUqxffsSE0HMvk89NFHhEvTfKGG7z6vaFi8hoHt94qcvPNInXqeL+NrFvn5X506ULKgGtSU72VmXRZWd6fzZuzwlZuMjMzVXz27Fnf7QYMGKDi1J++ufLA/PneF6PfVR8ktj59vOot7dt7NZdPnPCKod98s/fLJtzBsUweGzZ4lZXq1/fq2r/yisiKFd7fhYPJaxw0buw9QZmV5RU/r1pV5MknvYRmAPGTne2t5jNuXLxHgnCVKuU9zDN4sMg993j5kR06JOdyosmOY5k8ChUS+ec/vdVECxTwLu6sWydy7bXh7Y/JaxyMG8eXYzLr0cP7D+5JTfVuScJt1ap5aTtwH8cyOTRs+MtSWZEg5RkAAADO4MorgDy1cePGXP++YcOGRrt9+/Z5MRwAgGO48goAAABnMHkFAACAM1JyQlggOCUl5aCI7I7dcBBA5ZycnLRo7IjjGHccy+TBsUweHMvkwbFMDr7HMaTJKwAAABBPpA0AAADAGUxeAQAA4AwmrwAAAHAGk1cAAAA4g8krAAAAnMHkFQAAAM5g8goAAABnMHkFAACAM5i8AgAAwBn/D0rc/As23lzEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x288 with 12 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_test = next(iter(mnist_test.batch(12)))\n",
    "\n",
    "preds = model(batch_test[0])\n",
    "\n",
    "tf.print(preds.shape)\n",
    "preds = tf.argmax(preds, axis=1)\n",
    "print(preds)\n",
    "\n",
    "fig = plt.figure(figsize=(12, 4))\n",
    "for i in range(12):\n",
    "    ax = fig.add_subplot(2, 6, i+1)\n",
    "    ax.set_xticks([]); ax.set_yticks([])\n",
    "    img = batch_test[0][i, :, :, 0]\n",
    "    ax.imshow(img, cmap='gray_r')\n",
    "    ax.text(0.9, 0.1, '{}'.format(preds[i]), \n",
    "            size=15, color='blue',\n",
    "            horizontalalignment='center',\n",
    "            verticalalignment='center', \n",
    "            transform=ax.transAxes)\n",
    "    \n",
    "#plt.savefig('figures/15_13.png', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "if not os.path.exists('models'):\n",
    "    os.mkdir('models')\n",
    "\n",
    "\n",
    "model.save('models/mnist-cnn.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "\n",
    "Readers may ignore the next cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NbConvertApp] Converting notebook ch15_part1.ipynb to script\n",
      "[NbConvertApp] Writing 10824 bytes to ch15_part1.py\n"
     ]
    }
   ],
   "source": [
    "! python ../.convert_notebook_to_script.py --input ch15_part1.ipynb --output ch15_part1.py"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}