diff --git a/ch15/ch15_part1.ipynb b/ch15/ch15_part1.ipynb new file mode 100644 index 00000000..ca805a97 --- /dev/null +++ b/ch15/ch15_part1.ipynb @@ -0,0 +1,939 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "*Python Machine Learning 3rd Edition* by [Sebastian Raschka](https://sebastianraschka.com) & [Vahid Mirjalili](http://vahidmirjalili.com), Packt Publishing Ltd. 2019\n", + "\n", + "Code Repository: https://github.com/rasbt/python-machine-learning-book-3rd-edition\n", + "\n", + "Code License: [MIT License](https://github.com/rasbt/python-machine-learning-book-3rd-edition/blob/master/LICENSE.txt)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chapter 15: Classifying Images with Deep Convolutional Neural Networks (Part 1/2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that the optional watermark extension is a small IPython notebook plugin that I developed to make the code reproducible. You can just skip the following line(s)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sebastian Raschka & Vahid Mirjalili \n", + "last updated: 2019-11-01 \n", + "\n", + "numpy 1.17.2\n", + "scipy 1.2.1\n", + "matplotlib 3.1.0\n", + "tensorflow 2.0.0\n", + "tensorflow_datasets 1.3.0\n" + ] + } + ], + "source": [ + "%load_ext watermark\n", + "%watermark -a \"Sebastian Raschka & Vahid Mirjalili\" -u -d -p numpy,scipy,matplotlib,tensorflow,tensorflow_datasets" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The building blocks of convolutional neural networks\n", + "\n", + "### Understanding CNNs and feature hierarchies\n", + "\n", + "### Performing discrete convolutions\n", + "\n", + "### Discrete convolutions in one dimension\n", + "\n", + "### Padding inputs to control the size of the output feature maps\n", + "\n", + "### Determining the size of the convolution output" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TensorFlow version: 2.0.0\n", + "NumPy version: 1.17.2\n" + ] + } + ], + "source": [ + "import tensorflow as tf\n", + "import numpy as np\n", + "\n", + "print('TensorFlow version:', tf.__version__)\n", + "print('NumPy version: ', np.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conv1d Implementation: [ 5. 14. 16. 26. 24. 34. 19. 22.]\n", + "Numpy Results: [ 5 14 16 26 24 34 19 22]\n" + ] + } + ], + "source": [ + "def conv1d(x, w, p=0, s=1):\n", + " w_rot = np.array(w[::-1])\n", + " x_padded = np.array(x)\n", + " if p > 0:\n", + " zero_pad = np.zeros(shape=p)\n", + " x_padded = np.concatenate(\n", + " [zero_pad, x_padded, zero_pad])\n", + " res = []\n", + " for i in range(0, int(len(x)/s),s):\n", + " res.append(np.sum(\n", + " x_padded[i:i+w_rot.shape[0]] * w_rot))\n", + " return np.array(res)\n", + "\n", + "\n", + "## Testing:\n", + "x = [1, 3, 2, 4, 5, 6, 1, 3]\n", + "w = [1, 0, 3, 1, 2]\n", + "\n", + "print('Conv1d Implementation:',\n", + " conv1d(x, w, p=2, s=1))\n", + "\n", + "print('Numpy Results:',\n", + " np.convolve(x, w, mode='same')) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Performing a discrete convolution in 2D" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conv2d Implementation:\n", + " [[11. 25. 32. 13.]\n", + " [19. 25. 24. 13.]\n", + " [13. 28. 25. 17.]\n", + " [11. 17. 14. 9.]]\n", + "SciPy Results:\n", + " [[11 25 32 13]\n", + " [19 25 24 13]\n", + " [13 28 25 17]\n", + " [11 17 14 9]]\n" + ] + } + ], + "source": [ + "import scipy.signal\n", + "\n", + "\n", + "def conv2d(X, W, p=(0, 0), s=(1, 1)):\n", + " W_rot = np.array(W)[::-1,::-1]\n", + " X_orig = np.array(X)\n", + " n1 = X_orig.shape[0] + 2*p[0]\n", + " n2 = X_orig.shape[1] + 2*p[1]\n", + " X_padded = np.zeros(shape=(n1, n2))\n", + " X_padded[p[0]:p[0]+X_orig.shape[0],\n", + " p[1]:p[1]+X_orig.shape[1]] = X_orig\n", + "\n", + " res = []\n", + " for i in range(0, int((X_padded.shape[0] - \n", + " W_rot.shape[0])/s[0])+1, s[0]):\n", + " res.append([])\n", + " for j in range(0, int((X_padded.shape[1] - \n", + " W_rot.shape[1])/s[1])+1, s[1]):\n", + " X_sub = X_padded[i:i+W_rot.shape[0],\n", + " j:j+W_rot.shape[1]]\n", + " res[-1].append(np.sum(X_sub * W_rot))\n", + " return(np.array(res))\n", + "\n", + "X = [[1, 3, 2, 4], [5, 6, 1, 3], [1, 2, 0, 2], [3, 4, 3, 2]]\n", + "W = [[1, 0, 3], [1, 2, 1], [0, 1, 1]]\n", + "\n", + "print('Conv2d Implementation:\\n',\n", + " conv2d(X, W, p=(1, 1), s=(1, 1)))\n", + "\n", + "\n", + "print('SciPy Results:\\n',\n", + " scipy.signal.convolve2d(X, W, mode='same'))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Subsampling layers" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Putting everything together – implementing a CNN\n", + "\n", + "### Working with multiple input or color channels\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**TIP: Reading an image file**" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Image shape: (252, 221, 3)\n", + "Number of channels: 3\n", + "Image data type: \n", + "tf.Tensor(\n", + "[[[179 134 110]\n", + " [182 136 112]]\n", + "\n", + " [[180 135 111]\n", + " [182 137 113]]], shape=(2, 2, 3), dtype=uint8)\n" + ] + } + ], + "source": [ + "import tensorflow as tf\n", + "\n", + "\n", + "img_raw = tf.io.read_file('example-image.png')\n", + "img = tf.image.decode_image(img_raw)\n", + "print('Image shape:', img.shape)\n", + "print('Number of channels:', img.shape[2])\n", + "print('Image data type:', img.dtype)\n", + "print(img[100:102, 100:102, :])" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Image shape: (252, 221, 3)\n", + "Number of channels: 3\n", + "Image data type: uint8\n", + "[[[179 134 110]\n", + " [182 136 112]]\n", + "\n", + " [[180 135 111]\n", + " [182 137 113]]]\n" + ] + } + ], + "source": [ + "import imageio\n", + "\n", + "\n", + "img = imageio.imread('example-image.png')\n", + "print('Image shape:', img.shape)\n", + "print('Number of channels:', img.shape[2])\n", + "print('Image data type:', img.dtype)\n", + "print(img[100:102, 100:102, :])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**INFO-BOX: The rank of a grayscale image for input to a CNN**" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Rank: 3\n", + "Shape: TensorShape([252, 221, 1])\n" + ] + } + ], + "source": [ + "img_raw = tf.io.read_file('example-image-gray.png')\n", + "img = tf.image.decode_image(img_raw)\n", + "tf.print('Rank:', tf.rank(img))\n", + "tf.print('Shape:', img.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Rank: 2\n", + "Shape: (252, 221)\n", + "New Shape: TensorShape([252, 221, 1])\n" + ] + } + ], + "source": [ + "img = imageio.imread('example-image-gray.png')\n", + "tf.print('Rank:', tf.rank(img))\n", + "tf.print('Shape:', img.shape)\n", + "\n", + "img_reshaped = tf.reshape(img, (img.shape[0], img.shape[1], 1))\n", + "tf.print('New Shape:', img_reshaped.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Regularizing a neural network with dropout\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "from tensorflow import keras\n", + "\n", + "\n", + "conv_layer = keras.layers.Conv2D(\n", + " filters=16, kernel_size=(3, 3),\n", + " kernel_regularizer=keras.regularizers.l2(0.001))\n", + "\n", + "fc_layer = keras.layers.Dense(\n", + " units=16, kernel_regularizer=keras.regularizers.l2(0.001))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loss Functions for Classification\n", + "\n", + " * **`BinaryCrossentropy()`**\n", + " * `from_logits=False` \n", + " * `from_logits=True`\n", + "\n", + " * **`CategoricalCrossentropy()`**\n", + " * `from_logits=False`\n", + " * `from_logits=True`\n", + " \n", + " * **`SparseCategoricalCrossentropy()`**\n", + " * `from_logits=False`\n", + " * `from_logits=True`\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "BCE (w Probas): 0.3711 (w Logits): 0.3711\n", + "CCE (w Probas): 0.5996 (w Logits): 0.5996\n", + "Sparse CCE (w Probas): 0.5996 (w Logits): 0.5996\n" + ] + } + ], + "source": [ + "####### Binary Crossentropy\n", + "bce_probas = tf.keras.losses.BinaryCrossentropy(from_logits=False)\n", + "bce_logits = tf.keras.losses.BinaryCrossentropy(from_logits=True)\n", + "\n", + "logits = tf.constant([0.8])\n", + "probas = tf.keras.activations.sigmoid(logits)\n", + "\n", + "tf.print(\n", + " 'BCE (w Probas): {:.4f}'.format(\n", + " bce_probas(y_true=[1], y_pred=probas)),\n", + " '(w Logits): {:.4f}'.format(\n", + " bce_logits(y_true=[1], y_pred=logits)))\n", + "\n", + "\n", + "####### Categorical Crossentropy\n", + "cce_probas = tf.keras.losses.CategoricalCrossentropy(\n", + " from_logits=False)\n", + "cce_logits = tf.keras.losses.CategoricalCrossentropy(\n", + " from_logits=True)\n", + "\n", + "logits = tf.constant([[1.5, 0.8, 2.1]])\n", + "probas = tf.keras.activations.softmax(logits)\n", + "\n", + "tf.print(\n", + " 'CCE (w Probas): {:.4f}'.format(\n", + " cce_probas(y_true=[0, 0, 1], y_pred=probas)),\n", + " '(w Logits): {:.4f}'.format(\n", + " cce_logits(y_true=[0, 0, 1], y_pred=logits)))\n", + "\n", + "####### Sparse Categorical Crossentropy\n", + "sp_cce_probas = tf.keras.losses.SparseCategoricalCrossentropy(\n", + " from_logits=False)\n", + "sp_cce_logits = tf.keras.losses.SparseCategoricalCrossentropy(\n", + " from_logits=True)\n", + "\n", + "tf.print(\n", + " 'Sparse CCE (w Probas): {:.4f}'.format(\n", + " sp_cce_probas(y_true=[2], y_pred=probas)),\n", + " '(w Logits): {:.4f}'.format(\n", + " sp_cce_logits(y_true=[2], y_pred=logits)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Implementing a deep convolutional neural network using TensorFlow\n", + "\n", + "### The multilayer CNN architecture" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Loading and preprocessing the data" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow_datasets as tfds\n", + "import pandas as pd\n", + "\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['test', 'train'])\n" + ] + } + ], + "source": [ + "## MNIST dataset\n", + "\n", + "mnist_bldr = tfds.builder('mnist')\n", + "mnist_bldr.download_and_prepare()\n", + "datasets = mnist_bldr.as_dataset(shuffle_files=False)\n", + "print(datasets.keys())\n", + "mnist_train_orig, mnist_test_orig = datasets['train'], datasets['test']" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "BUFFER_SIZE = 10000\n", + "BATCH_SIZE = 64\n", + "NUM_EPOCHS = 20" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "mnist_train = mnist_train_orig.map(\n", + " lambda item: (tf.cast(item['image'], tf.float32)/255.0, \n", + " tf.cast(item['label'], tf.int32)))\n", + "\n", + "mnist_test = mnist_test_orig.map(\n", + " lambda item: (tf.cast(item['image'], tf.float32)/255.0, \n", + " tf.cast(item['label'], tf.int32)))\n", + "\n", + "tf.random.set_seed(1)\n", + "\n", + "mnist_train = mnist_train.shuffle(buffer_size=BUFFER_SIZE,\n", + " reshuffle_each_iteration=False)\n", + "\n", + "mnist_valid = mnist_train.take(10000).batch(BATCH_SIZE)\n", + "mnist_train = mnist_train.skip(10000).batch(BATCH_SIZE)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Implementing a CNN using the TensorFlow Keras API\n", + "\n", + "#### Configuring CNN layers in Keras\n", + "\n", + " * **Conv2D:** `tf.keras.layers.Conv2D`\n", + " * `filters`\n", + " * `kernel_size`\n", + " * `strides`\n", + " * `padding`\n", + " \n", + " \n", + " * **MaxPool2D:** `tf.keras.layers.MaxPool2D`\n", + " * `pool_size`\n", + " * `strides`\n", + " * `padding`\n", + " \n", + " \n", + " * **Dropout** `tf.keras.layers.Dropout2D`\n", + " * `rate`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Constructing a CNN in Keras" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "model = tf.keras.Sequential()\n", + "\n", + "model.add(tf.keras.layers.Conv2D(\n", + " filters=32, kernel_size=(5, 5),\n", + " strides=(1, 1), padding='same',\n", + " data_format='channels_last',\n", + " name='conv_1', activation='relu'))\n", + "\n", + "model.add(tf.keras.layers.MaxPool2D(\n", + " pool_size=(2, 2), name='pool_1'))\n", + " \n", + "model.add(tf.keras.layers.Conv2D(\n", + " filters=64, kernel_size=(5, 5),\n", + " strides=(1, 1), padding='same',\n", + " name='conv_2', activation='relu'))\n", + "\n", + "model.add(tf.keras.layers.MaxPool2D(pool_size=(2, 2), name='pool_2'))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "TensorShape([16, 7, 7, 64])" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.compute_output_shape(input_shape=(16, 28, 28, 1))" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "TensorShape([16, 3136])" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + " model.add(tf.keras.layers.Flatten())\n", + " \n", + "model.compute_output_shape(input_shape=(16, 28, 28, 1))" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "model.add(tf.keras.layers.Dense(\n", + " units=1024, name='fc_1', \n", + " activation='relu'))\n", + "\n", + "model.add(tf.keras.layers.Dropout(\n", + " rate=0.5))\n", + " \n", + "model.add(tf.keras.layers.Dense(\n", + " units=10, name='fc_2',\n", + " activation='softmax'))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "TensorShape([16, 10])" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tf.random.set_seed(1)\n", + "model.build(input_shape=(None, 28, 28, 1))\n", + "\n", + "model.compute_output_shape(input_shape=(16, 28, 28, 1))" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"sequential\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "conv_1 (Conv2D) multiple 832 \n", + "_________________________________________________________________\n", + "pool_1 (MaxPooling2D) multiple 0 \n", + "_________________________________________________________________\n", + "conv_2 (Conv2D) multiple 51264 \n", + "_________________________________________________________________\n", + "pool_2 (MaxPooling2D) multiple 0 \n", + "_________________________________________________________________\n", + "flatten (Flatten) multiple 0 \n", + "_________________________________________________________________\n", + "fc_1 (Dense) multiple 3212288 \n", + "_________________________________________________________________\n", + "dropout (Dropout) multiple 0 \n", + "_________________________________________________________________\n", + "fc_2 (Dense) multiple 10250 \n", + "=================================================================\n", + "Total params: 3,274,634\n", + "Trainable params: 3,274,634\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/20\n", + "782/782 [==============================] - 29s 38ms/step - loss: 0.1450 - accuracy: 0.9549 - val_loss: 0.0000e+00 - val_accuracy: 0.0000e+00\n", + "Epoch 2/20\n", + "782/782 [==============================] - 28s 35ms/step - loss: 0.0491 - accuracy: 0.9858 - val_loss: 0.0401 - val_accuracy: 0.9874\n", + "Epoch 3/20\n", + "782/782 [==============================] - 27s 35ms/step - loss: 0.0311 - accuracy: 0.9903 - val_loss: 0.0493 - val_accuracy: 0.9846\n", + "Epoch 4/20\n", + "782/782 [==============================] - 28s 35ms/step - loss: 0.0241 - accuracy: 0.9920 - val_loss: 0.0489 - val_accuracy: 0.9855\n", + "Epoch 5/20\n", + "782/782 [==============================] - 27s 34ms/step - loss: 0.0197 - accuracy: 0.9942 - val_loss: 0.0453 - val_accuracy: 0.9874\n", + "Epoch 6/20\n", + "782/782 [==============================] - 27s 35ms/step - loss: 0.0161 - accuracy: 0.9953 - val_loss: 0.0361 - val_accuracy: 0.9897\n", + "Epoch 7/20\n", + "782/782 [==============================] - 28s 35ms/step - loss: 0.0125 - accuracy: 0.9961 - val_loss: 0.0401 - val_accuracy: 0.9903\n", + "Epoch 8/20\n", + "782/782 [==============================] - 27s 34ms/step - loss: 0.0117 - accuracy: 0.9963 - val_loss: 0.0371 - val_accuracy: 0.9918\n", + "Epoch 9/20\n", + "782/782 [==============================] - 28s 35ms/step - loss: 0.0113 - accuracy: 0.9966 - val_loss: 0.0367 - val_accuracy: 0.9904\n", + "Epoch 10/20\n", + "782/782 [==============================] - 27s 35ms/step - loss: 0.0097 - accuracy: 0.9967 - val_loss: 0.0426 - val_accuracy: 0.9912\n", + "Epoch 11/20\n", + "782/782 [==============================] - 28s 35ms/step - loss: 0.0098 - accuracy: 0.9974 - val_loss: 0.0367 - val_accuracy: 0.9925\n", + "Epoch 12/20\n", + "782/782 [==============================] - 27s 35ms/step - loss: 0.0093 - accuracy: 0.9974 - val_loss: 0.0395 - val_accuracy: 0.9912\n", + "Epoch 13/20\n", + "782/782 [==============================] - 27s 35ms/step - loss: 0.0074 - accuracy: 0.9975 - val_loss: 0.0509 - val_accuracy: 0.9912\n", + "Epoch 14/20\n", + "782/782 [==============================] - 27s 35ms/step - loss: 0.0082 - accuracy: 0.9975 - val_loss: 0.0595 - val_accuracy: 0.9885\n", + "Epoch 15/20\n", + "782/782 [==============================] - 27s 35ms/step - loss: 0.0056 - accuracy: 0.9982 - val_loss: 0.0601 - val_accuracy: 0.9895\n", + "Epoch 16/20\n", + "782/782 [==============================] - 27s 35ms/step - loss: 0.0086 - accuracy: 0.9976 - val_loss: 0.0431 - val_accuracy: 0.9917\n", + "Epoch 17/20\n", + "782/782 [==============================] - 27s 34ms/step - loss: 0.0058 - accuracy: 0.9982 - val_loss: 0.0538 - val_accuracy: 0.9912\n", + "Epoch 18/20\n", + "782/782 [==============================] - 28s 35ms/step - loss: 0.0080 - accuracy: 0.9979 - val_loss: 0.0500 - val_accuracy: 0.9896\n", + "Epoch 19/20\n", + "782/782 [==============================] - 29s 37ms/step - loss: 0.0064 - accuracy: 0.9980 - val_loss: 0.0424 - val_accuracy: 0.9923\n", + "Epoch 20/20\n", + "782/782 [==============================] - 29s 37ms/step - loss: 0.0041 - accuracy: 0.9988 - val_loss: 0.0469 - val_accuracy: 0.9915\n" + ] + } + ], + "source": [ + "model.compile(optimizer=tf.keras.optimizers.Adam(),\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n", + " metrics=['accuracy']) # same as `tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy')`\n", + "\n", + "history = model.fit(mnist_train, epochs=NUM_EPOCHS, \n", + " validation_data=mnist_valid, \n", + " shuffle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtwAAAELCAYAAADqelFjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd3hU1dbA4d9KL5CEkgKhCiogUiOKgAqKoFIsIHAtiAULXOWqKCiXpthQQSzwoSLXijQBFUFUsCu9CIiiICQhoQSSAOnZ3x9nJqRMyqTMTMh6n2ceZk7dk4Qza/ZZe20xxqCUUkoppZSqGl7uboBSSimllFJnMw24lVJKKaWUqkIacCullFJKKVWFNOBWSimllFKqCmnArZRSSimlVBXycXcDqlL9+vVNs2bN3N0MpZQql02bNh01xoS7ux2upNdtpVR1VdI1+6wOuJs1a8bGjRvd3QyllCoXEfnH3W1wNb1uK6Wqq5Ku2ZpSopRSqlgiMk9EDovIb8WsFxGZJSJ7RWS7iHTKt264iPxpewx3XauVUsqzaMCtlFKqJPOBviWsvwY41/YYCcwGEJG6wCTgYqALMElE6lRpS5VSykNpwK2UUqpYxpjvgKQSNhkIvGssvwBhItIA6AOsMcYkGWOOA2soOXBXSqmzlgbcSimlKiIaOJjvdaxtWXHLixCRkSKyUUQ2HjlypMoaqpRS7qIBt1JKqYoQB8tMCcuLLjRmrjEmxhgTEx5eo4qyKKVqiLO6Somzlm2JY/rqPcSfSKNhWCBj+5zP9R0ddsgopZSyxAKN871uBMTbll9RaPk6l7VKKaWcVJVxoAbcNsu2xDF+6Q7SsnIAiDuRxvilOwA06D7LpKSkcPjwYbKystzdFFVD+fr6EhERQUhIiLubUhlWAKNFZAHWAMlkY8whEVkNPJNvoOTVwHh3NVIppUpS1XGgBtw201fvyfsh26Vl5TB99R4NuM8iKSkpJCYmEh0dTWBgICKO7norVXWMMaSlpREXFwfg8UG3iHyE1VNdX0RisSqP+AIYY+YAK4Frgb3AaWCEbV2SiDwFbLAdaqoxpqTBl0op5TbTV/9epXGgBtw28SfSnFquqqfDhw8THR1NUFCQu5uiaigRISgoiOjoaOLj4z0+4DbGDCtlvQFGFbNuHjCvKtqllHKOu9JmPTVdNyfX8HtCChv3H2fD/iTiTqQ73K6y4kANuG0ahgUS5+CH2jAs0A2tUVUlKyuLwED9nSr3CwwM1LQmpZRLuCtttiLnrUig7mjfvm2j2HrwBBv3J7F+/3G2/HOc1IxsABqEBhDo612khxsqLw7UgNtmbJ/zC/xRAAT6ejO2z/lubJWqCppGojyB/h0qpVwhJ9fwzMrdDtMlnv58Fz1bRRAa6Ftp50vPymFPQio74pKLPe+EZTs4diqTBqEBtkcg4bX98fayrosVCdQ/2RzL+E92kJ6Vm7fvfxZu5eGFkGurk3R+ZG0GdGjIRc3qclHzukSHBRY5J1RuHKgBt439F/jsF7tJTMkgLNCXyQMu8IjbHkoppZSqGpXdk1qWfavynMdPZbLl4HE2/3OCzQeOs+3gCU5lFu25BTh6MpP2U74kOiyQ1g1CaNOgNm0ahtC6QQiN6wThlS8AdnTOtMwcdiek8FtcMjtik/ktPoU/E1PJznVYATTPyYwcnvpsV4Fl3l5CZG1/okID2H0ohTRbwGxnBeq/sfGfJE5l5HAyI5tTGdmcysyx/rU9UtKzi5zPGAj292HWsA50alKHsCC/ItvYf4ZVlf4iVvqd64hIX+AVwBt4yxjzXKH1lwEzgXbAUGPM4kLrQ4DdwCfGmNElnSsmJsZs3LjRqfZlZOdw/oRVPNz7PB688lyn9lWeb/fu3bRu3drdzVAKKP3vUUQ2GWNiXNgktyvPdVup8iquV/PZGy8sU8pDefat7HP6+3gxoENDcnNhy4Hj/H30FGAFsK2iatOpSR0+2x7P8dNFU9jqBftxV4/m7D6Uyq74ZPYdPZXXC1zL34dWUbUJ8PXi131JZOWciRe9RQiv7ceRk5nk2HaoG+xH2+hQ2jYM4cLoUNpGhzJ07s8Oc6OjwwL4/MEexJ9IJyElzfo3OZ345DQSktP56a9jxf4M6gb7EezvTbCfD8H+1qOWvzdBfj7U8vdh/k/7He4nwL7nrivx51tRJV2zXdrDLSLewOtAb6warRtEZIUxJv/XnAPAHcCjxRzmKeDbqmqjv483dYP9SEhxnDyvlDuVJQ1h7dq1XHHFFRU6T1RUFHfffTdPP/10hY6zatUqrrnmGv78809atmxZoWMppVRlK65C2VOf7aJ2QMkh0lOf7XK47+QVO0lJzyIzO5fMnFyysg1ZOdbzzOxcFm486HC/8Ut38N0fRxARRMBLQBC8vKxrv2AF3IX3zcjOZdHGWOoG+9GpSRg3dW5EpyZ1aN84lCA/6z10blrHYZD/335tCgT5aZk57ElMZfehFHYfSmFXfAo/7j1WZMaqHGM4fjqLB65oQdvoUC6MDqVBaECRz6ixfVoVk6bRirAgP8KC/GjTsOjA8W7PfeNwXF10WCA/jutV9JeRz5pdiR45Js/VKSVdgL3GmL8BbHVbBwJ5AbcxZr9tXW7hnUWkMxAJrAKqrNcnMiSAxGQNuJXn+fnnn/Oep6Wl0atXLyZMmMB115351t6mTZsKn2flypVERERU+DhKKeVpTmdms/XgCTbtP+4wMAM4diqTu/5XvjstJ9KymLh8Z4Fl3l6Cr7fg6+3F6WLSO9KyctjwTxK5uVb5UAPkGoMx9txjU2xqiACbJlxVbKdMWdMlAv286dA4jA6Nw/KWNR/3ucNjZmbn8sjVJec3lzdNoyLj6jx1TJ6rA+5o4GC+17FYEyWUSkS8gJeA24ArS9huJDASoEmTJuVqZFSIv/ZwK490ySWX5D0/efIkAC1atCiwvDjp6ekEBASU6TydOnUqXwOVUqoCqiK3+VByGhv3H2fTP9Zj16GUvDQIHy9xmG8cXtuft4eX3K931/82ciQ1o8jyqBB/Pn+wB74+Xvh5e+Hr7ZU3GBBK7r39/rGSe2+L27dhWOnzSlzfMbpc+cgVreJWnvNWJJ+6qnOxy8vLxedz9NdQ1iTyB4CVxpiDJW1kjJlrjIkxxsSEh4c73UCAqNAAEjXgVqVYtiWObs99Q/Nxn9PtuW9YtiXO3U3KM2fOHESEzZs306NHDwIDA3n11VcxxvDII4/Qtm1bgoODady4McOHD+fIkSMF9o+KimLChAl5r4cOHUr37t1ZuXIlF1xwAbVq1eLyyy9nz549Trft5MmTPPDAA0RERBAYGMjFF1/M2rVrC2yzbt06Lr30UmrXrk1oaCidOnVi+fLleeuXLFlCx44dCQoKom7dunTt2pWffvrJ6bYopSpfea+N9vzkuBNpGM5UpijL/o72fWThNjpMWU3XZ7/h3x9t4eMNB6nl78P9l7fgnREXsW3i1bw4uD2Bvt4FjhXo682T17amXaOwEh9PXtva4b7jrmlNvVr+hAT4EuDrXSDYBqsH1tF+Ze29Le++5eWOc4IVOP84rhf7nruOH8f1cipgrsi+VcXVPdyxQON8rxsB8WXctyvQQ0QeAGoBfiJy0hgzrpLbSGRIAEdPZpKZnYufj6u/k6jqwF01TZ01ZMgQRo0axdSpU6lbty65ubkkJSUxYcIEGjRoQGJiItOnT+fqq69m8+bNJfaQ7N27lwkTJjB58mR8fX15+OGHGTZsGJs3b3aqTcOHD+err77i2WefpVmzZsyePZs+ffrwww8/0KVLF44dO0b//v0ZMmQIU6dOJScnh+3bt3P8+HEAdu3axdChQxk7diwvv/wyp0+fZuPGjXnrlVKVozy9zWW9Nmbn5HIiLYukU5kcO5nJ8dOZTFqx02Fu87gl21m6JY4se060LRfanhedlW04nJpO4Y7qHGPIyM5lUv82xDStS6sGtfH1LviZ7o6e1OrWe+upPcbVjasD7g3AuSLSHIgDhgL/KsuOxphb7M9F5A4gpiqCbYCoEOu2++HUdBrV0RkJz3ZTPt3JrvgUp/bZcuAEmTlFSxY9tng7H60/UObjtGkYwqT+Fzh1bmc8+uij3HvvvQWWvfPOO3nPc3Jy6Ny5My1btmTDhg106dKl2GMlJSXx66+/0rRpU8BKURk2bBj79++nWbNmZWrP1q1bWbp0KQsWLGDIkCEA9OnTh1atWjFt2jSWL1/O7t27OXXqFK+//jr+/v5529ht3ryZiIgInnnmmbxl+XPYlVIV50ynQm6uITkti+OnM3n6c8d1l8ct2c77v/xD0qlMkk5nkpyWRVmLpKVn55KcloW/txcBvl7UDvCxUjVsKRt+3l58vNHxze/0rFxGdGte4vHLm2pRkX3dcc6KcMc5zzYuDbiNMdkiMhpYjVUWcJ4xZqeITAU2GmNWiMhFwCdAHaC/iEwxxlRdROJAZKgVcCemaMCtHCscbJe23F0cBaIrVqzgmWeeYffu3aSknPmi8ccff5QYcJ933nl5wTacGZwZGxtb5oB7/fr1eHt7c+ONN+Yt8/b2ZtCgQcydOzfvPAEBAQwdOpQ777yTyy67jNDQ0Lzt27Vrx6FDh7j77rsZOnQol156KUFB+v9UqcpUXPWO8Ut3sGJbPMdPZ3LitBVklyV4Ts/Oxdfbi9YNQ6gb5Efd4KKPEe9scDh+KjoskOWjupV4/B/2HvXIyhRK2bl84htjzEpgZaFlE/M934CValLSMeYD86ugecCZHu6E5KKDIdTZpzw9zCUNevn43q6V0axKERkZWeD1jz/+yA033MDQoUN58sknCQ8PJysri8suu4z09JLHLYSFhRV47ednTRxQ2n75HTp0iDp16uDrW3BWs8jIyLyUkIiICFavXs3UqVO56aabAOjbty+vvvoqTZs2pV27dixdupTp06fTp08f/P39GTRoEDNnzqRu3bplbotSqqBjJzPYsP846/clFVu9Iy0rh4TkdOoE+9IwLJA6Qb7UCfKzHsG+PP3Zbo6dyiyyX3RYIB+NLHlw97hriishV30rUyhlpzNNOpAXcOvASVWM6nJxL5yTvWTJEpo0acIHH3yQt6w8Ax/Lq0GDBhw/fpysrKwCQXdiYiJ16tTJe92jRw/WrFnDqVOnWLNmDf/5z38YPnw469atA+D666/n+uuv58SJE3z66aeMGTMGLy8v5s+f77L3olR1UFIedvyJNNbvS2L9/iTW70ti72Gr8pG/jxd+Pl5kZhe9YxcdFsjKh3oUez5Byn1trG65zUo5QwNuB8KCfPHz8dJKJapY1fXinpaWltczbZc/+K5qXbp0IScnh08++YSbb74ZsPLIlyxZQvfu3YtsHxwczPXXX8+WLVuYPXt2kfVhYWHcdtttfPXVV+zatavIeqVqMkd52GMXb+P9X/aTkJJB7HGrF7u2vw8xzepwY6doLm5el7bRoXyxI6FcgXNFr43VLbdZqbLSgNsBESEqJIAEnfxGlaA6Xtx79+7NnDlzGDt2LH379uW7775jwYIFLjt/hw4duPHGGxk5ciRJSUk0bdqU2bNns3///rzA3z6ocuDAgTRq1IiDBw8yb948evWy6tPOmjWL7du307t3bxo0aMDvv//OsmXLuP/++132PpSqDl5Y/XuRPOysHMPmAyfoc0EUd3VvzkXN6tK6QUiR0nUV7W2ubtdGpaqaBtzFiAoJ0JQSdda58cYbeeqpp3jjjTd444036NGjB8uWLeOCC1w3Lvl///sfY8eO5b///S+pqam0b9+eVatWcdFFFwHWoMns7Gwef/xxjhw5QkREBAMGDMirStKhQwe++OILxowZw/Hjx2nYsCGjR49m8uTJLnsPSnkqYwy7DqWwfGs88Sccf4YZA7Nv7VzqsTRwVqryiClrXZ5qKCYmxmzcWL6pWf/90Ra2x57g27E9K7lVyp12795N69at3d0MpYDS/x5FZJMxpuTp7s4yFblu12T/HDvFiq3xLN8Wz97DJ/HxEny8hPRi8rB/HFfyjIZKKeeVdM3WHu5iRIX482VyOsaYUqdLVUoppaqSo8GP3VrW5/Pt8SzbGs/WgycA6NK8LtNuaMu1bRvw7R9HqsXgbqVqAg24ixEZEkCGrdh+WJBf6TsopZRSVcDR4MeHF27Nm1mxdYMQxl3Tiv7tGxKdr+50dR3crdTZSAPuYkSFnikNqAG3Ukopd3E0CU2ugVr+Pix94FLOi6xd7L6ah62UZ/BydwM81ZnJb3TgpFJKKfcwxhQ7Cc2pjOwSg22llOfQgLsYkSFnpndXSimlXO1wajr3vrep2PU6bblS1YcG3MWI1OndlVJKuYExhuVb47h6xnes++MIA9o1INC34Me1Dn5UqnrRHO5i+Pl4US/YT2txK6WUcpkjqRlMWLaD1TsT6dA4jBcHt6dlRK0Sp2hXSnk+DbhLEBkSoCklSimlqpwxhhXb4pm0YienM3MYf00r7u5xTt4MkDr4UanqTQPuEkSF6vTuSimlqtaR1Az+u+w3Vu1MsPVqt6NlhA6GdIvUBPj2BYhdD/f94O7WqMrgIb9TzeEugfZwK0/Tr18/LrzwwmLXjx49mjp16pCRUbaxB3v37kVEWLVqVd6yRo0aMW7cuBL327p1KyLCDz84d/GaM2cOK1asKLK8LOesLNnZ2YgIc+bMccn5lCqOvVf76hnf8s2ew4y7phVL7r9Ug213SE2Azx6GV9rDlvcgYYe7W6QqysN+p9rDXYKokACOncokIzsHfx9vdzdHKYYNG8att97Kzp07ueCCCwqsy8nJYfHixdx44434+/uX+xyffvop9evXr2hTHZozZw4xMTEMGDDAZedUylPkz8OODAkgorYf2+NSaN84jJe0V9s9/vkZVj4Kh3eDlzfkZLq7Raqi7D3aW94HciEny90tArSHu0RRoVbQcjhFK5UozzBw4ECCgoJYsGBBkXVr164lMTGRYcOGVegcHTt2pHHjxhU6RnU4p1KuZJ8tMu5EGgZrUrXtcSn0b9eAJfd1PXuDbXsv45zu7m7JGdkZsGMxzO8H7/SFxN/A5BQNtn+YASePuKeNnswTf6f5LRoBG9+GnAyPCbZBA+4SaS1uVWYuugDVqlWLfv368fHHHxdZt2DBAiIjI+nZsycAcXFxjBgxgubNmxMYGMh5553HpEmTyMoq+QLkKL3j1VdfpXHjxgQHBzNw4EASEhKK7Dd9+nRiYmIICQkhMjKSgQMH8tdff+Wt7969O9u2bePtt99GRBAR3n///WLPuWDBAtq2bYu/vz9NmjRh4sSJ5OScmW3vrbfeQkTYuXMnV111FcHBwbRu3Zrly5eX8lN0bNasWbRs2RJ/f3/OPfdcZs2aVWD9gQMHGDRoEOHh4QQGBtKyZUsmT56ct37Hjh306dOHOnXqUKtWLdq0aaNpKyqPo9kiATYfOIGP91n4Uexht/ML+GEmLLkLThyA7o9A+1vAJwC8C80q/dVkeLk1LLoDjv3l6Eg1iyf/Tu2MgcA61nPxth757VoBr18Mnz8Cvy2Fk4eLP1Ylf65rSkkJ8k/vrpRD9ltXWz8Ak+uS25HDhg1j4cKFbNq0ic6dOwOQlZXFJ598wi233IK3t3WBOXLkCPXr12fmzJmEhYXx+++/M2XKFI4ePcrrr79e5vMtWbKEBx98kFGjRtG/f3/Wrl3LPffcU2S72NhYHnzwQZo0aUJycjKzZ8+me/fu/PHHH9SuXZu5c+dy/fXX07p1a8aPHw9Ay5YtHZ5z5cqVDBs2jBEjRvDiiy+ydetWJk6cSFJSEq+99lqRn8fIkSN57LHHmDlzJkOGDGHfvn00aNCgzO9x9uzZjBkzhkceeYTevXvz9ddfM2bMGDIzM3n00UcBuPXWW8nJyeGtt94iJCSEv//+mz///BOwcnH79etH+/bt+fDDD/Hz8+P3338nJSWlzG1QZ689CanFzhYZX8zyais1AdZMgl3LXHZNLLYd374AB3+FHo/Apnfgkgfg/Gug023QKAbO6Qleti87V02Cb58veC0ftQE2zYftC8Db19ru2F8QEArBDlLgyjs4z0MG9RXL3r7N71o/G1P0i6PTx6qq9yoCrftB8x5wwQ1FP5/9a0FoI9i2ADa8Ze1T/zy4czUE1bV6xE8fq5LPdZcH3CLSF3gF8AbeMsY8V2j9ZcBMoB0w1Biz2La8AzAbCAFygGnGmKLdfJVIp3evQd65ruiyC66HLvdA5mn4YHDBddmZ4OsPsRtstyLz9Rrbj3XRndD2JkiOhaX3Fj3+paOti7+TrrnmGsLCwliwYEFewL169WqSkpIKpJN06NCBDh065L3u1q0bgYGB3Hfffbzyyiv4+JTtv/+0adPo169fXqDbp08fEhMTmT9/foHtXnnllbznOTk59O7dm/DwcD799FP+9a9/0aZNG4KCgggPD+eSSy4p8ZwTJ07kqquuYt68eQD07duX3NxcJk6cyJNPPlkgmH700Ue5/fbb895zVFQUn3/+OXfffXeZ3l92djZTpkzhrrvuYvr06QBcffXVHD9+nGnTpvHggw/i5+fH+vXr+eSTT7jmGut3Zr+TAJCYmMiBAwdYtWoVrVu3BuDKK68s0/nV2Wvf0VPM/OoPVmyLRwDjYJuzZrZIY+CfH2HBLZB+wvE2u5bD+deBdxWGHvnzd3OzrIBp8QgIbQLZts/ykIbWI7/akdDvZbj8cSvwjl0P4edB32eg95QzAfeqcfD3OmjdHzqPgGbd4WRi+QI0N3TYlMviEXDgF6uNhf3fZXDvd9bzfd9ZPcr1z4XgcCv4tavq9xq7EVLioc0A6PCvM8sL/05b9LIeOdlwaBv88wMk/Gb1iqcmWJ/fSX9Z+fy52ZXaRJcG3CLiDbwO9AZigQ0issIYsyvfZgeAO4BHC+1+GrjdGPOniDQENonIamNMMf+zKy400Bd/Hy9NKVFFHf0dMlJx/BFatfz9/bnhhhtYuHAhL7zwAiLCxx9/TNOmTQsEsrm5ucyYMYO33nqL/fv3k55+5u84NjaWZs2alXquzMxMtm3bxgMPPFBg+Y033lgk4P7pp5+YOHEiW7ZsISkpKW/5H3/84dT7y8rKYuvWrbzxxhsFlg8ZMoQnn3ySX375hRtuuCFv+dVXX533PCIigvr16xMbG1vm8x04cIDExEQGDy74pWrIkCG8+eab7Ny5k44dO9KhQwcef/xxDh8+TK9evQrknIeHhxMdHc29997L6NGjueKKK4iIiHDqfauzR9yJNGZ99SeLN8fi5+3FfZe3oHGdQJ76bHeBtJKzYrbI3Bz4/XP4cSbEbYLAutCwMxzeWTSwWng7hDaGa56HVg46OSoiNQHWPgPbPy563luWQIueVhBVGnvgnZ892AboPdXq9d72Efy2BPxDICvNCi7zn3PPF2ee1z8P6rWAzFNWUJp23Epn2LfO+gjJ9Zw84wKMgR2L4OIHILy142C5Rb6OhTWTIH6z9TwgFOqdC40usravyi8V2xfB8lFQ9xw4/9qiX+gc/k59oFFn62G3eIQVbGMqPdgG1/dwdwH2GmP+BhCRBcBAIC/gNsbst60r8FXKGPNHvufxInIYCAeqLOAWEasWtw6aPPuN+Lz4dX5BRdenJha9/VjcsUIblXz8chg2bBjvvPMOP//8M506dWL58uWMGjUKydej8NJLLzF+/HieeOIJevToQVhYGL/88gsPPvhggeC7JIcPHyY3N7dI8Fj49b59++jTpw+XXnopc+fOpUGDBvj5+dGnT58ynyv/OXNycoiMjCyw3P46fzAPEBYWVuC1n5+fU+c8dOhQgeMXd77FixfzxBNP8NBDD5GcnEzHjh156aWX6NmzJ97e3nz55ZdMmDCBESNGkJ6eTrdu3Xj11Vdp3759mduiqrfDqem8sfYvPvz1AAC3XdKUB3q2IKK2dbc0yM/n7Jst8mSirQe5EVz3EnS4BXwDHV8jh3wAv86xglSA5Dir4yKiVfnOfTrJ6lXf/4OV7pB12vF2515VvuM7EtHa+sJw1WSY0wOO/el4u4+Gnnl+1RToPsb6UpB/uSc7dRQ+GwO7P4W2g2DQ22d6ivP/Tq+adGafm9+Fo3vg6J+2xx+wcymcOuK4d7yicnNh7TT4/kVo2g1ufq9id08GzS/+c70SuDrgjgYO5nsdC1zs7EFEpAvgB1T5KIbIkAASNaVEFVb49qOLbwn26tWLyMhIFixYwKFDh0hNTS1SnWTRokUMHTqUqVOn5i3bvn27U+eJiIjAy8uLw4cLDiwp/PqLL74gIyODZcuWERho3SLPzMzkxAnnvw9HRETg7e1d5ByJiYkA1K1b1+ljlsSenlLa+Ro1asS7775LTk4O69evZ+LEiQwYMICDBw8SFhZGmzZtWLp0KZmZmXz//fc89thj9OvXjwMHDhT4IqSqv8LTrI/q2YIDSWn876f9ZObkMrhzI/595blEF0oXqdazRdpTAg78DO1uhsRdcNObVmrGnauhQYeCwY6jFI3W/ayH3U+zrAD8nJ5wyf3QsveZnOr857Tn+548AgEh4OMPv8yBVY9b2/kEQoP2Vmqfo571quAbCHd8Xvz1f+S6M89r29JXQqKt5aePWV8Q9qyyelIrkhNdFXZ/Cp+OgYwUq0e/62hruaPfaX5hja1Hy3xfcPK+eL1vpWLmvyuclQ6+AeVrY042LL7DamvH2+C6l8HHr9TdSlTFn+uuDrgdfeo4dU9eRBoA7wHDjSn6lUlERgIjAZo0aVKeNhYQFRLA1oNV1omuqrvSLkBVxNvbm8GDB7No0SLi4uJo3bo17dq1K7BNWlpakXrcH3zwgVPn8fPzo127dixfvrxATvTSpUuLnMvb27tAXviCBQvIzS34X7Qsvc++vr507NiRRYsWFRicuXDhQry9vUvN/3ZW06ZNiYyMZNGiRfTu3bvA+erUqVOk3rm3tzddu3Zl4sSJXHbZZRw4cKBAL7ufnx9XXnklY8aM4fbbbyclJYXQ0NBKbbNyH3t5P3tqSNyJNJ745DcABnZoyJirzqN5/WB3NrFyOcqJ/mqyFSRnpVmBZ6OY4vd3dDvf7rKxVq7vhrfgw5utlIBL/22lBdjPabKttJXXL4Yjv3aaFB4AACAASURBVMOtS6HlldC0K/T6r5VD3bDTmWCrpLuPla2kAK1hx6Lb+wacWd7yKsdtTYkvml/uSn9+BR/fan2BueEzq0e/sJJ+p8Vta/8ZbXnPNubJWNVO7vgc6jsePF8ibx+o0wz6PGMNhq3MTo0q+lx3dcAdC+QvttsIiC/rziISAnwOTDDG/OJoG2PMXGAuQExMTIUTbKNCA0jYmY4xRnupVPGcuQBVkmHDhvHaa6/xySefFOjFtuvduzezZ88mJiaGc845h3fffZf9+/c7fZ4nnniCm2++mdGjRzNgwADWrl3LV199VWCbK6+8kscee4wRI0YwYsQIduzYwYwZMwgJCSmwXatWrVi7di1ffvkldevW5ZxzznHYYz1lyhSuu+467r77bgYPHsy2bduYPHky9913n1PVR8rC29ubSZMmMWrUKOrUqcOVV17J2rVrefPNN3nhhRfw8/Pj2LFj9O/fn9tuu43zzjuPtLQ0XnzxRRo2bMj555/P5s2bGT9+PEOGDKF58+YkJSUxffp0OnfuXO2D7TIMdG8KzMNK8UsCbjXGxNrWPQ/Yk3WfquqB7q5QXHm/iNr+vDLUQZBVXeUf5JabXTCndeS30LBD8fuWVXB9uOxR6PaQNaDyp1fh5zdg1XgryM6f2xzaCNoNgXq24KxBe+tRmDs6Qcp7zvz7rRxr9dbO7QnDPoTozqXvX5lOHbV+Hy16wYBXof2wgrnrFVX4Z/TXN9CkK9Rtbq2P3WSlFvmV8mU1dpN1F6RhR7j66cprX0ltriSuDrg3AOeKSHMgDhgK/KvkXSwi4gd8ArxrjFlUdU0sKDIkgMzsXI6fzqJucAVvVyhVibp27UqzZs3Yv38/Q4cWzQucMmUKx44d44knnkBEGDRoEDNmzOD666936jyDBw9m5syZvPDCC8ybN49evXrx5ptv5lXrAKs6yNtvv83UqVNZsmQJHTt2ZMmSJUXONXHiROLi4hg8eDApKSm899573HrrrUXOee211/Lhhx8ybdo03n33XSIiInjssccK1L2uTPfffz+ZmZnMmjWLGTNm0KRJE2bMmMFDDz0EQFBQEG3atGHmzJkcPHiQ4OBgunbtypdffom/vz8NGzYkPDycp59+mvj4eOrUqUOvXr14/vnnq6S9rlLGge4vYl2X/ycivYBngdtE5DqgE9AB8Ae+FZEvjDHVulZicWX8jqRWg7E+JZVkO3XUqqucsMOaCGb3p1YPtqOb0JURbOfn7QsXDrImK0nY7jjf99Ylzh3TDZ0g5T5n7UgY8q6VpvPREHjnWrh+NrS9sfLbWFhGKnw5wfrC88AvUDsKOt1ededz9DPKSoMPB4N4WXc4Yu6yyvcVtmOxNTiyYUcY8UXl9mq7gBjj2ioLInItVtk/b2CeMWaaiEwFNhpjVojIRViBdR0gHUgwxlwgIrcC7wA78x3uDmPM1uLOFRMTYzZu3Fih9n6+/RCjPtzMygd70KZhSOk7KI+2e/fuvLJtSrlbaX+PIrLJGFPC/fqqJSJdgcnGmD621+MBjDHP5ttmJ9DHGBMr1m3AZGNMiIiMBfyNMU/btnsbWG2MWVjSOSvjul2Vukz7isMOguvosEB+HNfLDS0qA0cl2QbNs3KfW11r5cM+G32mbF7thlZpN/GycrYLp2ZMTq6idpaQDlJV5/Q0p45aKR0Hfrbyki+6q2rOk5oAn/0H/lxj3b249N/Q88ny51RX1IFfzvR8B9Wz8sa73AP+ta3BkV9OgF9eB98gGPMbBNdzTztLUdI12+V1uI0xK4GVhZZNzPd8A1aqSeH93gfer/IGFmKf3j0xJV0DbqVUTVOWge7bgJuw0k5uAGqLSD3b8kki8jIQBPQkX0Wq/Cp77E1VyczOxc/BrJAeW94vf6Cdk1mw53jxndDkUivg9vaBfjMhpAFEXlgwmPGUnOiaIrg+3L4c1ky0ShlWttQEWPc8bP7fmcGad66GJpU7NsZpTS6B2z6Bg+ut3/3XU6DxxdZA0/cGwvH9gJdVicZDg+3S6EyTpbBP766zTSqlaqCyDHR/FHhNRO4AvsNKF8w2xnxpu2P5E3AE+BlwWNy2ssfeVJVXvv6D2BNpjOjWjC93Jnp+eb+SJiy59zsIz1eOr8OwottA9cqJPlv4+FulB8Gqhb3uOeg8vGKDKQvPAJq/Moq7g+38Gnex0of2fW/VOd/6AWTb7yhVQWlBF9KAuxT2+qk626RSqgYqdaC7MSYeuBFARGoBNxljkm3rpgHTbOs+BIopWuz51u9L4o11fzEkpjGT+l/ApP4XlL6TO508bKWMnH8t7P2qaE+xowGHJalOOdFnk2N74efXrB7poR9CdKey72sMHN4Fv6+EH2dYE+9UF+ueKf7LYjVV9N6YKsDPx4v6tfx0tkmlVE2UN9DdNnB9KLAi/wYiUl9E7J8l47EqliAi3rbUEkSkHdAO+NJlLa9EKelZ/OfjrTSpG8TE/m3c3ZzS/fElzL7UmhCmdX94aLtVq9gnALx18H+1Uv9cuOtL8PK1BlP+trT0fQAO74ZZHay/g7VPQ92WEB1j9Z5Xh7+BQfOh84iz6m9We7jLIDIkQFNKziJa4lF5AlcPWC8PY0y2iIwGVnNmoPvO/APdgSuAZ0XEYKWUjLLt7gt8b/u/loJVLrDy50t2gYnLfiMhJZ3F93Ul2N+DPzaz0qzc3/VzIbItDP/0TB3lmpyiUd1FXgD3fGMNplw8ApJjoduD1rrUBGu2xb/XQdPu1p2LS+6DsCYQ0Qa6jYHzr7Gqj4Brc/Ir4izM5/fgK4fniAoJIK6YUlCqevH19SUtLY2goCB3N0XVcGlpafj6VmKd2ypShoHui4HFDvZLB6pBd3DJlm+NY9nWeP5z1Xl0bFLH3c0p2baPrGD7klFw5cSiFSc0RaP6qhUOw1fA549YQXVqAiy5B/754UzaRfpKqNPUeu4XDMM+Knqc6pYfX93aWwINuMsgMjSAzQeOu7sZqhJEREQQFxdHdHQ0gYGB2tOtXM4YQ1paGnFxcURGRrq7OaoEscdPM2HZb3RuWodRPVu4uzmO5ebC8X1QrwV0Gm71bDfu4u5Wqarg4w+9JpypPJOdQYExzGP3ln2ymur25au6tdcBDbjLICokgOOns0jPyiHA19vdzVEVYJ/5MD4+nqysrFK2Vqpq+Pr6EhkZWWQmTuU5cnINDy/chjEw4+YO+DgoB+h2KYdg2f3WZDGjN0JQXQ22z3YlVZ6pzJkhVaXTgLsMomylAQ+nZNCknqYiVHchISEa6CilSjT3u79Zvy+JFwe398zr/u7PYMW/rclq+j4LgR6e7qIqx6D5Z01Oc03jgV/ZPU9kqNbiVkqps1ZqAnz2MMzpDsBvccm8vGYP113YgJs6eVh97ZwsWHoPfHyLNQnIvd9B5zuq3TTXqpzsqRVaeaba0R7uMojSyW+UUursk/Q3fPcS/LY4r7cwLTOHBxdsoV6wP9NuaOtZ4zxSE6zezR2LQbyt3u3657q7VcodzqLBhDWFBtxlYA+4E3XyG6WUqv5SE2DlWNi9osiqaSt38feRU3xw98WEBXlIz+GRPbDoDjj2F2DOqslAVAWdBYMJawoNuMsgJNCHAF8v7eFWSqmzwUfDIH6zw1Xv/3KAe3o0p1vL+i5ulAO5ObD5Xfj8UaieJcyVUjaaw10GIkKUTn6jlFLVm32yoWELoGk3W/5rwcoOrRuE8Gif893QuEIO/Apv9oTPxkB0R7jgBs3XVaoa04C7jCJDAjSlRCmlqqtjf8GbvSB+i3UbfsRK28Cz2zE+AWTZbvjOucoXf3Lc3Fhgz0o4dRQGzYO71sDg+TpQTqlqTAPuMooK1R5upZSqlnZ/CnOvsCaISU85s9yW/7q4x0o+yr6C48EtaLp8EHwwCNJOuLaN2Rnww0z4a631+vLHYPQGaHvTmQokhStURF3o2jYqpcpNc7jLKCokgMMpGRhjPGvUulJKKcdysuHrKfDTLGjYCW7+H4Q1AWDZljimr95D/Ik0DNAq6iFue6iHNT36in/DvL5wy8K87avUH1/CqnGQ9Jc1LXuLntbU3MXRgXJKVTvaw11GkSEBZObkknRKi8wrpVS1sHm+FWxfdDfcuapAsD1+6Q7ibME2wP6jp1i+NR46/AtuXQop8fDWVRDneHBlhaUmwOK74dlG8OFgEC+4ZQn0faZqzqeUcivt4S6jBvkmv6lXy9/NrVFKKVWsrDTwDYROd0BYUzi3d4HV01fvIS2rYJ52enYu01fv4fqO0XDO5XDXl/DBYGtGv+hOlde21AT49gXruDlZYHKg91Nw8X3go3nZSp2tNOAuI/tsk4kp6VzQMNTNrVFKKVWEMfDza/Dr/8E930CtiCLBNkD8iTSHuxdYHtEKRq6FANv1Pu14xaZPT46HVY9bgyExVsk/u24Plv+4SqlqweUpJSLSV0T2iMheERnnYP1lIrJZRLJFZFChdcNF5E/bY7jrWp1vtsnkDFeeVimlVFmkJ1s90l9OsJ77BBS7acOwwLItD65vlQ08nQT/dzmsGl8wUC6Lnctgyd3wyoXWRDu52c4fQylV7bk04BYRb+B14BqgDTBMRNoU2uwAcAfwYaF96wKTgIuBLsAkEalAd4Nzwmv7I6LTuyullMf5+1uY0Rb2rrGmPM9IgYCQYjcf2+d8An29CywL9PVmbHH1twNCodV18Msb8PFtkHnqzLrUBPjsYZjT3ao08ve3VrURu98Ww9/r4PzroPkVWtJPqRrK1SklXYC9xpi/AURkATAQ2GXfwBiz37au8Ny1fYA1xpgk2/o1QF/go6pvNvh6e1G/lr/W4lZKKU9hz4fe9M6Z6c5N6b3H13eMJiMrh8eX7gAgOiyQsX3Ot/K3HfHyhr7PQp1mVjWR+ddB/1et82553+q1NjnwfHPIOgVevtDxVquHfMBr4B8CXrb+rdRE+PZ5K4fb5EKODsRXqiZwdcAdDRzM9zoWq8e6vPsWc3WsGjrbpFJKeZDFI+DAL2eCbSfENK8LwIwh7bmhY6Oy7XTxveBfG5aPhrmXgZdPwYC5wzBoeRU0625tBxAYVvAY9pJ+lz9uBd6x651uu1Kq+nF1wO2ogLVxsKzc+4rISGAkQJMmlVs/NTIkgNjjpyv1mEoppcpp0Pxy9xYn2jpPIkOKz/V2aMt7tgDfFD3fdS+V/ThaS1upGsXVgyZjgcb5XjcC4itzX2PMXGNMjDEmJjw8vNwNdSQq1F97uJVSylMUnnnRifxoe8Ad5WzAPWg+xNypudhKKae4OuDeAJwrIs1FxA8YCqwo476rgatFpI5tsOTVtmUuExUSwInTWaRn6QhzpZTyGOWY8txecSoq1MmAuwJBvlKq5nJpwG2MyQZGYwXKu4GFxpidIjJVRAYAiMhFIhILDAb+T0R22vZNAp7CCto3AFPtAyhdxX7rMVF7uZVSHk5E+olIzZpN2B4M3/dDqZsmpqRTO8CHIL9yZlaWI8hXStVcLp/4xhizElhZaNnEfM83YKWLONp3HjCvShtYAntPSEJyOk3rBburGUopVRbLgcMi8i4w3xiz290N8iQJyenO5287ornYSqkyqFm9HxWUN/mN9nArpTxfC2AucDPwm4j8LCL3iEjxBaprkISUdOfzt5VSqpw04HZC/undlVLKkxlj9htjJhljmgO9gb3ADOCQiLwnIj3d20L3OpxSST3cSilVBhpwO6G2vw9Bft46vbtSqloxxnxjjLkNOA/YBNwCfCUi+0TkPyLi8vRCd8rNNRxOzSAq1N/dTVFK1RAacDtBRIgKCdAebqVUtSIil4vIfGAP0BZ4HavS0yJgCvCu+1rnekdPZZCdazSlRCnlMjWqV6MyROpsk0qpakBEmgLDbY9mwDqsScGWGmPst+m+FpGfgffd0UZ3SbTdpYzQgFsp5SIacDspKjSA9ftcWo1QKaXK42+sycHmA/OMMfuK2W4nUKPmF08o76Q3SilVThpwOykyJIDDqenk5hq8vBzNNq+UUh6hP7DKGJNb0kbGmD+AGjWAMm+WSWcnvVFKqXLSHG4nRYX4k5VjSDqd6e6mKKVUSb4HIh2tEJEGIlLLxe3xGIkp6Xh7CfVr6aBJpZRraMDtpPyT3yillAd7G5hazLrJwFuua4pnSUhOJ7yWP956l1Ip5SIacDtJp3dXSlUTlwGfF7NupW19jZSQkp43r4JSSrmCBtxOyuvh1oBbKeXZQoHTxaxLB+q4sC0eJTElncjamk6ilHIdDbidFF7LHy+BRE0pUUp5tj+B64pZdy3wlwvb4lESktN1wKRSyqW0SomTfLy9qF/LX3u4lVKe7lVgjohkYpUGPAQ0wKrLPQq4331Nc5+0zBxS0rN1WnellEtpD3c5RIUGkJCi07srpTyXMeZNYBLwALAdOGL7dxQwwba+VCLSV0T2iMheERnnYH1TEflaRLaLyDoRaZRv3QsislNEdovILBFx+yjFRK3BrZRyAw24yyEyJEBTSpRSHs8Y8zTQECu15Hbbvw2NMc+VZX8R8caaBv4aoA0wTETaFNrsReBdY0w7rKooz9r2vRToBrTDmk7+IuDyir6nikrQGtxKKTfQlJJyiArR2SaVUtWDMSYZWFXO3bsAe40xfwOIyAJgILAr3zZtgP/Ynq8FltlPDQQAfoAAvkBiOdtRaew93JpSopRyJQ24yyEqNIDktCzSs3II8PV2d3OUUsohWwpHN+A8rOC3AGPMG6UcIho4mO91LHBxoW22ATcBrwA3ALVFpJ4x5mcRWYuVOy7Aa8aY3cW0cyQwEqBJkyalva0Ksc+hEBmiVUqUUq6jAXc52HtGEpLTaVY/2M2tUUqpokQkEvgaqwfaYAW92J7blRZwO8q5NoVePwq8JiJ3AN8BcUC2iLQEWgP2nO41InKZMea7Igc0Zi4wFyAmJqbw8StVQko6wX7e1A7wrcrTKKVUAU7lcItIhIg0z/daRGSkiMwUkf6V3zzPZB9so5VKlFIe7CUgGWiMFThfDDQD/otVMvC8Mhwj1ra/XSMgPv8Gxph4Y8yNxpiOwJO2ZclYvd2/GGNOGmNOAl8Al1TkDVWGwykZOumNUsrlnB00OZ8zuXoAU7B6SPoCn9h6OEpUhhHv/iLysW39ryLSzLbcV0T+JyI7bCPexzvZ9koTFWrditTZJpVSHuxyrKD7kO21GGMOGGOeAd6n9N5tgA3AuSLSXET8gKHAivwbiEh9EbF/lowH5tmeHwAuFxEfEfG1tcdhSokrJaSka4USpZTLORtwdwK+AbBdYO8HnjDGtAKmAWNK2rmMI97vAo4bY1oCM4DnbcsHA/7GmAuBzsC99mDc1fKnlCillIcKA44YY3KBFCAi37qfgEtLO4AxJhsYDazGCpYXGmN2ishUERlg2+wKYI+I/AFEYn0WACzGmlxnB1ae9zZjzKcVflcVlJCsAbdSyvWczeEOBY7ZnncG6gIf2F5/AzxSyv5lGfE+EJhse74YKzdQsPIGg0XEBwgEMrE+RFyudoAvwX7emlKilPJk+7AmugHYCdwCfGZ73R8oU6klY8xKYGWhZRPzPV+Mda0uvF8OcK/Tra5CubmGw6npmlKilHI5Z3u4Y7F6psGq5/q7MSbO9joUKC0CdTTiPbq4bWy9K8lAPawL+ims26MHgBeNMW6rzRcZGqApJUopT7YSuNr2/GngJhGJFZF9wINYM1HWKEmnM8nKMUTW1golSinXcraHex7wgohchRVw58+jvoTS8/PKMuK9uG26ADlYkzjUAb4Xka/sveV5O7uovFRUSICmlCilPJYxZly+51/YJqK5AesO4RpjzBdua5yb2K/ZOumNUsrVnAq4jTHPikgc1oxh/+bM4Biw0kveKuUQpY54z7dNrC19JBTr1ue/gFXGmCzgsIj8CMQABQJuV5WXigoJ4Fed/EYp5YFExB+rXN9nxphtAMaYjcBGtzbMzQ6n6qQ3Sin3cHpqd2PMu8aYfxtj3jbGmHzL7zPG/K+U3Usd8W57Pdz2fBDwje08B4BetlKEwVg96r872/7KYk8pyc2t0pKxSinlNGNMBlaJvjB3t8WTJCRnANrDrZRyPWfrcLcWkUvyvQ4SkWdEZJmI/Lu0/cs44v1toJ6I7AUeBuy3RV8HagG/YQXu7xhjtjvT/soUFRJAdq7h2KlMdzVBKaVK8ivW4HZlk5CSjgiE19IcbqWUazmbw/0GVjmpX2yvpwN3AN8Dz4tIgDFmekkHKMOI93SsEoCF9zvpaLm72G9JJqakE64DcJRSnucx4EMRycS65iZSaMyMMea0OxrmLonJ6dSv5Y+Pt9M3d5VSqkKcveq0BX4GayIa4FZgjDGmL/AEcGflNs9z2W9J6sBJpZSH+hVoAczCmlkyBUgt9KhRdNIbpZS7ONvDHcyZ2teX2F4vtb3eDDStpHZ5PJ3eXSnl4e6kaBWoGi0xJZ1GdYLc3QylVA3kbMD9N1ag/R1Weaktxhj7RDj1qUE9JvVr+eElOr27UsozGWPmu7sNniYxJZ2YZnXc3QylVA3kbMA9A5gtIoOBjsCIfOuuANw2iNHVfLy9CK/tryklSilVDaRn5XD8dJamlCil3MLZOtxvi8ifWHW4xxljvs63OgmYWZmN83RRIQGaUqKU8kgicoRSUkqMMREuao7bHU6xSgJqDW6llDs428ONMeY7rJSSwssnV0aDqpPIkAD2Hzvl7mYopZQjr1M04K4L9AJCsEqw1hj2zhENuJVS7uB0wC0iYcC9QHesi3cSVlnAucaYE5XbPM8WFRrAL38fK31DpZRyseI6QUREgIVAtksb5Gb2gFsnvVFKuYOzE9+0wJp4ZipWhZIDtn+nAttt62uMyJAAUtKzScvMcXdTlFKqTGwz976FNQlZjZGYrD3cSin3cbYO9wzgOHCOMaaXMWaYMaYXVq3XE8DLld1AT6alAZVS1dQ5gJ+7G+FKiSnpBPp6ExLg9I1dpZSqMGevPFcAw40xcfkXGmPiRGQK8E5lNaw6yD/5TfP6wW5ujVJKnSEiDzhY7Ae0Bm4BFrm2Re6VkJJOVGgAVkaNUkq5lrMBtwG8i1nnRQ2bZCH/9O5KKeVhXnOwLAOIBd4Apri2Oe6VmJJOZIi/u5uhlKqhnA241wJPicgGY8w/9oUi0hQrj/vrYvc8C+X1cGvArZTyMMYYZ1MGz2oJKel0aqKT3iil3MPZgHsM8A3wp4hsBhKBCKAzcBB4uHKb59lq+ftQy99HJ79RSikPZowhMSVDJ71RSrmNUz0gxpj9QCvgQWAn4Avswhrt3hVoUsnt83iRIf6aUqKU8jgiMk1E/q+YdXNE5ClXt8ldjp/OIjM7VyuUKKXcpjwT32QCc2yPPCJyE1Zt1+JyvM9KUaE626RSyiMNAyYWs+57rDTA/7quOe6TqDW4lVJupjl+FRQZEpBX31UppTxIQyCumHXxtvU1gs4yqZRyNw24KygqJIDDqRnk5taoAi1KKc+XAHQqZl0n4IgL2+JWZya90SolSin30IC7gqJCA8jONRw9leHupiilVH4LgYkicl3+hSJyLVYqyQK3tMoN7D3cEbW1h1sp5R465VYF5dXiTs7Qi7lSypNMBDoAn4rIMeAQ0ACoC3xJDcnfBiuHu34tP/x8tI9JKeUepQbcInKEsk1oUyPv1eWf3v1CQt3cGqWUshhj0oGrRaQP0BOoBxwDvjbGrHFr41wsITld87eVUm5Vlh7u16nEGSRFpC/wClY1k7eMMc8VWu8PvItV2/sYMMRWjhARaQf8HxAC5AIX2T5U3EYnv1FnldQE+PYFiF0P9/1Q9fupKmeMWQ2sdnc73CkxJYMGWqFEKeVGpQbcxpjJlXUyEfHGCuB7Y00vvEFEVhhjduXb7C7guDGmpYgMBZ4HhoiID/A+cJsxZpuI1AOyKqtt5VW/lj/eXqKVSlT1Zg+Yt34AJhdyMkvePicLMlLh+H7Y8n7Z91MuY7t+NjbGTHew7lHggDFmoetb5nqJKel0aBLm7mYopWowV+dwdwH2GmP+BhCRBcBArMlz7AYCk23PFwOviYgAVwPbjTHbAIwxx1zV6JJ4ewnhtfy1h1tVT/kD7dxs62G3+C7IPAmD5oFfMHz/Evz8hrUsO9/fu7efBtqeaRzwdjHrTgPjsQZWntUysnM4diqTSB1jo5RyI1ePIInGmgLeLta2zOE2xphsIBkr9/A8wIjIahHZLCKPOTqBiIwUkY0isvHIEddUvYoMDdDZJlX1tPB22DTPCqDzB9sA8Zsh9RBk2yrw1DsX2gyALiMhtAkg1nINtj3VucBvxazbbVt/1jucYv39RoXWyGFGSikP4eoebnGwrHB+eHHb+ADdgYuweme+FpFNxpivC2xozFxgLkBMTIxLimNHhfjz95FTrjiVUpUnaR+cToK6LSA5tmhKyINbCm7fZoD1AOg6Gr59XlNJPNtpoFEx6xoDNaKWaaJOeqOU8gCu7uGOxbrQ2zXCmvHM4Ta2vO1QIMm2/FtjzFFjzGlgJcVP6uBSUSE6vbuqZvZ9B2/2hFNH4LqX4aHt0PE28AmwUkRKUzsS+tn2az+06turyuMr4L8iEpF/oYiEA09ilQY86yXotO5KKQ/g6oB7A3CuiDQXET9gKLCi0DYrgOG254OAb4wxBmuUfTsRCbIF4pdTMPfbbSJDA0hNz+Z0ZnbpGyvlbhvegvdugOAIuOcbOOfyggF0x9sg6sKyHat2JAx4FfrPsr1uUHXtVs56HKgF/CUii0RklogsAv4CggCHaXlnm0R7Son2cCul3MilAbctJ3s0VvC8G1hojNkpIlNFxHavmreBeiKyF3gYa+APxpjjwMtYQftWYLMx5nNXtr84ebW4tVKJ8nQnj8DXU6FFL7h7DdRrUXC9PfB2trRfp9uhZW+rcklK4ZtWyh2MMQeA9sBrWHcNr7H9+yrWhDgJ7mud6ySmpOPv40Vo2utExQAAIABJREFUoK+7m6KUqsFcPtOkMWYlVjpI/mUT8z1PBwYXs+/7WKUBPUr+yW/OCa/l5tYo5UB6CvjXhlrhcNdXVqDt5V15xxeBa6fDG5fAmklw05uVd+ziaO3vUhljjmBVIwFARLyAK4DngBuxBqSf1eyT3ljFrpRSyj10avdKEGnLDdRKJcojHf4dPhoCMXdCt4cg/LyqOU/d5nDDHGjYsWqOb+dszXCFiFwMDANuBiKxxsUscGujXCQhJV3TSZRSbqcBdyU4k1JSIwb9q+pkzypYcjf4BkKTS6v+fBfcYP1rjBUMV2YvugbaThGRtlhB9lCgGZAJ+GGl6r1uS/Ery3FKmx24KTAPCMcK5G81xsSKSE9gRr5NWwFDjTHLKvK+nJWYkk67RjrpjVLKvVw9aPKsFOzvQ21/H+3hVp7DGPhhJnw0FOqdAyPXQeOLXHPu7Ez4YDCse670bZ2xeARstNUMLxxsG5dUAPV4InKOiDwhIjuAbcCjWONlbsequy3AFieCbfvswNcAbYBhItKm0GYvAu8aY9oBU4FnAYwxa40xHYwxHYBeWGUKXVoZxRhDYko6USFag1sp5V4acFeSyNAAHTSpPENqAiy6A76aBBdcDyNWQWjh+aWqkI8fBIbBDzPg6J+Vd9wBr0FwfRCvoqULX+0M6563aovXbHuBp4BU4F4gyhjTzxjzgW2Zs/JmBzbGZGKloQwstE0bwD4fwloH68GqOPWFraSry6SkZZOelas1uJVSbqcBdyXRWtzK7VITYMVD8Ep72GMblzzoHfALcn1brp4GvkHw+cOV0/uceQqWj7Ym6uk/q2jN8JCGsO4Z+OTeM/tk1cj/j/9g9WK3xRoceamtjGp5lWV24G3ATbbnNwC1RaTwYMyhwEfFnaSqZgjWGtxKKU+hAXcliQzR6d2Vm6QmwGcPw8wLYfP8gikX7qrMUDsSrvyvNcHOjsUVO1bmafhwCBz8BW56CzrdVrRm+B2fwZjf4JoXrH1OJ8GL51pT1/++0kpzyc/+M5vT/f/bu/OwKMv1gePfm0UERVEETNTUTHNNjeOx1CzLrSy3TK1OZVanc7LtlB2zfma2W53qtJm5lOlJc83K0krbyzIVyzUzS0DAFVABYXh+f7wzOAMz7LMA9+e65mLmXea952Xm4eaZ+32eysUWYIwxrYHewFvAJcD7QJqIvGF/XN7/fsoyO/B9QD8R2Yw1P0IyUFiyIiJnAF2whoP1FPcsY0yCMSYhJiamnCF6lqqzTCqlAoReNFlFmjYMIz0rF1uBITiogklOZYY50yHSaq+l4+GP7yh/LuVlCTdZFzh++yJ0uariyf+qO2Df1zByFnQeeXq5Y8xwh6gW1g3AlgfdroWfl8D29yC8MXQeBd3GweaFNfrCS2PMd8B3InIXVpI9DqsHegLWm+QWETlpjNlYhqcrdXZgY0wK1hCDiEh9YJQxJsNpk6uBFcaYvAq+pApLs5f56SglSil/04Tbk3ImsE0b1MVWYDh8PJfY8jbulRl9QUduUAOfgDcvg7yTEBQKBT7Pa9wLCoar5kJEdOV62vv+C9oNgq5Xl32fyDgY8hQMfBR+Wwcb51kXXG6aD5ha8TkxxhQAnwCfiMhtwGVYpR0jgGtEZLcxpkMpT1M4OzBWz/VY4BrnDUSkCXDEfrwHsEYscTYOp7HAfcnRwx2rF00qpfxME+6iKpjAxjlNflPmhDsrFb54Gjbb5/JxPtZ/OlpDuYVGQGwHq3cPrJEfMpKs+wcSId0+u32BTitfK+VkwvJbrET7+lVWb24g/fPVuI3105YHJw5BgzJO/W7Ls15L51EQ18m6VURwqJWsf/MCUAC22jl0p/2Cx5XAShGpBwzHSp5L2y9fRByzAwcDcx2zAwMbjTGrsGrFnxQRA3wJ3O7YX0RaYfWQf1GlL6iM0jJzaFyvDmEhVTg8pVJKVYAm3A4uibbN+oNfDjtTMwG48uVviI8KZ9Kg9gzvXsrIEEtuhD+/c7/urIut2tW8bGuGQIcDiZCyGY6nW3G6k58LIdqjUyuERVplFm0ugjMvgDb9oN+/rX/kkn7wd3Sn/e9qyD4KN39W+tjctnxYNsFKuKNaQouelT/+VW9a58Td5zv/lDWyircFSNmXMeYEsNB+K8v2pc0OvBRwW6hvjNlH8YssfSYtM0frt5VSAUETboel4+HP762ewXJauTmZV9f/Vvg4+Vg2Dyz/GcBz0p173OqVBAgKsYY6c+6RHPaK+/3G2S/0z0rznEA8czZ0GAqdRloJWHBouV+TCnD5p+B4qpWQXjzFdV3R2uZA0O1aK4neOBd63uJ5uwIbrLzNSrYHPVE1yTacPieOf0a2LLQ+b6YA5g6EUXOs6e69Qcu+/CY1M4c4LSdRSgUAHaXE4ao34bzx1lBjUr6vH59Zs4ucfNdEPTvPxjNrdrnf4Xg6vHk5/PE1DH0B7tlefJiz0jgSiLu2QvfrXfftMBR2fAALR8Gz7awa1pLU0BEbaqyCAlj5D3ijvzUaR3XQeRS07gefTbf+WXSnoMAa+u/nJXDJw3D+7e63qwznz8154yHqTDiyFxaMtHrWq5Ljc/V8Z9j8tvsJe5RXpWbk6gWTSqmAoAm3g8sf4htdE19HjbUHKceyy7WcfV9bE4KMfQcSxhdJnu3DnFUkbse+w1+FSfbnP6s/NGlnbbvjA1g9yRrRoqDgdELw4rlWQpD6c9mPq/zDGFgzBX5ZCr3+CRGN/R1R2YjA5f+xks61D7rfJnkjbF0EF02xLpT0Jsfn5u6tcNs3MHwmBIdYPeynTlTuuQtssPNDWHwd/DTPuohVE22fy7MVcPhErpaUKKUCgpaUFOX81fNnj8D2lfDDG9B1jMfSjGZR4SS7Sa6bRYW7LsjJhLoNrJrbM3tbx3J37MrE7RASBudcZt0cDv9qjdLwwywIrWclP0HBmgxUJ1//Bza8ZiXbfe7xdzTl06Qt9L7bmpQn9ziE1Xdd36KnVdscW3TmcC9zHk7w6/9A4iJrdJUzzi3f8+RkWmUjG2bC0X1w4SRo2lVLSfwkPSsXY3TSG6VUYNAebk8i46ye4sn74brlVrKdfQyO/lFs00mD2hMe6lqGEhYSxKRB7U8v2PmhNTHJnxtOP7+v9bkHJu2B6LMh74S99luTgGpjxwdWSUaXq62ZHP01qU1lXHgf3Pr56WTbGOtblgVXWSVNcZ38+7pa9LJ6uGdfCt+/VrZZMo2BNQ9aIwt9PBnqN4XRb0G/ya7fPpWnZExVmmMiMi0pUUoFAk24SxMUDPXssxR/9G+Y1Q9+/dRlk+Hd43lyZBfio8IRrHyhWcO6XHluM2uDH2dbXy9Hn+W9C7PKKiwSbvwQEiZYf/yD9EuOUgVKjftZF8PFD1oX1AZV049uSNjpf163LoHX+8LGOfDbZ4FR0tS6r1ViclZ/K3l+ZxycOHx6vfN7Ic0+JKcIHPsT2g+BW9bBhDXQabhVogKVKxlTFeaY9EZLSpRSgUCzrfK46N+Qtg0WXmWVnPT7d2HiM7x7fOGIJO9u3M/9S7ey+Mc/GZc1D75+HtoNtr6mrlPPn6/A4nbEhjzPwwzWVoEyukTKFms867oNoN/9/omhKmWlwltXwiGni4orMDqQ19SLhnGLYMPrVlnZkb1WHbbjveD4rLx2Ady5yfrdjH6r9H+CAnH0mBrs9LTuOkqJUsr/qmk3mZ80bgMT1sK54+CLp+B/o92OEjH6vOb8tXVjEj+abSXb542HMQsDI9l25kgA/rkBwhpYyxIX+zemQFB4MWlX/48ukfqLlZy+f5d/ju8NS8fDod3+jqJkItDrNrhlPSS+Y11Y/NOb1nvB8Y/pFS9ApH0in+r6jUMNlpqZQ53gIBrX0zIepZT/6V+J8qoTYdV2D30eDu6yJpkpQkR4YmQXVuT1YnazR6xtgwP4y4TGreDuRGjVF1bcCl89V7ba1ZrqnbFWmUN+rn9r3I/ugwWjrH/UBkz3XxxV7ao3IeGm6lHT/OE91kgjzom2w3k3WrPBqoCUlpFDbIMwpDpe66CUqnF8nnCLyGAR2SUie0Rkspv1YSKy2L5+g31qYOf1LUXkuIjc56uYixGxEoaJG62pqgtssOtjK0nNPABvXsFZX/6Lr6Km8djes/l890G/hVpmdRtaF4d2GW1dmPdxsV9N1QqUumhnJw5bwyamJFqTEgWFFE8IN71t/b69LXUbzOwLJ9Lhb8tPj6JRExStaQ7kxNt5fP5AjVG5lZqZoxdMKqUChk+7XUUkGHgFGAAkAT+KyCpjzHanzSYAR40xbUVkLPA0MMZp/fPAR76KuUSh9sY88R1473Y4ZyjsXW+NchAUQmxBPmfF1OOhlb+w9p4LiagTwL3cYE1vPWIWNIiH2A7eOUag1EUXlfoLzLsMTmVZY6NfPMVKrB017gX51m3VRGtYxcFPQisv/LPgOD8/zTtd1+yt34W/Fb2WIJCmondwd71DIL1vlUfpmbl0aNbA32EopRTg+x7unsAeY8xeY8wpYBEwrMg2w4C37PeXApeI/TtBERkO7AW2+SjesmlzMZzRDXZ+cHrSjAJr1ronRnQh6Wg2L376qx8DLIegIBjwCJw71nq851M4cahyz3nqBPy69vQkOxvnBsase8bAsf3W/ZhzoMso+Me3VoJVr4lrT2yPG6zRJUbNser2P7zPmjyoqmSlwgf3nK4bD6SLCL3NcZ5v+9rfkXimI41UK8YY7eFWSgUUX3e5xgP7nR4nAX/1tI0xJl9EMoBoEckG/o3VO+6xnEREbgVuBWjZsmXVRV6S5TdD6la3q/7aJpoxCS2Y/fXvDOsWT8fq1OOSkwFLb4KIaLh2afmGNDQG9m+wZuncthLyTgLGfSL5ej84fyJ0HV1loZcqZbM1dvKh3XDHJmsEkKHPu9+26OgS51wOmSnWPyc5GfDty3DBHdZzlFdOJvz+BXx4Lxz3MOW5Chw60ki1kJWbz8lTNh2hRCkVMHzdw+3u6pWiV+d52uYR4HljzPGSDmCMmWWMSTDGJMTExFQwzHIqpc7zgcvOISo8lAdW/IytoBpdjFi3oZVoZx+DOQMh6aey7bd7Lbx0HswdBL8sh45XWsOmeTpHoeHWRDxgJbKLr7NGhMhIKv7clan9zkqF5X+Hp1rBrIvh4E64aDKERpTveULDT//zsecz+HIGvNTDirlofXfReJ0vRv1kKsxobb3e3CxoeKZVO661wkpVio7BrZQKNL7u4U4CnK/+ag6keNgmSURCgIbAEaye8KtEZAYQBRSISI4x5mXvh12KUuo8oyLq8H9DO3L34i0s+P4PbriglX/jLY8WPWHCJ7BgJLx5OYyeZ03wAafrjfdvgL7/gthOEHuOlajXj7OWdRx+elbBjle4P0c3fXz6eEf/gORNsON963FMB2h7iXUx56b5FauhdcS5eQHY7KPK9L4L+t5rxVoZnUdCo1awZoo1dN8Pb8CgJyCmvVOtus0au3nZzbD3c/jHd1A/xipDuuBO6/U172nV0Gelaa2wUpWUqrNMKqUCjBgfDv9mT6B3A5cAycCPwDXGmG1O29wOdDHG3Ga/aHKkMebqIs8zDThujHm2pOMlJCSYjRs3VvGrKANH0pT0Q2FdqjGG6+f+wOY/j/Hpv/rRtGE1+0NwPB3+d7V1oeD5E63Xt3mBVavuKBPpfbdV/10Wbs5RIWOs3uc9n8LOD+HPDdbshBjXBLRxGwitZw3VeP17Vs/z1iWw7ytrKD1TAEk/2st9xHXfaRmVORvFGQPbV8LHD1qj2Jw8dPpCS4d6MdYMhhdPsZL0kpR0flStISI/GWMS/B2HL1VFu71k434mLd3KF5Mu4szoAJv/QJVLZmYm6enp5OXl+TsUVcuFhoYSGxtLgwaey0dLarN92sNtr8meCKwBgoG5xphtIjId2GiMWQXMAd4WkT1YPdtjfRljlXBT5ykiPDa8MwOf/5KHV/3C63+rZn9D68fCVfPgm/9aFz7m5+JSDfS3ldD6wrI/X0m1sCLWyByxHWDXahBzumfaWbMeVm34qROnyzAO7YLdH8Opk9aII74iAp1GwIaZ1j8IxSqlgHt3l32CFK0VVqrC0rOs9kJLSqq3zMxM0tLSiI+PJzw8XMdUV35jjCE7O5vk5GSAEpNuT3w+Tp0xZjWwusiyqU73c4ASr54zxkzzSnBedmZ0Pe669GxmfLyLtdtSGdipqb9DKp/3/gl/fu/+wsezLvbOMa9603OJxVVzim/f/yHrBlYpyedPQuIi35VnjJ7vOV6djVApn0jNyKFheCh1Q4P9HYqqhPT0dOLj44mIKOd1NkpVMREhIiKC+Ph4UlJSKpRwawbgY7f0bUP7uEgeXrWN47n5pe8QSPwxCUhlJkmJbApXvOjbCVaq06QuStVQOiRgzZCXl0d4uM7mqgJHeHh4hcubNOH2sdDgIJ4Y2YXUzByeW7vL3+GUjz+TycqMg+yPMZR13Gal/CYtM4e46nadjHJLy0hUIKnM+zHApz6smc47sxHX/rUlb327jxHd4+naPMrfIZWPP2cIrExtsz/qorUWWymfS83I4Zymkf4OQymlCmkPt5/cP/gcmtQPY/Kyn8m3VdNZBavDDIFKqVol31bAoeO5WlKilAoomnD7SYO6oUy7shPbD2TS49FPaD35Q3o/tY6Vm5P9HZpSSlVbh46fosCgJSXK70Sk1Nvnn39e6eM0bdqUhx56qPIBK6/SkhI/ys2zESSQmWNdPJl8LJsHlv8MwPDu8f4MTSmlqiXHpDdxkZpwK//67rvvCu9nZ2fTv39/HnroIS6//PLC5R07dqz0cVavXk1sbGyln0d5lybcfvTs2t0Unek9O8/GM2t2acKtlFIVkGqf1r3aTS6mapxevXoV3j9+/DgAZ511lstyT3Jycqhbt2zv4R49elQsQOVTWlLiRynHssu1XCmlVMnSHD3cWsOtnKzcnEzvp9YFZPnmzJkzERE2bdpE3759CQ8P56WXXsIYw7333kvnzp2pV68eLVq04IYbbuDgwYMu+xctKRk7dix9+vRh9erVdOrUifr169OvXz927Sp5ZLTMzEz+8Y9/0K5dOyIiImjTpg133XVX4T8LDvn5+Tz66KO0bduWsLAwWrRowa233uqyzZIlS0hISCA8PJwmTZowdOjQwkljaitNuP2oWZT78UXrh4WQk2fzcTRKKVX9pWbmEBosRNfT8e+VZeXmZB5Y/jPJx7IxnC7fDKSkG2DMmDGMGjWK1atXM3DgQAoKCjhy5AgPPfQQq1ev5rnnnmP79u0MHDgQY9zMZuxkz549PPTQQ0ybNo0FCxawf/9+xo0bV+I+WVlZBAcH8+STT/LRRx/x8MMPs3r1aq699lqX7W688UYee+wxrrvuOj788ENmzJhBVtbpmZ1nz57N1VdfTceOHVmyZAlz5syhdevWHD58uOInpwbQkhI/mjSoPQ8s/5lsp+Q6WISs3HyGvPgVjw/vzAVtm/gxQqWUql7SMnKIjaxLUJCO31wTPfL+NranZJZrn81/HuNUkdHAsvNs3L90K+/88GeZn6djswY8fEWnch27PO677z7+/ve/uyybN29e4X2bzcZ5551H27Zt+fHHH+nZs6fH5zpy5AgbNmzgzDPPBKwSlXHjxrFv3z5atWrldp/4+Hhefvnlwse9e/emefPmDBgwgLS0NOLi4khMTGThwoW8/vrrLr3ajmQ+Ly+PKVOmMG7cOObPn1+4ftiwYWU/ETWU9nD70fDu8Tw5sgvxUeEIEB8VznNXn8vbE3pSYAzXzN7Ave8mcuSED6YkV0qpGiAtK4e4BmH+DkMFkKLJdmnL/cX5YkqHVatW0atXLxo2bEhISAht27YFYPfu3SU+V7t27QqTbTh9cWZSUlKJ+82dO5dzzz2XevXqERoayqWXXooxhl9//RWAdevWERQUxPXXX+92/19++YWDBw8yfvz4Eo9TG2kPt58N7x7v9gLJNXdfyEvrfuX1L/aybmcaD17ekVE94nXWLaWUKkFqRg7t4nTSm5qqIj3MvZ9aR7Kba6Pio8JZ/PfzqyKsKhEXF+fy+JtvvmHEiBGMHTuWBx98kJiYGPLy8rjwwgvJyckp8bmiolwn1KtTxyqxKmm/d955hwkTJnDHHXfw1FNPER0dze+//87YsWML9zt8+DCNGjXyeEGno2zkjDPOKPnF1kLawx2g6oYGM2nQOay+qy9tYupz35JErnljA3sPHi99Z6WUqqXSMnP1gknlYtKg9oSHBrssCw8NZtKg9n6KyL2iHWrLli2jZcuWLFy4kCuuuIJevXp5dfi/JUuW0K9fP/773/8yZMgQevbsWSxxj46O5ujRox4T9+joaAAOHDjgtTirK024A1y7uEiW/P18Hh/RmV9SMhj84lf897Nfyc3XiyqVUsrZ8dx8jufm65CAyoW78s0nR3YJ+OF3s7OzC3umHRYuXOjV44WFuZZjFT3eJZdcQkFBAQsWLHD7HF26dCE2Npa33nrLa3FWV1pSUg0EBQnX/vVMBnSMY/r72/nPJ7tZlZjCkM5NWb4pmZRj2TSLCmfSoPYB34AopZS3FI7BrT3cqghP5ZuBbMCAAcycOZNJkyYxePBgvvzySxYtWuTV402aNIkZM2bQvXt3Vq1axddff+2yTdeuXbn++uuZOHEiKSkp9O7dm8OHD7Nq1SoWLFhASEgITz31FDfddBMhISGMHj0aYwyffvopN910E127dmXt2rVcdtllfPPNN/z1r3/12usJNJpwVyOxkXV5+ZoejDovnXsWbeGldXsK1+kslUqp2k7H4FY1yciRI3n00Ud59dVXefXVV+nbty8rV66kUyfvjJRyxx138Mcff/Dss8+Sk5PDkCFDmD9/Pn379nXZbs6cObRp04Z58+bx2GOPERcX53LB5/jx44mIiOCpp57inXfeITIykgsuuIAmTaxR1woKCrDZbKUObVjTSE1+wQkJCWbjxo3+DsMrzn/yMw5kFK+hio8K55vJ/f0QkVKqqonIT8aYBH/H4UuVabeXb0riX+8msu7efrSJqV/FkSlf27FjBx06dPB3GEq5KOl9WVKbrTXc1VSqm2QbrJ7ujJN5Po5GKaX8LzVTp3VXSgUmTbirKU+zVAL0eXod/1m7i2MndfxupVTliMhgEdklIntEZLKb9WeKyGcislVEPheR5k7rWorIWhHZISLbRaSVN2NNy8ghsm4IEXW0WlIpFVh8nnCXofEOE5HF9vUbHA20iAwQkZ9E5Gf7z1pdN1HSMEd92zXhv+v20Ofp9TynibdSqoJEJBh4BRgCdATGiUjHIps9C8w3xnQFpgNPOq2bDzxjjOkA9ATSvRlvamaOXjCplApIPu0GcGq8BwBJwI8issoYs91pswnAUWNMWxEZCzwNjAEOAVcYY1JEpDOwBqi1Vwc6Lox8Zs0ut6OU7EzN5KXP9vDSuj3M+2YfN17Qigl9WtOoXp2SnlYppZz1BPYYY/YCiMgiYBjg3GZ3BO6x318PrLRv2xEIMcZ8AmCM8fokAqmZuVpOopQKSL7+3q0sjfcwYJr9/lLgZRERY8xmp222AXVFJMwYk+v9sANTScMcndO0Aa9c24M7U7P477pfeeXzPcz75ndu7N2Km/u04YvdBz0m60opZRcP7Hd6nAQUHccrERgFvAiMACJFJBpoBxwTkeVAa+BTYLIxptgkAiJyK3ArQMuWLSscbFpGDmfHNqnw/kop5S2+TrjL0ngXbmOMyReRDCAaq4fbYRSw2V2yXVUNd03Rvmkkr1zTg132xPvVz3/jjS/3UmAgv8AaoUaHFFRKeSBulhUd2uo+rI6RG4EvgWQgH+vvS1+gO/AnsBi4EZhT7AmNmQXMAmuUkooEaiswHDyeS1yDsNI3VkopH/N1DXdZGu8StxGRTlhlJn93dwBjzCxjTIIxJiEmJqbCgdY0jsR7zd0XEhwkhcm2Q3aejWfW7PJTdEqpAJUEtHB63BxIcd7AGJNijBlpjOkOPGhflmHfd7MxZq8xJh+r1KSHtwI9fDwXW4HRGm6lVEDydcJdauPtvI2IhAANgSP2x82BFcD1xpjfvB5tDdQuLpKcvAK365KPZTP7q738cfiEj6NSSgWoH4GzRaS1iNQBxgKrnDcQkSYi4vhb8gAw12nfRiLi6Pnoj2v5YJVK1UlvlFIBzNclJYWNN9bXjmOBa4psswq4AfgOuApYZ4wxIhIFfAg8YIz5xocx1zjNosJJPpZdbHlIkPDYhzt47MMdnB1bn0s7xjGgYxzdmkcRFGR98bByc7LWfitVS9jL+iZiXaQeDMw1xmwTkenARmPMKuAi4EkRMVglJbfb97WJyH3AZyIiwE/AG96KtXBad71oUikVgHzaw23/WtHReO8A3nU03iJypX2zOUC0iOwB/gU4hg6cCLQF/k9Etthvsb6Mv6bwNKTgs6PP5ctJFzN1aEdiIsOY9eVeRr76LT2f+JR/L93Kox9sZ/LyrSQfy8ZwuvZ75eZk/7wQpZTXGWNWG2PaGWPOMsY8bl821Z5sY4xZaow5277Nzc7X1hhjPjHGdDXGdDHG3GiM8doYpY5p3bWkRAWKoUOH0qVLF4/rJ06cSKNGjcjNLdvYD3v27EFE+PjjjwuXNW/enMmTi42w7GLLli2ICF9//XXZArebOXMmq1atKra8LMdUxfl8dgBjzGpgdZFlU53u5wCj3ez3GPCY1wOsBUobUvCmPq25qU9rMk7m8fnudD7Znsbqnw+QlZtf7Lkctd/ay62U8qe0zFyCg4To+nrRpAoM48aN47rrrmPbtm106tTJZZ3NZmPp0qWMHDmSsLCKv2fff/99mjTxzsg8M2fOJCEhgSuvvNJluTePWZPpdFy1VElDCjo0jAhlWLd4hnWL51R+Ae0e+sjtdsnHsvnpj6N0axFFcJC7a16VUsq7UjNziKkfpm2QChjDhg0jIiKJjxzdAAAaqklEQVSCRYsW8eijj7qsW79+PWlpaYwbN65Sx+jevXul9q8ux6wJdGp3VSZ1QoKIL2E6+VGvfctfHv+UexZvYVViis5uqZTyqbTMHOK0fluVRVYqfPAvmNnHq4epX78+Q4cOZfHixcXWLVq0iLi4OC6++GIAkpOTGT9+PK1btyY8PJx27drx8MMPk5eXV+Ix3JV3vPTSS7Ro0YJ69eoxbNgwUlNTi+33zDPPkJCQQIMGDYiLi2PYsGH89tvpsSj69OlDYmIic+bMQUQQERYsWODxmIsWLaJz586EhYXRsmVLpk6dis12esj92bNnIyJs27aNSy+9lHr16tGhQwfee++9Us5i6bE6LFu2jL/85S+Eh4fTpEkTLr/8cvbvPz0SdWJiIpdffjkNGzYkMjKSXr16sW7dulKPX1W0h1uV2aRB7Xlg+c9k553+EIWHBvN/QztQv24o63em8/mudFZsTiZI4LwzG3HxObH0PyeW9nGRvLclRS+4VEp5RWpGDm1i6vk7DBXIslLhixmwZSGYArB5v2No3LhxvPvuu/z000+cd955AOTl5bFixQquvfZagoOt66kOHjxIkyZNeOGFF4iKimLnzp088sgjHDp0iFdeeaXMx1u2bBl33nknt99+O1dccQXr16/nlltuKbZdUlISd955Jy1btiQjI4PXXnuNPn36sHv3biIjI5k1axbDhw+nQ4cOPPDAAwC0bdvW7TFXr17NuHHjGD9+PM8++yxbtmxh6tSpHDlyhJdffrnY+bj11lu5//77eeGFFxgzZgy///47Z5xxhsfXVFqsAG+++Sbjx4/n2muv5eGHH6agoIDPPvuMQ4cO0aJFC7Zt20bv3r3p2LEjr7/+Oo0bN2bjxo38+eefZT63laUJtyqz0mq/rzy3GbYCQ2LSMdbvTGfdznRmfLyLGR/vIio8hKxcGzaXyXa2ujyvJzoyilKqNKmZOVxwVrS/w1C+MO/y4ss6DYeet8Cpk7CwyGVg+acgNAySfgRjA5tTr7Hjuf5yE3QeBRlJsNzNNB8XTIT2Q8od6pAhQ4iKimLRokWFCfeaNWs4cuSISzlJt27d6NatW+Hj3r17Ex4ezm233caLL75ISEjZ0rXHH3+coUOHFia6gwYNIi0tjTfffNNluxdffLHwvs1mY8CAAcTExPD+++9zzTXX0LFjRyIiIoiJiaFXr14lHnPq1KlceumlzJ1rjQg6ePBgCgoKmDp1Kg8++KBLMn3fffdx/fXXF77mpk2b8uGHH3LzzTd7fP7SYrXZbEyePJnRo0cX9sIDLrXn06ZNo3Hjxnz55ZfUrWt9EzZw4MASX1dV04RblUtptd/BQUKPlo3o0bIR9w5sT1pmDut3pjNt1bbCZNshO6+Af727hTe+2kt0/TCa1K9Dk/phRNerQ3T9MKLr1+GXpAxeXr+H3Hxr7PDyzoqpybpSNd/JU/lk5eRrSYly79BOyM2i+Dx73hcWFsaIESN49913mTFjBiLC4sWLOfPMM10S2YKCAp5//nlmz57Nvn37yMnJKVyXlJREq1atSj3WqVOnSExM5J///KfL8pEjRxZLuL/99lumTp3K5s2bOXLkSOHy3bt3l+v15eXlsWXLFl599VWX5WPGjOHBBx/k+++/Z8SIEYXLnZPc2NhYmjRpQlJSUonHKC3W7du3k5aWxvjx4z0+x7p167j55psLk21/0IRbeVVcg7qM7dmyMEkuqsBY2xw+nstv6cc5dDy3MLn2JDvPxv1Lt/LB1hQa1A2lQXgoDeqGEFk3lAbhIYXLNv1xtMLJuibqSlUfaZnWsGo6JGAtMf5Dz+vqRBRfn5UGXzztvpSk6LYNm5f8/BUwbtw45s2bx3fffUePHj147733uP3227GGp7c899xzPPDAA0yZMoW+ffsSFRXF999/z5133umSfJckPT2dgoICYmNdR0wu+vj3339n0KBBXHDBBcyaNYszzjiDOnXqMGjQoDIfy/mYNpuNuLg4l+WOx84JMkBUVJTL4zp16pR4zLLEevjwYYASy1KOHj1a4npf0IRb+YSnyXbio8KZe+NfCh8bYzh5ysbh46c4eDyXUa996/b5TtkKSDmWw67cLDKz88nMycOUofMiO8/GpKWJLP0piYYRoUSFh9IwPJSoiFCiwuvQMCKUn5OO8cZXv1e4V10p5VuOSW90lknlVmQcDP0P9Pu358Tbi/r3709cXByLFi3iwIEDZGVlFRudZMmSJYwdO5bp06cXLtu6dWu5jhMbG0tQUBDp6ekuy4s+/uijj8jNzWXlypWEh1uDIZw6dYpjx46V63iOYwYHBxc7RlpaGgCNGzcu93OWN9boaKuU7MCBAy5lOc4aNWrEgQMHKhVLZWnCrXzC0wWXkwa1d9lORKgXFkK9sBBaRkcQX0KivvquvoWPCwoMJ07lk5mTT2Z2HpnZeYyZ9b3bWPJshpOn8knJyCbjZB4Z2XnkF5ScrWfn2Xhg+c/8dvA4LRpF0LxxOC0aRXBGw7qEBLsO9lPR3vHa1Ktem16r8r40ndZdlUXRxDvpB58cNjg4mNGjR7NkyRKSk5Pp0KEDXbt2ddkmOzu72HjcCxcuLNdx6tSpQ9euXXnvvfdcaqKXL19e7FjBwcEudeGLFi2ioMD12+XSep8BQkND6d69O0uWLHG5OPPdd98lODi41Prv0pQl1o4dO9K0aVPeeusthgxxX2d/ySWXsGjRIqZPn16pcc8rQxNu5ROlXXDpSVkT9aAgIbJuKJF1QwuHLywpWV/+z96Fj40xnDhl49jJUxw7mcfQl9zPxpWdZ+PVz39zqUUPCRKaRYXTwp6AZ+XksXZ7Gnm20xeH/nvZVlIysrmoXSwGgzFgDBQYg8H6+cWug7z2xW+cculVL9tFpVC5BNbXye/Kzckuv1P9BkFVVmqmTuuuysGRePvQuHHjePnll1mxYoVLL7bDgAEDeO2110hISKBNmzbMnz+fffv2lfs4U6ZM4eqrr2bixIlceeWVrF+/nk8//dRlm0suuYT777+f8ePHM378eH7++Weef/55GjRo4LLdOeecw/r161m7di2NGzemTZs2bnusH3nkES6//HJuvvlmRo8eTWJiItOmTeO2226rdBlHWWINDg7m6aef5oYbbqBOnTqMGTMGgM8++4y//e1vdO/enUceeYSePXvSr18/7rnnHqKjo9m0aRNxcXHccMMN2Gw2wsLCmD59OlOmTKlUzJ5owq18piyT7bjbB8qfqEP5etXrh4VQPyyE5o1KTtQ/n3QRqRk5/HnkJPuPnGT/0ZPsP5LNn0dO8umONA4dL/4VZW5+QeFoLeWRnVfAfUsSWb45mZb2hL5l4wha2G8Nw0OByiWwvk5+jTE8sXqHy+8EfDNjqfaq11ypGTmFn2GlAtH5559Pq1at2LdvH2PHji22/pFHHuHw4cNMmTIFEeGqq67i+eefZ/jw4eU6zujRo3nhhReYMWMGc+fOpX///rzxxhsuPb/dunVjzpw5TJ8+nWXLltG9e3eWLVtW7FhTp04lOTmZ0aNHk5mZydtvv811111X7JiXXXYZ//vf/3j88ceZP38+sbGx3H///UybNq1csbtT1livv/56IiIieOKJJ1i8eDGRkZGcf/75xMTEANChQwe++uorJk+ezIQJEwgKCqJTp0488cQTgPW3yWazFevlr0piylL4Wk0lJCSYjRs3+jsM5UcVSbKKJqFgJepPjuxS6r6tJ3/o8Tr4mdf1sCYQAIJEELF+IjB+3o8en7NLfEP2Hz3JsZOuEyA0DA+lReNw9qQfJyeveCMRFR7Kvwa2I89myLcVkF9gyLMVkG8z5BVYPxf98CcnTtmK7RsfFc43k/uX+FrLKj0rh2/2HOKrXw/x9a+HSM/K9bjtEyO6cFH7GJqVMMlSRazcnMzk5VtdzlN4aBBPjuwa0BfQishPxpgEnxwsQFSk3f7Hgp/YnZbFZ/de5J2glF/s2LGDDh06+DsMpVyU9L4sqc3W7gBVo/m6V72ki0MHd/b81VpJverv32HNhpaRncf+IydJOnrS3sNu9az/kpfp9jmPZecx9b1txZaLQGhQECHBwkk3yTZYPd1/f3sjnZs1pFN8Azo3a0hskfpYT4loTp6NH34/wle/HuSrXw+xMzULgEYRofRu24Svfz3Esezis6cFizBlhdW7fk7TSC5qH8vF7WPocWYjQu118mVNfjNO5rH9QCbbD2Sy40Am721JLizzcXAMSznzi99o4jQsZZPIMGt4yvp1+CU5g1fW7SFHL6ANaGmZOVq/rZQKaJpwK+VGRRJ1KHsZS0X2axgeSsP4hnSOb+iyb++n1rlN1ps2COP9O/oSGiyEBAcREiSEBgcRHCSl7hseGsTutOOs2ZZWuKxJ/TA6NWtA5/gGnMjN550f9ruM5HLfkkReXb+HfUdOciq/gDrBQSS0asT9g9vTt20MnZo1IChIPH6D8MSIznSOb8j6Xems33mQ2V/tZeYXvxFZN4QLz46hQXgIKzYlF0l+t3IwK5f4RuHsOJDJ9hQrwU7JyHGJu2iy7VBgoHmjCA4dz+WPIyc4lHWqWLlLUdl5Nh79YDt9z25CdH3/XHyjXKVl5vLX1pUbDUEppbxJE26lqlBFe8e9Uas+eUgHYiJLTgg97eson8nKyWPHgSy2pWSwLSWTX5Iz+GbPIbejuuQXGPYeOsENF7Si79lN6Nm6MRF1ijcxpb3Ws+MiufXCs8jKyeObPYdYv/Mg63eluy1Fyc4r4PHVOwBr0qWzYurxl9aN6XBGA/stktjIuh7/sYiPCmf2Da7f/p3IzefQ8VwOHT/lcVjKwydOcd5jn9KicTjnNo+iWwvr1jm+IXVDgwu307px71vxUxLJx7JZvjmZDb8f0XOslApIWsOtVA3gy1FKcvJsdPi/j93Wqgvw+1Nupl2uJGMMbR5Y7bE+/v2JfTg7rr5LsuusonX5nhL1JvXrcOuFbdiy/xiJ+zMKtwkOEs5pGkm3FlHYjGHFpmSXiZzKei2Ag9Zwl2zl5mQmL9ta+K0HlP8cq8ClNdwqEGkNt1K1WEVLYCqyb93QYI+16lV9saODiJRYH9+leUM3e51W1cNSPnR5R5d907NySNyfQeL+Y2zZf4xViSlk5eQXez5fjMZSmzyzZpdLsg16jmsaY4zLjIxK+VNlOqk14VZKlVtFa9X9eUxvXkAbG1mXAR3rMqCjNZ1xQYHhrCnue+RT3PzToCrG07nUc1wzhIaGkp2dTUREhL9DUQqwJuIJDQ2t0L6acCulyq0yNefV6ZiO45b3GEFBnnvkvfUtQG2k57hmi42NJTk5mfj4eMLDw7WnW/mNMYbs7GySk5OJi4ur0HNowq2UqpDKlLFUp2NWlD++Baht9BzXbI7ZBFNSUsjLKz6UqFK+FBoaSlxcXLEZOcvK5wm3iAwGXgSCgdnGmKeKrA8D5gPnAYeBMcaYffZ1DwATABtwpzFmjQ9DV0qpMvNXj3xtoue45mvQoEGFExylAolPE24RCQZeAQYAScCPIrLKGLPdabMJwFFjTFsRGQs8DYwRkY7AWKAT0Az4VETaGWNKHjRXKaX8pDr1yFdXeo6VUtVBkI+P1xPYY4zZa4w5BSwChhXZZhjwlv3+UuASsQq3hgGLjDG5xpjfgT3251NKKaWUUipg+Trhjgf2Oz1Osi9zu40xJh/IAKLLuC8icquIbBSRjQcPHqzC0JVSSimllCo/Xyfc7i4xLjpylqdtyrIvxphZxpgEY0xCTExMBUJUSimllFKq6vg64U4CWjg9bg6keNpGREKAhsCRMu6rlFJKKaVUQPF1wv0jcLaItBaROlgXQa4qss0q4Ab7/auAdcaa2mcVMFZEwkSkNXA28IOP4lZKKaWUUqpCpDLTVFbogCKXAS9gDQs41xjzuIhMBzYaY1aJSF3gbaA7Vs/2WGPMXvu+DwI3AfnA3caYj0o51kHgD++9mnJrAhzydxBFBFpMGk/pAi2mQIsHAi+misZzpjGmVtXGBVi7HWjvIwi8mAItHgi8mDSe0gVaTFXeZvs84a7NRGSjMSbB33E4C7SYNJ7SBVpMgRYPBF5MgRaPKptA/L0FWkyBFg8EXkwaT+kCLSZvxOPrkhKllFJKKaVqFU24lVJKKaWU8iJNuH1rlr8DcCPQYtJ4ShdoMQVaPBB4MQVaPKpsAvH3FmgxBVo8EHgxaTylC7SYqjwereFWSimllFLKi7SHWymllFJKKS/ShFsppZRSSikv0oS7iolICxFZLyI7RGSbiNzlZpuLRCRDRLbYb1O9HNM+EfnZfqyNbtaLiPxXRPaIyFYR6eHleNo7vfYtIpIpIncX2car50hE5opIuoj84rSssYh8IiK/2n828rDvDfZtfhWRG9xtU4UxPSMiO+2/lxUiEuVh3xJ/x1UYzzQRSXb6vVzmYd/BIrLL/p6aXBXxlBDTYqd49onIFg/7euMcuf28+/u9pMouENts+zEDpt0OhDbbfoyAare1za5wTLWzzTbG6K0Kb8AZQA/7/UhgN9CxyDYXAR/4MKZ9QJMS1l8GfAQI0AvY4MPYgoFUrMHifXaOgAuBHsAvTstmAJPt9ycDT7vZrzGw1/6zkf1+Iy/GNBAIsd9/2l1MZfkdV2E804D7yvA7/Q1oA9QBEot+BqoypiLrnwOm+vAcuf28+/u9pLfK/w6LbOPTNtt+zIBst/3VZtuPEVDttrbZFYupyPpa02ZrD3cVM8YcMMZsst/PAnYA8f6NqlTDgPnG8j0QJSJn+OjYlwC/GWN8OrOcMeZLrJlMnQ0D3rLffwsY7mbXQcAnxpgjxpijwCfAYG/FZIxZa4zJtz/8HmheFceqaDxl1BPYY4zZa4w5BSzCOrdejUlEBLgaeKcqjlXGeDx93v36XlJlV03bbPBfu+2XNhsCr93WNrtyMdW2NlsTbi8SkVZYU9RvcLP6fBFJFJGPRKSTl0MxwFoR+UlEbnWzPh7Y7/Q4Cd/9wRmL5w+bL88RQJwx5gBYH0og1s02/jxXN2H1aLlT2u+4Kk20f10618PXbv46R32BNGPMrx7We/UcFfm8B/p7SbkRQG02BG67HUhtNgT2Z03b7JLVqjZbE24vEZH6wDLgbmNMZpHVm7C+jjsXeAlY6eVwehtjegBDgNtF5MKi4brZx+vjRYpIHeBKYImb1b4+R2Xlr3P1IJAPLPSwSWm/46ryGnAW0A04gPV1YFF+OUfAOEruKfHaOSrl8+5xNzfLdJxWPwmwNhsCsN2upm02+OdcaZtdulrVZmvC7QUiEor1i1xojFledL0xJtMYc9x+fzUQKiJNvBWPMSbF/jMdWIH19ZGzJKCF0+PmQIq34nEyBNhkjEkrusLX58guzfGVrP1nupttfH6u7BdmDAWuNfZCsqLK8DuuEsaYNGOMzRhTALzh4Tj+OEchwEhgsadtvHWOPHzeA/K9pNwLtDbbfpxAbLcDrc2GAPysaZtdutrYZmvCXcXsNUlzgB3GmP942KapfTtEpCfW7+Gwl+KpJyKRjvtYF3T8UmSzVcD1YukFZDi+WvEyj//d+vIcOVkFOK46vgF4z802a4CBItLI/tXcQPsyrxCRwcC/gSuNMSc9bFOW33FVxeNcIzrCw3F+BM4Wkdb2HrGxWOfWmy4Fdhpjktyt9NY5KuHzHnDvJeVeoLXZ9mMEarsdaG02BNhnTdvsMqt9bbap4JWeevN4BWwfrK8YtgJb7LfLgNuA2+zbTAS2YV0J/D1wgRfjaWM/TqL9mA/alzvHI8ArWFcp/wwk+OA8RWA1xg2dlvnsHGH90TgA5GH91zoBiAY+A361/2xs3zYBmO20703AHvttvJdj2oNVM+Z4L820b9sMWF3S79hL8bxtf49sxWqgzigaj/3xZVhXf/9WVfF4ism+/E3He8dpW1+cI0+fd7++l/RWJb9Dv7TZ9uMFXLuNn9ts+zECqt32EI+22aXEZF/+JrWszdap3ZVSSimllPIiLSlRSimllFLKizThVkoppZRSyos04VZKKaWUUsqLNOFWSimllFLKizThVkoppZRSyos04Va1gohMExHj4XadH+IxIjLR18dVSqnqQNtsVdOE+DsApXwoAxjsZvkeXweilFKqVNpmqxpDE25Vm+QbY773dxBKKaXKRNtsVWNoSYlSgIi0sn9leI2IvC0iWSKSLiIPu9m2v4hsEJEcEUkTkVdFpH6RbaJF5HUROWDfbpeI3F3kqYJF5AkROWg/1isiEubVF6qUUjWAttmqutEeblWriEix97wxJt/p4TPAB8BVwIXAwyJyyBjzin3/jsDHwCfAKKAF8BTWNLSD7duEA58DscAjwE6grf3m7F5gHXAd0BV4EvgDmFH5V6qUUtWfttmqptCp3VWtICLTgGI9H3at7T9/Bz4xxgx02u8N4DKghTGmQEQWAecB5xhjbPZtrgYWAxcYY74Tkb8DrwE9jDFbPMRjgK+MMRc6LVsJNDXG9KrES1VKqWpP22xV02hJiapNMoC/uLmlOG2zosg+y4FmQHP7457ACkfDbbcMyAf62B/3BzZ7aridrC3yeLvTcZRSqrbTNlvVGFpSomqTfGPMRncrRMRxN73IKsfjM4A/7T/TnDcwxthE5DDQ2L4oGjhQhniOFXl8Cqhbhv2UUqo20DZb1Rjaw62Uq1gPjw84/XTZRkSCsRrsI/ZFh7EaeaWUUt6lbbaqFjThVsrViCKPR2I12En2xxuAEfYG23mbEOBr++PPgO4i0tWbgSqllNI2W1UPWlKiapMQEXF3cct+p/udROR1rBq/C4EJwF3GmAL7+seAzcBKEXkNq37vaWCNMeY7+zbzgduBtfYLf3ZhXeTTzhgzuYpfk1JK1VTaZqsaQxNuVZs0BL5zs/z/gAX2+/cDQ7Ea7xzgUeBlx4bGmG0iMgR4AuvinEzgHft+jm1yRKQ/1tBT04EGwD7g1ap9OUopVaNpm61qDB0WUCmsSRSwhpi6whjzgX+jUUopVRJts1V1ozXcSimllFJKeZEm3EoppZRSSnmRlpQopZRSSinlRdrDrZRSSimllBdpwq2UUkoppZQXacKtlFJKKaWUF2nCrZRSSimllBdpwq2UUkoppZQX/T9aLkzxQ/RyZwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "hist = history.history\n", + "x_arr = np.arange(len(hist['loss'])) + 1\n", + "\n", + "fig = plt.figure(figsize=(12, 4))\n", + "ax = fig.add_subplot(1, 2, 1)\n", + "ax.plot(x_arr, hist['loss'], '-o', label='Train loss')\n", + "ax.plot(x_arr, hist['val_loss'], '--<', label='Validation loss')\n", + "ax.set_xlabel('Epoch', size=15)\n", + "ax.set_ylabel('Loss', size=15)\n", + "ax.legend(fontsize=15)\n", + "ax = fig.add_subplot(1, 2, 2)\n", + "ax.plot(x_arr, hist['accuracy'], '-o', label='Train acc.')\n", + "ax.plot(x_arr, hist['val_accuracy'], '--<', label='Validation acc.')\n", + "ax.legend(fontsize=15)\n", + "ax.set_xlabel('Epoch', size=15)\n", + "ax.set_ylabel('Accuracy', size=15)\n", + "\n", + "#plt.savefig('figures/15_12.png', dpi=300)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "500/500 [==============================] - 2s 5ms/step - loss: 0.0428 - accuracy: 0.9927\n", + "\n", + "Test Acc. 99.27%\n" + ] + } + ], + "source": [ + "test_results = model.evaluate(mnist_test.batch(20))\n", + "print('\\nTest Acc. {:.2f}%'.format(test_results[1]*100))" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TensorShape([12, 10])\n", + "tf.Tensor([6 2 3 7 2 2 3 4 7 6 6 9], shape=(12,), dtype=int64)\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAq8AAADoCAYAAADBow1gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de7zNVf748fchd4mcI6Ickmshl5HGtQvVkGiiaca1XAeZCt+QESW6YCgRTg0zk5lEUm6TLm4Rp9I0IhmHJHHcI0Ln98fn12qt1fls+3r2Xvu8no9HD+9lffZnr87H/ux1Pp/3571ScnJyBAAAAHBBgXgPAAAAAAgWk1cAAAA4g8krAAAAnMHkFQAAAM5g8goAAABnMHkFAACAMy4KZePU1NSc9PT0GA0FgWRlZUl2dnZKNPbFcYyvzMzM7JycnLRo7ItjGV8cy+TBsUweHMvkEGjeE9LkNT09XTZv3hydUSEkjRo1itq+OI7xlZKSsjta++JYxhfHMnlwLJMHxzI5BJr3kDYAAAAAZzB5BQAAgDOYvAIAAMAZTF4BAADgDCavAAAAcEZI1QZc9+WXX6p44MCBRl+hQoVUvGTJkjwbEwAAQLhq1apltLdt25brdqNHjzbaw4YNU3GJEiWiP7AY4sorAAAAnMHkFQAAAM5I+rSBU6dOqbhXr14qXrNmjbHdqFGj8mxMAAAA0TBgwACjPXjw4Fy3Gzt2rNF+9tlnVbx48WKj76abborS6GKDK68AAABwBpNXAAAAOIPJKwAAAJyR9DmvQ4YMUbGe51qjRg1juw4dOuTZmBC6zz//XMX9+/dX8fvvv29sV6VKFRW/8sorRl+TJk1iNLrkpf98P/30U6NPPyYzZszw3UdOTo7RTklJUXG/fv1UbJd7qVu3ropbtmwZ5IgBIH+ZOnVqWK87efKkijt16mT0vf322ypu3LhxeAOLIa68AgAAwBlMXgEAAOCMpEsbmD17ttGeNWtWrtvZJSMaNWoUszEhdB988IHR/s1vfqPiI0eO+L5u165dKu7WrZvR984776i4YsWKkQ4xaRw8eNBo9+7dW8WrV69W8bFjx3z3oacCXIi+7cyZM323K1WqlIpbtWpl9L344osqTktLC/q9ASDZ3H///Ub7f//7n4pvueUWFY8bN87YTk8FO378uNG3ceNGFZM2AAAAAESAySsAAACcweQVAAAAzkiKnNcffvhBxU8//bTvdtddd52KKY2VePScG7tsx+nTp1XctWtXFT/00EPGdm+88YaKR48ebfTdfPPNKt60aZPRV7JkyTBGnBy2bt1qtN99910VFy9eXMXVq1c3ths5cqSKA+WdBiqVpZfY2rZtm7GdntusH1cRkX379qm4bNmyRt/SpUt9xwLkJytXrjTabdq0UXFGRobRpy+lrpcjLFiwYIxGh2gZPnx4UNvdfvvtRltfAnbDhg1Gn35+t58R6dixY6hDjDquvAIAAMAZTF4BAADgjKRIG1i0aJGKv/jiC9/txowZo+IiRYrEckgIw4QJE1S8f/9+o+/vf/+7iu+9917ffeirMunlQkRE5s6dq2L7ltngwYNDG2wSsVevmjdvnorT09NVrP9so6Vt27a+fVu2bPHdbvPmzb6vW7BggYrvuusuFYdSzgtINOfPn1fxnj17VHzgwAFjuz/96U8q3rlzp9F31VVXqfijjz4y+vT0O/18a39XvvrqqyqmTJ1b9DQwEZG33npLxfXq1TP69u7dq+LJkycbfaQNAAAAACFg8goAAABnOJk2cOjQIaPds2dP3231W9Ht27eP2ZgQOf2JSb0yhIjIHXfcEdQ+9FvDL730ktGnP80+adIko09fjat06dJBvVeyCvZnHWv6bSw7jUR/anbFihVGX+fOnVWsr7jXq1evaA8xISxevFjFlSpVMvoaNmyY62vWrl1rtPUnzW3VqlVTcdWqVcMZIqJAr8ai3/4Phb2anh99dSWbvtqhncI1ZMiQsMaF+Lj00ktVPGjQIKNP/z7++OOPjT69bX9X5xWuvAIAAMAZTF4BAADgDCavAAAAcIaTOa9Tpkwx2t9//72KixYtavTpea6Uyklsl1xyiYrvvvvuqO///vvvzzUWMfOo83vOayKyc/XsvHc/derUicVw4urqq6822npJuAIFzOsRJUqUyHUf9nb6KmjHjh0z+goVKuT7Op2+AlvNmjWNPj1H2S7Nppdjw89q1apltO2V6n5il7KqXbu2iu3j9eOPPwbVd/ToURXv2rXL2E4vU6eXsxMROXv2rIqHDh2a63iRmC677DLfPj03VkSkTJkysR7OBXHlFQAAAM5g8goAAABnOJM2cOTIERXPnDnTdzv7Vot+CwX5W+XKleM9BFj0VX12795t9K1evVrFL774otEXaIWtpUuXqvjaa6+NdIgJ55///KfR1lfCsW/nlStXTsX6bWI79UC/Zbxu3Trf9165cqXR1ss36X16+S4Rc1Ume5UfvZzZPffc4/ve+Y1e2k/ETHtr1qyZirt3725sd99990X83jt27FDxwoULjb5HHnlExXqagEjwpbiQePTUHhHzHKH/exARef3111Ucr/JoXHkFAACAM5i8AgAAwBlMXgEAAOAMZ3JeT58+reJAeTUZGRlRf2/9/fQcL5Hwl+lD3tuwYYOK7XJYxYoVy+vhOG/r1q0qtnNSdXaJHz13Ty/JM2/ePN/X2WXu9FzOkSNHGn233nproGE7r0GDBgHb4dDzYZs3b+67XaA+3TPPPGO0hw0bpmJ7Kdqvv/46qH3mNzfeeKPR1n+GTZs2VXGpUqWi/t56vuPDDz9s9OmlswJ97uGWtLQ0o12/fn0V2zmven47Oa8AAADABTB5BQAAgDOcSRtYsGCBb59+y8tebSRYesmPvn37Gn36JXI7bWDRokUqvummm8J6b+SNTZs2qbhGjRpG3+WXX57Xw3GCfuu+c+fORt9rr70W8j5Egl/prm7duiru16+f0We3kVjOnz/v22cf/2ikPSSjVatWxXsIImKWsxMhVSBZ6eXyRH5ZBk1XuHDhWA/ngrjyCgAAAGcweQUAAIAzmLwCAADAGQmb83r48GGj/eSTT/pu27BhQxXby8P6sfM5unbtqmJ7+cVA9Nd99tlnKr700kuD3gdiQz8eIiLLli1Tcc+ePfN6OM677bbbjPaKFStUfPLkyaD3E2zOa3p6uoobNWoU9P4Rf4HKX1WvXt1ot27dOtbDAXAB06dPN9r6ErDVqlULuG08cOUVAAAAzmDyCgAAAGckbNrAtm3bjPY333zju629WlIw7BVgAqUK6PvXVwSyx7V9+3YV6yugIO/oZZkmTJhg9FWsWFHFo0ePzrMxuUy/xd+rVy+jT7+V9P333/vuY/DgwUZbL223b98+FX/33XfGdm+88YaKN27caPS1bdtWxfYKW/rqQMg7+spZ//73v323C+d8DeQH9jzn888/V/Hf/vY3o08/H1922WW++9RXAbXnL3pJu3fffdd3H+fOnfN9Xbxw5RUAAADOYPIKAAAAZyRs2kAo7rzzzqC2mzhxoorHjh3ru13NmjWN9vjx41XcqVMn39fpt0MRGfvpdf22sb5S1pIlS4ztrrzyShW/8sorRp9+/PUUAoSnRYsWQW2np9PY9NSAKVOmGH3vvfeeig8cOGD0zZs3L9dYJDFuaeUXp0+fVvHs2bNVHOiY2ykmffr0Cfl9y5cvb7QDnc8RGf1ziOjQ0yL1FLYPPvjA2G7v3r2++3jppZeCeq+OHTuqeMeOHUaf/vn98ssvffdxxRVXBGzHA7MtAAAAOIPJKwAAAJzB5BUAAADOSNicV7ucSrFixVRs50zpOTkDBgww+vbv36/iMWPGqFjP9RARadCggYpXrlxp9D366KO+49RXh2nSpInvdvDopazs0hwLFixQ8dKlS42+3bt3B7X/devW+fbp+T56WR8RkeLFiwe1f0TXHXfckWssIvL++++rePLkyUafnitra9WqlYoXL16s4ksuuSTcYSa9//3vfyp+4oknjL7s7GwV2+dG+zwajJ07dxrt2rVr5zoOEZG6devmug+/v0dgx48fV7G9ymTZsmVVrH++AuUk2ysv9e3bN9IhJiU7D18v77dw4cKYvveiRYsi3of+70bEXL3ymmuuiXj/4eDKKwAAAJzB5BUAAADOSNi0Af1WkohInTp1VLx582ajT78svmLFCqPvL3/5i4oD3eL69a9/reJhw4YZfRkZGb6vC9SHX9LLe9x3331Bv87v1oR++0LELFd2+eWXG316KZ9jx44Zff/617+CHgvyRsuWLVXcuHFjo09PAdLTTUREVq9ereLf//73KrZLy6SlpUVjmEnh4osvVrFdRk5vd+3a1ejTS9OtXbtWxQ8//LDvew0fPtxojxo1KrTBImjffvut0b7nnntUrK9uJyLSu3dvFeu3te30At3MmTONtr6aE35m/5z8UgXs1KYbbrghqP3r5zyRX5aajNSWLVuMtr66l/7/dt1110X1fQPhyisAAACcweQVAAAAzmDyCgAAAGckbM6rbc6cOSquV6+e0Xfu3DkV33rrrWHtf9q0aUFt16NHD6Nt51XC9MUXXxhtfSk8m57Xaucn6iVZ7r77bhXbOa8PPvigiu2SP3q5pUDL7iF3R48eNdp6ubFYfw7sUmZPPfWUinv27Gn06fnxy5YtU/FHH31kbNe2bdtoDtFpev5vuEutFilSJKjt9FxmxNbXX39ttPXyczb7WY9g9OvXz2h36dJFxXp+bbTYz8K4IthzTZkyZYy2Pd/wU7RoUaMdTnmsX/3qV0Zbz7/997//bfTpS7TfeOONKh48eLCx3WOPPRbyOILFlVcAAAA4g8krAAAAnOFM2sDVV1+t4nHjxhl9+q1ofQWncJUoUcJoT5gwQcUDBw6MeP/5iV5+RcS8jXXzzTcbffpqSPaKJPotLb0kj32LUy/DU7hwYd8+BEcvQ/X8888bfSVLllTxkiVL8mxMtlq1asXtvfGzzMzMeA8BFjud5/HHH1fxxIkTjb4TJ06EvH991UJ7/3ocLT/++GPU95kXqlSpYrS7deum4rlz56o4KyvL2E5Pw4gF/Tv4tddeM/r0NKD169cbffqcSF91b8qUKcZ2F1308xQz0Eql4eDKKwAAAJzB5BUAAADOYPIKAAAAZziT81qsWDEV28sJ1qxZU8V2eaTt27er+Pvvvw/qvfR8FBHyXCNhL8mbmpqq4ueee87o0/OZOnfubPQtX75cxXfccYeKo51HA9OaNWtUbJfZ0UtS6XlbIr/8DEWbnh+9Z88eo0/Pe69cubKK9aVMEX32cqN+9u/fH+OR4Cfly5c32iNGjFCxvdSv/pyBXgpp6tSpYb23vUTpJ598ouJGjRqpOFCJtQEDBoT13olGX7ZcROS2225TsX3ujLb+/fur2H7ORC8tapcj1LVu3dpo68vW6uXSXn75ZWM7vVSh/cyJ/UxKqLjyCgAAAGcweQUAAIAznEkbCOS3v/1trrGIyJtvvqnioUOHqnjnzp3Gdv/3f/+nYm5FR0YvuXL48GGj75ZbbvHtGzRokIrtFT06dOig4unTp0dlnLiwlJSUXGMRka1bt6q4b9++Rp+evtOsWTOjT/+MLly4UMWdOnUytnvggQd83/ubb75RsX17Ut9WX7WNklqJwb6Vjfi44oorfPvS09NVrK9oGAr7/D5//nwV6ykLF198cVj7d1mbNm1UrN/Wf+GFF3xfY6diVaxY0XfbIUOGqFhfPc8+j4ZLT/WYNWuWivVUMhHze+Ds2bNGH2kDAAAAyDeYvAIAAMAZSZE2EEi7du1yjRE7+m2gSy+91Oh75ZVXco1FRAoVKqRie2WRp556SsX2qjGIHT0dwH4a9W9/+5uK7VtCX375pYrtVXjsJ1J/kpGRYbT1qgGBbneVK1fOaLdv317FdjoD4qNo0aIq1iuOIHnZ5/5kqRwQDfrPRk+DczElTl9F6+GHHzb67HY0ceUVAAAAzmDyCgAAAGcweQUAAIAzkj7nNVGdOyfyzDMic+aI7NkjkpYmcvfdIpMnx3tk0dW2bVujra+GVKNGDaNvwoQJKr7++utjO7AoefVVkXnzRDIzRY4dE6lRQ+Thh0V+97t4jyw69PJS48ePN/r0fNJjx44ZfatXr/bd5+OPP67igwcPqtguyWOX2PLTokULo123bt2gXmdbsEBk0iSR7dtFTp4UqVxZpGtXkWHDRCKs6pIv6M8U6PnKIiIdO3ZUcV6ULGvVSsRaEE5Zv16kadOYDwFRkOzn1/wk2seSyWuc9OwpsmqVyJ//LFKzpshXX4loZTPhiEmTRKpU8X7pSE0VWbpU5N57RbKzRbSytXDAoUMirVuLDB0qUrq0yIcfiowZI7J/v4i1kjES3PTpIsePm383erTIxx+LNG4cnzEhdJxfk0e0jyWT1zhYvlxk/nyRLVtEateO92gQiSVLvA/iT268UWTfPu+DysnVLXZhgtatvQnQ88+LTJsmEqX63sgD9nn1hx9ENm8W6dJF5CK+9ZzB+TV5RPtY8jGOg4wM78Dlh4nr2LFjA7Zdl1vVn+uuE1m8OO/HktcqV67s2xfo1v3AgQNjMZyYKFvWm/jgwurXr59rnAiWLxc5coTbza7Jz+fXZBPtY8kDW3GwcaNI9eoiAweKlColUry4SKdO3m8hcN/69fnjF5Nkdf68yKlTImvXikydKtK/P1ddXTd/vkjFiiLNm8d7JIgU59fkEcmx5MprHOzfL/LyyyL16nkn1RMnvIdCOnYU2bCBL0qXrVrl/SZp1duHQ0qUEDlzxou7dRN5+un4jgeROXXKu2XZpw/nVtdxfk0ekR5LJq9xkJPj/bd4sXdbUkSkQgWRli1F3nlH5Kab4js+hCcry0tA79BBpEePeI8G4Vq/3pvwfPihyNix3h0SBxe+wf+3ZInId9+RMuA6zq/JIxrHkslrHJQpI1K16s8TVxGRZs28cjxbtzJ5ddHhwyK33SZy5ZUi2qqpcFCDBt6fzZp5eVrdu4s89JDIVVfFd1wIz/z5ItWqiTRqFO+RIFycX5NHtI4lOa9x4FfmMCdHpABHxDmnTom0a+c92PPWW95tZySHnyayu3bFdxwIz7FjIsuWcdXVZZxfk0c0jyVTpTho107k00+9+mY/Wb1a5OxZLw8W7jh3zltcYscO70uyXLl4jwjRtG6d92eVKvEdB8KzaJGXv8zk1U2cX5NHtI8laQNx0KeP9xRz+/YiI0Z4D2wNHy5y883erUq4Y8AAr9jyX/7i3Q7ZsOHnvuuuEylSJH5jQ2huvdX7DNapI1KwoDdxffZZrzYoKQNumj/fuyCQB4t6IQY4vyaPaB9LJq9xUKqU92DW4MEi99zj5bp26JB8S8PmBytXen8+8MAv+3btEklPz9PhIAKNG3tVQLKyvEL2VauKPPmkSL9+8R4ZwpGd7T3RPG5cvEeCcHF+TR7RPpZMXuOkWjXvtxC4LSsr3iNAtIwbx0QnmaSmeqlYcBfn1+QR7WNJzisAAACcweQVAAAAzmDyCgAAAGcweQUAAIAzUnJycoLfOCXloIjsjt1wEEDlnJyctGjsiOMYdxzL5MGxTB4cy+TBsUwOvscxpMkrAAAAEE+kDQAAAMAZTF4BAADgDCavAAAAcAaTVwAAADiDySsAAACcweQVAAAAzmDyCgAAAGcweQUAAIAzmLwCAADAGUxeAQAA4AwmrwAAAHAGk1cAAAA4g8krAAAAnMHkFQAAAM5g8goAAABnMHkFAACAM5i8AgAAwBlMXgEAAOAMJq8AAABwBpNXAAAAOIPJKwAAAJxxUSgbp6am5qSnp8doKAgkKytLsrOzU6KxL45jfGVmZmbn5OSkRWNfHMv44lgmD45l8uBYJodA856QJq/p6emyefPm6IwKIWnUqFHU9sVxjK+UlJTd0doXxzK+OJbJg2OZPDiWySHQvIe0AQAAADiDySsAAACcweQVAAAAzmDyCgAAAGcweQUAAIAzmLwCAADAGUxeAQAA4IyQ6rwCABCJDz74wGh37txZxXYx+CFDhqj4rrvuium4ALiDK68AAABwBpNXAAAAOIPJKwAAAJxBzisAIKZmzJih4kceecToO3bsmIorVKhg9NWrVy+2A0NEDh48aLRbtGih4u3btxt9OTk5Kh41apSKx40bF6PRIZlx5RUAAADOYPIKAAAAZziTNpCSkpJrfCG9e/dWcdOmTX23q1+/fq4xouvUqVMqLly4sNF30UWR/3M8c+aMigcNGmT0zZ49W8VvvPGG0deuXbuI3xuAZ8KECUb7scceU3GRIkWMvhdeeEHFXbt2NfqKFy8eg9EhEnqqwO2332706akCgb6nx48fr+KsrCyjb968eRGOEH52795ttGfNmqXirVu3Gn2vv/66ilNTU42+Tp06qVhP+0hLS4vKOIPBlVcAAAA4g8krAAAAnMHkFQAAAM5I2JxXO2dKz58pWLBg0PvJyMhQsZ7fYe/jyiuvVHHlypV99/frX//aaI8cOVLFRYsWDXpc+ZWed/r2228bfXYeajjeeustFc+ZM8fo0/8N/fGPfzT6yHkFQnfu3DkVP/jggyqeNm2asZ1e8mru3LlGX926dWM0OkTD6tWrjXbfvn1VHKgclv7dKCLy3//+V8V6PuWKFSuM7fbs2aNi/XsZ4Vm4cKGK+/fvb/RlZ2er2M5r1Y/zgQMHjD59LtWgQQMV9+nTJ7LBhoArrwAAAHAGk1cAAAA4I2HTBmrVqpWn76ffqtBj23vvvWe0b7nlFhXrq4vAo9+yEBEZPXq0ik+cOJHXw1H0kl0iInv37lVxpUqV8no4gJO6deum4vnz56u4fPnyxnbLly/37UPi0cth6aleIoHLYempAvZKajq9JNqiRYuMPv1WNmkD/k6ePKli+2e4du3aXPv0W/wiIh07dlRxoFv+mZmZRlvfZ/PmzYMccXRx5RUAAADOYPIKAAAAZyRs2kCHDh2M9q5du4J6nf3048SJE1X8zTffqHjHjh0RjO5n+hPtpA149HQAffUNuy8W9NsiFSpUMPr042+v8lO2bNmYjssl+i2hVatWqXjZsmXGdnrljVDSfFq2bKni999/P6jXtGrVymjr6TsrV640+qZOnaria6+9Nuhx4cLsfwMLFixQccmSJVX8n//8x9jOfpIZiU1/0lyvDCBirnpmr4al34YOpG3btiq20/RIFQiOfp7u3r270ad/3kaMGKHiBx54IKz3sj+/5cqVC2s/0cSVVwAAADiDySsAAACcweQVAAAAzkjYnFdboFWvAm3Xpk0bFb/88ssq7t27d1TGdd9990VlP8lk8eLFKt6yZYvvdnrZrGhJT09X8RVXXGH07du3T8UFCpi/txUrVizqY3HVhg0bVDx9+nTf7YLNQ7cVLlxYxT/88ENQr7FzlM+cOeO77Z133qninTt3hjg62PTjbK/Qc/78eRX/4x//UDE5ru55/PHHVaznudrlsDp16qTiYHNcbXqOvJ1Hzb+d3NkrnT3xxBMq1lc2ExEZNGiQisPNc9XZ86r9+/dHvM9IceUVAAAAzmDyCgAAAGc4kzYQLPtydufOnVW8Zs0aFdu3QoJll6S47LLLwtpPMtNvP9k/56ZNm6p4+PDhMR2H/d56O9zjnx+88cYbMd2/fss/2OMQKE3Advbs2ZDHBH8vvviiiu2yRvfff7+K27Vrl2djQuRGjRpltMePH69i/Ta0XQJy7ty5Eb93vFZlco2eKvDQQw8ZfYcPH1bx5MmTjb5opAp8/vnnKtb/bYiIbNu2TcWbNm2K+L3CwZVXAAAAOIPJKwAAAJzB5BUAAADOcDLn9dy5c0b7ueeeU7Gdj6MvU6jn1xUsWNDYrnTp0iouVaqU0afndT3yyCNhjDh/+eKLL1Rs5zRWqlRJxZSnSgz28o9ZWVlBva5JkyYqtpfiDUTPn9LLcoWSa7t+/XoVHzhwIOjX4cKWL19utKdNm+a7bfny5VWs58Hp51N7O8RPoDxG/Vydlpam4kmTJsV+YMiVvpS2/V06Y8YMFffp0yes/R88eFDF+pLAIub3gl2KKxGeGeHKKwAAAJzB5BUAAADOcDJtYO/evUZ76NChQb2ufv36KtZX4RExL8/rMYIza9aseA8BYTp+/LjR9itL1aFDB6OdkZGh4jJlyoT13jVr1lRxjx49gn7dLbfcouJVq1aF9d7I3W233Wa0A90i1Mvi6bG9Ilq1atVU3LBhQ6Nv9uzZKr7oIie/khLWyZMnjfbIkSNVbN8K1ukrUJYtWzb6A0Ou9LQOkcDlHfWVzoJlr9Kll9/KzMz0fW9b7dq1Q37vaOPKKwAAAJzB5BUAAADOYPIKAAAAZ+SrBKOpU6equFmzZnEcSfI5ceKEigPlUg0ZMiSm41i5cqWK9TJMtkBjzG/mzJkT1HZ2nni4ea7h+OijjwK2EZkVK1b49umfFTsfVl+GUi9fpi8pKyLy2Wef5RqLiKSnp6t4zJgxQY0XwdHLl4mILF68WMWBls/Wy2jpOckiIgMHDlSxvcQsIqMvYS8S+Hsq0M9+5syZKtaPa6CSV4Heq23btkZ73rx5vtvmFa68AgAAwBlMXgEAAOCMfJU20KtXLxXbt0CfeuqpvB5OUvEr6WHfmlq6dKmKr7/++qiPQ78FHqjURyKsEJIoChQwf4fVy1ddffXVKtZv7+a1w4cPG+0jR474bmvf4sKFBSoPqJe9mjhxotF37bXXqlj/uffr18/YTi+lp5fUEjFvUd97771GX/Xq1QMNG7nQyyHZxzXQbWL9c6+nd9llEPX0u5IlSxp9sU4LS3Z2+Sv9c2R/Z+mpOYFSQPSyVjVq1DC2s1dX1N11110qfvXVVwMNOy648goAAABnMHkFAACAM5xMG7BvVVxzzTUq3rJli+/rduzYoeJnnnnG6NPbdt8dd9yhYn2lmPzsiy++MNr79+8P6nUzZsxQ8dtvv230/elPf/J9XaVKlVTctGnTXGMRkZ07dwY1Dvzs3XffjfcQLmjBggVBbxvoyXnkrmjRor59derUUbGeJhBIhQoVjPbo0aNVfPToUaNvypQpKj537lxQ+4c/vcJAoNvJepqAiKeEns8AAAueSURBVMjmzZtVXLx4cd/967ey9RWaREgbiFRqaqrR/vbbb1W8aNGioPfTvHlzFdeqVUvFdlqCnjpiH3M7hSfRcOUVAAAAzmDyCgAAAGcweQUAAIAznMx5tfNC9FwQfYUlEbO0y65du1RcsGBB3/0PHz7caGdkZKj4zTffNPrKly+v4kB5Y8nGLmGj/5yrVq2q4gEDBhjbHTp0KNdYRKRLly4qtnO19HycSy+9VMVfffWVsV2wJbAaNGgQ1HaIHz1/XV8Z6EL0HHUE55NPPvHt08ulRUOpUqWiuj+Y9NJWgVZNqly5stEOlOeqC7TPgwcPqjgtLS2o/cGf/jPs06dPWPvQS9PZpbH070s7H7Zjx45hvV9e4corAAAAnMHkFQAAAM5wMm3Apq/8Y19a19tjx45V8V//+ldjuz179vjuf/v27SrWb4mLiLz33nsqbtGiRVDjTXZ9+/ZVccWKFY0+fYWt2bNnG31nz5713efJkydzjQPdwgpELyWCxKSXVdNLxtjKlStntO3VnXBhgX6+0WavlobI6LfqRUSys7NVHKhU1ogRI8J6v0ArKOopfOHe5kbkli9frmK9TJ39fTl37lwV/+EPf4j9wKKIK68AAABwBpNXAAAAOIPJKwAAAJyRFDmvwdJzP/S8TBGz1NO0adOC3ufQoUNVvHHjxghGl5zatWvn27799tuNPj13a86cOUHt/8cffzTaX375pYrtUly6YEtqIX70fPJAuc16bqyIuVw0gmPnTerCzSvXffrppyq2z696/nnt2rUjfq/8Zvfu3b7tQMcu3Lz/aPx7QGx1795dxfp3nV2+zOXndLjyCgAAAGcweQUAAIAz8lXagO6ll14y2votylDoZUkQGjulQNezZ8+w9qmv0rVgwQLf7aK9ahAit23bNqN99OhRFdtpHo0aNVJxoH9HCE6rVq18+86dOxfy/s6fP2+0n3/+eRXbx7Jz584h7x8/C1QO60LbBsP+XAYqlYW8o5eMfPLJJ42+AwcOqFg/RvY858orr4zN4PIAV14BAADgDCavAAAAcAaTVwAAADjDyZzX7777zmgHym3MyMhQ8Zo1a1Qcbq6OXSZE3z/cQZ5k4pk5c6bRDrRkqZ7zetFFTp7GEkqlSpVUXLp0aaNvyZIlKn7uueeMvv79+6v4xIkTKraX6P3Xv/6l4jvvvNN3HwidnbfYoEEDFW/evDmsfU6ePFnFU6ZMMfr070C79BJLwuadrl27qnjx4sVGn75k9rx581Rcq1at2A8sj3DlFQAAAM5g8goAAABnJNT9tr1796rYvoW4du1aFZ85c8bo27RpU1D711MFChYs6Ltd0aJFjbZ+Cd5OE6hXr15Q7428oa8a8+qrr/puN3bsWKOtr76GvLNz504Vz50713e7G264wWg//fTTMRtTfrdw4UKj3bZtWxUPHjzY6Bs1apSK9fJYdmpXkyZNVPzCCy8YfYHOxbgw+9Z97969VZyZmWn06d+B9nHWbylPmDAh19fYRowYEdpgETY9TUBEZNGiRSq2j5GeOnLFFVfEdmBxwpVXAAAAOIPJKwAAAJyRUGkDLVq0UPFXX31l9Om3pKJxm8l+onbQoEEqTk9PN/q6desW8fshbzRs2FDFgW53LV261GiTNhAfy5YtU/GRI0d8tytevLjRLlGiRMzGlN+1bt3aaK9YsULFdrrG6dOnVayvxGXf4uzRo4eKCxUqFI1hwof+xP+ePXuMvvHjx6v4rrvuMvr086VeUcA+j+ppJL///e8jGywCevzxx1X8+uuvG32BVjrT0zmSqcKAjiuvAAAAcAaTVwAAADiDySsAAACckVA5r8GyS1n96le/Cup1zZo1U7G9EkjFihUjH1iQFiwQmTRJZPt2kZMnRSpXFunaVWTYMJHChfNsGEnpmmuuyTUWEfnss89U3KVLl5i8/9dfi9So4R3XEydESpaMydskja+//jqo7Xr16hXjkYi0aiXy/vu5961fL9K0acyHkJD0HFg7HzZRnTsn8swzInPmiOzZI5KWJnL33SLawlH5gl42S0Tk0KFDKrbLUfo9I6CXQxP5ZZnBWMtPx1LPcRURefTRR1VsH58hQ4aouGPHjkafXjIykbz+usjo0d7c5/LLRQYNEnnwwfD25eTk1XWHDom0bi0ydKhI6dIiH34oMmaMyP79Itbqi3DM0KHehPXkyXiPBKGaPl3k+HHz70aPFvn4Y5HGjeMzJoSnZ0+RVatE/vxnkZo1Rb76SmTr1niPCuHgWCaHdetEOnUS6dXL+2Vk40aR4cNFChQQ0ebhQWPyGgd9+5rt1q29L83nnxeZNk0kwEPySGBr1ogsXy4yYoQ3iYVbatc22z/8ILJ5s0iXLiIXcaZ0xvLlIvPni2zZ8stjCrdwLJPH2LEizZqJzJ7ttdu0ETlyxPv7AQNCv+ucUKdkvQzLSevS1bp161Rsl7J65JFHYjquvFC2rPdlichcfPHFKh5i/Tp3//33x+x9z5/3boGMHu1dTUdwpk+fHtR2V111VYxH8kvLl3sn19/9Ls/fGhHIyBC58UYmOyIilStXNtr66mb2SmeJKL8dywIFzMeQ9FQBu7TZpEmT8mRM0fLJJyJ//KP5d23aeOkfH3wg0rJlaPvjga04On9e5NQpkbVrRaZOFenfn6uurpoxQ+T06V9+OOGu+fNFKlYUSdD0MfjYuFGkenWRgQNFSpUSKV7cu125b1+8R4ZQcSyTx+nTv7y6WqSI9+fnn4e+PyavcVSihPdf8+bebx0s1+6mQ4dEHn3UewiP+uvJ4dQpkSVLvJQBfqF0y/79Ii+/7F3pmT9f5KWXRDIzRTp2FNFq78MBHMvkUa2ayKZN5t99+KH35+HDoe8vodIG8pv1670vyQ8/9PI+Bg70HhqBW0aOFGnSROT22+M9EkTLkiUi331HyoCLcnK8/xYv9tKxREQqVPAuELzzjshNN8V3fAgexzJ59Ovn3V2eNUvkt7/15j3PPuv1hbNoakJNXu2cDl0yLtHaoIH3Z7NmIqmpIt27izz0kEgc0vuSUs+ePQO2o+G///XyslavFjl61Pu7U6e8P48d8z6UxYpF/W2dtmTJEhXry4vaGjVqpOIqVarEdEy2+fO9KwXaEOCIMmVEqlb9ebIj4p1jCxf2nlJnwuOO/HYs9WVdc2u7rFcv78G7/v1F+vTxUkAmTvSeFbnsstD3R9pAgvhpIrtrV3zHgdDs2CFy9qxXA7RMGe+/n/JeK1XyPphwy7FjIsuWcdXVVX5LuefkeGV54A6OZfIoWNArBXrwoMinn4p8+63I9dd7fT/9GYqEuvKan/1UTCGPLzAhQs2aibz7rvl3y5d7v1EuXepdNYBbFi0SOXOGyaur2rXzaoJmZ3t3tES8OyNnz4rUqxffsSE0HMvk89NFHhEvTfKGG7z6vaFi8hoHt94qcvPNInXqeL+NrFvn5X506ULKgGtSU72VmXRZWd6fzZuzwlZuMjMzVXz27Fnf7QYMGKDi1J++ufLA/PneF6PfVR8ktj59vOot7dt7NZdPnPCKod98s/fLJtzBsUweGzZ4lZXq1/fq2r/yisiKFd7fhYPJaxw0buw9QZmV5RU/r1pV5MknvYRmAPGTne2t5jNuXLxHgnCVKuU9zDN4sMg993j5kR06JOdyosmOY5k8ChUS+ec/vdVECxTwLu6sWydy7bXh7Y/JaxyMG8eXYzLr0cP7D+5JTfVuScJt1ap5aTtwH8cyOTRs+MtSWZEg5RkAAADO4MorgDy1cePGXP++YcOGRrt9+/Z5MRwAgGO48goAAABnMHkFAACAM1JyQlggOCUl5aCI7I7dcBBA5ZycnLRo7IjjGHccy+TBsUweHMvkwbFMDr7HMaTJKwAAABBPpA0AAADAGUxeAQAA4AwmrwAAAHAGk1cAAAA4g8krAAAAnMHkFQAAAM5g8goAAABnMHkFAACAM5i8AgAAwBn/D0rc/As23lzEAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "batch_test = next(iter(mnist_test.batch(12)))\n", + "\n", + "preds = model(batch_test[0])\n", + "\n", + "tf.print(preds.shape)\n", + "preds = tf.argmax(preds, axis=1)\n", + "print(preds)\n", + "\n", + "fig = plt.figure(figsize=(12, 4))\n", + "for i in range(12):\n", + " ax = fig.add_subplot(2, 6, i+1)\n", + " ax.set_xticks([]); ax.set_yticks([])\n", + " img = batch_test[0][i, :, :, 0]\n", + " ax.imshow(img, cmap='gray_r')\n", + " ax.text(0.9, 0.1, '{}'.format(preds[i]), \n", + " size=15, color='blue',\n", + " horizontalalignment='center',\n", + " verticalalignment='center', \n", + " transform=ax.transAxes)\n", + " \n", + "#plt.savefig('figures/15_13.png', dpi=300)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "if not os.path.exists('models'):\n", + " os.mkdir('models')\n", + "\n", + "\n", + "model.save('models/mnist-cnn.h5')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "----\n", + "\n", + "Readers may ignore the next cell." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[NbConvertApp] Converting notebook ch15_part1.ipynb to script\n", + "[NbConvertApp] Writing 10824 bytes to ch15_part1.py\n" + ] + } + ], + "source": [ + "! python ../.convert_notebook_to_script.py --input ch15_part1.ipynb --output ch15_part1.py" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/ch15/ch15_part1.py b/ch15/ch15_part1.py new file mode 100644 index 00000000..06278c66 --- /dev/null +++ b/ch15/ch15_part1.py @@ -0,0 +1,451 @@ +# coding: utf-8 + + +import tensorflow as tf +import numpy as np +import scipy.signal +import imageio +from tensorflow import keras +import tensorflow_datasets as tfds +import pandas as pd +import matplotlib.pyplot as plt +import os + +# *Python Machine Learning 3rd Edition* by [Sebastian Raschka](https://sebastianraschka.com) & [Vahid Mirjalili](http://vahidmirjalili.com), Packt Publishing Ltd. 2019 +# +# Code Repository: https://github.com/rasbt/python-machine-learning-book-3rd-edition +# +# Code License: [MIT License](https://github.com/rasbt/python-machine-learning-book-3rd-edition/blob/master/LICENSE.txt) + +# # Chapter 15: Classifying Images with Deep Convolutional Neural Networks (Part 1/2) + +# Note that the optional watermark extension is a small IPython notebook plugin that I developed to make the code reproducible. You can just skip the following line(s). + + + + + +# ## The building blocks of convolutional neural networks +# +# ### Understanding CNNs and feature hierarchies +# +# ### Performing discrete convolutions +# +# ### Discrete convolutions in one dimension +# +# ### Padding inputs to control the size of the output feature maps +# +# ### Determining the size of the convolution output + + + + +print('TensorFlow version:', tf.__version__) +print('NumPy version: ', np.__version__) + + + + +def conv1d(x, w, p=0, s=1): + w_rot = np.array(w[::-1]) + x_padded = np.array(x) + if p > 0: + zero_pad = np.zeros(shape=p) + x_padded = np.concatenate( + [zero_pad, x_padded, zero_pad]) + res = [] + for i in range(0, int(len(x)/s),s): + res.append(np.sum( + x_padded[i:i+w_rot.shape[0]] * w_rot)) + return np.array(res) + + +## Testing: +x = [1, 3, 2, 4, 5, 6, 1, 3] +w = [1, 0, 3, 1, 2] + +print('Conv1d Implementation:', + conv1d(x, w, p=2, s=1)) + +print('Numpy Results:', + np.convolve(x, w, mode='same')) + + +# ### Performing a discrete convolution in 2D + + + + + +def conv2d(X, W, p=(0, 0), s=(1, 1)): + W_rot = np.array(W)[::-1,::-1] + X_orig = np.array(X) + n1 = X_orig.shape[0] + 2*p[0] + n2 = X_orig.shape[1] + 2*p[1] + X_padded = np.zeros(shape=(n1, n2)) + X_padded[p[0]:p[0]+X_orig.shape[0], + p[1]:p[1]+X_orig.shape[1]] = X_orig + + res = [] + for i in range(0, int((X_padded.shape[0] - + W_rot.shape[0])/s[0])+1, s[0]): + res.append([]) + for j in range(0, int((X_padded.shape[1] - + W_rot.shape[1])/s[1])+1, s[1]): + X_sub = X_padded[i:i+W_rot.shape[0], + j:j+W_rot.shape[1]] + res[-1].append(np.sum(X_sub * W_rot)) + return(np.array(res)) + +X = [[1, 3, 2, 4], [5, 6, 1, 3], [1, 2, 0, 2], [3, 4, 3, 2]] +W = [[1, 0, 3], [1, 2, 1], [0, 1, 1]] + +print('Conv2d Implementation:\n', + conv2d(X, W, p=(1, 1), s=(1, 1))) + + +print('SciPy Results:\n', + scipy.signal.convolve2d(X, W, mode='same')) + + +# ## Subsampling layers + +# ## Putting everything together – implementing a CNN +# +# ### Working with multiple input or color channels +# +# + +# **TIP: Reading an image file** + + + + + +img_raw = tf.io.read_file('example-image.png') +img = tf.image.decode_image(img_raw) +print('Image shape:', img.shape) +print('Number of channels:', img.shape[2]) +print('Image data type:', img.dtype) +print(img[100:102, 100:102, :]) + + + + + + +img = imageio.imread('example-image.png') +print('Image shape:', img.shape) +print('Number of channels:', img.shape[2]) +print('Image data type:', img.dtype) +print(img[100:102, 100:102, :]) + + +# **INFO-BOX: The rank of a grayscale image for input to a CNN** + + + +img_raw = tf.io.read_file('example-image-gray.png') +img = tf.image.decode_image(img_raw) +tf.print('Rank:', tf.rank(img)) +tf.print('Shape:', img.shape) + + + + +img = imageio.imread('example-image-gray.png') +tf.print('Rank:', tf.rank(img)) +tf.print('Shape:', img.shape) + +img_reshaped = tf.reshape(img, (img.shape[0], img.shape[1], 1)) +tf.print('New Shape:', img_reshaped.shape) + + +# ## Regularizing a neural network with dropout +# +# + + + + + +conv_layer = keras.layers.Conv2D( + filters=16, kernel_size=(3, 3), + kernel_regularizer=keras.regularizers.l2(0.001)) + +fc_layer = keras.layers.Dense( + units=16, kernel_regularizer=keras.regularizers.l2(0.001)) + + +# ## Loss Functions for Classification +# +# * **`BinaryCrossentropy()`** +# * `from_logits=False` +# * `from_logits=True` +# +# * **`CategoricalCrossentropy()`** +# * `from_logits=False` +# * `from_logits=True` +# +# * **`SparseCategoricalCrossentropy()`** +# * `from_logits=False` +# * `from_logits=True` +# + + + +####### Binary Crossentropy +bce_probas = tf.keras.losses.BinaryCrossentropy(from_logits=False) +bce_logits = tf.keras.losses.BinaryCrossentropy(from_logits=True) + +logits = tf.constant([0.8]) +probas = tf.keras.activations.sigmoid(logits) + +tf.print( + 'BCE (w Probas): {:.4f}'.format( + bce_probas(y_true=[1], y_pred=probas)), + '(w Logits): {:.4f}'.format( + bce_logits(y_true=[1], y_pred=logits))) + + +####### Categorical Crossentropy +cce_probas = tf.keras.losses.CategoricalCrossentropy( + from_logits=False) +cce_logits = tf.keras.losses.CategoricalCrossentropy( + from_logits=True) + +logits = tf.constant([[1.5, 0.8, 2.1]]) +probas = tf.keras.activations.softmax(logits) + +tf.print( + 'CCE (w Probas): {:.4f}'.format( + cce_probas(y_true=[0, 0, 1], y_pred=probas)), + '(w Logits): {:.4f}'.format( + cce_logits(y_true=[0, 0, 1], y_pred=logits))) + +####### Sparse Categorical Crossentropy +sp_cce_probas = tf.keras.losses.SparseCategoricalCrossentropy( + from_logits=False) +sp_cce_logits = tf.keras.losses.SparseCategoricalCrossentropy( + from_logits=True) + +tf.print( + 'Sparse CCE (w Probas): {:.4f}'.format( + sp_cce_probas(y_true=[2], y_pred=probas)), + '(w Logits): {:.4f}'.format( + sp_cce_logits(y_true=[2], y_pred=logits))) + + +# ## Implementing a deep convolutional neural network using TensorFlow +# +# ### The multilayer CNN architecture + +# ### Loading and preprocessing the data + + + + + + + + +## MNIST dataset + +mnist_bldr = tfds.builder('mnist') +mnist_bldr.download_and_prepare() +datasets = mnist_bldr.as_dataset(shuffle_files=False) +print(datasets.keys()) +mnist_train_orig, mnist_test_orig = datasets['train'], datasets['test'] + + + + +BUFFER_SIZE = 10000 +BATCH_SIZE = 64 +NUM_EPOCHS = 20 + + + + +mnist_train = mnist_train_orig.map( + lambda item: (tf.cast(item['image'], tf.float32)/255.0, + tf.cast(item['label'], tf.int32))) + +mnist_test = mnist_test_orig.map( + lambda item: (tf.cast(item['image'], tf.float32)/255.0, + tf.cast(item['label'], tf.int32))) + +tf.random.set_seed(1) + +mnist_train = mnist_train.shuffle(buffer_size=BUFFER_SIZE, + reshuffle_each_iteration=False) + +mnist_valid = mnist_train.take(10000).batch(BATCH_SIZE) +mnist_train = mnist_train.skip(10000).batch(BATCH_SIZE) + + +# ### Implementing a CNN using the TensorFlow Keras API +# +# #### Configuring CNN layers in Keras +# +# * **Conv2D:** `tf.keras.layers.Conv2D` +# * `filters` +# * `kernel_size` +# * `strides` +# * `padding` +# +# +# * **MaxPool2D:** `tf.keras.layers.MaxPool2D` +# * `pool_size` +# * `strides` +# * `padding` +# +# +# * **Dropout** `tf.keras.layers.Dropout2D` +# * `rate` + +# ### Constructing a CNN in Keras + + + +model = tf.keras.Sequential() + +model.add(tf.keras.layers.Conv2D( + filters=32, kernel_size=(5, 5), + strides=(1, 1), padding='same', + data_format='channels_last', + name='conv_1', activation='relu')) + +model.add(tf.keras.layers.MaxPool2D( + pool_size=(2, 2), name='pool_1')) + +model.add(tf.keras.layers.Conv2D( + filters=64, kernel_size=(5, 5), + strides=(1, 1), padding='same', + name='conv_2', activation='relu')) + +model.add(tf.keras.layers.MaxPool2D(pool_size=(2, 2), name='pool_2')) + + + + +model.compute_output_shape(input_shape=(16, 28, 28, 1)) + + + + +model.add(tf.keras.layers.Flatten()) + +model.compute_output_shape(input_shape=(16, 28, 28, 1)) + + + + +model.add(tf.keras.layers.Dense( + units=1024, name='fc_1', + activation='relu')) + +model.add(tf.keras.layers.Dropout( + rate=0.5)) + +model.add(tf.keras.layers.Dense( + units=10, name='fc_2', + activation='softmax')) + + + + +tf.random.set_seed(1) +model.build(input_shape=(None, 28, 28, 1)) + +model.compute_output_shape(input_shape=(16, 28, 28, 1)) + + + + +model.summary() + + + + +model.compile(optimizer=tf.keras.optimizers.Adam(), + loss=tf.keras.losses.SparseCategoricalCrossentropy(), + metrics=['accuracy']) # same as `tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy')` + +history = model.fit(mnist_train, epochs=NUM_EPOCHS, + validation_data=mnist_valid, + shuffle=True) + + + + +hist = history.history +x_arr = np.arange(len(hist['loss'])) + 1 + +fig = plt.figure(figsize=(12, 4)) +ax = fig.add_subplot(1, 2, 1) +ax.plot(x_arr, hist['loss'], '-o', label='Train loss') +ax.plot(x_arr, hist['val_loss'], '--<', label='Validation loss') +ax.set_xlabel('Epoch', size=15) +ax.set_ylabel('Loss', size=15) +ax.legend(fontsize=15) +ax = fig.add_subplot(1, 2, 2) +ax.plot(x_arr, hist['accuracy'], '-o', label='Train acc.') +ax.plot(x_arr, hist['val_accuracy'], '--<', label='Validation acc.') +ax.legend(fontsize=15) +ax.set_xlabel('Epoch', size=15) +ax.set_ylabel('Accuracy', size=15) + +#plt.savefig('figures/15_12.png', dpi=300) +plt.show() + + + + +test_results = model.evaluate(mnist_test.batch(20)) +print('\nTest Acc. {:.2f}%'.format(test_results[1]*100)) + + + + +batch_test = next(iter(mnist_test.batch(12))) + +preds = model(batch_test[0]) + +tf.print(preds.shape) +preds = tf.argmax(preds, axis=1) +print(preds) + +fig = plt.figure(figsize=(12, 4)) +for i in range(12): + ax = fig.add_subplot(2, 6, i+1) + ax.set_xticks([]); ax.set_yticks([]) + img = batch_test[0][i, :, :, 0] + ax.imshow(img, cmap='gray_r') + ax.text(0.9, 0.1, '{}'.format(preds[i]), + size=15, color='blue', + horizontalalignment='center', + verticalalignment='center', + transform=ax.transAxes) + +#plt.savefig('figures/15_13.png', dpi=300) +plt.show() + + + + + +if not os.path.exists('models'): + os.mkdir('models') + + +model.save('models/mnist-cnn.h5') + + +# ---- +# +# Readers may ignore the next cell. + + + + diff --git a/ch15/ch15-notebook.ipynb b/ch15/ch15_part2.ipynb similarity index 95% rename from ch15/ch15-notebook.ipynb rename to ch15/ch15_part2.ipynb index 95a00c65..16b610d2 100644 --- a/ch15/ch15-notebook.ipynb +++ b/ch15/ch15_part2.ipynb @@ -15,7 +15,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Chapter 15: Classifying Images with Deep Convolutional Neural Networks" + "# Chapter 15: Classifying Images with Deep Convolutional Neural Networks (Part 2/2)" ] }, { @@ -35,7 +35,7 @@ "output_type": "stream", "text": [ "Sebastian Raschka & Vahid Mirjalili \n", - "last updated: 2019-10-31 \n", + "last updated: 2019-11-01 \n", "\n", "numpy 1.17.2\n", "scipy 1.2.1\n", @@ -50,841 +50,18 @@ "%watermark -a \"Sebastian Raschka & Vahid Mirjalili\" -u -d -p numpy,scipy,matplotlib,tensorflow,tensorflow_datasets" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## The building blocks of convolutional neural networks\n", - "\n", - "### Understanding CNNs and feature hierarchies\n", - "\n", - "### Performing discrete convolutions\n", - "\n", - "### Discrete convolutions in one dimension\n", - "\n", - "### Padding inputs to control the size of the output feature maps\n", - "\n", - "### Determining the size of the convolution output" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "TensorFlow version: 2.0.0\n", - "NumPy version: 1.17.2\n" - ] - } - ], - "source": [ - "import tensorflow as tf\n", - "import numpy as np\n", - "\n", - "print('TensorFlow version:', tf.__version__)\n", - "print('NumPy version: ', np.__version__)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Conv1d Implementation: [ 5. 14. 16. 26. 24. 34. 19. 22.]\n", - "Numpy Results: [ 5 14 16 26 24 34 19 22]\n" - ] - } - ], - "source": [ - "def conv1d(x, w, p=0, s=1):\n", - " w_rot = np.array(w[::-1])\n", - " x_padded = np.array(x)\n", - " if p > 0:\n", - " zero_pad = np.zeros(shape=p)\n", - " x_padded = np.concatenate(\n", - " [zero_pad, x_padded, zero_pad])\n", - " res = []\n", - " for i in range(0, int(len(x)/s),s):\n", - " res.append(np.sum(\n", - " x_padded[i:i+w_rot.shape[0]] * w_rot))\n", - " return np.array(res)\n", - "\n", - "\n", - "## Testing:\n", - "x = [1, 3, 2, 4, 5, 6, 1, 3]\n", - "w = [1, 0, 3, 1, 2]\n", - "\n", - "print('Conv1d Implementation:',\n", - " conv1d(x, w, p=2, s=1))\n", - "\n", - "print('Numpy Results:',\n", - " np.convolve(x, w, mode='same')) " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Performing a discrete convolution in 2D" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Conv2d Implementation:\n", - " [[11. 25. 32. 13.]\n", - " [19. 25. 24. 13.]\n", - " [13. 28. 25. 17.]\n", - " [11. 17. 14. 9.]]\n", - "SciPy Results:\n", - " [[11 25 32 13]\n", - " [19 25 24 13]\n", - " [13 28 25 17]\n", - " [11 17 14 9]]\n" - ] - } - ], - "source": [ - "import scipy.signal\n", - "\n", - "\n", - "def conv2d(X, W, p=(0, 0), s=(1, 1)):\n", - " W_rot = np.array(W)[::-1,::-1]\n", - " X_orig = np.array(X)\n", - " n1 = X_orig.shape[0] + 2*p[0]\n", - " n2 = X_orig.shape[1] + 2*p[1]\n", - " X_padded = np.zeros(shape=(n1, n2))\n", - " X_padded[p[0]:p[0]+X_orig.shape[0],\n", - " p[1]:p[1]+X_orig.shape[1]] = X_orig\n", - "\n", - " res = []\n", - " for i in range(0, int((X_padded.shape[0] - \n", - " W_rot.shape[0])/s[0])+1, s[0]):\n", - " res.append([])\n", - " for j in range(0, int((X_padded.shape[1] - \n", - " W_rot.shape[1])/s[1])+1, s[1]):\n", - " X_sub = X_padded[i:i+W_rot.shape[0],\n", - " j:j+W_rot.shape[1]]\n", - " res[-1].append(np.sum(X_sub * W_rot))\n", - " return(np.array(res))\n", - "\n", - "X = [[1, 3, 2, 4], [5, 6, 1, 3], [1, 2, 0, 2], [3, 4, 3, 2]]\n", - "W = [[1, 0, 3], [1, 2, 1], [0, 1, 1]]\n", - "\n", - "print('Conv2d Implementation:\\n',\n", - " conv2d(X, W, p=(1, 1), s=(1, 1)))\n", - "\n", - "\n", - "print('SciPy Results:\\n',\n", - " scipy.signal.convolve2d(X, W, mode='same'))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Subsampling layers" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Putting everything together – implementing a CNN\n", - "\n", - "### Working with multiple input or color channels\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**TIP: Reading an image file**" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Image shape: (252, 221, 3)\n", - "Number of channels: 3\n", - "Image data type: \n", - "tf.Tensor(\n", - "[[[179 134 110]\n", - " [182 136 112]]\n", - "\n", - " [[180 135 111]\n", - " [182 137 113]]], shape=(2, 2, 3), dtype=uint8)\n" - ] - } - ], - "source": [ - "import tensorflow as tf\n", - "\n", - "\n", - "img_raw = tf.io.read_file('example-image.png')\n", - "img = tf.image.decode_image(img_raw)\n", - "print('Image shape:', img.shape)\n", - "print('Number of channels:', img.shape[2])\n", - "print('Image data type:', img.dtype)\n", - "print(img[100:102, 100:102, :])" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Image shape: (252, 221, 3)\n", - "Number of channels: 3\n", - "Image data type: uint8\n", - "[[[179 134 110]\n", - " [182 136 112]]\n", - "\n", - " [[180 135 111]\n", - " [182 137 113]]]\n" - ] - } - ], - "source": [ - "import imageio\n", - "\n", - "\n", - "img = imageio.imread('example-image.png')\n", - "print('Image shape:', img.shape)\n", - "print('Number of channels:', img.shape[2])\n", - "print('Image data type:', img.dtype)\n", - "print(img[100:102, 100:102, :])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**INFO-BOX: The rank of a grayscale image for input to a CNN**" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Rank: 3\n", - "Shape: TensorShape([252, 221, 1])\n" - ] - } - ], - "source": [ - "img_raw = tf.io.read_file('example-image-gray.png')\n", - "img = tf.image.decode_image(img_raw)\n", - "tf.print('Rank:', tf.rank(img))\n", - "tf.print('Shape:', img.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Rank: 2\n", - "Shape: (252, 221)\n", - "New Shape: TensorShape([252, 221, 1])\n" - ] - } - ], - "source": [ - "img = imageio.imread('example-image-gray.png')\n", - "tf.print('Rank:', tf.rank(img))\n", - "tf.print('Shape:', img.shape)\n", - "\n", - "img_reshaped = tf.reshape(img, (img.shape[0], img.shape[1], 1))\n", - "tf.print('New Shape:', img_reshaped.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Regularizing a neural network with dropout\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow import keras\n", - "\n", - "\n", - "conv_layer = keras.layers.Conv2D(\n", - " filters=16, kernel_size=(3, 3),\n", - " kernel_regularizer=keras.regularizers.l2(0.001))\n", - "\n", - "fc_layer = keras.layers.Dense(\n", - " units=16, kernel_regularizer=keras.regularizers.l2(0.001))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loss Functions for Classification\n", - "\n", - " * **`BinaryCrossentropy()`**\n", - " * `from_logits=False` \n", - " * `from_logits=True`\n", - "\n", - " * **`CategoricalCrossentropy()`**\n", - " * `from_logits=False`\n", - " * `from_logits=True`\n", - " \n", - " * **`SparseCategoricalCrossentropy()`**\n", - " * `from_logits=False`\n", - " * `from_logits=True`\n" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "BCE (w Probas): 0.3711 (w Logits): 0.3711\n", - "CCE (w Probas): 0.5996 (w Logits): 0.5996\n", - "Sparse CCE (w Probas): 0.5996 (w Logits): 0.5996\n" - ] - } - ], - "source": [ - "####### Binary Crossentropy\n", - "bce_probas = tf.keras.losses.BinaryCrossentropy(from_logits=False)\n", - "bce_logits = tf.keras.losses.BinaryCrossentropy(from_logits=True)\n", - "\n", - "logits = tf.constant([0.8])\n", - "probas = tf.keras.activations.sigmoid(logits)\n", - "\n", - "tf.print(\n", - " 'BCE (w Probas): {:.4f}'.format(\n", - " bce_probas(y_true=[1], y_pred=probas)),\n", - " '(w Logits): {:.4f}'.format(\n", - " bce_logits(y_true=[1], y_pred=logits)))\n", - "\n", - "\n", - "####### Categorical Crossentropy\n", - "cce_probas = tf.keras.losses.CategoricalCrossentropy(\n", - " from_logits=False)\n", - "cce_logits = tf.keras.losses.CategoricalCrossentropy(\n", - " from_logits=True)\n", - "\n", - "logits = tf.constant([[1.5, 0.8, 2.1]])\n", - "probas = tf.keras.activations.softmax(logits)\n", - "\n", - "tf.print(\n", - " 'CCE (w Probas): {:.4f}'.format(\n", - " cce_probas(y_true=[0, 0, 1], y_pred=probas)),\n", - " '(w Logits): {:.4f}'.format(\n", - " cce_logits(y_true=[0, 0, 1], y_pred=logits)))\n", - "\n", - "####### Sparse Categorical Crossentropy\n", - "sp_cce_probas = tf.keras.losses.SparseCategoricalCrossentropy(\n", - " from_logits=False)\n", - "sp_cce_logits = tf.keras.losses.SparseCategoricalCrossentropy(\n", - " from_logits=True)\n", - "\n", - "tf.print(\n", - " 'Sparse CCE (w Probas): {:.4f}'.format(\n", - " sp_cce_probas(y_true=[2], y_pred=probas)),\n", - " '(w Logits): {:.4f}'.format(\n", - " sp_cce_logits(y_true=[2], y_pred=logits)))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Implementing a deep convolutional neural network using TensorFlow\n", - "\n", - "### The multilayer CNN architecture" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Loading and preprocessing the data" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow_datasets as tfds\n", - "import pandas as pd\n", - "\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "dict_keys(['test', 'train'])\n" - ] - } - ], - "source": [ - "## MNIST dataset\n", - "\n", - "mnist_bldr = tfds.builder('mnist')\n", - "mnist_bldr.download_and_prepare()\n", - "datasets = mnist_bldr.as_dataset(shuffle_files=False)\n", - "print(datasets.keys())\n", - "mnist_train_orig, mnist_test_orig = datasets['train'], datasets['test']" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "BUFFER_SIZE = 10000\n", - "BATCH_SIZE = 64\n", - "NUM_EPOCHS = 20" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "mnist_train = mnist_train_orig.map(\n", - " lambda item: (tf.cast(item['image'], tf.float32)/255.0, \n", - " tf.cast(item['label'], tf.int32)))\n", - "\n", - "mnist_test = mnist_test_orig.map(\n", - " lambda item: (tf.cast(item['image'], tf.float32)/255.0, \n", - " tf.cast(item['label'], tf.int32)))\n", - "\n", - "tf.random.set_seed(1)\n", - "\n", - "mnist_train = mnist_train.shuffle(buffer_size=BUFFER_SIZE,\n", - " reshuffle_each_iteration=False)\n", - "\n", - "mnist_valid = mnist_train.take(10000).batch(BATCH_SIZE)\n", - "mnist_train = mnist_train.skip(10000).batch(BATCH_SIZE)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Implementing a CNN using the TensorFlow Keras API\n", - "\n", - "#### Configuring CNN layers in Keras\n", - "\n", - " * **Conv2D:** `tf.keras.layers.Conv2D`\n", - " * `filters`\n", - " * `kernel_size`\n", - " * `strides`\n", - " * `padding`\n", - " \n", - " \n", - " * **MaxPool2D:** `tf.keras.layers.MaxPool2D`\n", - " * `pool_size`\n", - " * `strides`\n", - " * `padding`\n", - " \n", - " \n", - " * **Dropout** `tf.keras.layers.Dropout2D`\n", - " * `rate`" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Constructing a CNN in Keras" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "model = tf.keras.Sequential()\n", - "\n", - "model.add(tf.keras.layers.Conv2D(\n", - " filters=32, kernel_size=(5, 5),\n", - " strides=(1, 1), padding='same',\n", - " data_format='channels_last',\n", - " name='conv_1', activation='relu'))\n", - "\n", - "model.add(tf.keras.layers.MaxPool2D(\n", - " pool_size=(2, 2), name='pool_1'))\n", - " \n", - "model.add(tf.keras.layers.Conv2D(\n", - " filters=64, kernel_size=(5, 5),\n", - " strides=(1, 1), padding='same',\n", - " name='conv_2', activation='relu'))\n", - "\n", - "model.add(tf.keras.layers.MaxPool2D(pool_size=(2, 2), name='pool_2'))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "TensorShape([16, 7, 7, 64])" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.compute_output_shape(input_shape=(16, 28, 28, 1))" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "TensorShape([16, 3136])" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - " model.add(tf.keras.layers.Flatten())\n", - " \n", - "model.compute_output_shape(input_shape=(16, 28, 28, 1))" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "model.add(tf.keras.layers.Dense(\n", - " units=1024, name='fc_1', \n", - " activation='relu'))\n", - "\n", - "model.add(tf.keras.layers.Dropout(\n", - " rate=0.5))\n", - " \n", - "model.add(tf.keras.layers.Dense(\n", - " units=10, name='fc_2',\n", - " activation='softmax'))" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "TensorShape([16, 10])" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tf.random.set_seed(1)\n", - "model.build(input_shape=(None, 28, 28, 1))\n", - "\n", - "model.compute_output_shape(input_shape=(16, 28, 28, 1))" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Model: \"sequential\"\n", - "_________________________________________________________________\n", - "Layer (type) Output Shape Param # \n", - "=================================================================\n", - "conv_1 (Conv2D) multiple 832 \n", - "_________________________________________________________________\n", - "pool_1 (MaxPooling2D) multiple 0 \n", - "_________________________________________________________________\n", - "conv_2 (Conv2D) multiple 51264 \n", - "_________________________________________________________________\n", - "pool_2 (MaxPooling2D) multiple 0 \n", - "_________________________________________________________________\n", - "flatten (Flatten) multiple 0 \n", - "_________________________________________________________________\n", - "fc_1 (Dense) multiple 3212288 \n", - "_________________________________________________________________\n", - "dropout (Dropout) multiple 0 \n", - "_________________________________________________________________\n", - "fc_2 (Dense) multiple 10250 \n", - "=================================================================\n", - "Total params: 3,274,634\n", - "Trainable params: 3,274,634\n", - "Non-trainable params: 0\n", - "_________________________________________________________________\n" - ] - } - ], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoch 1/20\n", - "782/782 [==============================] - 29s 38ms/step - loss: 0.1450 - accuracy: 0.9549 - val_loss: 0.0000e+00 - val_accuracy: 0.0000e+00\n", - "Epoch 2/20\n", - "782/782 [==============================] - 27s 34ms/step - loss: 0.0491 - accuracy: 0.9858 - val_loss: 0.0401 - val_accuracy: 0.9874\n", - "Epoch 3/20\n", - "782/782 [==============================] - 27s 35ms/step - loss: 0.0311 - accuracy: 0.9903 - val_loss: 0.0493 - val_accuracy: 0.9846\n", - "Epoch 4/20\n", - "782/782 [==============================] - 28s 35ms/step - loss: 0.0241 - accuracy: 0.9920 - val_loss: 0.0489 - val_accuracy: 0.9855\n", - "Epoch 5/20\n", - "782/782 [==============================] - 27s 35ms/step - loss: 0.0197 - accuracy: 0.9942 - val_loss: 0.0453 - val_accuracy: 0.9874\n", - "Epoch 6/20\n", - "782/782 [==============================] - 27s 34ms/step - loss: 0.0161 - accuracy: 0.9953 - val_loss: 0.0361 - val_accuracy: 0.9897\n", - "Epoch 7/20\n", - "782/782 [==============================] - 27s 35ms/step - loss: 0.0125 - accuracy: 0.9961 - val_loss: 0.0401 - val_accuracy: 0.9903\n", - "Epoch 8/20\n", - "782/782 [==============================] - 27s 35ms/step - loss: 0.0117 - accuracy: 0.9963 - val_loss: 0.0371 - val_accuracy: 0.9918\n", - "Epoch 9/20\n", - "782/782 [==============================] - 28s 35ms/step - loss: 0.0113 - accuracy: 0.9966 - val_loss: 0.0367 - val_accuracy: 0.9904\n", - "Epoch 10/20\n", - "782/782 [==============================] - 28s 36ms/step - loss: 0.0097 - accuracy: 0.9967 - val_loss: 0.0426 - val_accuracy: 0.9912\n", - "Epoch 11/20\n", - "782/782 [==============================] - 27s 35ms/step - loss: 0.0098 - accuracy: 0.9974 - val_loss: 0.0367 - val_accuracy: 0.9925\n", - "Epoch 12/20\n", - "782/782 [==============================] - 28s 35ms/step - loss: 0.0093 - accuracy: 0.9974 - val_loss: 0.0395 - val_accuracy: 0.9912\n", - "Epoch 13/20\n", - "782/782 [==============================] - 27s 34ms/step - loss: 0.0074 - accuracy: 0.9975 - val_loss: 0.0509 - val_accuracy: 0.9912\n", - "Epoch 14/20\n", - "782/782 [==============================] - 27s 35ms/step - loss: 0.0082 - accuracy: 0.9975 - val_loss: 0.0595 - val_accuracy: 0.9885\n", - "Epoch 15/20\n", - "782/782 [==============================] - 27s 35ms/step - loss: 0.0056 - accuracy: 0.9982 - val_loss: 0.0601 - val_accuracy: 0.9895\n", - "Epoch 16/20\n", - "782/782 [==============================] - 27s 35ms/step - loss: 0.0086 - accuracy: 0.9976 - val_loss: 0.0431 - val_accuracy: 0.9917\n", - "Epoch 17/20\n", - "782/782 [==============================] - 28s 35ms/step - loss: 0.0058 - accuracy: 0.9982 - val_loss: 0.0538 - val_accuracy: 0.9912\n", - "Epoch 18/20\n", - "782/782 [==============================] - 28s 35ms/step - loss: 0.0080 - accuracy: 0.9979 - val_loss: 0.0500 - val_accuracy: 0.9896\n", - "Epoch 19/20\n", - "782/782 [==============================] - 27s 35ms/step - loss: 0.0064 - accuracy: 0.9980 - val_loss: 0.0424 - val_accuracy: 0.9923\n", - "Epoch 20/20\n", - "782/782 [==============================] - 27s 35ms/step - loss: 0.0041 - accuracy: 0.9988 - val_loss: 0.0469 - val_accuracy: 0.9915\n" - ] - } - ], - "source": [ - "model.compile(optimizer=tf.keras.optimizers.Adam(),\n", - " loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n", - " metrics=['accuracy']) # same as `tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy')`\n", - "\n", - "history = model.fit(mnist_train, epochs=NUM_EPOCHS, \n", - " validation_data=mnist_valid, \n", - " shuffle=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtwAAAELCAYAAADqelFjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd3hU1dbA4d9KL5CEkgKhCiogUiOKgAqKoFIsIHAtiAULXOWqKCiXpthQQSzwoSLXijQBFUFUsCu9CIiiICQhoQSSAOnZ3x9nJqRMyqTMTMh6n2ceZk7dk4Qza/ZZe20xxqCUUkoppZSqGl7uboBSSimllFJnMw24lVJKKaWUqkIacCullFJKKVWFNOBWSimllFKqCmnArZRSSimlVBXycXcDqlL9+vVNs2bN3N0MpZQql02bNh01xoS7ux2upNdtpVR1VdI1+6wOuJs1a8bGjRvd3QyllCoXEfnH3W1wNb1uK6Wqq5Ku2ZpSopRSqlgiMk9EDovIb8WsFxGZJSJ7RWS7iHTKt264iPxpewx3XauVUsqzaMCtlFKqJPOBviWsvwY41/YYCcwGEJG6wCTgYqALMElE6lRpS5VSykNpwK2UUqpYxpjvgKQSNhkIvGssvwBhItIA6AOsMcYkGWOOA2soOXBXSqmzlgbcSimlKiIaOJjvdaxtWXHLixCRkSKyUUQ2HjlypMoaqpRS7qIBt1JKqYoQB8tMCcuLLjRmrjEmxhgTEx5eo4qyKKVqiLO6Somzlm2JY/rqPcSfSKNhWCBj+5zP9R0ddsgopZSyxAKN871uBMTbll9RaPk6l7VKKaWcVJVxoAbcNsu2xDF+6Q7SsnIAiDuRxvilOwA06D7LpKSkcPjwYbKystzdFFVD+fr6EhERQUhIiLubUhlWAKNFZAHWAMlkY8whEVkNPJNvoOTVwHh3NVIppUpS1XGgBtw201fvyfsh26Vl5TB99R4NuM8iKSkpJCYmEh0dTWBgICKO7norVXWMMaSlpREXFwfg8UG3iHyE1VNdX0RisSqP+AIYY+YAK4Frgb3AaWCEbV2SiDwFbLAdaqoxpqTBl0op5TbTV/9epXGgBtw28SfSnFquqqfDhw8THR1NUFCQu5uiaigRISgoiOjoaOLj4z0+4DbGDCtlvQFGFbNuHjCvKtqllHKOu9JmPTVdNyfX8HtCChv3H2fD/iTiTqQ73K6y4kANuG0ahgUS5+CH2jAs0A2tUVUlKyuLwED9nSr3CwwM1LQmpZRLuCtttiLnrUig7mjfvm2j2HrwBBv3J7F+/3G2/HOc1IxsABqEBhDo612khxsqLw7UgNtmbJ/zC/xRAAT6ejO2z/lubJWqCppGojyB/h0qpVwhJ9fwzMrdDtMlnv58Fz1bRRAa6Ftp50vPymFPQio74pKLPe+EZTs4diqTBqEBtkcg4bX98fayrosVCdQ/2RzL+E92kJ6Vm7fvfxZu5eGFkGurk3R+ZG0GdGjIRc3qclHzukSHBRY5J1RuHKgBt439F/jsF7tJTMkgLNCXyQMu8IjbHkoppZSqGpXdk1qWfavynMdPZbLl4HE2/3OCzQeOs+3gCU5lFu25BTh6MpP2U74kOiyQ1g1CaNOgNm0ahtC6QQiN6wThlS8AdnTOtMwcdiek8FtcMjtik/ktPoU/E1PJznVYATTPyYwcnvpsV4Fl3l5CZG1/okID2H0ohTRbwGxnBeq/sfGfJE5l5HAyI5tTGdmcysyx/rU9UtKzi5zPGAj292HWsA50alKHsCC/ItvYf4ZVlf4iVvqd64hIX+AVwBt4yxjzXKH1lwEzgXbAUGPM4kLrQ4DdwCfGmNElnSsmJsZs3LjRqfZlZOdw/oRVPNz7PB688lyn9lWeb/fu3bRu3drdzVAKKP3vUUQ2GWNiXNgktyvPdVup8iquV/PZGy8sU8pDefat7HP6+3gxoENDcnNhy4Hj/H30FGAFsK2iatOpSR0+2x7P8dNFU9jqBftxV4/m7D6Uyq74ZPYdPZXXC1zL34dWUbUJ8PXi131JZOWciRe9RQiv7ceRk5nk2HaoG+xH2+hQ2jYM4cLoUNpGhzJ07s8Oc6OjwwL4/MEexJ9IJyElzfo3OZ345DQSktP56a9jxf4M6gb7EezvTbCfD8H+1qOWvzdBfj7U8vdh/k/7He4nwL7nrivx51tRJV2zXdrDLSLewOtAb6warRtEZIUxJv/XnAPAHcCjxRzmKeDbqmqjv483dYP9SEhxnDyvlDuVJQ1h7dq1XHHFFRU6T1RUFHfffTdPP/10hY6zatUqrrnmGv78809atmxZoWMppVRlK65C2VOf7aJ2QMkh0lOf7XK47+QVO0lJzyIzO5fMnFyysg1ZOdbzzOxcFm486HC/8Ut38N0fRxARRMBLQBC8vKxrv2AF3IX3zcjOZdHGWOoG+9GpSRg3dW5EpyZ1aN84lCA/6z10blrHYZD/335tCgT5aZk57ElMZfehFHYfSmFXfAo/7j1WZMaqHGM4fjqLB65oQdvoUC6MDqVBaECRz6ixfVoVk6bRirAgP8KC/GjTsOjA8W7PfeNwXF10WCA/jutV9JeRz5pdiR45Js/VKSVdgL3GmL8BbHVbBwJ5AbcxZr9tXW7hnUWkMxAJrAKqrNcnMiSAxGQNuJXn+fnnn/Oep6Wl0atXLyZMmMB115351t6mTZsKn2flypVERERU+DhKKeVpTmdms/XgCTbtP+4wMAM4diqTu/5XvjstJ9KymLh8Z4Fl3l6Cr7fg6+3F6WLSO9KyctjwTxK5uVb5UAPkGoMx9txjU2xqiACbJlxVbKdMWdMlAv286dA4jA6Nw/KWNR/3ucNjZmbn8sjVJec3lzdNoyLj6jx1TJ6rA+5o4GC+17FYEyWUSkS8gJeA24ArS9huJDASoEmTJuVqZFSIv/ZwK490ySWX5D0/efIkAC1atCiwvDjp6ekEBASU6TydOnUqXwOVUqoCqiK3+VByGhv3H2fTP9Zj16GUvDQIHy9xmG8cXtuft4eX3K931/82ciQ1o8jyqBB/Pn+wB74+Xvh5e+Hr7ZU3GBBK7r39/rGSe2+L27dhWOnzSlzfMbpc+cgVreJWnvNWJJ+6qnOxy8vLxedz9NdQ1iTyB4CVxpiDJW1kjJlrjIkxxsSEh4c73UCAqNAAEjXgVqVYtiWObs99Q/Nxn9PtuW9YtiXO3U3KM2fOHESEzZs306NHDwIDA3n11VcxxvDII4/Qtm1bgoODady4McOHD+fIkSMF9o+KimLChAl5r4cOHUr37t1ZuXIlF1xwAbVq1eLyyy9nz549Trft5MmTPPDAA0RERBAYGMjFF1/M2rVrC2yzbt06Lr30UmrXrk1oaCidOnVi+fLleeuXLFlCx44dCQoKom7dunTt2pWffvrJ6bYopSpfea+N9vzkuBNpGM5UpijL/o72fWThNjpMWU3XZ7/h3x9t4eMNB6nl78P9l7fgnREXsW3i1bw4uD2Bvt4FjhXo682T17amXaOwEh9PXtva4b7jrmlNvVr+hAT4EuDrXSDYBqsH1tF+Ze29Le++5eWOc4IVOP84rhf7nruOH8f1cipgrsi+VcXVPdyxQON8rxsB8WXctyvQQ0QeAGoBfiJy0hgzrpLbSGRIAEdPZpKZnYufj6u/k6jqwF01TZ01ZMgQRo0axdSpU6lbty65ubkkJSUxYcIEGjRoQGJiItOnT+fqq69m8+bNJfaQ7N27lwkTJjB58mR8fX15+OGHGTZsGJs3b3aqTcOHD+err77i2WefpVmzZsyePZs+ffrwww8/0KVLF44dO0b//v0ZMmQIU6dOJScnh+3bt3P8+HEAdu3axdChQxk7diwvv/wyp0+fZuPGjXnrlVKVozy9zWW9Nmbn5HIiLYukU5kcO5nJ8dOZTFqx02Fu87gl21m6JY4se060LRfanhedlW04nJpO4Y7qHGPIyM5lUv82xDStS6sGtfH1LviZ7o6e1OrWe+upPcbVjasD7g3AuSLSHIgDhgL/KsuOxphb7M9F5A4gpiqCbYCoEOu2++HUdBrV0RkJz3ZTPt3JrvgUp/bZcuAEmTlFSxY9tng7H60/UObjtGkYwqT+Fzh1bmc8+uij3HvvvQWWvfPOO3nPc3Jy6Ny5My1btmTDhg106dKl2GMlJSXx66+/0rRpU8BKURk2bBj79++nWbNmZWrP1q1bWbp0KQsWLGDIkCEA9OnTh1atWjFt2jSWL1/O7t27OXXqFK+//jr+/v5529ht3ryZiIgInnnmmbxl+XPYlVIV50ynQm6uITkti+OnM3n6c8d1l8ct2c77v/xD0qlMkk5nkpyWRVmLpKVn55KcloW/txcBvl7UDvCxUjVsKRt+3l58vNHxze/0rFxGdGte4vHLm2pRkX3dcc6KcMc5zzYuDbiNMdkiMhpYjVUWcJ4xZqeITAU2GmNWiMhFwCdAHaC/iEwxxlRdROJAZKgVcCemaMCtHCscbJe23F0cBaIrVqzgmWeeYffu3aSknPmi8ccff5QYcJ933nl5wTacGZwZGxtb5oB7/fr1eHt7c+ONN+Yt8/b2ZtCgQcydOzfvPAEBAQwdOpQ777yTyy67jNDQ0Lzt27Vrx6FDh7j77rsZOnQol156KUFB+v9UqcpUXPWO8Ut3sGJbPMdPZ3LitBVklyV4Ts/Oxdfbi9YNQ6gb5Efd4KKPEe9scDh+KjoskOWjupV4/B/2HvXIyhRK2bl84htjzEpgZaFlE/M934CValLSMeYD86ugecCZHu6E5KKDIdTZpzw9zCUNevn43q6V0axKERkZWeD1jz/+yA033MDQoUN58sknCQ8PJysri8suu4z09JLHLYSFhRV47ednTRxQ2n75HTp0iDp16uDrW3BWs8jIyLyUkIiICFavXs3UqVO56aabAOjbty+vvvoqTZs2pV27dixdupTp06fTp08f/P39GTRoEDNnzqRu3bplbotSqqBjJzPYsP846/clFVu9Iy0rh4TkdOoE+9IwLJA6Qb7UCfKzHsG+PP3Zbo6dyiyyX3RYIB+NLHlw97hriishV30rUyhlpzNNOpAXcOvASVWM6nJxL5yTvWTJEpo0acIHH3yQt6w8Ax/Lq0GDBhw/fpysrKwCQXdiYiJ16tTJe92jRw/WrFnDqVOnWLNmDf/5z38YPnw469atA+D666/n+uuv58SJE3z66aeMGTMGLy8v5s+f77L3olR1UFIedvyJNNbvS2L9/iTW70ti72Gr8pG/jxd+Pl5kZhe9YxcdFsjKh3oUez5Byn1trG65zUo5QwNuB8KCfPHz8dJKJapY1fXinpaWltczbZc/+K5qXbp0IScnh08++YSbb74ZsPLIlyxZQvfu3YtsHxwczPXXX8+WLVuYPXt2kfVhYWHcdtttfPXVV+zatavIeqVqMkd52GMXb+P9X/aTkJJB7HGrF7u2vw8xzepwY6doLm5el7bRoXyxI6FcgXNFr43VLbdZqbLSgNsBESEqJIAEnfxGlaA6Xtx79+7NnDlzGDt2LH379uW7775jwYIFLjt/hw4duPHGGxk5ciRJSUk0bdqU2bNns3///rzA3z6ocuDAgTRq1IiDBw8yb948evWy6tPOmjWL7du307t3bxo0aMDvv//OsmXLuP/++132PpSqDl5Y/XuRPOysHMPmAyfoc0EUd3VvzkXN6tK6QUiR0nUV7W2ubtdGpaqaBtzFiAoJ0JQSdda58cYbeeqpp3jjjTd444036NGjB8uWLeOCC1w3Lvl///sfY8eO5b///S+pqam0b9+eVatWcdFFFwHWoMns7Gwef/xxjhw5QkREBAMGDMirStKhQwe++OILxowZw/Hjx2nYsCGjR49m8uTJLnsPSnkqYwy7DqWwfGs88Sccf4YZA7Nv7VzqsTRwVqryiClrXZ5qKCYmxmzcWL6pWf/90Ra2x57g27E9K7lVyp12795N69at3d0MpYDS/x5FZJMxpuTp7s4yFblu12T/HDvFiq3xLN8Wz97DJ/HxEny8hPRi8rB/HFfyjIZKKeeVdM3WHu5iRIX482VyOsaYUqdLVUoppaqSo8GP3VrW5/Pt8SzbGs/WgycA6NK8LtNuaMu1bRvw7R9HqsXgbqVqAg24ixEZEkCGrdh+WJBf6TsopZRSVcDR4MeHF27Nm1mxdYMQxl3Tiv7tGxKdr+50dR3crdTZSAPuYkSFnikNqAG3Ukopd3E0CU2ugVr+Pix94FLOi6xd7L6ah62UZ/BydwM81ZnJb3TgpFJKKfcwxhQ7Cc2pjOwSg22llOfQgLsYkSFnpndXSimlXO1wajr3vrep2PU6bblS1YcG3MWI1OndlVJKuYExhuVb47h6xnes++MIA9o1INC34Me1Dn5UqnrRHO5i+Pl4US/YT2txK6WUcpkjqRlMWLaD1TsT6dA4jBcHt6dlRK0Sp2hXSnk+DbhLEBkSoCklSimlqpwxhhXb4pm0YienM3MYf00r7u5xTt4MkDr4UanqTQPuEkSF6vTuSimlqtaR1Az+u+w3Vu1MsPVqt6NlhA6GdIvUBPj2BYhdD/f94O7WqMrgIb9TzeEugfZwK0/Tr18/LrzwwmLXjx49mjp16pCRUbaxB3v37kVEWLVqVd6yRo0aMW7cuBL327p1KyLCDz84d/GaM2cOK1asKLK8LOesLNnZ2YgIc+bMccn5lCqOvVf76hnf8s2ew4y7phVL7r9Ug213SE2Azx6GV9rDlvcgYYe7W6QqysN+p9rDXYKokACOncokIzsHfx9vdzdHKYYNG8att97Kzp07ueCCCwqsy8nJYfHixdx44434+/uX+xyffvop9evXr2hTHZozZw4xMTEMGDDAZedUylPkz8OODAkgorYf2+NSaN84jJe0V9s9/vkZVj4Kh3eDlzfkZLq7Raqi7D3aW94HciEny90tArSHu0RRoVbQcjhFK5UozzBw4ECCgoJYsGBBkXVr164lMTGRYcOGVegcHTt2pHHjxhU6RnU4p1KuZJ8tMu5EGgZrUrXtcSn0b9eAJfd1PXuDbXsv45zu7m7JGdkZsGMxzO8H7/SFxN/A5BQNtn+YASePuKeNnswTf6f5LRoBG9+GnAyPCbZBA+4SaS1uVWYuugDVqlWLfv368fHHHxdZt2DBAiIjI+nZsycAcXFxjBgxgubNmxMYGMh5553HpEmTyMoq+QLkKL3j1VdfpXHjxgQHBzNw4EASEhKK7Dd9+nRiYmIICQkhMjKSgQMH8tdff+Wt7969O9u2bePtt99GRBAR3n///WLPuWDBAtq2bYu/vz9NmjRh4sSJ5OScmW3vrbfeQkTYuXMnV111FcHBwbRu3Zrly5eX8lN0bNasWbRs2RJ/f3/OPfdcZs2aVWD9gQMHGDRoEOHh4QQGBtKyZUsmT56ct37Hjh306dOHOnXqUKtWLdq0aaNpKyqPo9kiATYfOIGP91n4Uexht/ML+GEmLLkLThyA7o9A+1vAJwC8C80q/dVkeLk1LLoDjv3l6Eg1iyf/Tu2MgcA61nPxth757VoBr18Mnz8Cvy2Fk4eLP1Ylf65rSkkJ8k/vrpRD9ltXWz8Ak+uS25HDhg1j4cKFbNq0ic6dOwOQlZXFJ598wi233IK3t3WBOXLkCPXr12fmzJmEhYXx+++/M2XKFI4ePcrrr79e5vMtWbKEBx98kFGjRtG/f3/Wrl3LPffcU2S72NhYHnzwQZo0aUJycjKzZ8+me/fu/PHHH9SuXZu5c+dy/fXX07p1a8aPHw9Ay5YtHZ5z5cqVDBs2jBEjRvDiiy+ydetWJk6cSFJSEq+99lqRn8fIkSN57LHHmDlzJkOGDGHfvn00aNCgzO9x9uzZjBkzhkceeYTevXvz9ddfM2bMGDIzM3n00UcBuPXWW8nJyeGtt94iJCSEv//+mz///BOwcnH79etH+/bt+fDDD/Hz8+P3338nJSWlzG1QZ689CanFzhYZX8zyais1AdZMgl3LXHZNLLYd374AB3+FHo/Apnfgkgfg/Gug023QKAbO6Qleti87V02Cb58veC0ftQE2zYftC8Db19ru2F8QEArBDlLgyjs4z0MG9RXL3r7N71o/G1P0i6PTx6qq9yoCrftB8x5wwQ1FP5/9a0FoI9i2ADa8Ze1T/zy4czUE1bV6xE8fq5LPdZcH3CLSF3gF8AbeMsY8V2j9ZcBMoB0w1Biz2La8AzAbCAFygGnGmKLdfJVIp3evQd65ruiyC66HLvdA5mn4YHDBddmZ4OsPsRtstyLz9Rrbj3XRndD2JkiOhaX3Fj3+paOti7+TrrnmGsLCwliwYEFewL169WqSkpIKpJN06NCBDh065L3u1q0bgYGB3Hfffbzyyiv4+JTtv/+0adPo169fXqDbp08fEhMTmT9/foHtXnnllbznOTk59O7dm/DwcD799FP+9a9/0aZNG4KCgggPD+eSSy4p8ZwTJ07kqquuYt68eQD07duX3NxcJk6cyJNPPlkgmH700Ue5/fbb895zVFQUn3/+OXfffXeZ3l92djZTpkzhrrvuYvr06QBcffXVHD9+nGnTpvHggw/i5+fH+vXr+eSTT7jmGut3Zr+TAJCYmMiBAwdYtWoVrVu3BuDKK68s0/nV2Wvf0VPM/OoPVmyLRwDjYJuzZrZIY+CfH2HBLZB+wvE2u5bD+deBdxWGHvnzd3OzrIBp8QgIbQLZts/ykIbWI7/akdDvZbj8cSvwjl0P4edB32eg95QzAfeqcfD3OmjdHzqPgGbd4WRi+QI0N3TYlMviEXDgF6uNhf3fZXDvd9bzfd9ZPcr1z4XgcCv4tavq9xq7EVLioc0A6PCvM8sL/05b9LIeOdlwaBv88wMk/Gb1iqcmWJ/fSX9Z+fy52ZXaRJcG3CLiDbwO9AZigQ0issIYsyvfZgeAO4BHC+1+GrjdGPOniDQENonIamNMMf+zKy400Bd/Hy9NKVFFHf0dMlJx/BFatfz9/bnhhhtYuHAhL7zwAiLCxx9/TNOmTQsEsrm5ucyYMYO33nqL/fv3k55+5u84NjaWZs2alXquzMxMtm3bxgMPPFBg+Y033lgk4P7pp5+YOHEiW7ZsISkpKW/5H3/84dT7y8rKYuvWrbzxxhsFlg8ZMoQnn3ySX375hRtuuCFv+dVXX533PCIigvr16xMbG1vm8x04cIDExEQGDy74pWrIkCG8+eab7Ny5k44dO9KhQwcef/xxDh8+TK9evQrknIeHhxMdHc29997L6NGjueKKK4iIiHDqfauzR9yJNGZ99SeLN8fi5+3FfZe3oHGdQJ76bHeBtJKzYrbI3Bz4/XP4cSbEbYLAutCwMxzeWTSwWng7hDaGa56HVg46OSoiNQHWPgPbPy563luWQIueVhBVGnvgnZ892AboPdXq9d72Efy2BPxDICvNCi7zn3PPF2ee1z8P6rWAzFNWUJp23Epn2LfO+gjJ9Zw84wKMgR2L4OIHILy142C5Rb6OhTWTIH6z9TwgFOqdC40usravyi8V2xfB8lFQ9xw4/9qiX+gc/k59oFFn62G3eIQVbGMqPdgG1/dwdwH2GmP+BhCRBcBAIC/gNsbst60r8FXKGPNHvufxInIYCAeqLOAWEasWtw6aPPuN+Lz4dX5BRdenJha9/VjcsUIblXz8chg2bBjvvPMOP//8M506dWL58uWMGjUKydej8NJLLzF+/HieeOIJevToQVhYGL/88gsPPvhggeC7JIcPHyY3N7dI8Fj49b59++jTpw+XXnopc+fOpUGDBvj5+dGnT58ynyv/OXNycoiMjCyw3P46fzAPEBYWVuC1n5+fU+c8dOhQgeMXd77FixfzxBNP8NBDD5GcnEzHjh156aWX6NmzJ97e3nz55ZdMmDCBESNGkJ6eTrdu3Xj11Vdp3759mduiqrfDqem8sfYvPvz1AAC3XdKUB3q2IKK2dbc0yM/n7Jst8mSirQe5EVz3EnS4BXwDHV8jh3wAv86xglSA5Dir4yKiVfnOfTrJ6lXf/4OV7pB12vF2515VvuM7EtHa+sJw1WSY0wOO/el4u4+Gnnl+1RToPsb6UpB/uSc7dRQ+GwO7P4W2g2DQ22d6ivP/Tq+adGafm9+Fo3vg6J+2xx+wcymcOuK4d7yicnNh7TT4/kVo2g1ufq9id08GzS/+c70SuDrgjgYO5nsdC1zs7EFEpAvgB1T5KIbIkAASNaVEFVb49qOLbwn26tWLyMhIFixYwKFDh0hNTS1SnWTRokUMHTqUqVOn5i3bvn27U+eJiIjAy8uLw4cLDiwp/PqLL74gIyODZcuWERho3SLPzMzkxAnnvw9HRETg7e1d5ByJiYkA1K1b1+ljlsSenlLa+Ro1asS7775LTk4O69evZ+LEiQwYMICDBw8SFhZGmzZtWLp0KZmZmXz//fc89thj9OvXjwMHDhT4IqSqv8LTrI/q2YIDSWn876f9ZObkMrhzI/595blEF0oXqdazRdpTAg78DO1uhsRdcNObVmrGnauhQYeCwY6jFI3W/ayH3U+zrAD8nJ5wyf3QsveZnOr857Tn+548AgEh4OMPv8yBVY9b2/kEQoP2Vmqfo571quAbCHd8Xvz1f+S6M89r29JXQqKt5aePWV8Q9qyyelIrkhNdFXZ/Cp+OgYwUq0e/62hruaPfaX5hja1Hy3xfcPK+eL1vpWLmvyuclQ6+AeVrY042LL7DamvH2+C6l8HHr9TdSlTFn+uuDrgdfeo4dU9eRBoA7wHDjSn6lUlERgIjAZo0aVKeNhYQFRLA1oNV1omuqrvSLkBVxNvbm8GDB7No0SLi4uJo3bo17dq1K7BNWlpakXrcH3zwgVPn8fPzo127dixfvrxATvTSpUuLnMvb27tAXviCBQvIzS34X7Qsvc++vr507NiRRYsWFRicuXDhQry9vUvN/3ZW06ZNiYyMZNGiRfTu3bvA+erUqVOk3rm3tzddu3Zl4sSJXHbZZRw4cKBAL7ufnx9XXnklY8aM4fbbbyclJYXQ0NBKbbNyH3t5P3tqSNyJNJ745DcABnZoyJirzqN5/WB3NrFyOcqJ/mqyFSRnpVmBZ6OY4vd3dDvf7rKxVq7vhrfgw5utlIBL/22lBdjPabKttJXXL4Yjv3aaFB4AACAASURBVMOtS6HlldC0K/T6r5VD3bDTmWCrpLuPla2kAK1hx6Lb+wacWd7yKsdtTYkvml/uSn9+BR/fan2BueEzq0e/sJJ+p8Vta/8ZbXnPNubJWNVO7vgc6jsePF8ibx+o0wz6PGMNhq3MTo0q+lx3dcAdC+QvttsIiC/rziISAnwOTDDG/OJoG2PMXGAuQExMTIUTbKNCA0jYmY4xRnupVPGcuQBVkmHDhvHaa6/xySefFOjFtuvduzezZ88mJiaGc845h3fffZf9+/c7fZ4nnniCm2++mdGjRzNgwADWrl3LV199VWCbK6+8kscee4wRI0YwYsQIduzYwYwZMwgJCSmwXatWrVi7di1ffvkldevW5ZxzznHYYz1lyhSuu+467r77bgYPHsy2bduYPHky9913n1PVR8rC29ubSZMmMWrUKOrUqcOVV17J2rVrefPNN3nhhRfw8/Pj2LFj9O/fn9tuu43zzjuPtLQ0XnzxRRo2bMj555/P5s2bGT9+PEOGDKF58+YkJSUxffp0OnfuXO2D7TIMdG8KzMNK8UsCbjXGxNrWPQ/Yk3WfquqB7q5QXHm/iNr+vDLUQZBVXeUf5JabXTCndeS30LBD8fuWVXB9uOxR6PaQNaDyp1fh5zdg1XgryM6f2xzaCNoNgXq24KxBe+tRmDs6Qcp7zvz7rRxr9dbO7QnDPoTozqXvX5lOHbV+Hy16wYBXof2wgrnrFVX4Z/TXN9CkK9Rtbq2P3WSlFvmV8mU1dpN1F6RhR7j66cprX0ltriSuDrg3AOeKSHMgDhgK/KvkXSwi4gd8ArxrjFlUdU0sKDIkgMzsXI6fzqJucAVvVyhVibp27UqzZs3Yv38/Q4cWzQucMmUKx44d44knnkBEGDRoEDNmzOD666936jyDBw9m5syZvPDCC8ybN49evXrx5ptv5lXrAKs6yNtvv83UqVNZsmQJHTt2ZMmSJUXONXHiROLi4hg8eDApKSm899573HrrrUXOee211/Lhhx8ybdo03n33XSIiInjssccK1L2uTPfffz+ZmZnMmjWLGTNm0KRJE2bMmMFDDz0EQFBQEG3atGHmzJkcPHiQ4OBgunbtypdffom/vz8NGzYkPDycp59+mvj4eOrUqUOvXr14/vnnq6S9rlLGge4vYl2X/ycivYBngdtE5DqgE9AB8Ae+FZEvjDHVulZicWX8jqRWg7E+JZVkO3XUqqucsMOaCGb3p1YPtqOb0JURbOfn7QsXDrImK0nY7jjf99Ylzh3TDZ0g5T5n7UgY8q6VpvPREHjnWrh+NrS9sfLbWFhGKnw5wfrC88AvUDsKOt1ededz9DPKSoMPB4N4WXc4Yu6yyvcVtmOxNTiyYUcY8UXl9mq7gBjj2ioLInItVtk/b2CeMWaaiEwFNhpjVojIRViBdR0gHUgwxlwgIrcC7wA78x3uDmPM1uLOFRMTYzZu3Fih9n6+/RCjPtzMygd70KZhSOk7KI+2e/fuvLJtSrlbaX+PIrLJGFPC/fqqJSJdgcnGmD621+MBjDHP5ttmJ9DHGBMr1m3AZGNMiIiMBfyNMU/btnsbWG2MWVjSOSvjul2Vukz7isMOguvosEB+HNfLDS0qA0cl2QbNs3KfW11r5cM+G32mbF7thlZpN/GycrYLp2ZMTq6idpaQDlJV5/Q0p45aKR0Hfrbyki+6q2rOk5oAn/0H/lxj3b249N/Q88ny51RX1IFfzvR8B9Wz8sa73AP+ta3BkV9OgF9eB98gGPMbBNdzTztLUdI12+V1uI0xK4GVhZZNzPd8A1aqSeH93gfer/IGFmKf3j0xJV0DbqVUTVOWge7bgJuw0k5uAGqLSD3b8kki8jIQBPQkX0Wq/Cp77E1VyczOxc/BrJAeW94vf6Cdk1mw53jxndDkUivg9vaBfjMhpAFEXlgwmPGUnOiaIrg+3L4c1ky0ShlWttQEWPc8bP7fmcGad66GJpU7NsZpTS6B2z6Bg+ut3/3XU6DxxdZA0/cGwvH9gJdVicZDg+3S6EyTpbBP766zTSqlaqCyDHR/FHhNRO4AvsNKF8w2xnxpu2P5E3AE+BlwWNy2ssfeVJVXvv6D2BNpjOjWjC93Jnp+eb+SJiy59zsIz1eOr8OwottA9cqJPlv4+FulB8Gqhb3uOeg8vGKDKQvPAJq/Moq7g+38Gnex0of2fW/VOd/6AWTb7yhVQWlBF9KAuxT2+qk626RSqgYqdaC7MSYeuBFARGoBNxljkm3rpgHTbOs+BIopWuz51u9L4o11fzEkpjGT+l/ApP4XlL6TO508bKWMnH8t7P2qaE+xowGHJalOOdFnk2N74efXrB7poR9CdKey72sMHN4Fv6+EH2dYE+9UF+ueKf7LYjVV9N6YKsDPx4v6tfx0tkmlVE2UN9DdNnB9KLAi/wYiUl9E7J8l47EqliAi3rbUEkSkHdAO+NJlLa9EKelZ/OfjrTSpG8TE/m3c3ZzS/fElzL7UmhCmdX94aLtVq9gnALx18H+1Uv9cuOtL8PK1BlP+trT0fQAO74ZZHay/g7VPQ92WEB1j9Z5Xh7+BQfOh84iz6m9We7jLIDIkQFNKziJa4lF5AlcPWC8PY0y2iIwGVnNmoPvO/APdgSuAZ0XEYKWUjLLt7gt8b/u/loJVLrDy50t2gYnLfiMhJZ3F93Ul2N+DPzaz0qzc3/VzIbItDP/0TB3lmpyiUd1FXgD3fGMNplw8ApJjoduD1rrUBGu2xb/XQdPu1p2LS+6DsCYQ0Qa6jYHzr7Gqj4Brc/Ir4izM5/fgK4fniAoJIK6YUlCqevH19SUtLY2goCB3N0XVcGlpafj6VmKd2ypShoHui4HFDvZLB6pBd3DJlm+NY9nWeP5z1Xl0bFLH3c0p2baPrGD7klFw5cSiFSc0RaP6qhUOw1fA549YQXVqAiy5B/754UzaRfpKqNPUeu4XDMM+Knqc6pYfX93aWwINuMsgMjSAzQeOu7sZqhJEREQQFxdHdHQ0gYGB2tOtXM4YQ1paGnFxcURGRrq7OaoEscdPM2HZb3RuWodRPVu4uzmO5ebC8X1QrwV0Gm71bDfu4u5Wqarg4w+9JpypPJOdQYExzGP3ln2ymur25au6tdcBDbjLICokgOOns0jPyiHA19vdzVEVYJ/5MD4+nqysrFK2Vqpq+Pr6EhkZWWQmTuU5cnINDy/chjEw4+YO+DgoB+h2KYdg2f3WZDGjN0JQXQ22z3YlVZ6pzJkhVaXTgLsMomylAQ+nZNCknqYiVHchISEa6CilSjT3u79Zvy+JFwe398zr/u7PYMW/rclq+j4LgR6e7qIqx6D5Z01Oc03jgV/ZPU9kqNbiVkqps1ZqAnz2MMzpDsBvccm8vGYP113YgJs6eVh97ZwsWHoPfHyLNQnIvd9B5zuq3TTXqpzsqRVaeaba0R7uMojSyW+UUursk/Q3fPcS/LY4r7cwLTOHBxdsoV6wP9NuaOtZ4zxSE6zezR2LQbyt3u3657q7VcodzqLBhDWFBtxlYA+4E3XyG6WUqv5SE2DlWNi9osiqaSt38feRU3xw98WEBXlIz+GRPbDoDjj2F2DOqslAVAWdBYMJawoNuMsgJNCHAF8v7eFWSqmzwUfDIH6zw1Xv/3KAe3o0p1vL+i5ulAO5ObD5Xfj8UaieJcyVUjaaw10GIkKUTn6jlFLVm32yoWELoGk3W/5rwcoOrRuE8Gif893QuEIO/Apv9oTPxkB0R7jgBs3XVaoa04C7jCJDAjSlRCmlqqtjf8GbvSB+i3UbfsRK28Cz2zE+AWTZbvjOucoXf3Lc3Fhgz0o4dRQGzYO71sDg+TpQTqlqTAPuMooK1R5upZSqlnZ/CnOvsCaISU85s9yW/7q4x0o+yr6C48EtaLp8EHwwCNJOuLaN2Rnww0z4a631+vLHYPQGaHvTmQokhStURF3o2jYqpcpNc7jLKCokgMMpGRhjPGvUulJKKcdysuHrKfDTLGjYCW7+H4Q1AWDZljimr95D/Ik0DNAq6iFue6iHNT36in/DvL5wy8K87avUH1/CqnGQ9Jc1LXuLntbU3MXRgXJKVTvaw11GkSEBZObkknRKi8wrpVS1sHm+FWxfdDfcuapAsD1+6Q7ibME2wP6jp1i+NR46/AtuXQop8fDWVRDneHBlhaUmwOK74dlG8OFgEC+4ZQn0faZqzqeUcivt4S6jBvkmv6lXy9/NrVFKKVWsrDTwDYROd0BYUzi3d4HV01fvIS2rYJ52enYu01fv4fqO0XDO5XDXl/DBYGtGv+hOlde21AT49gXruDlZYHKg91Nw8X3go3nZSp2tNOAuI/tsk4kp6VzQMNTNrVFKKVWEMfDza/Dr/8E930CtiCLBNkD8iTSHuxdYHtEKRq6FANv1Pu14xaZPT46HVY9bgyExVsk/u24Plv+4SqlqweUpJSLSV0T2iMheERnnYP1lIrJZRLJFZFChdcNF5E/bY7jrWp1vtsnkDFeeVimlVFmkJ1s90l9OsJ77BBS7acOwwLItD65vlQ08nQT/dzmsGl8wUC6Lnctgyd3wyoXWRDu52c4fQylV7bk04BYRb+B14BqgDTBMRNoU2uwAcAfwYaF96wKTgIuBLsAkEalAd4Nzwmv7I6LTuyullMf5+1uY0Rb2rrGmPM9IgYCQYjcf2+d8An29CywL9PVmbHH1twNCodV18Msb8PFtkHnqzLrUBPjsYZjT3ao08ve3VrURu98Ww9/r4PzroPkVWtJPqRrK1SklXYC9xpi/AURkATAQ2GXfwBiz37au8Ny1fYA1xpgk2/o1QF/go6pvNvh6e1G/lr/W4lZKKU9hz4fe9M6Z6c5N6b3H13eMJiMrh8eX7gAgOiyQsX3Ot/K3HfHyhr7PQp1mVjWR+ddB/1et82553+q1NjnwfHPIOgVevtDxVquHfMBr4B8CXrb+rdRE+PZ5K4fb5EKODsRXqiZwdcAdDRzM9zoWq8e6vPsWc3WsGjrbpFJKeZDFI+DAL2eCbSfENK8LwIwh7bmhY6Oy7XTxveBfG5aPhrmXgZdPwYC5wzBoeRU0625tBxAYVvAY9pJ+lz9uBd6x651uu1Kq+nF1wO2ogLVxsKzc+4rISGAkQJMmlVs/NTIkgNjjpyv1mEoppcpp0Pxy9xYn2jpPIkOKz/V2aMt7tgDfFD3fdS+V/ThaS1upGsXVgyZjgcb5XjcC4itzX2PMXGNMjDEmJjw8vNwNdSQq1F97uJVSylMUnnnRifxoe8Ad5WzAPWg+xNypudhKKae4OuDeAJwrIs1FxA8YCqwo476rgatFpI5tsOTVtmUuExUSwInTWaRn6QhzpZTyGOWY8txecSoq1MmAuwJBvlKq5nJpwG2MyQZGYwXKu4GFxpidIjJVRAYAiMhFIhILDAb+T0R22vZNAp7CCto3AFPtAyhdxX7rMVF7uZVSHk5E+olIzZpN2B4M3/dDqZsmpqRTO8CHIL9yZlaWI8hXStVcLp/4xhizElhZaNnEfM83YKWLONp3HjCvShtYAntPSEJyOk3rBburGUopVRbLgcMi8i4w3xiz290N8iQJyenO5287ornYSqkyqFm9HxWUN/mN9nArpTxfC2AucDPwm4j8LCL3iEjxBaprkISUdOfzt5VSqpw04HZC/undlVLKkxlj9htjJhljmgO9gb3ADOCQiLwnIj3d20L3OpxSST3cSilVBhpwO6G2vw9Bft46vbtSqloxxnxjjLkNOA/YBNwCfCUi+0TkPyLi8vRCd8rNNRxOzSAq1N/dTVFK1RAacDtBRIgKCdAebqVUtSIil4vIfGAP0BZ4HavS0yJgCvCu+1rnekdPZZCdazSlRCnlMjWqV6MyROpsk0qpakBEmgLDbY9mwDqsScGWGmPst+m+FpGfgffd0UZ3SbTdpYzQgFsp5SIacDspKjSA9ftcWo1QKaXK42+sycHmA/OMMfuK2W4nUKPmF08o76Q3SilVThpwOykyJIDDqenk5hq8vBzNNq+UUh6hP7DKGJNb0kbGmD+AGjWAMm+WSWcnvVFKqXLSHG4nRYX4k5VjSDqd6e6mKKVUSb4HIh2tEJEGIlLLxe3xGIkp6Xh7CfVr6aBJpZRraMDtpPyT3yillAd7G5hazLrJwFuua4pnSUhOJ7yWP956l1Ip5SIacDtJp3dXSlUTlwGfF7NupW19jZSQkp43r4JSSrmCBtxOyuvh1oBbKeXZQoHTxaxLB+q4sC0eJTElncjamk6ilHIdDbidFF7LHy+BRE0pUUp5tj+B64pZdy3wlwvb4lESktN1wKRSyqW0SomTfLy9qF/LX3u4lVKe7lVgjohkYpUGPAQ0wKrLPQq4331Nc5+0zBxS0rN1WnellEtpD3c5RIUGkJCi07srpTyXMeZNYBLwALAdOGL7dxQwwba+VCLSV0T2iMheERnnYH1TEflaRLaLyDoRaZRv3QsislNEdovILBFx+yjFRK3BrZRyAw24yyEyJEBTSpRSHs8Y8zTQECu15Hbbvw2NMc+VZX8R8caaBv4aoA0wTETaFNrsReBdY0w7rKooz9r2vRToBrTDmk7+IuDyir6nikrQGtxKKTfQlJJyiArR2SaVUtWDMSYZWFXO3bsAe40xfwOIyAJgILAr3zZtgP/Ynq8FltlPDQQAfoAAvkBiOdtRaew93JpSopRyJQ24yyEqNIDktCzSs3II8PV2d3OUUsohWwpHN+A8rOC3AGPMG6UcIho4mO91LHBxoW22ATcBrwA3ALVFpJ4x5mcRWYuVOy7Aa8aY3cW0cyQwEqBJkyalva0Ksc+hEBmiVUqUUq6jAXc52HtGEpLTaVY/2M2tUUqpokQkEvgaqwfaYAW92J7blRZwO8q5NoVePwq8JiJ3AN8BcUC2iLQEWgP2nO41InKZMea7Igc0Zi4wFyAmJqbw8StVQko6wX7e1A7wrcrTKKVUAU7lcItIhIg0z/daRGSkiMwUkf6V3zzPZB9so5VKlFIe7CUgGWiMFThfDDQD/otVMvC8Mhwj1ra/XSMgPv8Gxph4Y8yNxpiOwJO2ZclYvd2/GGNOGmNOAl8Al1TkDVWGwykZOumNUsrlnB00OZ8zuXoAU7B6SPoCn9h6OEpUhhHv/iLysW39ryLSzLbcV0T+JyI7bCPexzvZ9koTFWrditTZJpVSHuxyrKD7kO21GGMOGGOeAd6n9N5tgA3AuSLSXET8gKHAivwbiEh9EbF/lowH5tmeHwAuFxEfEfG1tcdhSokrJaSka4USpZTLORtwdwK+AbBdYO8HnjDGtAKmAWNK2rmMI97vAo4bY1oCM4DnbcsHA/7GmAuBzsC99mDc1fKnlCillIcKA44YY3KBFCAi37qfgEtLO4AxJhsYDazGCpYXGmN2ishUERlg2+wKYI+I/AFEYn0WACzGmlxnB1ae9zZjzKcVflcVlJCsAbdSyvWczeEOBY7ZnncG6gIf2F5/AzxSyv5lGfE+EJhse74YKzdQsPIGg0XEBwgEMrE+RFyudoAvwX7emlKilPJk+7AmugHYCdwCfGZ73R8oU6klY8xKYGWhZRPzPV+Mda0uvF8OcK/Tra5CubmGw6npmlKilHI5Z3u4Y7F6psGq5/q7MSbO9joUKC0CdTTiPbq4bWy9K8lAPawL+ims26MHgBeNMW6rzRcZGqApJUopT7YSuNr2/GngJhGJFZF9wINYM1HWKEmnM8nKMUTW1golSinXcraHex7wgohchRVw58+jvoTS8/PKMuK9uG26ADlYkzjUAb4Xka/sveV5O7uovFRUSICmlCilPJYxZly+51/YJqK5AesO4RpjzBdua5yb2K/ZOumNUsrVnAq4jTHPikgc1oxh/+bM4Biw0kveKuUQpY54z7dNrC19JBTr1ue/gFXGmCzgsIj8CMQABQJuV5WXigoJ4Fed/EYp5YFExB+rXN9nxphtAMaYjcBGtzbMzQ6n6qQ3Sin3cHpqd2PMu8aYfxtj3jbGmHzL7zPG/K+U3Usd8W57Pdz2fBDwje08B4BetlKEwVg96r872/7KYk8pyc2t0pKxSinlNGNMBlaJvjB3t8WTJCRnANrDrZRyPWfrcLcWkUvyvQ4SkWdEZJmI/Lu0/cs44v1toJ6I7AUeBuy3RV8HagG/YQXu7xhjtjvT/soUFRJAdq7h2KlMdzVBKaVK8ivW4HZlk5CSjgiE19IcbqWUazmbw/0GVjmpX2yvpwN3AN8Dz4tIgDFmekkHKMOI93SsEoCF9zvpaLm72G9JJqakE64DcJRSnucx4EMRycS65iZSaMyMMea0OxrmLonJ6dSv5Y+Pt9M3d5VSqkKcveq0BX4GayIa4FZgjDGmL/AEcGflNs9z2W9J6sBJpZSH+hVoAczCmlkyBUgt9KhRdNIbpZS7ONvDHcyZ2teX2F4vtb3eDDStpHZ5PJ3eXSnl4e6kaBWoGi0xJZ1GdYLc3QylVA3kbMD9N1ag/R1Weaktxhj7RDj1qUE9JvVr+eElOr27UsozGWPmu7sNniYxJZ2YZnXc3QylVA3kbMA9A5gtIoOBjsCIfOuuANw2iNHVfLy9CK/tryklSilVDaRn5XD8dJamlCil3MLZOtxvi8ifWHW4xxljvs63OgmYWZmN83RRIQGaUqKU8kgicoRSUkqMMREuao7bHU6xSgJqDW6llDs428ONMeY7rJSSwssnV0aDqpPIkAD2Hzvl7mYopZQjr1M04K4L9AJCsEqw1hj2zhENuJVS7uB0wC0iYcC9QHesi3cSVlnAucaYE5XbPM8WFRrAL38fK31DpZRyseI6QUREgIVAtksb5Gb2gFsnvVFKuYOzE9+0wJp4ZipWhZIDtn+nAttt62uMyJAAUtKzScvMcXdTlFKqTGwz976FNQlZjZGYrD3cSin3cbYO9wzgOHCOMaaXMWaYMaYXVq3XE8DLld1AT6alAZVS1dQ5gJ+7G+FKiSnpBPp6ExLg9I1dpZSqMGevPFcAw40xcfkXGmPiRGQK8E5lNaw6yD/5TfP6wW5ujVJKnSEiDzhY7Ae0Bm4BFrm2Re6VkJJOVGgAVkaNUkq5lrMBtwG8i1nnRQ2bZCH/9O5KKeVhXnOwLAOIBd4Apri2Oe6VmJJOZIi/u5uhlKqhnA241wJPicgGY8w/9oUi0hQrj/vrYvc8C+X1cGvArZTyMMYYZ1MGz2oJKel0aqKT3iil3MPZgHsM8A3wp4hsBhKBCKAzcBB4uHKb59lq+ftQy99HJ79RSikPZowhMSVDJ71RSrmNUz0gxpj9QCvgQWAn4Avswhrt3hVoUsnt83iRIf6aUqKU8jgiMk1E/q+YdXNE5ClXt8ldjp/OIjM7VyuUKKXcpjwT32QCc2yPPCJyE1Zt1+JyvM9KUaE626RSyiMNAyYWs+57rDTA/7quOe6TqDW4lVJupjl+FRQZEpBX31UppTxIQyCumHXxtvU1gs4yqZRyNw24KygqJIDDqRnk5taoAi1KKc+XAHQqZl0n4IgL2+JWZya90SolSin30IC7gqJCA8jONRw9leHupiilVH4LgYkicl3+hSJyLVYqyQK3tMoN7D3cEbW1h1sp5R465VYF5dXiTs7Qi7lSypNMBDoAn4rIMeAQ0ACoC3xJDcnfBiuHu34tP/x8tI9JKeUepQbcInKEsk1oUyPv1eWf3v1CQt3cGqWUshhj0oGrRaQP0BOoBxwDvjbGrHFr41wsITld87eVUm5Vlh7u16nEGSRFpC/wClY1k7eMMc8VWu8PvItV2/sYMMRWjhARaQf8HxAC5AIX2T5U3EYnv1FnldQE+PYFiF0P9/1Q9fupKmeMWQ2sdnc73CkxJYMGWqFEKeVGpQbcxpjJlXUyEfHGCuB7Y00vvEFEVhhjduXb7C7guDGmpYgMBZ4HhoiID/A+cJsxZpuI1AOyKqtt5VW/lj/eXqKVSlT1Zg+Yt34AJhdyMkvePicLMlLh+H7Y8n7Z91MuY7t+NjbGTHew7lHggDFmoetb5nqJKel0aBLm7mYopWowV+dwdwH2GmP+BhCRBcBArMlz7AYCk23PFwOviYgAVwPbjTHbAIwxx1zV6JJ4ewnhtfy1h1tVT/kD7dxs62G3+C7IPAmD5oFfMHz/Evz8hrUsO9/fu7efBtqeaRzwdjHrTgPjsQZWntUysnM4diqTSB1jo5RyI1ePIInGmgLeLta2zOE2xphsIBkr9/A8wIjIahHZLCKPOTqBiIwUkY0isvHIEddUvYoMDdDZJlX1tPB22DTPCqDzB9sA8Zsh9RBk2yrw1DsX2gyALiMhtAkg1nINtj3VucBvxazbbVt/1jucYv39RoXWyGFGSikP4eoebnGwrHB+eHHb+ADdgYuweme+FpFNxpivC2xozFxgLkBMTIxLimNHhfjz95FTrjiVUpUnaR+cToK6LSA5tmhKyINbCm7fZoD1AOg6Gr59XlNJPNtpoFEx6xoDNaKWaaJOeqOU8gCu7uGOxbrQ2zXCmvHM4Ta2vO1QIMm2/FtjzFFjzGlgJcVP6uBSUSE6vbuqZvZ9B2/2hFNH4LqX4aHt0PE28AmwUkRKUzsS+tn2az+06turyuMr4L8iEpF/oYiEA09ilQY86yXotO5KKQ/g6oB7A3CuiDQXET9gKLCi0DYrgOG254OAb4wxBmuUfTsRCbIF4pdTMPfbbSJDA0hNz+Z0ZnbpGyvlbhvegvdugOAIuOcbOOfyggF0x9sg6sKyHat2JAx4FfrPsr1uUHXtVs56HKgF/CUii0RklogsAv4CggCHaXlnm0R7Son2cCul3MilAbctJ3s0VvC8G1hojNkpIlNFxHavmreBeiKyF3gYa+APxpjjwMtYQftWYLMx5nNXtr84ebW4tVKJ8nQnj8DXU6FFL7h7DdRrUXC9PfB2trRfp9uhZW+rcklK4ZtWyh2MMQeA9sBrWHcNr7H9+yrWhDgJ7mud6ySmpOPv40Vo2utExQAAIABJREFUoK+7m6KUqsFcPtOkMWYlVjpI/mUT8z1PBwYXs+/7WKUBPUr+yW/OCa/l5tYo5UB6CvjXhlrhcNdXVqDt5V15xxeBa6fDG5fAmklw05uVd+ziaO3vUhljjmBVIwFARLyAK4DngBuxBqSf1eyT3ljFrpRSyj10avdKEGnLDdRKJcojHf4dPhoCMXdCt4cg/LyqOU/d5nDDHGjYsWqOb+dszXCFiFwMDANuBiKxxsUscGujXCQhJV3TSZRSbqcBdyU4k1JSIwb9q+pkzypYcjf4BkKTS6v+fBfcYP1rjBUMV2YvugbaThGRtlhB9lCgGZAJ+GGl6r1uS/Ery3FKmx24KTAPCMcK5G81xsSKSE9gRr5NWwFDjTHLKvK+nJWYkk67RjrpjVLKvVw9aPKsFOzvQ21/H+3hVp7DGPhhJnw0FOqdAyPXQeOLXHPu7Ez4YDCse670bZ2xeARstNUMLxxsG5dUAPV4InKOiDwhIjuAbcCjWONlbsequy3AFieCbfvswNcAbYBhItKm0GYvAu8aY9oBU4FnAYwxa40xHYwxHYBeWGUKXVoZxRhDYko6USFag1sp5V4acFeSyNAAHTSpPENqAiy6A76aBBdcDyNWQWjh+aWqkI8fBIbBDzPg6J+Vd9wBr0FwfRCvoqULX+0M6563aovXbHuBp4BU4F4gyhjTzxjzgW2Zs/JmBzbGZGKloQwstE0bwD4fwloH68GqOPWFraSry6SkZZOelas1uJVSbqcBdyXRWtzK7VITYMVD8Ep72GMblzzoHfALcn1brp4GvkHw+cOV0/uceQqWj7Ym6uk/q2jN8JCGsO4Z+OTeM/tk1cj/j/9g9WK3xRoceamtjGp5lWV24G3ATbbnNwC1RaTwYMyhwEfFnaSqZgjWGtxKKU+hAXcliQzR6d2Vm6QmwGcPw8wLYfP8gikX7qrMUDsSrvyvNcHOjsUVO1bmafhwCBz8BW56CzrdVrRm+B2fwZjf4JoXrH1OJ8GL51pT1/++0kpzyc/+M5vT/f/bu/OwKMv1gePfm0UERVEETNTUTHNNjeOx1CzLrSy3TK1OZVanc7LtlB2zfma2W53qtJm5lOlJc83K0krbyzIVyzUzS0DAFVABYXh+f7wzOAMz7LMA9+e65mLmXea952Xm4eaZ+32eysUWYIwxrYHewFvAJcD7QJqIvGF/XN7/fsoyO/B9QD8R2Yw1P0IyUFiyIiJnAF2whoP1FPcsY0yCMSYhJiamnCF6lqqzTCqlAoReNFlFmjYMIz0rF1uBITiogklOZYY50yHSaq+l4+GP7yh/LuVlCTdZFzh++yJ0uariyf+qO2Df1zByFnQeeXq5Y8xwh6gW1g3AlgfdroWfl8D29yC8MXQeBd3GweaFNfrCS2PMd8B3InIXVpI9DqsHegLWm+QWETlpjNlYhqcrdXZgY0wK1hCDiEh9YJQxJsNpk6uBFcaYvAq+pApLs5f56SglSil/04Tbk3ImsE0b1MVWYDh8PJfY8jbulRl9QUduUAOfgDcvg7yTEBQKBT7Pa9wLCoar5kJEdOV62vv+C9oNgq5Xl32fyDgY8hQMfBR+Wwcb51kXXG6aD5ha8TkxxhQAnwCfiMhtwGVYpR0jgGtEZLcxpkMpT1M4OzBWz/VY4BrnDUSkCXDEfrwHsEYscTYOp7HAfcnRwx2rF00qpfxME+6iKpjAxjlNflPmhDsrFb54Gjbb5/JxPtZ/OlpDuYVGQGwHq3cPrJEfMpKs+wcSId0+u32BTitfK+VkwvJbrET7+lVWb24g/fPVuI3105YHJw5BgzJO/W7Ls15L51EQ18m6VURwqJWsf/MCUAC22jl0p/2Cx5XAShGpBwzHSp5L2y9fRByzAwcDcx2zAwMbjTGrsGrFnxQRA3wJ3O7YX0RaYfWQf1GlL6iM0jJzaFyvDmEhVTg8pVJKVYAm3A4uibbN+oNfDjtTMwG48uVviI8KZ9Kg9gzvXsrIEEtuhD+/c7/urIut2tW8bGuGQIcDiZCyGY6nW3G6k58LIdqjUyuERVplFm0ugjMvgDb9oN+/rX/kkn7wd3Sn/e9qyD4KN39W+tjctnxYNsFKuKNaQouelT/+VW9a58Td5zv/lDWyircFSNmXMeYEsNB+K8v2pc0OvBRwW6hvjNlH8YssfSYtM0frt5VSAUETboel4+HP762ewXJauTmZV9f/Vvg4+Vg2Dyz/GcBz0p173OqVBAgKsYY6c+6RHPaK+/3G2S/0z0rznEA8czZ0GAqdRloJWHBouV+TCnD5p+B4qpWQXjzFdV3R2uZA0O1aK4neOBd63uJ5uwIbrLzNSrYHPVE1yTacPieOf0a2LLQ+b6YA5g6EUXOs6e69Qcu+/CY1M4c4LSdRSgUAHaXE4ao34bzx1lBjUr6vH59Zs4ucfNdEPTvPxjNrdrnf4Xg6vHk5/PE1DH0B7tlefJiz0jgSiLu2QvfrXfftMBR2fAALR8Gz7awa1pLU0BEbaqyCAlj5D3ijvzUaR3XQeRS07gefTbf+WXSnoMAa+u/nJXDJw3D+7e63qwznz8154yHqTDiyFxaMtHrWq5Ljc/V8Z9j8tvsJe5RXpWbk6gWTSqmAoAm3g8sf4htdE19HjbUHKceyy7WcfV9bE4KMfQcSxhdJnu3DnFUkbse+w1+FSfbnP6s/NGlnbbvjA1g9yRrRoqDgdELw4rlWQpD6c9mPq/zDGFgzBX5ZCr3+CRGN/R1R2YjA5f+xks61D7rfJnkjbF0EF02xLpT0Jsfn5u6tcNs3MHwmBIdYPeynTlTuuQtssPNDWHwd/DTPuohVE22fy7MVcPhErpaUKKUCgpaUFOX81fNnj8D2lfDDG9B1jMfSjGZR4SS7Sa6bRYW7LsjJhLoNrJrbM3tbx3J37MrE7RASBudcZt0cDv9qjdLwwywIrWclP0HBmgxUJ1//Bza8ZiXbfe7xdzTl06Qt9L7bmpQn9ziE1Xdd36KnVdscW3TmcC9zHk7w6/9A4iJrdJUzzi3f8+RkWmUjG2bC0X1w4SRo2lVLSfwkPSsXY3TSG6VUYNAebk8i46ye4sn74brlVrKdfQyO/lFs00mD2hMe6lqGEhYSxKRB7U8v2PmhNTHJnxtOP7+v9bkHJu2B6LMh74S99luTgGpjxwdWSUaXq62ZHP01qU1lXHgf3Pr56WTbGOtblgVXWSVNcZ38+7pa9LJ6uGdfCt+/VrZZMo2BNQ9aIwt9PBnqN4XRb0G/ya7fPpWnZExVmmMiMi0pUUoFAk24SxMUDPXssxR/9G+Y1Q9+/dRlk+Hd43lyZBfio8IRrHyhWcO6XHluM2uDH2dbXy9Hn+W9C7PKKiwSbvwQEiZYf/yD9EuOUgVKjftZF8PFD1oX1AZV049uSNjpf163LoHX+8LGOfDbZ4FR0tS6r1ViclZ/K3l+ZxycOHx6vfN7Ic0+JKcIHPsT2g+BW9bBhDXQabhVogKVKxlTFeaY9EZLSpRSgUCzrfK46N+Qtg0WXmWVnPT7d2HiM7x7fOGIJO9u3M/9S7ey+Mc/GZc1D75+HtoNtr6mrlPPn6/A4nbEhjzPwwzWVoEyukTKFms867oNoN/9/omhKmWlwltXwiGni4orMDqQ19SLhnGLYMPrVlnZkb1WHbbjveD4rLx2Ady5yfrdjH6r9H+CAnH0mBrs9LTuOkqJUsr/qmk3mZ80bgMT1sK54+CLp+B/o92OEjH6vOb8tXVjEj+abSXb542HMQsDI9l25kgA/rkBwhpYyxIX+zemQFB4MWlX/48ukfqLlZy+f5d/ju8NS8fDod3+jqJkItDrNrhlPSS+Y11Y/NOb1nvB8Y/pFS9ApH0in+r6jUMNlpqZQ53gIBrX0zIepZT/6V+J8qoTYdV2D30eDu6yJpkpQkR4YmQXVuT1YnazR6xtgwP4y4TGreDuRGjVF1bcCl89V7ba1ZrqnbFWmUN+rn9r3I/ugwWjrH/UBkz3XxxV7ao3IeGm6lHT/OE91kgjzom2w3k3WrPBqoCUlpFDbIMwpDpe66CUqnF8nnCLyGAR2SUie0Rkspv1YSKy2L5+g31qYOf1LUXkuIjc56uYixGxEoaJG62pqgtssOtjK0nNPABvXsFZX/6Lr6Km8djes/l890G/hVpmdRtaF4d2GW1dmPdxsV9N1QqUumhnJw5bwyamJFqTEgWFFE8IN71t/b69LXUbzOwLJ9Lhb8tPj6JRExStaQ7kxNt5fP5AjVG5lZqZoxdMKqUChk+7XUUkGHgFGAAkAT+KyCpjzHanzSYAR40xbUVkLPA0MMZp/fPAR76KuUSh9sY88R1473Y4ZyjsXW+NchAUQmxBPmfF1OOhlb+w9p4LiagTwL3cYE1vPWIWNIiH2A7eOUag1EUXlfoLzLsMTmVZY6NfPMVKrB017gX51m3VRGtYxcFPQisv/LPgOD8/zTtd1+yt34W/Fb2WIJCmondwd71DIL1vlUfpmbl0aNbA32EopRTg+x7unsAeY8xeY8wpYBEwrMg2w4C37PeXApeI/TtBERkO7AW2+SjesmlzMZzRDXZ+cHrSjAJr1ronRnQh6Wg2L376qx8DLIegIBjwCJw71nq851M4cahyz3nqBPy69vQkOxvnBsase8bAsf3W/ZhzoMso+Me3VoJVr4lrT2yPG6zRJUbNser2P7zPmjyoqmSlwgf3nK4bD6SLCL3NcZ5v+9rfkXimI41UK8YY7eFWSgUUX3e5xgP7nR4nAX/1tI0xJl9EMoBoEckG/o3VO+6xnEREbgVuBWjZsmXVRV6S5TdD6la3q/7aJpoxCS2Y/fXvDOsWT8fq1OOSkwFLb4KIaLh2afmGNDQG9m+wZuncthLyTgLGfSL5ej84fyJ0HV1loZcqZbM1dvKh3XDHJmsEkKHPu9+26OgS51wOmSnWPyc5GfDty3DBHdZzlFdOJvz+BXx4Lxz3MOW5Chw60ki1kJWbz8lTNh2hRCkVMHzdw+3u6pWiV+d52uYR4HljzPGSDmCMmWWMSTDGJMTExFQwzHIqpc7zgcvOISo8lAdW/IytoBpdjFi3oZVoZx+DOQMh6aey7bd7Lbx0HswdBL8sh45XWsOmeTpHoeHWRDxgJbKLr7NGhMhIKv7clan9zkqF5X+Hp1rBrIvh4E64aDKERpTveULDT//zsecz+HIGvNTDirlofXfReJ0vRv1kKsxobb3e3CxoeKZVO661wkpVio7BrZQKNL7u4U4CnK/+ag6keNgmSURCgIbAEaye8KtEZAYQBRSISI4x5mXvh12KUuo8oyLq8H9DO3L34i0s+P4PbriglX/jLY8WPWHCJ7BgJLx5OYyeZ03wAafrjfdvgL7/gthOEHuOlajXj7OWdRx+elbBjle4P0c3fXz6eEf/gORNsON963FMB2h7iXUx56b5FauhdcS5eQHY7KPK9L4L+t5rxVoZnUdCo1awZoo1dN8Pb8CgJyCmvVOtus0au3nZzbD3c/jHd1A/xipDuuBO6/U172nV0Gelaa2wUpWUqrNMKqUCjBgfDv9mT6B3A5cAycCPwDXGmG1O29wOdDHG3Ga/aHKkMebqIs8zDThujHm2pOMlJCSYjRs3VvGrKANH0pT0Q2FdqjGG6+f+wOY/j/Hpv/rRtGE1+0NwPB3+d7V1oeD5E63Xt3mBVavuKBPpfbdV/10Wbs5RIWOs3uc9n8LOD+HPDdbshBjXBLRxGwitZw3VeP17Vs/z1iWw7ytrKD1TAEk/2st9xHXfaRmVORvFGQPbV8LHD1qj2Jw8dPpCS4d6MdYMhhdPsZL0kpR0flStISI/GWMS/B2HL1VFu71k434mLd3KF5Mu4szoAJv/QJVLZmYm6enp5OXl+TsUVcuFhoYSGxtLgwaey0dLarN92sNtr8meCKwBgoG5xphtIjId2GiMWQXMAd4WkT1YPdtjfRljlXBT5ykiPDa8MwOf/5KHV/3C63+rZn9D68fCVfPgm/9aFz7m5+JSDfS3ldD6wrI/X0m1sCLWyByxHWDXahBzumfaWbMeVm34qROnyzAO7YLdH8Opk9aII74iAp1GwIaZ1j8IxSqlgHt3l32CFK0VVqrC0rOs9kJLSqq3zMxM0tLSiI+PJzw8XMdUV35jjCE7O5vk5GSAEpNuT3w+Tp0xZjWwusiyqU73c4ASr54zxkzzSnBedmZ0Pe669GxmfLyLtdtSGdipqb9DKp/3/gl/fu/+wsezLvbOMa9603OJxVVzim/f/yHrBlYpyedPQuIi35VnjJ7vOV6djVApn0jNyKFheCh1Q4P9HYqqhPT0dOLj44mIKOd1NkpVMREhIiKC+Ph4UlJSKpRwawbgY7f0bUP7uEgeXrWN47n5pe8QSPwxCUhlJkmJbApXvOjbCVaq06QuStVQOiRgzZCXl0d4uM7mqgJHeHh4hcubNOH2sdDgIJ4Y2YXUzByeW7vL3+GUjz+TycqMg+yPMZR13Gal/CYtM4e46nadjHJLy0hUIKnM+zHApz6smc47sxHX/rUlb327jxHd4+naPMrfIZWPP2cIrExtsz/qorUWWymfS83I4Zymkf4OQymlCmkPt5/cP/gcmtQPY/Kyn8m3VdNZBavDDIFKqVol31bAoeO5WlKilAoomnD7SYO6oUy7shPbD2TS49FPaD35Q3o/tY6Vm5P9HZpSSlVbh46fosCgJSXK70Sk1Nvnn39e6eM0bdqUhx56qPIBK6/SkhI/ys2zESSQmWNdPJl8LJsHlv8MwPDu8f4MTSmlqiXHpDdxkZpwK//67rvvCu9nZ2fTv39/HnroIS6//PLC5R07dqz0cVavXk1sbGyln0d5lybcfvTs2t0Unek9O8/GM2t2acKtlFIVkGqf1r3aTS6mapxevXoV3j9+/DgAZ511lstyT3Jycqhbt2zv4R49elQsQOVTWlLiRynHssu1XCmlVMnSHD3cWsOtnKzcnEzvp9YFZPnmzJkzERE2bdpE3759CQ8P56WXXsIYw7333kvnzp2pV68eLVq04IYbbuDgwYMu+xctKRk7dix9+vRh9erVdOrUifr169OvXz927Sp5ZLTMzEz+8Y9/0K5dOyIiImjTpg133XVX4T8LDvn5+Tz66KO0bduWsLAwWrRowa233uqyzZIlS0hISCA8PJwmTZowdOjQwkljaitNuP2oWZT78UXrh4WQk2fzcTRKKVX9pWbmEBosRNfT8e+VZeXmZB5Y/jPJx7IxnC7fDKSkG2DMmDGMGjWK1atXM3DgQAoKCjhy5AgPPfQQq1ev5rnnnmP79u0MHDgQY9zMZuxkz549PPTQQ0ybNo0FCxawf/9+xo0bV+I+WVlZBAcH8+STT/LRRx/x8MMPs3r1aq699lqX7W688UYee+wxrrvuOj788ENmzJhBVtbpmZ1nz57N1VdfTceOHVmyZAlz5syhdevWHD58uOInpwbQkhI/mjSoPQ8s/5lsp+Q6WISs3HyGvPgVjw/vzAVtm/gxQqWUql7SMnKIjaxLUJCO31wTPfL+NranZJZrn81/HuNUkdHAsvNs3L90K+/88GeZn6djswY8fEWnch27PO677z7+/ve/uyybN29e4X2bzcZ5551H27Zt+fHHH+nZs6fH5zpy5AgbNmzgzDPPBKwSlXHjxrFv3z5atWrldp/4+Hhefvnlwse9e/emefPmDBgwgLS0NOLi4khMTGThwoW8/vrrLr3ajmQ+Ly+PKVOmMG7cOObPn1+4ftiwYWU/ETWU9nD70fDu8Tw5sgvxUeEIEB8VznNXn8vbE3pSYAzXzN7Ave8mcuSED6YkV0qpGiAtK4e4BmH+DkMFkKLJdmnL/cX5YkqHVatW0atXLxo2bEhISAht27YFYPfu3SU+V7t27QqTbTh9cWZSUlKJ+82dO5dzzz2XevXqERoayqWXXooxhl9//RWAdevWERQUxPXXX+92/19++YWDBw8yfvz4Eo9TG2kPt58N7x7v9gLJNXdfyEvrfuX1L/aybmcaD17ekVE94nXWLaWUKkFqRg7t4nTSm5qqIj3MvZ9aR7Kba6Pio8JZ/PfzqyKsKhEXF+fy+JtvvmHEiBGMHTuWBx98kJiYGPLy8rjwwgvJyckp8bmiolwn1KtTxyqxKmm/d955hwkTJnDHHXfw1FNPER0dze+//87YsWML9zt8+DCNGjXyeEGno2zkjDPOKPnF1kLawx2g6oYGM2nQOay+qy9tYupz35JErnljA3sPHi99Z6WUqqXSMnP1gknlYtKg9oSHBrssCw8NZtKg9n6KyL2iHWrLli2jZcuWLFy4kCuuuIJevXp5dfi/JUuW0K9fP/773/8yZMgQevbsWSxxj46O5ujRox4T9+joaAAOHDjgtTirK024A1y7uEiW/P18Hh/RmV9SMhj84lf897Nfyc3XiyqVUsrZ8dx8jufm65CAyoW78s0nR3YJ+OF3s7OzC3umHRYuXOjV44WFuZZjFT3eJZdcQkFBAQsWLHD7HF26dCE2Npa33nrLa3FWV1pSUg0EBQnX/vVMBnSMY/r72/nPJ7tZlZjCkM5NWb4pmZRj2TSLCmfSoPYB34AopZS3FI7BrT3cqghP5ZuBbMCAAcycOZNJkyYxePBgvvzySxYtWuTV402aNIkZM2bQvXt3Vq1axddff+2yTdeuXbn++uuZOHEiKSkp9O7dm8OHD7Nq1SoWLFhASEgITz31FDfddBMhISGMHj0aYwyffvopN910E127dmXt2rVcdtllfPPNN/z1r3/12usJNJpwVyOxkXV5+ZoejDovnXsWbeGldXsK1+kslUqp2k7H4FY1yciRI3n00Ud59dVXefXVV+nbty8rV66kUyfvjJRyxx138Mcff/Dss8+Sk5PDkCFDmD9/Pn379nXZbs6cObRp04Z58+bx2GOPERcX53LB5/jx44mIiOCpp57inXfeITIykgsuuIAmTaxR1woKCrDZbKUObVjTSE1+wQkJCWbjxo3+DsMrzn/yMw5kFK+hio8K55vJ/f0QkVKqqonIT8aYBH/H4UuVabeXb0riX+8msu7efrSJqV/FkSlf27FjBx06dPB3GEq5KOl9WVKbrTXc1VSqm2QbrJ7ujJN5Po5GKaX8LzVTp3VXSgUmTbirKU+zVAL0eXod/1m7i2MndfxupVTliMhgEdklIntEZLKb9WeKyGcislVEPheR5k7rWorIWhHZISLbRaSVN2NNy8ghsm4IEXW0WlIpFVh8nnCXofEOE5HF9vUbHA20iAwQkZ9E5Gf7z1pdN1HSMEd92zXhv+v20Ofp9TynibdSqoJEJBh4BRgCdATGiUjHIps9C8w3xnQFpgNPOq2bDzxjjOkA9ATSvRlvamaOXjCplApIPu0GcGq8BwBJwI8issoYs91pswnAUWNMWxEZCzwNjAEOAVcYY1JEpDOwBqi1Vwc6Lox8Zs0ut6OU7EzN5KXP9vDSuj3M+2YfN17Qigl9WtOoXp2SnlYppZz1BPYYY/YCiMgiYBjg3GZ3BO6x318PrLRv2xEIMcZ8AmCM8fokAqmZuVpOopQKSL7+3q0sjfcwYJr9/lLgZRERY8xmp222AXVFJMwYk+v9sANTScMcndO0Aa9c24M7U7P477pfeeXzPcz75ndu7N2Km/u04YvdBz0m60opZRcP7Hd6nAQUHccrERgFvAiMACJFJBpoBxwTkeVAa+BTYLIxptgkAiJyK3ArQMuWLSscbFpGDmfHNqnw/kop5S2+TrjL0ngXbmOMyReRDCAaq4fbYRSw2V2yXVUNd03Rvmkkr1zTg132xPvVz3/jjS/3UmAgv8AaoUaHFFRKeSBulhUd2uo+rI6RG4EvgWQgH+vvS1+gO/AnsBi4EZhT7AmNmQXMAmuUkooEaiswHDyeS1yDsNI3VkopH/N1DXdZGu8StxGRTlhlJn93dwBjzCxjTIIxJiEmJqbCgdY0jsR7zd0XEhwkhcm2Q3aejWfW7PJTdEqpAJUEtHB63BxIcd7AGJNijBlpjOkOPGhflmHfd7MxZq8xJh+r1KSHtwI9fDwXW4HRGm6lVEDydcJdauPtvI2IhAANgSP2x82BFcD1xpjfvB5tDdQuLpKcvAK365KPZTP7q738cfiEj6NSSgWoH4GzRaS1iNQBxgKrnDcQkSYi4vhb8gAw12nfRiLi6Pnoj2v5YJVK1UlvlFIBzNclJYWNN9bXjmOBa4psswq4AfgOuApYZ4wxIhIFfAg8YIz5xocx1zjNosJJPpZdbHlIkPDYhzt47MMdnB1bn0s7xjGgYxzdmkcRFGR98bByc7LWfitVS9jL+iZiXaQeDMw1xmwTkenARmPMKuAi4EkRMVglJbfb97WJyH3AZyIiwE/AG96KtXBad71oUikVgHzaw23/WtHReO8A3nU03iJypX2zOUC0iOwB/gU4hg6cCLQF/k9Etthvsb6Mv6bwNKTgs6PP5ctJFzN1aEdiIsOY9eVeRr76LT2f+JR/L93Kox9sZ/LyrSQfy8ZwuvZ75eZk/7wQpZTXGWNWG2PaGWPOMsY8bl821Z5sY4xZaow5277Nzc7X1hhjPjHGdDXGdDHG3GiM8doYpY5p3bWkRAWKoUOH0qVLF4/rJ06cSKNGjcjNLdvYD3v27EFE+PjjjwuXNW/enMmTi42w7GLLli2ICF9//XXZArebOXMmq1atKra8LMdUxfl8dgBjzGpgdZFlU53u5wCj3ez3GPCY1wOsBUobUvCmPq25qU9rMk7m8fnudD7Znsbqnw+QlZtf7Lkctd/ay62U8qe0zFyCg4To+nrRpAoM48aN47rrrmPbtm106tTJZZ3NZmPp0qWMHDmSsLCKv2fff/99mjTxzsg8M2fOJCEhgSuvvNJluTePWZPpdFy1VElDCjo0jAhlWLd4hnWL51R+Ae0e+sjtdsnHsvnpj6N0axFFcJC7a16VUsq7UjNziKkfpm2QChjDhg0jIiKJjxzdAAAaqklEQVSCRYsW8eijj7qsW79+PWlpaYwbN65Sx+jevXul9q8ux6wJdGp3VSZ1QoKIL2E6+VGvfctfHv+UexZvYVViis5uqZTyqbTMHOK0fluVRVYqfPAvmNnHq4epX78+Q4cOZfHixcXWLVq0iLi4OC6++GIAkpOTGT9+PK1btyY8PJx27drx8MMPk5eXV+Ix3JV3vPTSS7Ro0YJ69eoxbNgwUlNTi+33zDPPkJCQQIMGDYiLi2PYsGH89tvpsSj69OlDYmIic+bMQUQQERYsWODxmIsWLaJz586EhYXRsmVLpk6dis12esj92bNnIyJs27aNSy+9lHr16tGhQwfee++9Us5i6bE6LFu2jL/85S+Eh4fTpEkTLr/8cvbvPz0SdWJiIpdffjkNGzYkMjKSXr16sW7dulKPX1W0h1uV2aRB7Xlg+c9k553+EIWHBvN/QztQv24o63em8/mudFZsTiZI4LwzG3HxObH0PyeW9nGRvLclRS+4VEp5RWpGDm1i6vk7DBXIslLhixmwZSGYArB5v2No3LhxvPvuu/z000+cd955AOTl5bFixQquvfZagoOt66kOHjxIkyZNeOGFF4iKimLnzp088sgjHDp0iFdeeaXMx1u2bBl33nknt99+O1dccQXr16/nlltuKbZdUlISd955Jy1btiQjI4PXXnuNPn36sHv3biIjI5k1axbDhw+nQ4cOPPDAAwC0bdvW7TFXr17NuHHjGD9+PM8++yxbtmxh6tSpHDlyhJdffrnY+bj11lu5//77eeGFFxgzZgy///47Z5xxhsfXVFqsAG+++Sbjx4/n2muv5eGHH6agoIDPPvuMQ4cO0aJFC7Zt20bv3r3p2LEjr7/+Oo0bN2bjxo38+eefZT63laUJtyqz0mq/rzy3GbYCQ2LSMdbvTGfdznRmfLyLGR/vIio8hKxcGzaXyXa2ujyvJzoyilKqNKmZOVxwVrS/w1C+MO/y4ss6DYeet8Cpk7CwyGVg+acgNAySfgRjA5tTr7Hjuf5yE3QeBRlJsNzNNB8XTIT2Q8od6pAhQ4iKimLRokWFCfeaNWs4cuSISzlJt27d6NatW+Hj3r17Ex4ezm233caLL75ISEjZ0rXHH3+coUOHFia6gwYNIi0tjTfffNNluxdffLHwvs1mY8CAAcTExPD+++9zzTXX0LFjRyIiIoiJiaFXr14lHnPq1KlceumlzJ1rjQg6ePBgCgoKmDp1Kg8++KBLMn3fffdx/fXXF77mpk2b8uGHH3LzzTd7fP7SYrXZbEyePJnRo0cX9sIDLrXn06ZNo3Hjxnz55ZfUrWt9EzZw4MASX1dV04RblUtptd/BQUKPlo3o0bIR9w5sT1pmDut3pjNt1bbCZNshO6+Af727hTe+2kt0/TCa1K9Dk/phRNerQ3T9MKLr1+GXpAxeXr+H3Hxr7PDyzoqpybpSNd/JU/lk5eRrSYly79BOyM2i+Dx73hcWFsaIESN49913mTFjBiLC4sWLOfPMM10S2YKCAp5//nlmz57Nvn37yMnJKVyXlJREq1atSj3WqVOnSExM5J///KfL8pEjRxZLuL/99lumTp3K5s2bOXLkSOHy3bt3l+v15eXlsWXLFl599VWX5WPGjOHBBx/k+++/Z8SIEYXLnZPc2NhYmjRpQlJSUonHKC3W7du3k5aWxvjx4z0+x7p167j55psLk21/0IRbeVVcg7qM7dmyMEkuqsBY2xw+nstv6cc5dDy3MLn2JDvPxv1Lt/LB1hQa1A2lQXgoDeqGEFk3lAbhIYXLNv1xtMLJuibqSlUfaZnWsGo6JGAtMf5Dz+vqRBRfn5UGXzztvpSk6LYNm5f8/BUwbtw45s2bx3fffUePHj147733uP3227GGp7c899xzPPDAA0yZMoW+ffsSFRXF999/z5133umSfJckPT2dgoICYmNdR0wu+vj3339n0KBBXHDBBcyaNYszzjiDOnXqMGjQoDIfy/mYNpuNuLg4l+WOx84JMkBUVJTL4zp16pR4zLLEevjwYYASy1KOHj1a4npf0IRb+YSnyXbio8KZe+NfCh8bYzh5ysbh46c4eDyXUa996/b5TtkKSDmWw67cLDKz88nMycOUofMiO8/GpKWJLP0piYYRoUSFh9IwPJSoiFCiwuvQMCKUn5OO8cZXv1e4V10p5VuOSW90lknlVmQcDP0P9Pu358Tbi/r3709cXByLFi3iwIEDZGVlFRudZMmSJYwdO5bp06cXLtu6dWu5jhMbG0tQUBDp6ekuy4s+/uijj8jNzWXlypWEh1uDIZw6dYpjx46V63iOYwYHBxc7RlpaGgCNGzcu93OWN9boaKuU7MCBAy5lOc4aNWrEgQMHKhVLZWnCrXzC0wWXkwa1d9lORKgXFkK9sBBaRkcQX0KivvquvoWPCwoMJ07lk5mTT2Z2HpnZeYyZ9b3bWPJshpOn8knJyCbjZB4Z2XnkF5ScrWfn2Xhg+c/8dvA4LRpF0LxxOC0aRXBGw7qEBLsO9lPR3vHa1Ktem16r8r40ndZdlUXRxDvpB58cNjg4mNGjR7NkyRKSk5Pp0KEDXbt2ddkmOzu72HjcCxcuLNdx6tSpQ9euXXnvvfdcaqKXL19e7FjBwcEudeGLFi2ioMD12+XSep8BQkND6d69O0uWLHG5OPPdd98lODi41Prv0pQl1o4dO9K0aVPeeusthgxxX2d/ySWXsGjRIqZPn16pcc8rQxNu5ROlXXDpSVkT9aAgIbJuKJF1QwuHLywpWV/+z96Fj40xnDhl49jJUxw7mcfQl9zPxpWdZ+PVz39zqUUPCRKaRYXTwp6AZ+XksXZ7Gnm20xeH/nvZVlIysrmoXSwGgzFgDBQYg8H6+cWug7z2xW+cculVL9tFpVC5BNbXye/Kzckuv1P9BkFVVmqmTuuuysGRePvQuHHjePnll1mxYoVLL7bDgAEDeO2110hISKBNmzbMnz+fffv2lfs4U6ZM4eqrr2bixIlceeWVrF+/nk8//dRlm0suuYT777+f8ePHM378eH7++Weef/55GjRo4LLdOeecw/r161m7di2NGzemTZs2bnusH3nkES6//HJuvvlmRo8eTWJiItOmTeO2226rdBlHWWINDg7m6aef5oYbbqBOnTqMGTMGgM8++4y//e1vdO/enUceeYSePXvSr18/7rnnHqKjo9m0aRNxcXHccMMN2Gw2wsLCmD59OlOmTKlUzJ5owq18piyT7bjbB8qfqEP5etXrh4VQPyyE5o1KTtQ/n3QRqRk5/HnkJPuPnGT/0ZPsP5LNn0dO8umONA4dL/4VZW5+QeFoLeWRnVfAfUsSWb45mZb2hL5l4wha2G8Nw0OByiWwvk5+jTE8sXqHy+8EfDNjqfaq11ypGTmFn2GlAtH5559Pq1at2LdvH2PHji22/pFHHuHw4cNMmTIFEeGqq67i+eefZ/jw4eU6zujRo3nhhReYMWMGc+fOpX///rzxxhsuPb/dunVjzpw5TJ8+nWXLltG9e3eWLVtW7FhTp04lOTmZ0aNHk5mZydtvv811111X7JiXXXYZ//vf/3j88ceZP38+sbGx3H///UybNq1csbtT1livv/56IiIieOKJJ1i8eDGRkZGcf/75xMTEANChQwe++uorJk+ezIQJEwgKCqJTp0488cQTgPW3yWazFevlr0piylL4Wk0lJCSYjRs3+jsM5UcVSbKKJqFgJepPjuxS6r6tJ3/o8Tr4mdf1sCYQAIJEELF+IjB+3o8en7NLfEP2Hz3JsZOuEyA0DA+lReNw9qQfJyeveCMRFR7Kvwa2I89myLcVkF9gyLMVkG8z5BVYPxf98CcnTtmK7RsfFc43k/uX+FrLKj0rh2/2HOKrXw/x9a+HSM/K9bjtEyO6cFH7GJqVMMlSRazcnMzk5VtdzlN4aBBPjuwa0BfQishPxpgEnxwsQFSk3f7Hgp/YnZbFZ/de5J2glF/s2LGDDh06+DsMpVyU9L4sqc3W7gBVo/m6V72ki0MHd/b81VpJverv32HNhpaRncf+IydJOnrS3sNu9az/kpfp9jmPZecx9b1txZaLQGhQECHBwkk3yTZYPd1/f3sjnZs1pFN8Azo3a0hskfpYT4loTp6NH34/wle/HuSrXw+xMzULgEYRofRu24Svfz3Esezis6cFizBlhdW7fk7TSC5qH8vF7WPocWYjQu118mVNfjNO5rH9QCbbD2Sy40Am721JLizzcXAMSznzi99o4jQsZZPIMGt4yvp1+CU5g1fW7SFHL6ANaGmZOVq/rZQKaJpwK+VGRRJ1KHsZS0X2axgeSsP4hnSOb+iyb++n1rlN1ps2COP9O/oSGiyEBAcREiSEBgcRHCSl7hseGsTutOOs2ZZWuKxJ/TA6NWtA5/gGnMjN550f9ruM5HLfkkReXb+HfUdOciq/gDrBQSS0asT9g9vTt20MnZo1IChIPH6D8MSIznSOb8j6Xems33mQ2V/tZeYXvxFZN4QLz46hQXgIKzYlF0l+t3IwK5f4RuHsOJDJ9hQrwU7JyHGJu2iy7VBgoHmjCA4dz+WPIyc4lHWqWLlLUdl5Nh79YDt9z25CdH3/XHyjXKVl5vLX1pUbDUEppbxJE26lqlBFe8e9Uas+eUgHYiJLTgg97eson8nKyWPHgSy2pWSwLSWTX5Iz+GbPIbejuuQXGPYeOsENF7Si79lN6Nm6MRF1ijcxpb3Ws+MiufXCs8jKyeObPYdYv/Mg63eluy1Fyc4r4PHVOwBr0qWzYurxl9aN6XBGA/stktjIuh7/sYiPCmf2Da7f/p3IzefQ8VwOHT/lcVjKwydOcd5jn9KicTjnNo+iWwvr1jm+IXVDgwu307px71vxUxLJx7JZvjmZDb8f0XOslApIWsOtVA3gy1FKcvJsdPi/j93Wqgvw+1Nupl2uJGMMbR5Y7bE+/v2JfTg7rr5LsuusonX5nhL1JvXrcOuFbdiy/xiJ+zMKtwkOEs5pGkm3FlHYjGHFpmSXiZzKei2Ag9Zwl2zl5mQmL9ta+K0HlP8cq8ClNdwqEGkNt1K1WEVLYCqyb93QYI+16lV9saODiJRYH9+leUM3e51W1cNSPnR5R5d907NySNyfQeL+Y2zZf4xViSlk5eQXez5fjMZSmzyzZpdLsg16jmsaY4zLjIxK+VNlOqk14VZKlVtFa9X9eUxvXkAbG1mXAR3rMqCjNZ1xQYHhrCnue+RT3PzToCrG07nUc1wzhIaGkp2dTUREhL9DUQqwJuIJDQ2t0L6acCulyq0yNefV6ZiO45b3GEFBnnvkvfUtQG2k57hmi42NJTk5mfj4eMLDw7WnW/mNMYbs7GySk5OJi4ur0HNowq2UqpDKlLFUp2NWlD++Baht9BzXbI7ZBFNSUsjLKz6UqFK+FBoaSlxcXLEZOcvK5wm3iAwGXgSCgdnGmKeKrA8D5gPnAYeBMcaYffZ1DwATABtwpzFmjQ9DV0qpMvNXj3xtoue45mvQoEGFExylAolPE24RCQZeAQYAScCPIrLKGLPdabMJwFFjTFsRGQs8DYwRkY7AWKAT0Az4VETaGWNKHjRXKaX8pDr1yFdXeo6VUtVBkI+P1xPYY4zZa4w5BSwChhXZZhjwlv3+UuASsQq3hgGLjDG5xpjfgT3251NKKaWUUipg+Trhjgf2Oz1Osi9zu40xJh/IAKLLuC8icquIbBSRjQcPHqzC0JVSSimllCo/Xyfc7i4xLjpylqdtyrIvxphZxpgEY0xCTExMBUJUSimllFKq6vg64U4CWjg9bg6keNpGREKAhsCRMu6rlFJKKaVUQPF1wv0jcLaItBaROlgXQa4qss0q4Ab7/auAdcaa2mcVMFZEwkSkNXA28IOP4lZKKaWUUqpCpDLTVFbogCKXAS9gDQs41xjzuIhMBzYaY1aJSF3gbaA7Vs/2WGPMXvu+DwI3AfnA3caYj0o51kHgD++9mnJrAhzydxBFBFpMGk/pAi2mQIsHAi+misZzpjGmVtXGBVi7HWjvIwi8mAItHgi8mDSe0gVaTFXeZvs84a7NRGSjMSbB33E4C7SYNJ7SBVpMgRYPBF5MgRaPKptA/L0FWkyBFg8EXkwaT+kCLSZvxOPrkhKllFJKKaVqFU24lVJKKaWU8iJNuH1rlr8DcCPQYtJ4ShdoMQVaPBB4MQVaPKpsAvH3FmgxBVo8EHgxaTylC7SYqjwereFWSimllFLKi7SHWymllFJKKS/ShFsppZRSSikv0oS7iolICxFZLyI7RGSbiNzlZpuLRCRDRLbYb1O9HNM+EfnZfqyNbtaLiPxXRPaIyFYR6eHleNo7vfYtIpIpIncX2car50hE5opIuoj84rSssYh8IiK/2n828rDvDfZtfhWRG9xtU4UxPSMiO+2/lxUiEuVh3xJ/x1UYzzQRSXb6vVzmYd/BIrLL/p6aXBXxlBDTYqd49onIFg/7euMcuf28+/u9pMouENts+zEDpt0OhDbbfoyAare1za5wTLWzzTbG6K0Kb8AZQA/7/UhgN9CxyDYXAR/4MKZ9QJMS1l8GfAQI0AvY4MPYgoFUrMHifXaOgAuBHsAvTstmAJPt9ycDT7vZrzGw1/6zkf1+Iy/GNBAIsd9/2l1MZfkdV2E804D7yvA7/Q1oA9QBEot+BqoypiLrnwOm+vAcuf28+/u9pLfK/w6LbOPTNtt+zIBst/3VZtuPEVDttrbZFYupyPpa02ZrD3cVM8YcMMZsst/PAnYA8f6NqlTDgPnG8j0QJSJn+OjYlwC/GWN8OrOcMeZLrJlMnQ0D3rLffwsY7mbXQcAnxpgjxpijwCfAYG/FZIxZa4zJtz/8HmheFceqaDxl1BPYY4zZa4w5BSzCOrdejUlEBLgaeKcqjlXGeDx93v36XlJlV03bbPBfu+2XNhsCr93WNrtyMdW2NlsTbi8SkVZYU9RvcLP6fBFJFJGPRKSTl0MxwFoR+UlEbnWzPh7Y7/Q4Cd/9wRmL5w+bL88RQJwx5gBYH0og1s02/jxXN2H1aLlT2u+4Kk20f10618PXbv46R32BNGPMrx7We/UcFfm8B/p7SbkRQG02BG67HUhtNgT2Z03b7JLVqjZbE24vEZH6wDLgbmNMZpHVm7C+jjsXeAlY6eVwehtjegBDgNtF5MKi4brZx+vjRYpIHeBKYImb1b4+R2Xlr3P1IJAPLPSwSWm/46ryGnAW0A04gPV1YFF+OUfAOEruKfHaOSrl8+5xNzfLdJxWPwmwNhsCsN2upm02+OdcaZtdulrVZmvC7QUiEor1i1xojFledL0xJtMYc9x+fzUQKiJNvBWPMSbF/jMdWIH19ZGzJKCF0+PmQIq34nEyBNhkjEkrusLX58guzfGVrP1nupttfH6u7BdmDAWuNfZCsqLK8DuuEsaYNGOMzRhTALzh4Tj+OEchwEhgsadtvHWOPHzeA/K9pNwLtDbbfpxAbLcDrc2GAPysaZtdutrYZmvCXcXsNUlzgB3GmP942KapfTtEpCfW7+Gwl+KpJyKRjvtYF3T8UmSzVcD1YukFZDi+WvEyj//d+vIcOVkFOK46vgF4z802a4CBItLI/tXcQPsyrxCRwcC/gSuNMSc9bFOW33FVxeNcIzrCw3F+BM4Wkdb2HrGxWOfWmy4Fdhpjktyt9NY5KuHzHnDvJeVeoLXZ9mMEarsdaG02BNhnTdvsMqt9bbap4JWeevN4BWwfrK8YtgJb7LfLgNuA2+zbTAS2YV0J/D1wgRfjaWM/TqL9mA/alzvHI8ArWFcp/wwk+OA8RWA1xg2dlvnsHGH90TgA5GH91zoBiAY+A361/2xs3zYBmO20703AHvttvJdj2oNVM+Z4L820b9sMWF3S79hL8bxtf49sxWqgzigaj/3xZVhXf/9WVfF4ism+/E3He8dpW1+cI0+fd7++l/RWJb9Dv7TZ9uMFXLuNn9ts+zECqt32EI+22aXEZF/+JrWszdap3ZVSSimllPIiLSlRSimllFLKizThVkoppZRSyos04VZKKaWUUsqLNOFWSimllFLKizThVkoppZRSyos04Va1gohMExHj4XadH+IxIjLR18dVSqnqQNtsVdOE+DsApXwoAxjsZvkeXweilFKqVNpmqxpDE25Vm+QbY773dxBKKaXKRNtsVWNoSYlSgIi0sn9leI2IvC0iWSKSLiIPu9m2v4hsEJEcEUkTkVdFpH6RbaJF5HUROWDfbpeI3F3kqYJF5AkROWg/1isiEubVF6qUUjWAttmqutEeblWriEix97wxJt/p4TPAB8BVwIXAwyJyyBjzin3/jsDHwCfAKKAF8BTWNLSD7duEA58DscAjwE6grf3m7F5gHXAd0BV4EvgDmFH5V6qUUtWfttmqptCp3VWtICLTgGI9H3at7T9/Bz4xxgx02u8N4DKghTGmQEQWAecB5xhjbPZtrgYWAxcYY74Tkb8DrwE9jDFbPMRjgK+MMRc6LVsJNDXG9KrES1VKqWpP22xV02hJiapNMoC/uLmlOG2zosg+y4FmQHP7457ACkfDbbcMyAf62B/3BzZ7aridrC3yeLvTcZRSqrbTNlvVGFpSomqTfGPMRncrRMRxN73IKsfjM4A/7T/TnDcwxthE5DDQ2L4oGjhQhniOFXl8Cqhbhv2UUqo20DZb1Rjaw62Uq1gPjw84/XTZRkSCsRrsI/ZFh7EaeaWUUt6lbbaqFjThVsrViCKPR2I12En2xxuAEfYG23mbEOBr++PPgO4i0tWbgSqllNI2W1UPWlKiapMQEXF3cct+p/udROR1rBq/C4EJwF3GmAL7+seAzcBKEXkNq37vaWCNMeY7+zbzgduBtfYLf3ZhXeTTzhgzuYpfk1JK1VTaZqsaQxNuVZs0BL5zs/z/gAX2+/cDQ7Ea7xzgUeBlx4bGmG0iMgR4AuvinEzgHft+jm1yRKQ/1tBT04EGwD7g1ap9OUopVaNpm61qDB0WUCmsSRSwhpi6whjzgX+jUUopVRJts1V1ozXcSimllFJKeZEm3EoppZRSSnmRlpQopZRSSinlRdrDrZRSSimllBdpwq2UUkoppZQXacKtlFJKKaWUF2nCrZRSSimllBdpwq2UUkoppZQX/T9aLkzxQ/RyZwAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "hist = history.history\n", - "x_arr = np.arange(len(hist['loss'])) + 1\n", - "\n", - "fig = plt.figure(figsize=(12, 4))\n", - "ax = fig.add_subplot(1, 2, 1)\n", - "ax.plot(x_arr, hist['loss'], '-o', label='Train loss')\n", - "ax.plot(x_arr, hist['val_loss'], '--<', label='Validation loss')\n", - "ax.set_xlabel('Epoch', size=15)\n", - "ax.set_ylabel('Loss', size=15)\n", - "ax.legend(fontsize=15)\n", - "ax = fig.add_subplot(1, 2, 2)\n", - "ax.plot(x_arr, hist['accuracy'], '-o', label='Train acc.')\n", - "ax.plot(x_arr, hist['val_accuracy'], '--<', label='Validation acc.')\n", - "ax.legend(fontsize=15)\n", - "ax.set_xlabel('Epoch', size=15)\n", - "ax.set_ylabel('Accuracy', size=15)\n", - "\n", - "#plt.savefig('figs/B13208-15_12.png', dpi=300)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "500/500 [==============================] - 3s 5ms/step - loss: 0.0428 - accuracy: 0.9927\n", - "\n", - "Test Acc. 99.27%\n" - ] - } - ], - "source": [ - "test_results = model.evaluate(mnist_test.batch(20))\n", - "print('\\nTest Acc. {:.2f}%'.format(test_results[1]*100))" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "TensorShape([12, 10])\n", - "tf.Tensor([6 2 3 7 2 2 3 4 7 6 6 9], shape=(12,), dtype=int64)\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAq8AAADoCAYAAADBow1gAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de7zNVf748fchd4mcI6Ickmshl5HGtQvVkGiiaca1XAeZCt+QESW6YCgRTg0zk5lEUm6TLm4Rp9I0IhmHJHHcI0Ln98fn12qt1fls+3r2Xvu8no9HD+9lffZnr87H/ux1Pp/3571ScnJyBAAAAHBBgXgPAAAAAAgWk1cAAAA4g8krAAAAnMHkFQAAAM5g8goAAABnMHkFAACAMy4KZePU1NSc9PT0GA0FgWRlZUl2dnZKNPbFcYyvzMzM7JycnLRo7ItjGV8cy+TBsUweHMvkEGjeE9LkNT09XTZv3hydUSEkjRo1itq+OI7xlZKSsjta++JYxhfHMnlwLJMHxzI5BJr3kDYAAAAAZzB5BQAAgDOYvAIAAMAZTF4BAADgDCavAAAAcEZI1QZc9+WXX6p44MCBRl+hQoVUvGTJkjwbEwAAQLhq1apltLdt25brdqNHjzbaw4YNU3GJEiWiP7AY4sorAAAAnMHkFQAAAM5I+rSBU6dOqbhXr14qXrNmjbHdqFGj8mxMAAAA0TBgwACjPXjw4Fy3Gzt2rNF+9tlnVbx48WKj76abborS6GKDK68AAABwBpNXAAAAOIPJKwAAAJyR9DmvQ4YMUbGe51qjRg1juw4dOuTZmBC6zz//XMX9+/dX8fvvv29sV6VKFRW/8sorRl+TJk1iNLrkpf98P/30U6NPPyYzZszw3UdOTo7RTklJUXG/fv1UbJd7qVu3ropbtmwZ5IgBIH+ZOnVqWK87efKkijt16mT0vf322ypu3LhxeAOLIa68AgAAwBlMXgEAAOCMpEsbmD17ttGeNWtWrtvZJSMaNWoUszEhdB988IHR/s1vfqPiI0eO+L5u165dKu7WrZvR984776i4YsWKkQ4xaRw8eNBo9+7dW8WrV69W8bFjx3z3oacCXIi+7cyZM323K1WqlIpbtWpl9L344osqTktLC/q9ASDZ3H///Ub7f//7n4pvueUWFY8bN87YTk8FO378uNG3ceNGFZM2AAAAAESAySsAAACcweQVAAAAzkiKnNcffvhBxU8//bTvdtddd52KKY2VePScG7tsx+nTp1XctWtXFT/00EPGdm+88YaKR48ebfTdfPPNKt60aZPRV7JkyTBGnBy2bt1qtN99910VFy9eXMXVq1c3ths5cqSKA+WdBiqVpZfY2rZtm7GdntusH1cRkX379qm4bNmyRt/SpUt9xwLkJytXrjTabdq0UXFGRobRpy+lrpcjLFiwYIxGh2gZPnx4UNvdfvvtRltfAnbDhg1Gn35+t58R6dixY6hDjDquvAIAAMAZTF4BAADgjKRIG1i0aJGKv/jiC9/txowZo+IiRYrEckgIw4QJE1S8f/9+o+/vf/+7iu+9917ffeirMunlQkRE5s6dq2L7ltngwYNDG2wSsVevmjdvnorT09NVrP9so6Vt27a+fVu2bPHdbvPmzb6vW7BggYrvuusuFYdSzgtINOfPn1fxnj17VHzgwAFjuz/96U8q3rlzp9F31VVXqfijjz4y+vT0O/18a39XvvrqqyqmTJ1b9DQwEZG33npLxfXq1TP69u7dq+LJkycbfaQNAAAAACFg8goAAABnOJk2cOjQIaPds2dP3231W9Ht27eP2ZgQOf2JSb0yhIjIHXfcEdQ+9FvDL730ktGnP80+adIko09fjat06dJBvVeyCvZnHWv6bSw7jUR/anbFihVGX+fOnVWsr7jXq1evaA8xISxevFjFlSpVMvoaNmyY62vWrl1rtPUnzW3VqlVTcdWqVcMZIqJAr8ai3/4Phb2anh99dSWbvtqhncI1ZMiQsMaF+Lj00ktVPGjQIKNP/z7++OOPjT69bX9X5xWuvAIAAMAZTF4BAADgDCavAAAAcIaTOa9Tpkwx2t9//72KixYtavTpea6Uyklsl1xyiYrvvvvuqO///vvvzzUWMfOo83vOayKyc/XsvHc/derUicVw4urqq6822npJuAIFzOsRJUqUyHUf9nb6KmjHjh0z+goVKuT7Op2+AlvNmjWNPj1H2S7Nppdjw89q1apltO2V6n5il7KqXbu2iu3j9eOPPwbVd/ToURXv2rXL2E4vU6eXsxMROXv2rIqHDh2a63iRmC677DLfPj03VkSkTJkysR7OBXHlFQAAAM5g8goAAABnOJM2cOTIERXPnDnTdzv7Vot+CwX5W+XKleM9BFj0VX12795t9K1evVrFL774otEXaIWtpUuXqvjaa6+NdIgJ55///KfR1lfCsW/nlStXTsX6bWI79UC/Zbxu3Trf9165cqXR1ss36X16+S4Rc1Ume5UfvZzZPffc4/ve+Y1e2k/ETHtr1qyZirt3725sd99990X83jt27FDxwoULjb5HHnlExXqagEjwpbiQePTUHhHzHKH/exARef3111Ucr/JoXHkFAACAM5i8AgAAwBlMXgEAAOAMZ3JeT58+reJAeTUZGRlRf2/9/fQcL5Hwl+lD3tuwYYOK7XJYxYoVy+vhOG/r1q0qtnNSdXaJHz13Ty/JM2/ePN/X2WXu9FzOkSNHGn233nproGE7r0GDBgHb4dDzYZs3b+67XaA+3TPPPGO0hw0bpmJ7Kdqvv/46qH3mNzfeeKPR1n+GTZs2VXGpUqWi/t56vuPDDz9s9OmlswJ97uGWtLQ0o12/fn0V2zmven47Oa8AAADABTB5BQAAgDOcSRtYsGCBb59+y8tebSRYesmPvn37Gn36JXI7bWDRokUqvummm8J6b+SNTZs2qbhGjRpG3+WXX57Xw3GCfuu+c+fORt9rr70W8j5Egl/prm7duiru16+f0We3kVjOnz/v22cf/2ikPSSjVatWxXsIImKWsxMhVSBZ6eXyRH5ZBk1XuHDhWA/ngrjyCgAAAGcweQUAAIAzmLwCAADAGQmb83r48GGj/eSTT/pu27BhQxXby8P6sfM5unbtqmJ7+cVA9Nd99tlnKr700kuD3gdiQz8eIiLLli1Tcc+ePfN6OM677bbbjPaKFStUfPLkyaD3E2zOa3p6uoobNWoU9P4Rf4HKX1WvXt1ot27dOtbDAXAB06dPN9r6ErDVqlULuG08cOUVAAAAzmDyCgAAAGckbNrAtm3bjPY333zju629WlIw7BVgAqUK6PvXVwSyx7V9+3YV6yugIO/oZZkmTJhg9FWsWFHFo0ePzrMxuUy/xd+rVy+jT7+V9P333/vuY/DgwUZbL223b98+FX/33XfGdm+88YaKN27caPS1bdtWxfYKW/rqQMg7+spZ//73v323C+d8DeQH9jzn888/V/Hf/vY3o08/H1922WW++9RXAbXnL3pJu3fffdd3H+fOnfN9Xbxw5RUAAADOYPIKAAAAZyRs2kAo7rzzzqC2mzhxoorHjh3ru13NmjWN9vjx41XcqVMn39fpt0MRGfvpdf22sb5S1pIlS4ztrrzyShW/8sorRp9+/PUUAoSnRYsWQW2np9PY9NSAKVOmGH3vvfeeig8cOGD0zZs3L9dYJDFuaeUXp0+fVvHs2bNVHOiY2ykmffr0Cfl9y5cvb7QDnc8RGf1ziOjQ0yL1FLYPPvjA2G7v3r2++3jppZeCeq+OHTuqeMeOHUaf/vn98ssvffdxxRVXBGzHA7MtAAAAOIPJKwAAAJzB5BUAAADOSNicV7ucSrFixVRs50zpOTkDBgww+vbv36/iMWPGqFjP9RARadCggYpXrlxp9D366KO+49RXh2nSpInvdvDopazs0hwLFixQ8dKlS42+3bt3B7X/devW+fbp+T56WR8RkeLFiwe1f0TXHXfckWssIvL++++rePLkyUafnitra9WqlYoXL16s4ksuuSTcYSa9//3vfyp+4oknjL7s7GwV2+dG+zwajJ07dxrt2rVr5zoOEZG6devmug+/v0dgx48fV7G9ymTZsmVVrH++AuUk2ysv9e3bN9IhJiU7D18v77dw4cKYvveiRYsi3of+70bEXL3ymmuuiXj/4eDKKwAAAJzB5BUAAADOSNi0Af1WkohInTp1VLx582ajT78svmLFCqPvL3/5i4oD3eL69a9/reJhw4YZfRkZGb6vC9SHX9LLe9x3331Bv87v1oR++0LELFd2+eWXG316KZ9jx44Zff/617+CHgvyRsuWLVXcuHFjo09PAdLTTUREVq9ereLf//73KrZLy6SlpUVjmEnh4osvVrFdRk5vd+3a1ejTS9OtXbtWxQ8//LDvew0fPtxojxo1KrTBImjffvut0b7nnntUrK9uJyLSu3dvFeu3te30At3MmTONtr6aE35m/5z8UgXs1KYbbrghqP3r5zyRX5aajNSWLVuMtr66l/7/dt1110X1fQPhyisAAACcweQVAAAAzmDyCgAAAGckbM6rbc6cOSquV6+e0Xfu3DkV33rrrWHtf9q0aUFt16NHD6Nt51XC9MUXXxhtfSk8m57Xaucn6iVZ7r77bhXbOa8PPvigiu2SP3q5pUDL7iF3R48eNdp6ubFYfw7sUmZPPfWUinv27Gn06fnxy5YtU/FHH31kbNe2bdtoDtFpev5vuEutFilSJKjt9FxmxNbXX39ttPXyczb7WY9g9OvXz2h36dJFxXp+bbTYz8K4IthzTZkyZYy2Pd/wU7RoUaMdTnmsX/3qV0Zbz7/997//bfTpS7TfeOONKh48eLCx3WOPPRbyOILFlVcAAAA4g8krAAAAnOFM2sDVV1+t4nHjxhl9+q1ofQWncJUoUcJoT5gwQcUDBw6MeP/5iV5+RcS8jXXzzTcbffpqSPaKJPotLb0kj32LUy/DU7hwYd8+BEcvQ/X8888bfSVLllTxkiVL8mxMtlq1asXtvfGzzMzMeA8BFjud5/HHH1fxxIkTjb4TJ06EvH991UJ7/3ocLT/++GPU95kXqlSpYrS7deum4rlz56o4KyvL2E5Pw4gF/Tv4tddeM/r0NKD169cbffqcSF91b8qUKcZ2F1308xQz0Eql4eDKKwAAAJzB5BUAAADOYPIKAAAAZziT81qsWDEV28sJ1qxZU8V2eaTt27er+Pvvvw/qvfR8FBHyXCNhL8mbmpqq4ueee87o0/OZOnfubPQtX75cxXfccYeKo51HA9OaNWtUbJfZ0UtS6XlbIr/8DEWbnh+9Z88eo0/Pe69cubKK9aVMEX32cqN+9u/fH+OR4Cfly5c32iNGjFCxvdSv/pyBXgpp6tSpYb23vUTpJ598ouJGjRqpOFCJtQEDBoT13olGX7ZcROS2225TsX3ujLb+/fur2H7ORC8tapcj1LVu3dpo68vW6uXSXn75ZWM7vVSh/cyJ/UxKqLjyCgAAAGcweQUAAIAznEkbCOS3v/1trrGIyJtvvqnioUOHqnjnzp3Gdv/3f/+nYm5FR0YvuXL48GGj75ZbbvHtGzRokIrtFT06dOig4unTp0dlnLiwlJSUXGMRka1bt6q4b9++Rp+evtOsWTOjT/+MLly4UMWdOnUytnvggQd83/ubb75RsX17Ut9WX7WNklqJwb6Vjfi44oorfPvS09NVrK9oGAr7/D5//nwV6ykLF198cVj7d1mbNm1UrN/Wf+GFF3xfY6diVaxY0XfbIUOGqFhfPc8+j4ZLT/WYNWuWivVUMhHze+Ds2bNGH2kDAAAAyDeYvAIAAMAZSZE2EEi7du1yjRE7+m2gSy+91Oh75ZVXco1FRAoVKqRie2WRp556SsX2qjGIHT0dwH4a9W9/+5uK7VtCX375pYrtVXjsJ1J/kpGRYbT1qgGBbneVK1fOaLdv317FdjoD4qNo0aIq1iuOIHnZ5/5kqRwQDfrPRk+DczElTl9F6+GHHzb67HY0ceUVAAAAzmDyCgAAAGcweQUAAIAzkj7nNVGdOyfyzDMic+aI7NkjkpYmcvfdIpMnx3tk0dW2bVujra+GVKNGDaNvwoQJKr7++utjO7AoefVVkXnzRDIzRY4dE6lRQ+Thh0V+97t4jyw69PJS48ePN/r0fNJjx44ZfatXr/bd5+OPP67igwcPqtguyWOX2PLTokULo123bt2gXmdbsEBk0iSR7dtFTp4UqVxZpGtXkWHDRCKs6pIv6M8U6PnKIiIdO3ZUcV6ULGvVSsRaEE5Zv16kadOYDwFRkOzn1/wk2seSyWuc9OwpsmqVyJ//LFKzpshXX4loZTPhiEmTRKpU8X7pSE0VWbpU5N57RbKzRbSytXDAoUMirVuLDB0qUrq0yIcfiowZI7J/v4i1kjES3PTpIsePm383erTIxx+LNG4cnzEhdJxfk0e0jyWT1zhYvlxk/nyRLVtEateO92gQiSVLvA/iT268UWTfPu+DysnVLXZhgtatvQnQ88+LTJsmEqX63sgD9nn1hx9ENm8W6dJF5CK+9ZzB+TV5RPtY8jGOg4wM78Dlh4nr2LFjA7Zdl1vVn+uuE1m8OO/HktcqV67s2xfo1v3AgQNjMZyYKFvWm/jgwurXr59rnAiWLxc5coTbza7Jz+fXZBPtY8kDW3GwcaNI9eoiAweKlColUry4SKdO3m8hcN/69fnjF5Nkdf68yKlTImvXikydKtK/P1ddXTd/vkjFiiLNm8d7JIgU59fkEcmx5MprHOzfL/LyyyL16nkn1RMnvIdCOnYU2bCBL0qXrVrl/SZp1duHQ0qUEDlzxou7dRN5+un4jgeROXXKu2XZpw/nVtdxfk0ekR5LJq9xkJPj/bd4sXdbUkSkQgWRli1F3nlH5Kab4js+hCcry0tA79BBpEePeI8G4Vq/3pvwfPihyNix3h0SBxe+wf+3ZInId9+RMuA6zq/JIxrHkslrHJQpI1K16s8TVxGRZs28cjxbtzJ5ddHhwyK33SZy5ZUi2qqpcFCDBt6fzZp5eVrdu4s89JDIVVfFd1wIz/z5ItWqiTRqFO+RIFycX5NHtI4lOa9x4FfmMCdHpABHxDmnTom0a+c92PPWW95tZySHnyayu3bFdxwIz7FjIsuWcdXVZZxfk0c0jyVTpTho107k00+9+mY/Wb1a5OxZLw8W7jh3zltcYscO70uyXLl4jwjRtG6d92eVKvEdB8KzaJGXv8zk1U2cX5NHtI8laQNx0KeP9xRz+/YiI0Z4D2wNHy5y883erUq4Y8AAr9jyX/7i3Q7ZsOHnvuuuEylSJH5jQ2huvdX7DNapI1KwoDdxffZZrzYoKQNumj/fuyCQB4t6IQY4vyaPaB9LJq9xUKqU92DW4MEi99zj5bp26JB8S8PmBytXen8+8MAv+3btEklPz9PhIAKNG3tVQLKyvEL2VauKPPmkSL9+8R4ZwpGd7T3RPG5cvEeCcHF+TR7RPpZMXuOkWjXvtxC4LSsr3iNAtIwbx0QnmaSmeqlYcBfn1+QR7WNJzisAAACcweQVAAAAzmDyCgAAAGcweQUAAIAzUnJycoLfOCXloIjsjt1wEEDlnJyctGjsiOMYdxzL5MGxTB4cy+TBsUwOvscxpMkrAAAAEE+kDQAAAMAZTF4BAADgDCavAAAAcAaTVwAAADiDySsAAACcweQVAAAAzmDyCgAAAGcweQUAAIAzmLwCAADAGUxeAQAA4AwmrwAAAHAGk1cAAAA4g8krAAAAnMHkFQAAAM5g8goAAABnMHkFAACAM5i8AgAAwBlMXgEAAOAMJq8AAABwBpNXAAAAOIPJKwAAAJxxUSgbp6am5qSnp8doKAgkKytLsrOzU6KxL45jfGVmZmbn5OSkRWNfHMv44lgmD45l8uBYJodA856QJq/p6emyefPm6IwKIWnUqFHU9sVxjK+UlJTd0doXxzK+OJbJg2OZPDiWySHQvIe0AQAAADiDySsAAACcweQVAAAAzmDyCgAAAGcweQUAAIAzmLwCAADAGUxeAQAA4IyQ6rwCABCJDz74wGh37txZxXYx+CFDhqj4rrvuium4ALiDK68AAABwBpNXAAAAOIPJKwAAAJxBzisAIKZmzJih4kceecToO3bsmIorVKhg9NWrVy+2A0NEDh48aLRbtGih4u3btxt9OTk5Kh41apSKx40bF6PRIZlx5RUAAADOYPIKAAAAZziTNpCSkpJrfCG9e/dWcdOmTX23q1+/fq4xouvUqVMqLly4sNF30UWR/3M8c+aMigcNGmT0zZ49W8VvvPGG0deuXbuI3xuAZ8KECUb7scceU3GRIkWMvhdeeEHFXbt2NfqKFy8eg9EhEnqqwO2332706akCgb6nx48fr+KsrCyjb968eRGOEH52795ttGfNmqXirVu3Gn2vv/66ilNTU42+Tp06qVhP+0hLS4vKOIPBlVcAAAA4g8krAAAAnMHkFQAAAM5I2JxXO2dKz58pWLBg0PvJyMhQsZ7fYe/jyiuvVHHlypV99/frX//aaI8cOVLFRYsWDXpc+ZWed/r2228bfXYeajjeeustFc+ZM8fo0/8N/fGPfzT6yHkFQnfu3DkVP/jggyqeNm2asZ1e8mru3LlGX926dWM0OkTD6tWrjXbfvn1VHKgclv7dKCLy3//+V8V6PuWKFSuM7fbs2aNi/XsZ4Vm4cKGK+/fvb/RlZ2er2M5r1Y/zgQMHjD59LtWgQQMV9+nTJ7LBhoArrwAAAHAGk1cAAAA4I2HTBmrVqpWn76ffqtBj23vvvWe0b7nlFhXrq4vAo9+yEBEZPXq0ik+cOJHXw1H0kl0iInv37lVxpUqV8no4gJO6deum4vnz56u4fPnyxnbLly/37UPi0cth6aleIoHLYempAvZKajq9JNqiRYuMPv1WNmkD/k6ePKli+2e4du3aXPv0W/wiIh07dlRxoFv+mZmZRlvfZ/PmzYMccXRx5RUAAADOYPIKAAAAZyRs2kCHDh2M9q5du4J6nf3048SJE1X8zTffqHjHjh0RjO5n+hPtpA149HQAffUNuy8W9NsiFSpUMPr042+v8lO2bNmYjssl+i2hVatWqXjZsmXGdnrljVDSfFq2bKni999/P6jXtGrVymjr6TsrV640+qZOnaria6+9Nuhx4cLsfwMLFixQccmSJVX8n//8x9jOfpIZiU1/0lyvDCBirnpmr4al34YOpG3btiq20/RIFQiOfp7u3r270ad/3kaMGKHiBx54IKz3sj+/5cqVC2s/0cSVVwAAADiDySsAAACcweQVAAAAzkjYnFdboFWvAm3Xpk0bFb/88ssq7t27d1TGdd9990VlP8lk8eLFKt6yZYvvdnrZrGhJT09X8RVXXGH07du3T8UFCpi/txUrVizqY3HVhg0bVDx9+nTf7YLNQ7cVLlxYxT/88ENQr7FzlM+cOeO77Z133qninTt3hjg62PTjbK/Qc/78eRX/4x//UDE5ru55/PHHVaznudrlsDp16qTiYHNcbXqOvJ1Hzb+d3NkrnT3xxBMq1lc2ExEZNGiQisPNc9XZ86r9+/dHvM9IceUVAAAAzmDyCgAAAGc4kzYQLPtydufOnVW8Zs0aFdu3QoJll6S47LLLwtpPMtNvP9k/56ZNm6p4+PDhMR2H/d56O9zjnx+88cYbMd2/fss/2OMQKE3Advbs2ZDHBH8vvviiiu2yRvfff7+K27Vrl2djQuRGjRpltMePH69i/Ta0XQJy7ty5Eb93vFZlco2eKvDQQw8ZfYcPH1bx5MmTjb5opAp8/vnnKtb/bYiIbNu2TcWbNm2K+L3CwZVXAAAAOIPJKwAAAJzB5BUAAADOcDLn9dy5c0b7ueeeU7Gdj6MvU6jn1xUsWNDYrnTp0iouVaqU0afndT3yyCNhjDh/+eKLL1Rs5zRWqlRJxZSnSgz28o9ZWVlBva5JkyYqtpfiDUTPn9LLcoWSa7t+/XoVHzhwIOjX4cKWL19utKdNm+a7bfny5VWs58Hp51N7O8RPoDxG/Vydlpam4kmTJsV+YMiVvpS2/V06Y8YMFffp0yes/R88eFDF+pLAIub3gl2KKxGeGeHKKwAAAJzB5BUAAADOcDJtYO/evUZ76NChQb2ufv36KtZX4RExL8/rMYIza9aseA8BYTp+/LjR9itL1aFDB6OdkZGh4jJlyoT13jVr1lRxjx49gn7dLbfcouJVq1aF9d7I3W233Wa0A90i1Mvi6bG9Ilq1atVU3LBhQ6Nv9uzZKr7oIie/khLWyZMnjfbIkSNVbN8K1ukrUJYtWzb6A0Ou9LQOkcDlHfWVzoJlr9Kll9/KzMz0fW9b7dq1Q37vaOPKKwAAAJzB5BUAAADOYPIKAAAAZ+SrBKOpU6equFmzZnEcSfI5ceKEigPlUg0ZMiSm41i5cqWK9TJMtkBjzG/mzJkT1HZ2nni4ea7h+OijjwK2EZkVK1b49umfFTsfVl+GUi9fpi8pKyLy2Wef5RqLiKSnp6t4zJgxQY0XwdHLl4mILF68WMWBls/Wy2jpOckiIgMHDlSxvcQsIqMvYS8S+Hsq0M9+5syZKtaPa6CSV4Heq23btkZ73rx5vtvmFa68AgAAwBlMXgEAAOCMfJU20KtXLxXbt0CfeuqpvB5OUvEr6WHfmlq6dKmKr7/++qiPQ78FHqjURyKsEJIoChQwf4fVy1ddffXVKtZv7+a1w4cPG+0jR474bmvf4sKFBSoPqJe9mjhxotF37bXXqlj/uffr18/YTi+lp5fUEjFvUd97771GX/Xq1QMNG7nQyyHZxzXQbWL9c6+nd9llEPX0u5IlSxp9sU4LS3Z2+Sv9c2R/Z+mpOYFSQPSyVjVq1DC2s1dX1N11110qfvXVVwMNOy648goAAABnMHkFAACAM5xMG7BvVVxzzTUq3rJli+/rduzYoeJnnnnG6NPbdt8dd9yhYn2lmPzsiy++MNr79+8P6nUzZsxQ8dtvv230/elPf/J9XaVKlVTctGnTXGMRkZ07dwY1Dvzs3XffjfcQLmjBggVBbxvoyXnkrmjRor59derUUbGeJhBIhQoVjPbo0aNVfPToUaNvypQpKj537lxQ+4c/vcJAoNvJepqAiKeEns8AAAueSURBVMjmzZtVXLx4cd/967ey9RWaREgbiFRqaqrR/vbbb1W8aNGioPfTvHlzFdeqVUvFdlqCnjpiH3M7hSfRcOUVAAAAzmDyCgAAAGcweQUAAIAznMx5tfNC9FwQfYUlEbO0y65du1RcsGBB3/0PHz7caGdkZKj4zTffNPrKly+v4kB5Y8nGLmGj/5yrVq2q4gEDBhjbHTp0KNdYRKRLly4qtnO19HycSy+9VMVfffWVsV2wJbAaNGgQ1HaIHz1/XV8Z6EL0HHUE55NPPvHt08ulRUOpUqWiuj+Y9NJWgVZNqly5stEOlOeqC7TPgwcPqjgtLS2o/cGf/jPs06dPWPvQS9PZpbH070s7H7Zjx45hvV9e4corAAAAnMHkFQAAAM5wMm3Apq/8Y19a19tjx45V8V//+ldjuz179vjuf/v27SrWb4mLiLz33nsqbtGiRVDjTXZ9+/ZVccWKFY0+fYWt2bNnG31nz5713efJkydzjQPdwgpELyWCxKSXVdNLxtjKlStntO3VnXBhgX6+0WavlobI6LfqRUSys7NVHKhU1ogRI8J6v0ArKOopfOHe5kbkli9frmK9TJ39fTl37lwV/+EPf4j9wKKIK68AAABwBpNXAAAAOIPJKwAAAJyRFDmvwdJzP/S8TBGz1NO0adOC3ufQoUNVvHHjxghGl5zatWvn27799tuNPj13a86cOUHt/8cffzTaX375pYrtUly6YEtqIX70fPJAuc16bqyIuVw0gmPnTerCzSvXffrppyq2z696/nnt2rUjfq/8Zvfu3b7tQMcu3Lz/aPx7QGx1795dxfp3nV2+zOXndLjyCgAAAGcweQUAAIAz8lXagO6ll14y2votylDoZUkQGjulQNezZ8+w9qmv0rVgwQLf7aK9ahAit23bNqN99OhRFdtpHo0aNVJxoH9HCE6rVq18+86dOxfy/s6fP2+0n3/+eRXbx7Jz584h7x8/C1QO60LbBsP+XAYqlYW8o5eMfPLJJ42+AwcOqFg/RvY858orr4zN4PIAV14BAADgDCavAAAAcAaTVwAAADjDyZzX7777zmgHym3MyMhQ8Zo1a1Qcbq6OXSZE3z/cQZ5k4pk5c6bRDrRkqZ7zetFFTp7GEkqlSpVUXLp0aaNvyZIlKn7uueeMvv79+6v4xIkTKraX6P3Xv/6l4jvvvNN3HwidnbfYoEEDFW/evDmsfU6ePFnFU6ZMMfr070C79BJLwuadrl27qnjx4sVGn75k9rx581Rcq1at2A8sj3DlFQAAAM5g8goAAABnJNT9tr1796rYvoW4du1aFZ85c8bo27RpU1D711MFChYs6Ltd0aJFjbZ+Cd5OE6hXr15Q7428oa8a8+qrr/puN3bsWKOtr76GvLNz504Vz50713e7G264wWg//fTTMRtTfrdw4UKj3bZtWxUPHjzY6Bs1apSK9fJYdmpXkyZNVPzCCy8YfYHOxbgw+9Z97969VZyZmWn06d+B9nHWbylPmDAh19fYRowYEdpgETY9TUBEZNGiRSq2j5GeOnLFFVfEdmBxwpVXAAAAOIPJKwAAAJyRUGkDLVq0UPFXX31l9Om3pKJxm8l+onbQoEEqTk9PN/q6desW8fshbzRs2FDFgW53LV261GiTNhAfy5YtU/GRI0d8tytevLjRLlGiRMzGlN+1bt3aaK9YsULFdrrG6dOnVayvxGXf4uzRo4eKCxUqFI1hwof+xP+ePXuMvvHjx6v4rrvuMvr086VeUcA+j+ppJL///e8jGywCevzxx1X8+uuvG32BVjrT0zmSqcKAjiuvAAAAcAaTVwAAADiDySsAAACckVA5r8GyS1n96le/Cup1zZo1U7G9EkjFihUjH1iQFiwQmTRJZPt2kZMnRSpXFunaVWTYMJHChfNsGEnpmmuuyTUWEfnss89U3KVLl5i8/9dfi9So4R3XEydESpaMydskja+//jqo7Xr16hXjkYi0aiXy/vu5961fL9K0acyHkJD0HFg7HzZRnTsn8swzInPmiOzZI5KWJnL33SLawlH5gl42S0Tk0KFDKrbLUfo9I6CXQxP5ZZnBWMtPx1LPcRURefTRR1VsH58hQ4aouGPHjkafXjIykbz+usjo0d7c5/LLRQYNEnnwwfD25eTk1XWHDom0bi0ydKhI6dIiH34oMmaMyP79Itbqi3DM0KHehPXkyXiPBKGaPl3k+HHz70aPFvn4Y5HGjeMzJoSnZ0+RVatE/vxnkZo1Rb76SmTr1niPCuHgWCaHdetEOnUS6dXL+2Vk40aR4cNFChQQ0ebhQWPyGgd9+5rt1q29L83nnxeZNk0kwEPySGBr1ogsXy4yYoQ3iYVbatc22z/8ILJ5s0iXLiIXcaZ0xvLlIvPni2zZ8stjCrdwLJPH2LEizZqJzJ7ttdu0ETlyxPv7AQNCv+ucUKdkvQzLSevS1bp161Rsl7J65JFHYjquvFC2rPdlichcfPHFKh5i/Tp3//33x+x9z5/3boGMHu1dTUdwpk+fHtR2V111VYxH8kvLl3sn19/9Ls/fGhHIyBC58UYmOyIilStXNtr66mb2SmeJKL8dywIFzMeQ9FQBu7TZpEmT8mRM0fLJJyJ//KP5d23aeOkfH3wg0rJlaPvjga04On9e5NQpkbVrRaZOFenfn6uurpoxQ+T06V9+OOGu+fNFKlYUSdD0MfjYuFGkenWRgQNFSpUSKV7cu125b1+8R4ZQcSyTx+nTv7y6WqSI9+fnn4e+PyavcVSihPdf8+bebx0s1+6mQ4dEHn3UewiP+uvJ4dQpkSVLvJQBfqF0y/79Ii+/7F3pmT9f5KWXRDIzRTp2FNFq78MBHMvkUa2ayKZN5t99+KH35+HDoe8vodIG8pv1670vyQ8/9PI+Bg70HhqBW0aOFGnSROT22+M9EkTLkiUi331HyoCLcnK8/xYv9tKxREQqVPAuELzzjshNN8V3fAgexzJ59Ovn3V2eNUvkt7/15j3PPuv1hbNoakJNXu2cDl0yLtHaoIH3Z7NmIqmpIt27izz0kEgc0vuSUs+ePQO2o+G///XyslavFjl61Pu7U6e8P48d8z6UxYpF/W2dtmTJEhXry4vaGjVqpOIqVarEdEy2+fO9KwXaEOCIMmVEqlb9ebIj4p1jCxf2nlJnwuOO/HYs9WVdc2u7rFcv78G7/v1F+vTxUkAmTvSeFbnsstD3R9pAgvhpIrtrV3zHgdDs2CFy9qxXA7RMGe+/n/JeK1XyPphwy7FjIsuWcdXVVX5LuefkeGV54A6OZfIoWNArBXrwoMinn4p8+63I9dd7fT/9GYqEuvKan/1UTCGPLzAhQs2aibz7rvl3y5d7v1EuXepdNYBbFi0SOXOGyaur2rXzaoJmZ3t3tES8OyNnz4rUqxffsSE0HMvk89NFHhEvTfKGG7z6vaFi8hoHt94qcvPNInXqeL+NrFvn5X506ULKgGtSU72VmXRZWd6fzZuzwlZuMjMzVXz27Fnf7QYMGKDi1J++ufLA/PneF6PfVR8ktj59vOot7dt7NZdPnPCKod98s/fLJtzBsUweGzZ4lZXq1/fq2r/yisiKFd7fhYPJaxw0buw9QZmV5RU/r1pV5MknvYRmAPGTne2t5jNuXLxHgnCVKuU9zDN4sMg993j5kR06JOdyosmOY5k8ChUS+ec/vdVECxTwLu6sWydy7bXh7Y/JaxyMG8eXYzLr0cP7D+5JTfVuScJt1ap5aTtwH8cyOTRs+MtSWZEg5RkAAADO4MorgDy1cePGXP++YcOGRrt9+/Z5MRwAgGO48goAAABnMHkFAACAM1JyQlggOCUl5aCI7I7dcBBA5ZycnLRo7IjjGHccy+TBsUweHMvkwbFMDr7HMaTJKwAAABBPpA0AAADAGUxeAQAA4AwmrwAAAHAGk1cAAAA4g8krAAAAnMHkFQAAAM5g8goAAABnMHkFAACAM5i8AgAAwBn/D0rc/As23lzEAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "batch_test = next(iter(mnist_test.batch(12)))\n", - "\n", - "preds = model(batch_test[0])\n", - "\n", - "tf.print(preds.shape)\n", - "preds = tf.argmax(preds, axis=1)\n", - "print(preds)\n", - "\n", - "fig = plt.figure(figsize=(12, 4))\n", - "for i in range(12):\n", - " ax = fig.add_subplot(2, 6, i+1)\n", - " ax.set_xticks([]); ax.set_yticks([])\n", - " img = batch_test[0][i, :, :, 0]\n", - " ax.imshow(img, cmap='gray_r')\n", - " ax.text(0.9, 0.1, '{}'.format(preds[i]), \n", - " size=15, color='blue',\n", - " horizontalalignment='center',\n", - " verticalalignment='center', \n", - " transform=ax.transAxes)\n", - " \n", - "#plt.savefig('figs/B13208-15_13.png', dpi=300)\n", - "plt.show()" - ] - }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "import os\n", - "\n", - "if not os.path.exists('models'):\n", - " os.mkdir('models')\n", - "\n", + "import tensorflow as tf\n", + "import tensorflow_datasets as tfds\n", + "import numpy as np\n", "\n", - "model.save('models/mnist-cnn.h5')" + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" ] }, { @@ -903,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -940,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -969,7 +146,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -1039,13 +216,13 @@ " img_center_crop, size=(218, 178))\n", "ax.imshow(img_resized.numpy().astype('uint8'))\n", "\n", - "#plt.savefig('figs/B13208-15_14.png', dpi=300)\n", + "# plt.savefig('figures/15_14.png', dpi=300)\n", "plt.show()" ] }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -1092,13 +269,13 @@ " if i == 0:\n", " ax.set_title('Step 3: Resize', size=15)\n", "\n", - "#plt.savefig('figs/B13208-15_15.png', dpi=300)\n", + "# plt.savefig('figures/15_15.png', dpi=300)\n", "plt.show()" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -1129,7 +306,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -1158,13 +335,13 @@ " ax.set_yticks([])\n", " ax.imshow(example[0])\n", " \n", - "#plt.savefig('figs/B13208-15_16.png', dpi=300)\n", + "#plt.savefig('figures/15_16.png', dpi=300)\n", "plt.show()" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -1201,7 +378,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -1227,7 +404,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -1236,7 +413,7 @@ "TensorShape([None, 8, 8, 256])" ] }, - "execution_count": 34, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -1247,7 +424,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -1256,19 +433,19 @@ "TensorShape([None, 256])" ] }, - "execution_count": 35, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model.add(tf.keras.layers.GlobalAveragePooling2D())\n", - "model.compute_output_shape(input_shape=(None, 64, 64, 3))\n" + "model.compute_output_shape(input_shape=(None, 64, 64, 3))" ] }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -1277,38 +454,38 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Model: \"sequential_1\"\n", + "Model: \"sequential\"\n", "_________________________________________________________________\n", "Layer (type) Output Shape Param # \n", "=================================================================\n", - "conv2d_1 (Conv2D) multiple 896 \n", + "conv2d (Conv2D) multiple 896 \n", "_________________________________________________________________\n", "max_pooling2d (MaxPooling2D) multiple 0 \n", "_________________________________________________________________\n", - "dropout_1 (Dropout) multiple 0 \n", + "dropout (Dropout) multiple 0 \n", "_________________________________________________________________\n", - "conv2d_2 (Conv2D) multiple 18496 \n", + "conv2d_1 (Conv2D) multiple 18496 \n", "_________________________________________________________________\n", "max_pooling2d_1 (MaxPooling2 multiple 0 \n", "_________________________________________________________________\n", - "dropout_2 (Dropout) multiple 0 \n", + "dropout_1 (Dropout) multiple 0 \n", "_________________________________________________________________\n", - "conv2d_3 (Conv2D) multiple 73856 \n", + "conv2d_2 (Conv2D) multiple 73856 \n", "_________________________________________________________________\n", "max_pooling2d_2 (MaxPooling2 multiple 0 \n", "_________________________________________________________________\n", - "conv2d_4 (Conv2D) multiple 295168 \n", + "conv2d_3 (Conv2D) multiple 295168 \n", "_________________________________________________________________\n", "global_average_pooling2d (Gl multiple 0 \n", "_________________________________________________________________\n", - "dense_1 (Dense) multiple 257 \n", + "dense (Dense) multiple 257 \n", "=================================================================\n", "Total params: 388,673\n", "Trainable params: 388,673\n", @@ -1327,7 +504,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -1336,25 +513,25 @@ "text": [ "Train for 500.0 steps\n", "Epoch 1/20\n", - "500/500 [==============================] - 30s 60ms/step - loss: 0.6232 - accuracy: 0.6235 - val_loss: 0.5808 - val_accuracy: 0.7320\n", + "500/500 [==============================] - 29s 59ms/step - loss: 0.6232 - accuracy: 0.6235 - val_loss: 0.5808 - val_accuracy: 0.7320\n", "Epoch 2/20\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.4629 - accuracy: 0.7109 - val_loss: 0.4583 - val_accuracy: 0.7850\n", "Epoch 3/20\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.5221 - accuracy: 0.7604 - val_loss: 0.3920 - val_accuracy: 0.7980\n", "Epoch 4/20\n", - "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.3831 - accuracy: 0.7881 - val_loss: 0.3481 - val_accuracy: 0.8100\n", + "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.3831 - accuracy: 0.7881 - val_loss: 0.3481 - val_accuracy: 0.8100\n", "Epoch 5/20\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.3772 - accuracy: 0.8281 - val_loss: 0.3495 - val_accuracy: 0.7830\n", "Epoch 6/20\n", - "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.3005 - accuracy: 0.8473 - val_loss: 0.2049 - val_accuracy: 0.9040\n", + "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.3005 - accuracy: 0.8473 - val_loss: 0.2049 - val_accuracy: 0.9040\n", "Epoch 7/20\n", - "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.2753 - accuracy: 0.8633 - val_loss: 0.2625 - val_accuracy: 0.8340\n", + "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.2753 - accuracy: 0.8633 - val_loss: 0.2625 - val_accuracy: 0.8340\n", "Epoch 8/20\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.2201 - accuracy: 0.8734 - val_loss: 0.2092 - val_accuracy: 0.8950\n", "Epoch 9/20\n", - "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.2053 - accuracy: 0.8850 - val_loss: 0.1748 - val_accuracy: 0.9180\n", + "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.2053 - accuracy: 0.8850 - val_loss: 0.1748 - val_accuracy: 0.9180\n", "Epoch 10/20\n", - "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.2377 - accuracy: 0.8919 - val_loss: 0.1615 - val_accuracy: 0.9300\n", + "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.2377 - accuracy: 0.8919 - val_loss: 0.1615 - val_accuracy: 0.9300\n", "Epoch 11/20\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.2083 - accuracy: 0.8929 - val_loss: 0.1703 - val_accuracy: 0.9200\n", "Epoch 12/20\n", @@ -1362,19 +539,19 @@ "Epoch 13/20\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.2624 - accuracy: 0.9052 - val_loss: 0.1393 - val_accuracy: 0.9450\n", "Epoch 14/20\n", - "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.2423 - accuracy: 0.9067 - val_loss: 0.1843 - val_accuracy: 0.9000\n", + "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.2423 - accuracy: 0.9067 - val_loss: 0.1843 - val_accuracy: 0.9000\n", "Epoch 15/20\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.2244 - accuracy: 0.9094 - val_loss: 0.1436 - val_accuracy: 0.9320\n", "Epoch 16/20\n", - "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 27s 53ms/step - loss: 0.2531 - accuracy: 0.9129 - val_loss: 0.1420 - val_accuracy: 0.9410\n", + "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.2531 - accuracy: 0.9129 - val_loss: 0.1420 - val_accuracy: 0.9410\n", "Epoch 17/20\n", - "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.1811 - accuracy: 0.9172 - val_loss: 0.1225 - val_accuracy: 0.9500\n", + "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.1811 - accuracy: 0.9172 - val_loss: 0.1225 - val_accuracy: 0.9500\n", "Epoch 18/20\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.3324 - accuracy: 0.9182 - val_loss: 0.1136 - val_accuracy: 0.9610\n", "Epoch 19/20\n", - "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.1236 - accuracy: 0.9214 - val_loss: 0.1230 - val_accuracy: 0.9590\n", + "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.1236 - accuracy: 0.9214 - val_loss: 0.1230 - val_accuracy: 0.9590\n", "Epoch 20/20\n", - "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.1587 - accuracy: 0.9210 - val_loss: 0.1201 - val_accuracy: 0.9580\n" + "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.1587 - accuracy: 0.9210 - val_loss: 0.1201 - val_accuracy: 0.9580\n" ] } ], @@ -1389,7 +566,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -1424,13 +601,13 @@ "ax.set_xlabel('Epoch', size=15)\n", "ax.set_ylabel('Accuracy', size=15)\n", "\n", - "#plt.savefig('figs/B13208-15_18.png', dpi=300)\n", + "#plt.savefig('figures/15_18.png', dpi=300)\n", "plt.show()" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -1450,7 +627,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -1459,25 +636,25 @@ "text": [ "Train for 500.0 steps\n", "Epoch 21/30\n", - "500/500 [==============================] - 27s 54ms/step - loss: 0.1741 - accuracy: 0.9249 - val_loss: 0.1222 - val_accuracy: 0.9460\n", + "500/500 [==============================] - 27s 53ms/step - loss: 0.1741 - accuracy: 0.9249 - val_loss: 0.1222 - val_accuracy: 0.9460\n", "Epoch 22/30\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.2248 - accuracy: 0.9246 - val_loss: 0.1197 - val_accuracy: 0.9530\n", "Epoch 23/30\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.1961 - accuracy: 0.9271 - val_loss: 0.1516 - val_accuracy: 0.9270\n", "Epoch 24/30\n", - "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.2566 - accuracy: 0.9303 - val_loss: 0.1216 - val_accuracy: 0.9490\n", + "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.2566 - accuracy: 0.9303 - val_loss: 0.1216 - val_accuracy: 0.9490\n", "Epoch 25/30\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.2437 - accuracy: 0.9302 - val_loss: 0.1146 - val_accuracy: 0.9530\n", "Epoch 26/30\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.1298 - accuracy: 0.9308 - val_loss: 0.1254 - val_accuracy: 0.9480\n", "Epoch 27/30\n", - "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.1122 - accuracy: 0.9301 - val_loss: 0.1152 - val_accuracy: 0.9580\n", + "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.1122 - accuracy: 0.9301 - val_loss: 0.1152 - val_accuracy: 0.9580\n", "Epoch 28/30\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.2245 - accuracy: 0.9317 - val_loss: 0.1070 - val_accuracy: 0.9630\n", "Epoch 29/30\n", "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.1584 - accuracy: 0.9341 - val_loss: 0.1037 - val_accuracy: 0.9530\n", "Epoch 30/30\n", - "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 53ms/step - loss: 0.2083 - accuracy: 0.9354 - val_loss: 0.1041 - val_accuracy: 0.9500\n" + "500/32 [====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 26s 52ms/step - loss: 0.2083 - accuracy: 0.9354 - val_loss: 0.1041 - val_accuracy: 0.9500\n" ] } ], @@ -1489,7 +666,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -1530,7 +707,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -1550,7 +727,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -1588,13 +765,13 @@ " verticalalignment='center', \n", " transform=ax.transAxes)\n", " \n", - "#plt.savefig('figs/B13208-15_19.png', dpi=300)\n", + "#plt.savefig('figures/figures-15_19.png', dpi=300)\n", "plt.show()" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -1625,7 +802,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -1669,15 +846,15 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Count of labels: Counter({1: 13, 0: 13, 9: 12, 4: 11, 5: 11, 2: 10, 6: 9, 7: 8, 3: 7, 8: 6})\n", - "Count of labels: Counter({1: 13, 0: 13, 9: 12, 4: 11, 5: 11, 2: 10, 6: 9, 7: 8, 3: 7, 8: 6})\n" + "Count of labels: Counter({1: 13, 0: 13, 9: 12, 4: 11, 5: 11, 2: 10, 6: 9, 7: 8, 3: 7, 8: 6})\n", + "Count of labels: Counter({1: 13, 0: 13, 9: 12, 4: 11, 5: 11, 2: 10, 6: 9, 7: 8, 3: 7, 8: 6})\n" ] } ], @@ -1690,13 +867,13 @@ " counter.update([example[1].numpy()])\n", " return counter\n", " \n", - "print('Count of labels: ', count_labels(mnist_valid))\n", - "print('Count of labels: ', count_labels(mnist_valid))" + "print('Count of labels:', count_labels(mnist_valid))\n", + "print('Count of labels:', count_labels(mnist_valid))" ] }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -1727,8 +904,7 @@ " reshuffle_each_iteration=False)\n", "\n", "mnist_valid = mnist_train.take(100)#.batch(BATCH_SIZE)\n", - "mnist_train = mnist_train.skip(100)#.batch(BATCH_SIZE)\n", - "\n" + "mnist_train = mnist_train.skip(100)#.batch(BATCH_SIZE)" ] }, { @@ -1740,15 +916,15 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Count of labels: Counter({1: 13, 0: 13, 9: 12, 4: 11, 5: 11, 2: 10, 6: 9, 7: 8, 3: 7, 8: 6})\n", - "Count of labels: Counter({1: 13, 0: 13, 9: 12, 4: 11, 5: 11, 2: 10, 6: 9, 7: 8, 3: 7, 8: 6})\n" + "Count of labels: Counter({1: 13, 0: 13, 9: 12, 4: 11, 5: 11, 2: 10, 6: 9, 7: 8, 3: 7, 8: 6})\n", + "Count of labels: Counter({1: 13, 0: 13, 9: 12, 4: 11, 5: 11, 2: 10, 6: 9, 7: 8, 3: 7, 8: 6})\n" ] } ], @@ -1761,8 +937,8 @@ " counter.update([example[1].numpy()])\n", " return counter\n", " \n", - "print('Count of labels: ', count_labels(mnist_valid))\n", - "print('Count of labels: ', count_labels(mnist_valid))" + "print('Count of labels:', count_labels(mnist_valid))\n", + "print('Count of labels:', count_labels(mnist_valid))" ] }, { @@ -1776,20 +952,20 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[NbConvertApp] Converting notebook ch15-notebook.ipynb to script\n", - "[NbConvertApp] Writing 22676 bytes to ch15.py\n" + "[NbConvertApp] Converting notebook ch15_part2.ipynb to script\n", + "[NbConvertApp] Writing 13062 bytes to ch15_part2.py\n" ] } ], "source": [ - "! python ../.convert_notebook_to_script.py --input ch15-notebook.ipynb --output ch15.py" + "! python ../.convert_notebook_to_script.py --input ch15_part2.ipynb --output ch15_part2.py" ] } ], diff --git a/ch15/ch15.py b/ch15/ch15_part2.py similarity index 54% rename from ch15/ch15.py rename to ch15/ch15_part2.py index 1573da54..7dc02bac 100644 --- a/ch15/ch15.py +++ b/ch15/ch15_part2.py @@ -2,14 +2,10 @@ import tensorflow as tf -import numpy as np -import scipy.signal -import imageio -from tensorflow import keras import tensorflow_datasets as tfds -import pandas as pd +import numpy as np import matplotlib.pyplot as plt -import os +import pandas as pd from collections import Counter # *Python Machine Learning 3rd Edition* by [Sebastian Raschka](https://sebastianraschka.com) & [Vahid Mirjalili](http://vahidmirjalili.com), Packt Publishing Ltd. 2019 @@ -18,7 +14,7 @@ # # Code License: [MIT License](https://github.com/rasbt/python-machine-learning-book-3rd-edition/blob/master/LICENSE.txt) -# # Chapter 15: Classifying Images with Deep Convolutional Neural Networks +# # Chapter 15: Classifying Images with Deep Convolutional Neural Networks (Part 2/2) # Note that the optional watermark extension is a small IPython notebook plugin that I developed to make the code reproducible. You can just skip the following line(s). @@ -26,421 +22,9 @@ -# ## The building blocks of convolutional neural networks -# -# ### Understanding CNNs and feature hierarchies -# -# ### Performing discrete convolutions -# -# ### Discrete convolutions in one dimension -# -# ### Padding inputs to control the size of the output feature maps -# -# ### Determining the size of the convolution output - - - - -print('TensorFlow version:', tf.__version__) -print('NumPy version: ', np.__version__) - - - - -def conv1d(x, w, p=0, s=1): - w_rot = np.array(w[::-1]) - x_padded = np.array(x) - if p > 0: - zero_pad = np.zeros(shape=p) - x_padded = np.concatenate( - [zero_pad, x_padded, zero_pad]) - res = [] - for i in range(0, int(len(x)/s),s): - res.append(np.sum( - x_padded[i:i+w_rot.shape[0]] * w_rot)) - return np.array(res) - - -## Testing: -x = [1, 3, 2, 4, 5, 6, 1, 3] -w = [1, 0, 3, 1, 2] - -print('Conv1d Implementation:', - conv1d(x, w, p=2, s=1)) - -print('Numpy Results:', - np.convolve(x, w, mode='same')) - - -# ### Performing a discrete convolution in 2D - - - - - -def conv2d(X, W, p=(0, 0), s=(1, 1)): - W_rot = np.array(W)[::-1,::-1] - X_orig = np.array(X) - n1 = X_orig.shape[0] + 2*p[0] - n2 = X_orig.shape[1] + 2*p[1] - X_padded = np.zeros(shape=(n1, n2)) - X_padded[p[0]:p[0]+X_orig.shape[0], - p[1]:p[1]+X_orig.shape[1]] = X_orig - - res = [] - for i in range(0, int((X_padded.shape[0] - - W_rot.shape[0])/s[0])+1, s[0]): - res.append([]) - for j in range(0, int((X_padded.shape[1] - - W_rot.shape[1])/s[1])+1, s[1]): - X_sub = X_padded[i:i+W_rot.shape[0], - j:j+W_rot.shape[1]] - res[-1].append(np.sum(X_sub * W_rot)) - return(np.array(res)) - -X = [[1, 3, 2, 4], [5, 6, 1, 3], [1, 2, 0, 2], [3, 4, 3, 2]] -W = [[1, 0, 3], [1, 2, 1], [0, 1, 1]] - -print('Conv2d Implementation:\n', - conv2d(X, W, p=(1, 1), s=(1, 1))) - - -print('SciPy Results:\n', - scipy.signal.convolve2d(X, W, mode='same')) - - -# ## Subsampling layers - -# ## Putting everything together – implementing a CNN -# -# ### Working with multiple input or color channels -# -# - -# **TIP: Reading an image file** - - - - - -img_raw = tf.io.read_file('example-image.png') -img = tf.image.decode_image(img_raw) -print('Image shape:', img.shape) -print('Number of channels:', img.shape[2]) -print('Image data type:', img.dtype) -print(img[100:102, 100:102, :]) - - - - - - -img = imageio.imread('example-image.png') -print('Image shape:', img.shape) -print('Number of channels:', img.shape[2]) -print('Image data type:', img.dtype) -print(img[100:102, 100:102, :]) - - -# **INFO-BOX: The rank of a grayscale image for input to a CNN** - - - -img_raw = tf.io.read_file('example-image-gray.png') -img = tf.image.decode_image(img_raw) -tf.print('Rank:', tf.rank(img)) -tf.print('Shape:', img.shape) - - - - -img = imageio.imread('example-image-gray.png') -tf.print('Rank:', tf.rank(img)) -tf.print('Shape:', img.shape) - -img_reshaped = tf.reshape(img, (img.shape[0], img.shape[1], 1)) -tf.print('New Shape:', img_reshaped.shape) - - -# ## Regularizing a neural network with dropout -# -# - - - - - -conv_layer = keras.layers.Conv2D( - filters=16, kernel_size=(3, 3), - kernel_regularizer=keras.regularizers.l2(0.001)) - -fc_layer = keras.layers.Dense( - units=16, kernel_regularizer=keras.regularizers.l2(0.001)) - - -# ## Loss Functions for Classification -# -# * **`BinaryCrossentropy()`** -# * `from_logits=False` -# * `from_logits=True` -# -# * **`CategoricalCrossentropy()`** -# * `from_logits=False` -# * `from_logits=True` -# -# * **`SparseCategoricalCrossentropy()`** -# * `from_logits=False` -# * `from_logits=True` -# - - - -####### Binary Crossentropy -bce_probas = tf.keras.losses.BinaryCrossentropy(from_logits=False) -bce_logits = tf.keras.losses.BinaryCrossentropy(from_logits=True) - -logits = tf.constant([0.8]) -probas = tf.keras.activations.sigmoid(logits) - -tf.print( - 'BCE (w Probas): {:.4f}'.format( - bce_probas(y_true=[1], y_pred=probas)), - '(w Logits): {:.4f}'.format( - bce_logits(y_true=[1], y_pred=logits))) - - -####### Categorical Crossentropy -cce_probas = tf.keras.losses.CategoricalCrossentropy( - from_logits=False) -cce_logits = tf.keras.losses.CategoricalCrossentropy( - from_logits=True) - -logits = tf.constant([[1.5, 0.8, 2.1]]) -probas = tf.keras.activations.softmax(logits) - -tf.print( - 'CCE (w Probas): {:.4f}'.format( - cce_probas(y_true=[0, 0, 1], y_pred=probas)), - '(w Logits): {:.4f}'.format( - cce_logits(y_true=[0, 0, 1], y_pred=logits))) - -####### Sparse Categorical Crossentropy -sp_cce_probas = tf.keras.losses.SparseCategoricalCrossentropy( - from_logits=False) -sp_cce_logits = tf.keras.losses.SparseCategoricalCrossentropy( - from_logits=True) - -tf.print( - 'Sparse CCE (w Probas): {:.4f}'.format( - sp_cce_probas(y_true=[2], y_pred=probas)), - '(w Logits): {:.4f}'.format( - sp_cce_logits(y_true=[2], y_pred=logits))) - - -# ## Implementing a deep convolutional neural network using TensorFlow -# -# ### The multilayer CNN architecture - -# ### Loading and preprocessing the data - - - - - - - - -## MNIST dataset - -mnist_bldr = tfds.builder('mnist') -mnist_bldr.download_and_prepare() -datasets = mnist_bldr.as_dataset(shuffle_files=False) -print(datasets.keys()) -mnist_train_orig, mnist_test_orig = datasets['train'], datasets['test'] - - - - -BUFFER_SIZE = 10000 -BATCH_SIZE = 64 -NUM_EPOCHS = 20 - - - - -mnist_train = mnist_train_orig.map( - lambda item: (tf.cast(item['image'], tf.float32)/255.0, - tf.cast(item['label'], tf.int32))) - -mnist_test = mnist_test_orig.map( - lambda item: (tf.cast(item['image'], tf.float32)/255.0, - tf.cast(item['label'], tf.int32))) - -tf.random.set_seed(1) - -mnist_train = mnist_train.shuffle(buffer_size=BUFFER_SIZE, - reshuffle_each_iteration=False) - -mnist_valid = mnist_train.take(10000).batch(BATCH_SIZE) -mnist_train = mnist_train.skip(10000).batch(BATCH_SIZE) - - -# ### Implementing a CNN using the TensorFlow Keras API -# -# #### Configuring CNN layers in Keras -# -# * **Conv2D:** `tf.keras.layers.Conv2D` -# * `filters` -# * `kernel_size` -# * `strides` -# * `padding` -# -# -# * **MaxPool2D:** `tf.keras.layers.MaxPool2D` -# * `pool_size` -# * `strides` -# * `padding` -# -# -# * **Dropout** `tf.keras.layers.Dropout2D` -# * `rate` - -# ### Constructing a CNN in Keras - - - -model = tf.keras.Sequential() - -model.add(tf.keras.layers.Conv2D( - filters=32, kernel_size=(5, 5), - strides=(1, 1), padding='same', - data_format='channels_last', - name='conv_1', activation='relu')) - -model.add(tf.keras.layers.MaxPool2D( - pool_size=(2, 2), name='pool_1')) - -model.add(tf.keras.layers.Conv2D( - filters=64, kernel_size=(5, 5), - strides=(1, 1), padding='same', - name='conv_2', activation='relu')) - -model.add(tf.keras.layers.MaxPool2D(pool_size=(2, 2), name='pool_2')) - - - - -model.compute_output_shape(input_shape=(16, 28, 28, 1)) - - - - -model.add(tf.keras.layers.Flatten()) - -model.compute_output_shape(input_shape=(16, 28, 28, 1)) - - - - -model.add(tf.keras.layers.Dense( - units=1024, name='fc_1', - activation='relu')) - -model.add(tf.keras.layers.Dropout( - rate=0.5)) - -model.add(tf.keras.layers.Dense( - units=10, name='fc_2', - activation='softmax')) - - - - -tf.random.set_seed(1) -model.build(input_shape=(None, 28, 28, 1)) - -model.compute_output_shape(input_shape=(16, 28, 28, 1)) - - - - -model.summary() - - - - -model.compile(optimizer=tf.keras.optimizers.Adam(), - loss=tf.keras.losses.SparseCategoricalCrossentropy(), - metrics=['accuracy']) # same as `tf.keras.metrics.SparseCategoricalAccuracy(name='accuracy')` - -history = model.fit(mnist_train, epochs=NUM_EPOCHS, - validation_data=mnist_valid, - shuffle=True) - - - - -hist = history.history -x_arr = np.arange(len(hist['loss'])) + 1 - -fig = plt.figure(figsize=(12, 4)) -ax = fig.add_subplot(1, 2, 1) -ax.plot(x_arr, hist['loss'], '-o', label='Train loss') -ax.plot(x_arr, hist['val_loss'], '--<', label='Validation loss') -ax.set_xlabel('Epoch', size=15) -ax.set_ylabel('Loss', size=15) -ax.legend(fontsize=15) -ax = fig.add_subplot(1, 2, 2) -ax.plot(x_arr, hist['accuracy'], '-o', label='Train acc.') -ax.plot(x_arr, hist['val_accuracy'], '--<', label='Validation acc.') -ax.legend(fontsize=15) -ax.set_xlabel('Epoch', size=15) -ax.set_ylabel('Accuracy', size=15) - -#plt.savefig('figs/B13208-15_12.png', dpi=300) -plt.show() - - - - -test_results = model.evaluate(mnist_test.batch(20)) -print('\nTest Acc. {:.2f}%'.format(test_results[1]*100)) - - - - -batch_test = next(iter(mnist_test.batch(12))) - -preds = model(batch_test[0]) - -tf.print(preds.shape) -preds = tf.argmax(preds, axis=1) -print(preds) - -fig = plt.figure(figsize=(12, 4)) -for i in range(12): - ax = fig.add_subplot(2, 6, i+1) - ax.set_xticks([]); ax.set_yticks([]) - img = batch_test[0][i, :, :, 0] - ax.imshow(img, cmap='gray_r') - ax.text(0.9, 0.1, '{}'.format(preds[i]), - size=15, color='blue', - horizontalalignment='center', - verticalalignment='center', - transform=ax.transAxes) - -#plt.savefig('figs/B13208-15_13.png', dpi=300) -plt.show() - - - - -if not os.path.exists('models'): - os.mkdir('models') -model.save('models/mnist-cnn.h5') # ## Gender classification from face images using CNN @@ -536,7 +120,7 @@ def count_items(ds): img_center_crop, size=(218, 178)) ax.imshow(img_resized.numpy().astype('uint8')) -#plt.savefig('figs/B13208-15_14.png', dpi=300) +#plt.savefig('figures/15_14.png', dpi=300) plt.show() @@ -572,7 +156,7 @@ def count_items(ds): if i == 0: ax.set_title('Step 3: Resize', size=15) -#plt.savefig('figs/B13208-15_15.png', dpi=300) +#plt.savefig('figures/15_15.png', dpi=300) plt.show() @@ -619,7 +203,7 @@ def preprocess(example, size=(64, 64), mode='train'): ax.set_yticks([]) ax.imshow(example[0]) -#plt.savefig('figs/B13208-15_16.png', dpi=300) +#plt.savefig('figures/15_16.png', dpi=300) plt.show() @@ -720,7 +304,7 @@ def preprocess(example, size=(64, 64), mode='train'): ax.set_xlabel('Epoch', size=15) ax.set_ylabel('Accuracy', size=15) -#plt.savefig('figs/B13208-15_18.png', dpi=300) +#plt.savefig('figures/15_18.png', dpi=300) plt.show() @@ -796,7 +380,7 @@ def preprocess(example, size=(64, 64), mode='train'): verticalalignment='center', transform=ax.transAxes) -#plt.savefig('figs/B13208-15_19.png', dpi=300) +#plt.savefig('figures/figures-15_19.png', dpi=300) plt.show() @@ -858,8 +442,8 @@ def count_labels(ds): counter.update([example[1].numpy()]) return counter -print('Count of labels: ', count_labels(mnist_valid)) -print('Count of labels: ', count_labels(mnist_valid)) +print('Count of labels:', count_labels(mnist_valid)) +print('Count of labels:', count_labels(mnist_valid)) @@ -902,8 +486,8 @@ def count_labels(ds): counter.update([example[1].numpy()]) return counter -print('Count of labels: ', count_labels(mnist_valid)) -print('Count of labels: ', count_labels(mnist_valid)) +print('Count of labels:', count_labels(mnist_valid)) +print('Count of labels:', count_labels(mnist_valid)) # ---- diff --git a/ch15/figures/15_12.png b/ch15/figures/15_12.png new file mode 100644 index 00000000..d86ae552 Binary files /dev/null and b/ch15/figures/15_12.png differ diff --git a/ch15/figures/15_13.png b/ch15/figures/15_13.png new file mode 100644 index 00000000..fbcd5052 Binary files /dev/null and b/ch15/figures/15_13.png differ diff --git a/ch15/figures/15_14.png b/ch15/figures/15_14.png new file mode 100644 index 00000000..38eb84df Binary files /dev/null and b/ch15/figures/15_14.png differ diff --git a/ch15/figures/15_15.png b/ch15/figures/15_15.png new file mode 100644 index 00000000..81a132b2 Binary files /dev/null and b/ch15/figures/15_15.png differ diff --git a/ch15/figures/15_16.png b/ch15/figures/15_16.png new file mode 100644 index 00000000..2fba8ff3 Binary files /dev/null and b/ch15/figures/15_16.png differ diff --git a/ch15/figures/15_18.png b/ch15/figures/15_18.png new file mode 100644 index 00000000..132d4b14 Binary files /dev/null and b/ch15/figures/15_18.png differ diff --git a/ch15/figures/figures-15_19.png b/ch15/figures/figures-15_19.png new file mode 100644 index 00000000..5171cdfc Binary files /dev/null and b/ch15/figures/figures-15_19.png differ