{
"cells": [
{
"cell_type": "markdown",
"id": "b65f2a05",
"metadata": {},
"source": [
"# Chapter 13: Going Deeper -- the Mechanics of PyTorch (Part 3/3)"
]
},
{
"cell_type": "markdown",
"id": "ddec26f4",
"metadata": {},
"source": [
"## Higher-level PyTorch APIs: a short introduction to PyTorch-Ignite "
]
},
{
"cell_type": "markdown",
"id": "7722db5f",
"metadata": {},
"source": [
"### Setting up the PyTorch model"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "220a5ea0",
"metadata": {},
"outputs": [],
"source": [
"import torch \n",
"import torch.nn as nn \n",
"from torch.utils.data import DataLoader \n",
" \n",
"from torchvision.datasets import MNIST \n",
"from torchvision import transforms\n",
" \n",
" \n",
"image_path = './' \n",
"torch.manual_seed(1) \n",
" \n",
"transform = transforms.Compose([ \n",
" transforms.ToTensor() \n",
"]) \n",
" \n",
" \n",
"mnist_train_dataset = MNIST( \n",
" root=image_path, \n",
" train=True,\n",
" transform=transform, \n",
" download=True\n",
") \n",
" \n",
"mnist_val_dataset = MNIST( \n",
" root=image_path, \n",
" train=False, \n",
" transform=transform, \n",
" download=False \n",
") \n",
" \n",
"batch_size = 64\n",
"train_loader = DataLoader( \n",
" mnist_train_dataset, batch_size, shuffle=True \n",
") \n",
" \n",
"val_loader = DataLoader( \n",
" mnist_val_dataset, batch_size, shuffle=False \n",
") \n",
" \n",
" \n",
"def get_model(image_shape=(1, 28, 28), hidden_units=(32, 16)): \n",
" input_size = image_shape[0] * image_shape[1] * image_shape[2] \n",
" all_layers = [nn.Flatten()]\n",
" for hidden_unit in hidden_units: \n",
" layer = nn.Linear(input_size, hidden_unit) \n",
" all_layers.append(layer) \n",
" all_layers.append(nn.ReLU()) \n",
" input_size = hidden_unit \n",
" \n",
" all_layers.append(nn.Linear(hidden_units[-1], 10)) \n",
" all_layers.append(nn.Softmax(dim=1)) \n",
" model = nn.Sequential(*all_layers)\n",
" return model \n",
" \n",
" \n",
"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
" \n",
"model = get_model().to(device)\n",
"loss_fn = nn.CrossEntropyLoss()\n",
"optimizer = torch.optim.Adam(model.parameters(), lr=0.001)"
]
},
{
"cell_type": "markdown",
"id": "99dcabd0",
"metadata": {},
"source": [
"### Setting up training and validation engines with PyTorch-Ignite"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "c972bfa2",
"metadata": {},
"outputs": [],
"source": [
"from ignite.engine import Events, create_supervised_trainer, create_supervised_evaluator\n",
"from ignite.metrics import Accuracy, Loss\n",
" \n",
" \n",
"trainer = create_supervised_trainer(\n",
" model, optimizer, loss_fn, device=device\n",
")\n",
" \n",
"val_metrics = {\n",
" \"accuracy\": Accuracy(),\n",
" \"loss\": Loss(loss_fn)\n",
"}\n",
" \n",
"evaluator = create_supervised_evaluator(\n",
" model, metrics=val_metrics, device=device\n",
")\n"
]
},
{
"cell_type": "markdown",
"id": "aa17c608",
"metadata": {},
"source": [
"### Creating event handlers for logging and validation"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "0edafc78",
"metadata": {},
"outputs": [],
"source": [
"# How many batches to wait before logging training status\n",
"log_interval = 100\n",
" \n",
"@trainer.on(Events.ITERATION_COMPLETED(every=log_interval))\n",
"def log_training_loss():\n",
" e = trainer.state.epoch\n",
" max_e = trainer.state.max_epochs\n",
" i = trainer.state.iteration\n",
" batch_loss = trainer.state.output\n",
" print(f\"Epoch[{e}/{max_e}], Iter[{i}] Loss: {batch_loss:.2f}\")\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "1944b444",
"metadata": {},
"outputs": [],
"source": [
"@trainer.on(Events.EPOCH_COMPLETED)\n",
"def log_validation_results():\n",
" eval_state = evaluator.run(val_loader)\n",
" metrics = eval_state.metrics\n",
" e = trainer.state.epoch\n",
" max_e = trainer.state.max_epochs\n",
" acc = metrics['accuracy']\n",
" avg_loss = metrics['loss']\n",
" print(f\"Validation Results - Epoch[{e}/{max_e}] Avg Accuracy: {acc:.2f} Avg Loss: {avg_loss:.2f}\")"
]
},
{
"cell_type": "markdown",
"id": "11b8cdfa",
"metadata": {},
"source": [
"### Setting up training checkpoints and saving the best model"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "e451c03b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<ignite.engine.events.RemovableEventHandle at 0x1060993d0>"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from ignite.handlers import Checkpoint, DiskSaver\n",
" \n",
"# We will save in the checkpoint the following:\n",
"to_save = {\"model\": model, \"optimizer\": optimizer, \"trainer\": trainer}\n",
" \n",
"# We will save checkpoints to the local disk\n",
"output_path = \"./output\"\n",
"save_handler = DiskSaver(dirname=output_path, require_empty=False)\n",
" \n",
"# Set up the handler:\n",
"checkpoint_handler = Checkpoint(\n",
" to_save, save_handler, filename_prefix=\"training\")\n",
"\n",
"# Attach the handler to the trainer\n",
"trainer.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "e3c8def7",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<ignite.engine.events.RemovableEventHandle at 0x106088430>"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Store best model by validation accuracy\n",
"best_model_handler = Checkpoint(\n",
" {\"model\": model},\n",
" save_handler,\n",
" filename_prefix=\"best\",\n",
" n_saved=1,\n",
" score_name=\"accuracy\",\n",
" score_function=Checkpoint.get_default_score_fn(\"accuracy\"),\n",
")\n",
" \n",
"evaluator.add_event_handler(Events.COMPLETED, best_model_handler)\n"
]
},
{
"cell_type": "markdown",
"id": "7092b069",
"metadata": {},
"source": [
"### Setting up TensorBoard as an experiment tracking system"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "9dc0368b",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<ignite.engine.events.RemovableEventHandle at 0x16605ff70>"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from ignite.contrib.handlers import TensorboardLogger, global_step_from_engine\n",
" \n",
" \n",
"tb_logger = TensorboardLogger(log_dir=output_path)\n",
" \n",
"# Attach handler to plot trainer's loss every 100 iterations\n",
"tb_logger.attach_output_handler(\n",
" trainer,\n",
" event_name=Events.ITERATION_COMPLETED(every=100),\n",
" tag=\"training\",\n",
" output_transform=lambda loss: {\"batch_loss\": loss},\n",
")\n",
" \n",
"# Attach handler for plotting both evaluators' metrics after every epoch completes\n",
"tb_logger.attach_output_handler(\n",
" evaluator,\n",
" event_name=Events.EPOCH_COMPLETED,\n",
" tag=\"validation\",\n",
" metric_names=\"all\",\n",
" global_step_transform=global_step_from_engine(trainer),\n",
")"
]
},
{
"cell_type": "markdown",
"id": "ad26fb6b",
"metadata": {},
"source": [
"### Executing the PyTorch-Ignite model training code"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "7f4d38cf",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch[1/5], Iter[100] Loss: 1.87\n",
"Epoch[1/5], Iter[200] Loss: 1.82\n",
"Epoch[1/5], Iter[300] Loss: 1.67\n",
"Epoch[1/5], Iter[400] Loss: 1.55\n",
"Epoch[1/5], Iter[500] Loss: 1.65\n",
"Epoch[1/5], Iter[600] Loss: 1.59\n",
"Epoch[1/5], Iter[700] Loss: 1.59\n",
"Epoch[1/5], Iter[800] Loss: 1.56\n",
"Epoch[1/5], Iter[900] Loss: 1.63\n",
"Validation Results - Epoch[1/5] Avg Accuracy: 0.91 Avg Loss: 1.56\n",
"Epoch[2/5], Iter[1000] Loss: 1.61\n",
"Epoch[2/5], Iter[1100] Loss: 1.56\n",
"Epoch[2/5], Iter[1200] Loss: 1.54\n",
"Epoch[2/5], Iter[1300] Loss: 1.54\n",
"Epoch[2/5], Iter[1400] Loss: 1.51\n",
"Epoch[2/5], Iter[1500] Loss: 1.53\n",
"Epoch[2/5], Iter[1600] Loss: 1.50\n",
"Epoch[2/5], Iter[1700] Loss: 1.50\n",
"Epoch[2/5], Iter[1800] Loss: 1.52\n",
"Validation Results - Epoch[2/5] Avg Accuracy: 0.92 Avg Loss: 1.54\n",
"Epoch[3/5], Iter[1900] Loss: 1.61\n",
"Epoch[3/5], Iter[2000] Loss: 1.60\n",
"Epoch[3/5], Iter[2100] Loss: 1.54\n",
"Epoch[3/5], Iter[2200] Loss: 1.51\n",
"Epoch[3/5], Iter[2300] Loss: 1.48\n",
"Epoch[3/5], Iter[2400] Loss: 1.56\n",
"Epoch[3/5], Iter[2500] Loss: 1.57\n",
"Epoch[3/5], Iter[2600] Loss: 1.52\n",
"Epoch[3/5], Iter[2700] Loss: 1.54\n",
"Epoch[3/5], Iter[2800] Loss: 1.54\n",
"Validation Results - Epoch[3/5] Avg Accuracy: 0.93 Avg Loss: 1.53\n",
"Epoch[4/5], Iter[2900] Loss: 1.53\n",
"Epoch[4/5], Iter[3000] Loss: 1.49\n",
"Epoch[4/5], Iter[3100] Loss: 1.51\n",
"Epoch[4/5], Iter[3200] Loss: 1.51\n",
"Epoch[4/5], Iter[3300] Loss: 1.54\n",
"Epoch[4/5], Iter[3400] Loss: 1.50\n",
"Epoch[4/5], Iter[3500] Loss: 1.58\n",
"Epoch[4/5], Iter[3600] Loss: 1.59\n",
"Epoch[4/5], Iter[3700] Loss: 1.50\n",
"Validation Results - Epoch[4/5] Avg Accuracy: 0.94 Avg Loss: 1.53\n",
"Epoch[5/5], Iter[3800] Loss: 1.52\n",
"Epoch[5/5], Iter[3900] Loss: 1.60\n",
"Epoch[5/5], Iter[4000] Loss: 1.50\n",
"Epoch[5/5], Iter[4100] Loss: 1.50\n",
"Epoch[5/5], Iter[4200] Loss: 1.51\n",
"Epoch[5/5], Iter[4300] Loss: 1.57\n",
"Epoch[5/5], Iter[4400] Loss: 1.56\n",
"Epoch[5/5], Iter[4500] Loss: 1.55\n",
"Epoch[5/5], Iter[4600] Loss: 1.50\n",
"Validation Results - Epoch[5/5] Avg Accuracy: 0.94 Avg Loss: 1.52\n"
]
},
{
"data": {
"text/plain": [
"State:\n",
"\titeration: 4690\n",
"\tepoch: 5\n",
"\tepoch_length: 938\n",
"\tmax_epochs: 5\n",
"\toutput: 1.5042390823364258\n",
"\tbatch: <class 'list'>\n",
"\tmetrics: <class 'dict'>\n",
"\tdataloader: <class 'torch.utils.data.dataloader.DataLoader'>\n",
"\tseed: <class 'NoneType'>\n",
"\ttimes: <class 'dict'>"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"trainer.run(train_loader, max_epochs=5)"
]
},
{
"cell_type": "markdown",
"id": "523acf34",
"metadata": {},
"source": [
"---\n",
"\n",
"Readers may ignore the next cell."
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "29befadd",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[NbConvertApp] Converting notebook ch13_part3.ipynb to script\n",
"[NbConvertApp] Writing 4864 bytes to ch13_part3.py\n"
]
}
],
"source": [
"! python ../.convert_notebook_to_script.py --input ch13_part3.ipynb --output ch13_part3.py"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}