notebooks/gpt2/text_classification_full_finetune.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1938fc31",
   "metadata": {
    "papermill": {
     "duration": 0.00364,
     "end_time": "2024-03-23T23:53:56.999258",
     "exception": false,
     "start_time": "2024-03-23T23:53:56.995618",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Linear probing of sentiment classification in a transformer trained on causal_lm\n",
    "In this notebook we'll try to find out if and how a transformer trained to do causal_lm process information about the sentiment of a sentence. We'll use the imdb dataset for this."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c98bc134",
   "metadata": {},
   "source": [
    "**GPU Requirements:** Full finetuning is expensive. With a consumer GPU you probably will only be able to run GPT-2. Check the QLoRA notebooks to run bigger models with limited VRAM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "39794270-9b0e-4aa8-8e5e-9a791972482d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:53:57.006384Z",
     "iopub.status.busy": "2024-03-23T23:53:57.006036Z",
     "iopub.status.idle": "2024-03-23T23:54:00.955520Z",
     "shell.execute_reply": "2024-03-23T23:54:00.954935Z"
    },
    "papermill": {
     "duration": 3.954752,
     "end_time": "2024-03-23T23:54:00.957170",
     "exception": false,
     "start_time": "2024-03-23T23:53:57.002418",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformer_heads import load_headed\n",
    "from datasets import load_dataset\n",
    "from transformers import (\n",
    "    AutoTokenizer,\n",
    "    MistralForCausalLM,\n",
    "    Trainer,\n",
    "    BitsAndBytesConfig,\n",
    "    TrainingArguments,\n",
    "    GPT2Model,\n",
    "    GPT2LMHeadModel,\n",
    ")\n",
    "from transformer_heads.util.helpers import DataCollatorWithPadding, get_model_params\n",
    "from peft import LoraConfig\n",
    "from transformer_heads.config import HeadConfig\n",
    "from transformer_heads.util.model import print_trainable_parameters\n",
    "from transformer_heads.util.evaluate import (\n",
    "    evaluate_head_wise,\n",
    "    get_top_n_preds,\n",
    "    get_some_preds,\n",
    ")\n",
    "import torch\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bbb6f926",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:54:00.976032Z",
     "iopub.status.busy": "2024-03-23T23:54:00.975696Z",
     "iopub.status.idle": "2024-03-23T23:54:00.978733Z",
     "shell.execute_reply": "2024-03-23T23:54:00.978222Z"
    },
    "papermill": {
     "duration": 0.007695,
     "end_time": "2024-03-23T23:54:00.979772",
     "exception": false,
     "start_time": "2024-03-23T23:54:00.972077",
     "status": "completed"
    },
    "tags": [
     "injected-parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "model_path = \"gpt2\"\n",
    "train_epochs = 1\n",
    "eval_epochs = 1\n",
    "logging_steps = 100\n",
    "full_finetune = True\n",
    "num_heads = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "711228b7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:54:00.986495Z",
     "iopub.status.busy": "2024-03-23T23:54:00.986290Z",
     "iopub.status.idle": "2024-03-23T23:54:01.473549Z",
     "shell.execute_reply": "2024-03-23T23:54:01.472858Z"
    },
    "papermill": {
     "duration": 0.492116,
     "end_time": "2024-03-23T23:54:01.474889",
     "exception": false,
     "start_time": "2024-03-23T23:54:00.982773",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'vocab_size': 50257, 'n_positions': 1024, 'n_embd': 768, 'n_layer': 12, 'n_head': 12, 'n_inner': None, 'activation_function': 'gelu_new', 'resid_pdrop': 0.1, 'embd_pdrop': 0.1, 'attn_pdrop': 0.1, 'layer_norm_epsilon': 1e-05, 'initializer_range': 0.02, 'summary_type': 'cls_index', 'summary_use_proj': True, 'summary_activation': None, 'summary_first_dropout': 0.1, 'summary_proj_to_labels': True, 'scale_attn_weights': True, 'use_cache': True, 'scale_attn_by_inverse_layer_idx': False, 'reorder_and_upcast_attn': False, 'bos_token_id': 50256, 'eos_token_id': 50256, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': None, 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['GPT2LMHeadModel'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'pad_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': {'text-generation': {'do_sample': True, 'max_length': 50}}, 'problem_type': None, '_name_or_path': 'gpt2', 'transformers_version': '4.37.2', 'model_type': 'gpt2', 'n_ctx': 1024, 'model_class': <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>, 'hidden_size': 768}\n"
     ]
    }
   ],
   "source": [
    "model_params = get_model_params(model_path)\n",
    "model_class = model_params[\"model_class\"]\n",
    "hidden_size = model_params[\"hidden_size\"]\n",
    "vocab_size = model_params[\"vocab_size\"]\n",
    "print(model_params)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3c60716",
   "metadata": {
    "papermill": {
     "duration": 0.003147,
     "end_time": "2024-03-23T23:54:01.481404",
     "exception": false,
     "start_time": "2024-03-23T23:54:01.478257",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "We are doing text classification, so we have to set pred_for_sequence to True for this task. In the imdb dataset, we only have two labels, 0 for negative and 1 for positive. So we have to set num_outputs to 2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d37267c8-b10b-479b-b808-a7e5ffead033",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:54:01.488334Z",
     "iopub.status.busy": "2024-03-23T23:54:01.488150Z",
     "iopub.status.idle": "2024-03-23T23:54:01.492234Z",
     "shell.execute_reply": "2024-03-23T23:54:01.491545Z"
    },
    "papermill": {
     "duration": 0.008872,
     "end_time": "2024-03-23T23:54:01.493342",
     "exception": false,
     "start_time": "2024-03-23T23:54:01.484470",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "head_configs = [\n",
    "    HeadConfig(\n",
    "        name=f\"imdb_head_{(1+(i-1)*2)}\",\n",
    "        layer_hook=-(1 + (i - 1) * 2),\n",
    "        in_size=hidden_size,\n",
    "        output_activation=\"linear\",\n",
    "        pred_for_sequence=True,\n",
    "        loss_fct=\"cross_entropy\",\n",
    "        num_outputs=2,\n",
    "    )\n",
    "    for i in range(1, num_heads + 2)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "487ed6ab-0386-4ab8-8cf5-e896952303f7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:54:01.500411Z",
     "iopub.status.busy": "2024-03-23T23:54:01.500223Z",
     "iopub.status.idle": "2024-03-23T23:54:08.312995Z",
     "shell.execute_reply": "2024-03-23T23:54:08.312513Z"
    },
    "papermill": {
     "duration": 6.817763,
     "end_time": "2024-03-23T23:54:08.314264",
     "exception": false,
     "start_time": "2024-03-23T23:54:01.496501",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "dd = load_dataset(\"imdb\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "58cccf74-e147-4612-afd9-7fadaea8d7f8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:54:08.323482Z",
     "iopub.status.busy": "2024-03-23T23:54:08.323109Z",
     "iopub.status.idle": "2024-03-23T23:54:08.338410Z",
     "shell.execute_reply": "2024-03-23T23:54:08.337716Z"
    },
    "papermill": {
     "duration": 0.020748,
     "end_time": "2024-03-23T23:54:08.339792",
     "exception": false,
     "start_time": "2024-03-23T23:54:08.319044",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': 'I love sci-fi and am willing to put up with a lot. Sci-fi movies/TV are usually underfunded, under-appreciated and misunderstood. I tried to like this, I really did, but it is to good TV sci-fi as Babylon 5 is to Star Trek (the original). Silly prosthetics, cheap cardboard sets, stilted dialogues, CG that doesn\\'t match the background, and painfully one-dimensional characters cannot be overcome with a \\'sci-fi\\' setting. (I\\'m sure there are those of you out there who think Babylon 5 is good sci-fi TV. It\\'s not. It\\'s clich\u00e9d and uninspiring.) While US viewers might like emotion and character development, sci-fi is a genre that does not take itself seriously (cf. Star Trek). It may treat important issues, yet not as a serious philosophy. It\\'s really difficult to care about the characters here as they are not simply foolish, just missing a spark of life. Their actions and reactions are wooden and predictable, often painful to watch. The makers of Earth KNOW it\\'s rubbish as they have to always say \"Gene Roddenberry\\'s Earth...\" otherwise people would not continue watching. Roddenberry\\'s ashes must be turning in their orbit as this dull, cheap, poorly edited (watching it without advert breaks really brings this home) trudging Trabant of a show lumbers into space. Spoiler. So, kill off a main character. And then bring him back as another actor. Jeeez! Dallas all over again.',\n",
       " 'label': 0}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd[\"test\"][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96bdc467",
   "metadata": {
    "papermill": {
     "duration": 0.003422,
     "end_time": "2024-03-23T23:54:08.346816",
     "exception": false,
     "start_time": "2024-03-23T23:54:08.343394",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "In the *tokenize_function*, we set the *label* entry in the dataset for each of our heads."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2ea1074e-b0b4-46b0-bc8f-dbb513a0386f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:54:08.354547Z",
     "iopub.status.busy": "2024-03-23T23:54:08.354295Z",
     "iopub.status.idle": "2024-03-23T23:54:31.064446Z",
     "shell.execute_reply": "2024-03-23T23:54:31.063810Z"
    },
    "papermill": {
     "duration": 22.71565,
     "end_time": "2024-03-23T23:54:31.065777",
     "exception": false,
     "start_time": "2024-03-23T23:54:08.350127",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2574b075674f4c019f95b2dad6b4f04e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/25000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87eff4934b214aadae26d10b7147585f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/25000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9550cfcfcb594350803bb46e78820043",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/50000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
    "if tokenizer.pad_token_id is None:\n",
    "    tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "\n",
    "def tokenize_function(examples):\n",
    "    out = tokenizer(examples[\"text\"], padding=False, truncation=True)\n",
    "    for hc in head_configs:\n",
    "        out[hc.name] = examples[\"label\"]\n",
    "    return out\n",
    "\n",
    "\n",
    "for split in dd.keys():\n",
    "    dd[split] = dd[split].filter(function=lambda example: len(example[\"text\"]) > 10)\n",
    "    dd[split] = dd[split].shuffle()\n",
    "    dd[split] = dd[split].map(tokenize_function, batched=True)\n",
    "\n",
    "dd.set_format(\n",
    "    type=\"torch\",\n",
    "    columns=[\"input_ids\", \"attention_mask\"] + [x.name for x in head_configs],\n",
    ")\n",
    "for split in dd.keys():\n",
    "    dd[split] = dd[split].remove_columns([\"text\", \"label\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8ea0bde2-594c-4b3c-bd8e-fd08777c10dd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:54:31.076158Z",
     "iopub.status.busy": "2024-03-23T23:54:31.075864Z",
     "iopub.status.idle": "2024-03-23T23:54:32.432385Z",
     "shell.execute_reply": "2024-03-23T23:54:32.431781Z"
    },
    "papermill": {
     "duration": 1.362815,
     "end_time": "2024-03-23T23:54:32.433664",
     "exception": false,
     "start_time": "2024-03-23T23:54:31.070849",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of TransformerWithHeads were not initialized from the model checkpoint at gpt2 and are newly initialized: ['heads.imdb_head_1.lins.0.weight', 'heads.imdb_head_3.lins.0.weight', 'heads.imdb_head_5.lins.0.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "quantization_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    load_in_8bit=False,\n",
    "    llm_int8_threshold=6.0,\n",
    "    llm_int8_has_fp16_weight=False,\n",
    "    bnb_4bit_compute_dtype=torch.float32,\n",
    "    bnb_4bit_use_double_quant=True,\n",
    "    bnb_4bit_quant_type=\"nf4\",\n",
    ")\n",
    "\n",
    "model = load_headed(\n",
    "    model_class,\n",
    "    model_path,\n",
    "    head_configs=head_configs,\n",
    "    quantization_config=None if full_finetune else quantization_config,\n",
    "    freeze_base_model=not full_finetune,\n",
    "    device_map={\"\": torch.cuda.current_device()},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "828af630-a734-4d0e-9d13-e86bf5f66cdb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:54:32.443170Z",
     "iopub.status.busy": "2024-03-23T23:54:32.442869Z",
     "iopub.status.idle": "2024-03-23T23:54:32.446995Z",
     "shell.execute_reply": "2024-03-23T23:54:32.446435Z"
    },
    "papermill": {
     "duration": 0.01005,
     "end_time": "2024-03-23T23:54:32.448155",
     "exception": false,
     "start_time": "2024-03-23T23:54:32.438105",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all params: 81977088 || trainable params: 4608 || trainable%: 0.005621082807918232\n",
      "params by dtype: defaultdict(<class 'int'>, {torch.float32: 39509760, torch.uint8: 42467328})\n",
      "trainable params by dtype: defaultdict(<class 'int'>, {torch.float32: 4608})\n"
     ]
    }
   ],
   "source": [
    "print_trainable_parameters(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8723801c",
   "metadata": {
    "papermill": {
     "duration": 0.003852,
     "end_time": "2024-03-23T23:54:32.455949",
     "exception": false,
     "start_time": "2024-03-23T23:54:32.452097",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Our heads are linear layers with only two outputs. Thus we have a very low amount of trainable parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e3c107a0-6c81-4d08-af0f-ef690e6fc92e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:54:32.464820Z",
     "iopub.status.busy": "2024-03-23T23:54:32.464392Z",
     "iopub.status.idle": "2024-03-23T23:54:32.698557Z",
     "shell.execute_reply": "2024-03-23T23:54:32.697937Z"
    },
    "papermill": {
     "duration": 0.239756,
     "end_time": "2024-03-23T23:54:32.699678",
     "exception": false,
     "start_time": "2024-03-23T23:54:32.459922",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Predicting:   0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Predicting:  20%|\u2588\u2588        | 1/5 [00:00<00:00,  8.46it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Predicting: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 5/5 [00:00<00:00, 22.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              review  label  ground_truth\n",
      "0  Seriously i thought it was a spoof when i saw ...      0             0\n",
      "1  i am amazed anyone likes this film. i never wa...      1             0\n",
      "2  1940's cartoon, banned nowadays probably becau...      1             0\n",
      "3  OK, this movie, was the worst display I have s...      1             0\n",
      "4  Maybe this was *An Important Movie* and that's...      1             0\n",
      "5  MONKEY is surely one of the best shows to have...      1             1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "ins, preds, ground_truths = get_some_preds(\n",
    "    model, dd[\"test\"], tokenizer, n=5, classification=True\n",
    ")\n",
    "print(\n",
    "    pd.DataFrame(\n",
    "        list(zip(ins, preds[\"imdb_head_5\"], ground_truths[\"imdb_head_5\"])),\n",
    "        columns=[\"review\", \"label\", \"ground_truth\"],\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "279113e4",
   "metadata": {
    "papermill": {
     "duration": 0.004182,
     "end_time": "2024-03-23T23:54:32.708270",
     "exception": false,
     "start_time": "2024-03-23T23:54:32.704088",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Untrained heads give fairly random outputs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "58359743-0cf4-43cd-97ae-48862d563b89",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:54:32.717925Z",
     "iopub.status.busy": "2024-03-23T23:54:32.717471Z",
     "iopub.status.idle": "2024-03-23T23:54:32.720738Z",
     "shell.execute_reply": "2024-03-23T23:54:32.720204Z"
    },
    "papermill": {
     "duration": 0.009379,
     "end_time": "2024-03-23T23:54:32.721830",
     "exception": false,
     "start_time": "2024-03-23T23:54:32.712451",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "collator = DataCollatorWithPadding(\n",
    "    feature_name_to_padding_value={\n",
    "        \"input_ids\": tokenizer.pad_token_id,\n",
    "        \"attention_mask\": 0,\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "193c099f-28be-40e6-ba93-22f326f6cd83",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:54:32.731082Z",
     "iopub.status.busy": "2024-03-23T23:54:32.730880Z",
     "iopub.status.idle": "2024-03-23T23:59:49.786491Z",
     "shell.execute_reply": "2024-03-23T23:59:49.785623Z"
    },
    "papermill": {
     "duration": 317.061648,
     "end_time": "2024-03-23T23:59:49.787715",
     "exception": false,
     "start_time": "2024-03-23T23:54:32.726067",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Evaluating: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 3125/3125 [05:17<00:00,  9.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5.763098739700317, {'imdb_head_1': 2.0209639251464604, 'imdb_head_3': 2.0075390841564538, 'imdb_head_5': 1.734595729969144})\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print(evaluate_head_wise(model, dd[\"test\"], collator, epochs=eval_epochs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ac8e4faa-5e10-44af-a201-121565372a63",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:59:49.954799Z",
     "iopub.status.busy": "2024-03-23T23:59:49.954403Z",
     "iopub.status.idle": "2024-03-24T00:18:28.075741Z",
     "shell.execute_reply": "2024-03-24T00:18:28.075049Z"
    },
    "papermill": {
     "duration": 1118.205537,
     "end_time": "2024-03-24T00:18:28.076943",
     "exception": false,
     "start_time": "2024-03-23T23:59:49.871406",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mykeller\u001b[0m (\u001b[33mchm-hci\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: wandb version 0.16.4 is available!  To upgrade, please run:\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:  $ pip install wandb --upgrade\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/raven/u/ykeller/transformer_heads/wandb/run-20240324_005951-apsvwyou\u001b[0m\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mworthy-puddle-213\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u2b50\ufe0f View project at \u001b[34m\u001b[4mhttps://wandb.ai/chm-hci/huggingface\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \ud83d\ude80 View run at \u001b[34m\u001b[4mhttps://wandb.ai/chm-hci/huggingface/runs/apsvwyou\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [3125/3125 18:31, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>2.804900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>2.311600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>2.212700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>2.241200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>2.140100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>2.114100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>2.065600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>1.963600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>2.049300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>1.967300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>1.991200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>1.947400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1300</td>\n",
       "      <td>1.922400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>1.861900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>1.910100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>1.823200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1700</td>\n",
       "      <td>1.814900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>1.867700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1900</td>\n",
       "      <td>1.832600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>1.851100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2100</td>\n",
       "      <td>1.782800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2200</td>\n",
       "      <td>1.821200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2300</td>\n",
       "      <td>1.811400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2400</td>\n",
       "      <td>1.756600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>1.795800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2600</td>\n",
       "      <td>1.814100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2700</td>\n",
       "      <td>1.796000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2800</td>\n",
       "      <td>1.769400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2900</td>\n",
       "      <td>1.765000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>1.769300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3100</td>\n",
       "      <td>1.816100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory imdb_linear_probe/checkpoint-500 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory imdb_linear_probe/checkpoint-1000 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory imdb_linear_probe/checkpoint-1500 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory imdb_linear_probe/checkpoint-2000 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory imdb_linear_probe/checkpoint-2500 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory imdb_linear_probe/checkpoint-3000 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=3125, training_loss=1.9467576330566407, metrics={'train_runtime': 1117.5754, 'train_samples_per_second': 22.37, 'train_steps_per_second': 2.796, 'total_flos': 8517017967280128.0, 'train_loss': 1.9467576330566407, 'epoch': 1.0})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args = TrainingArguments(\n",
    "    output_dir=\"imdb_linear_probe\",\n",
    "    learning_rate=0.0002,\n",
    "    num_train_epochs=train_epochs,  # To speed things up set to 0.1, set to 1 for better performance\n",
    "    logging_steps=logging_steps,\n",
    "    do_eval=False,\n",
    "    remove_unused_columns=False,\n",
    ")\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    args=args,\n",
    "    train_dataset=dd[\"train\"],\n",
    "    data_collator=collator,\n",
    ")\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8c239044-aef1-4509-a723-819a3a107928",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-24T00:18:28.249338Z",
     "iopub.status.busy": "2024-03-24T00:18:28.249028Z",
     "iopub.status.idle": "2024-03-24T00:24:00.541069Z",
     "shell.execute_reply": "2024-03-24T00:24:00.540395Z"
    },
    "papermill": {
     "duration": 332.378205,
     "end_time": "2024-03-24T00:24:00.542239",
     "exception": false,
     "start_time": "2024-03-24T00:18:28.164034",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Evaluating: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 3125/3125 [05:32<00:00,  9.40it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n",
      "\r\n",
      "Evaluating:   0%|          | 2/3125 [00:00<02:55, 17.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1.791820015411377, {'imdb_head_1': 0.6188445170402527, 'imdb_head_3': 0.5862952603769302, 'imdb_head_5': 0.58668023624897})\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print(evaluate_head_wise(model, dd[\"test\"], collator, epochs=eval_epochs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "38a9a41a-d8a5-49d3-8e7c-f81bfbe730df",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-24T00:24:00.881310Z",
     "iopub.status.busy": "2024-03-24T00:24:00.880958Z",
     "iopub.status.idle": "2024-03-24T00:24:01.048705Z",
     "shell.execute_reply": "2024-03-24T00:24:01.048184Z"
    },
    "papermill": {
     "duration": 0.337665,
     "end_time": "2024-03-24T00:24:01.049946",
     "exception": false,
     "start_time": "2024-03-24T00:24:00.712281",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Predicting:   0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Predicting:  80%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588  | 4/5 [00:00<00:00, 38.08it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Predicting: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 5/5 [00:00<00:00, 31.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              review  label  ground_truth\n",
      "0  Seriously i thought it was a spoof when i saw ...      0             0\n",
      "1  i am amazed anyone likes this film. i never wa...      1             0\n",
      "2  1940's cartoon, banned nowadays probably becau...      1             0\n",
      "3  OK, this movie, was the worst display I have s...      0             0\n",
      "4  Maybe this was *An Important Movie* and that's...      0             0\n",
      "5  MONKEY is surely one of the best shows to have...      1             1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "ins, preds, ground_truths = get_some_preds(\n",
    "    model, dd[\"test\"], tokenizer, n=5, classification=True\n",
    ")\n",
    "print(\n",
    "    pd.DataFrame(\n",
    "        list(zip(ins, preds[\"imdb_head_5\"], ground_truths[\"imdb_head_5\"])),\n",
    "        columns=[\"review\", \"label\", \"ground_truth\"],\n",
    "    )\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sh_finetuning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1808.043963,
   "end_time": "2024-03-24T00:24:03.959601",
   "environment_variables": {},
   "exception": null,
   "input_path": "notebooks/base/text_classification.ipynb",
   "output_path": "notebooks/gpt2/text_classification_full_finetune.ipynb",
   "parameters": {
    "eval_epochs": 1,
    "full_finetune": true,
    "logging_steps": 100,
    "model_path": "gpt2",
    "num_heads": 2,
    "train_epochs": 1
   },
   "start_time": "2024-03-23T23:53:55.915638",
   "version": "2.5.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0a2a3240a92d4a6790ee7c05db7b97ab": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_987bac1ec192417bbebde082fe902f91",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_f882ed1ac6d24d179c49104f8aac8927",
       "tabbable": null,
       "tooltip": null,
       "value": "\u200725000/25000\u2007[00:05&lt;00:00,\u20074694.51\u2007examples/s]"
      }
     },
     "0a358a15ff524d7cb00558dbdd20d3be": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_35ee1ae5358749fdab0b3edf70925911",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_4d4a54a6e3ac41179e943ac6bd298652",
       "tabbable": null,
       "tooltip": null,
       "value": "\u200750000/50000\u2007[00:11&lt;00:00,\u20074593.08\u2007examples/s]"
      }
     },
     "0b815b57059f42a4919565e414c3648a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_bbf586cdbee04c97ae78ae4c88147e0d",
       "max": 50000,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_13ae1d8ef9034470935b371cc61b6401",
       "tabbable": null,
       "tooltip": null,
       "value": 50000
      }
     },
     "130fd67ae19446a6be91771585168cb4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "13ae1d8ef9034470935b371cc61b6401": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "2574b075674f4c019f95b2dad6b4f04e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_463e7b9de34f4bcbae0f84d16396be71",
        "IPY_MODEL_b19d9859741b4c0d8e8cbad243ea298f",
        "IPY_MODEL_9e432069607f4e60976b3938ecc93c07"
       ],
       "layout": "IPY_MODEL_26319bfd201d4928980e8ba704b03690",
       "tabbable": null,
       "tooltip": null
      }
     },
     "26319bfd201d4928980e8ba704b03690": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "30681602464448aab15100e616649a5e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "35ee1ae5358749fdab0b3edf70925911": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "463e7b9de34f4bcbae0f84d16396be71": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_ab379f26c90b4cb8a8bb70a3d61830c2",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_9ea1e64a70174b77b6807607cd1360fa",
       "tabbable": null,
       "tooltip": null,
       "value": "Map:\u2007100%"
      }
     },
     "4d4a54a6e3ac41179e943ac6bd298652": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "655f0777542448f5aa2c343dc4bde735": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_130fd67ae19446a6be91771585168cb4",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_fd28e4f947c64213bd48b0d18474bdf5",
       "tabbable": null,
       "tooltip": null,
       "value": "Map:\u2007100%"
      }
     },
     "77a6ffc0ee6046a3aff0e3679bce7763": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "87eff4934b214aadae26d10b7147585f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_655f0777542448f5aa2c343dc4bde735",
        "IPY_MODEL_b86bf7c699f241a0942cedc69bfb08c8",
        "IPY_MODEL_0a2a3240a92d4a6790ee7c05db7b97ab"
       ],
       "layout": "IPY_MODEL_c91f33783e4e434884cba9557d1f1f3a",
       "tabbable": null,
       "tooltip": null
      }
     },
     "8a7e1ee877f74d80ac8eb93804c47590": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "9550cfcfcb594350803bb46e78820043": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_d311e147d1ec4178b9b2e891dd333c3d",
        "IPY_MODEL_0b815b57059f42a4919565e414c3648a",
        "IPY_MODEL_0a358a15ff524d7cb00558dbdd20d3be"
       ],
       "layout": "IPY_MODEL_f69a57d625504ab2a73551878f75fd84",
       "tabbable": null,
       "tooltip": null
      }
     },
     "9695dee1f5a64e22a694123638d3c6f3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "987bac1ec192417bbebde082fe902f91": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9e01889b059e4aaa82bb703d78ef25cd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "9e432069607f4e60976b3938ecc93c07": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_f55478e7023249d58347a9a869705027",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_dac0ea6d28be4c5fb15adbe7c005c6e3",
       "tabbable": null,
       "tooltip": null,
       "value": "\u200725000/25000\u2007[00:05&lt;00:00,\u20074508.61\u2007examples/s]"
      }
     },
     "9ea1e64a70174b77b6807607cd1360fa": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "ab379f26c90b4cb8a8bb70a3d61830c2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b19d9859741b4c0d8e8cbad243ea298f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_9695dee1f5a64e22a694123638d3c6f3",
       "max": 25000,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_9e01889b059e4aaa82bb703d78ef25cd",
       "tabbable": null,
       "tooltip": null,
       "value": 25000
      }
     },
     "b86bf7c699f241a0942cedc69bfb08c8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_77a6ffc0ee6046a3aff0e3679bce7763",
       "max": 25000,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_30681602464448aab15100e616649a5e",
       "tabbable": null,
       "tooltip": null,
       "value": 25000
      }
     },
     "bbf586cdbee04c97ae78ae4c88147e0d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c91f33783e4e434884cba9557d1f1f3a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d311e147d1ec4178b9b2e891dd333c3d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_f575d82df8ce4bf1a4f4bc54ee1858d7",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_8a7e1ee877f74d80ac8eb93804c47590",
       "tabbable": null,
       "tooltip": null,
       "value": "Map:\u2007100%"
      }
     },
     "dac0ea6d28be4c5fb15adbe7c005c6e3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "f55478e7023249d58347a9a869705027": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f575d82df8ce4bf1a4f4bc54ee1858d7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f69a57d625504ab2a73551878f75fd84": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f882ed1ac6d24d179c49104f8aac8927": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "fd28e4f947c64213bd48b0d18474bdf5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}