08-blurr_hf_fastai_imdb_sequence_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[?25l\r\u001b[K     |████                            | 10 kB 29.3 MB/s eta 0:00:01\r\u001b[K     |████████                        | 20 kB 29.0 MB/s eta 0:00:01\r\u001b[K     |████████████                    | 30 kB 11.6 MB/s eta 0:00:01\r\u001b[K     |████████████████                | 40 kB 9.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 51 kB 5.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 61 kB 5.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 71 kB 5.7 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 81 kB 3.9 MB/s \n",
      "\u001b[K     |████████████████████████████████| 43 kB 1.9 MB/s \n",
      "\u001b[K     |████████████████████████████████| 2.8 MB 38.6 MB/s \n",
      "\u001b[K     |████████████████████████████████| 264 kB 48.5 MB/s \n",
      "\u001b[K     |████████████████████████████████| 186 kB 48.8 MB/s \n",
      "\u001b[K     |████████████████████████████████| 1.2 MB 38.2 MB/s \n",
      "\u001b[K     |████████████████████████████████| 56 kB 3.7 MB/s \n",
      "\u001b[K     |████████████████████████████████| 3.3 MB 39.5 MB/s \n",
      "\u001b[K     |████████████████████████████████| 50 kB 6.3 MB/s \n",
      "\u001b[K     |████████████████████████████████| 895 kB 37.4 MB/s \n",
      "\u001b[K     |████████████████████████████████| 636 kB 30.7 MB/s \n",
      "\u001b[K     |████████████████████████████████| 243 kB 51.3 MB/s \n",
      "\u001b[K     |████████████████████████████████| 119 kB 52.3 MB/s \n",
      "\u001b[K     |████████████████████████████████| 46 kB 3.9 MB/s \n",
      "\u001b[K     |████████████████████████████████| 51 kB 301 kB/s \n",
      "\u001b[?25h  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
     ]
    }
   ],
   "source": [
    "%pip install ohmeow-blurr -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import *\n",
    "from fastai.text.all import *\n",
    "\n",
    "from blurr.utils import *\n",
    "from blurr.data.core import *\n",
    "from blurr.modeling.core import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='573440' class='' max='571827' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.28% [573440/571827 00:00<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "path = untar_data(URLs.IMDB_SAMPLE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#1) [Path('/root/.fastai/data/imdb_sample/texts.csv')]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(path/'texts.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "      <th>is_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>negative</td>\n",
       "      <td>Un-bleeping-believable! Meg Ryan doesn't even look her usual pert lovable self in this, which normally makes me forgive her shallow ticky acting schtick. Hard to believe she was the producer on this dog. Plus Kevin Kline: what kind of suicide trip has his career been on? Whoosh... Banzai!!! Finally this was directed by the guy who did Big Chill? Must be a replay of Jonestown - hollywood style. Wooofff!</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>positive</td>\n",
       "      <td>This is a extremely well-made film. The acting, script and camera-work are all first-rate. The music is good, too, though it is mostly early in the film, when things are still relatively cheery. There are no really superstars in the cast, though several faces will be familiar. The entire cast does an excellent job with the script.&lt;br /&gt;&lt;br /&gt;But it is hard to watch, because there is no good end to a situation like the one presented. It is now fashionable to blame the British for setting Hindus and Muslims against each other, and then cruelly separating them into two countries. There is som...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>negative</td>\n",
       "      <td>Every once in a long while a movie will come along that will be so awful that I feel compelled to warn people. If I labor all my days and I can save but one soul from watching this movie, how great will be my joy.&lt;br /&gt;&lt;br /&gt;Where to begin my discussion of pain. For starters, there was a musical montage every five minutes. There was no character development. Every character was a stereotype. We had swearing guy, fat guy who eats donuts, goofy foreign guy, etc. The script felt as if it were being written as the movie was being shot. The production value was so incredibly low that it felt li...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>positive</td>\n",
       "      <td>Name just says it all. I watched this movie with my dad when it came out and having served in Korea he had great admiration for the man. The disappointing thing about this film is that it only concentrate on a short period of the man's life - interestingly enough the man's entire life would have made such an epic bio-pic that it is staggering to imagine the cost for production.&lt;br /&gt;&lt;br /&gt;Some posters elude to the flawed characteristics about the man, which are cheap shots. The theme of the movie \"Duty, Honor, Country\" are not just mere words blathered from the lips of a high-brassed offic...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>negative</td>\n",
       "      <td>This movie succeeds at being one of the most unique movies you've seen. However this comes from the fact that you can't make heads or tails of this mess. It almost seems as a series of challenges set up to determine whether or not you are willing to walk out of the movie and give up the money you just paid. If you don't want to feel slighted you'll sit through this horrible film and develop a real sense of pity for the actors involved, they've all seen better days, but then you realize they actually got paid quite a bit of money to do this and you'll lose pity for them just like you've alr...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      label  ... is_valid\n",
       "0  negative  ...    False\n",
       "1  positive  ...    False\n",
       "2  negative  ...    False\n",
       "3  positive  ...    False\n",
       "4  negative  ...    False\n",
       "\n",
       "[5 rows x 3 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_cls = AutoModelForSequenceClassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['albert', 'albert_fast', 'bart', 'bart_fast', 'barthez', 'barthez_fast', 'beit', 'bert', 'bert_fast', 'bert_generation', 'bert_japanese', 'bertweet', 'big_bird', 'big_bird_fast', 'bigbird_pegasus', 'blenderbot', 'blenderbot_small', 'byt5', 'camembert', 'camembert_fast', 'canine', 'clip', 'clip_fast', 'convbert', 'convbert_fast', 'cpm', 'ctrl', 'deberta', 'deberta_fast', 'deberta_v2', 'deit', 'detr', 'distilbert', 'distilbert_fast', 'dpr', 'dpr_fast', 'electra', 'electra_fast', 'encoder_decoder', 'flaubert', 'fsmt', 'funnel', 'funnel_fast', 'gpt2', 'gpt2_fast', 'gpt_neo', 'herbert', 'herbert_fast', 'hubert', 'ibert', 'layoutlm', 'layoutlm_fast', 'layoutlmv2', 'layoutlmv2_fast', 'led', 'led_fast', 'longformer', 'longformer_fast', 'luke', 'lxmert', 'lxmert_fast', 'm2m_100', 'marian', 'mbart', 'mbart50', 'mbart50_fast', 'mbart_fast', 'megatron_bert', 'mmbt', 'mobilebert', 'mobilebert_fast', 'mpnet', 'mpnet_fast', 'mt5', 'openai', 'openai_fast', 'pegasus', 'pegasus_fast', 'phobert', 'prophetnet', 'rag', 'reformer', 'reformer_fast', 'rembert', 'rembert_fast', 'retribert', 'retribert_fast', 'roberta', 'roberta_fast', 'roformer', 'roformer_fast', 'speech_to_text', 'splinter', 'splinter_fast', 'squeezebert', 'squeezebert_fast', 't5', 't5_fast', 'tapas', 'tf_albert', 'tf_auto', 'tf_bart', 'tf_bert', 'tf_blenderbot', 'tf_blenderbot_small', 'tf_camembert', 'tf_convbert', 'tf_ctrl', 'tf_deberta', 'tf_deberta_v2', 'tf_distilbert', 'tf_dpr', 'tf_electra', 'tf_flaubert', 'tf_funnel', 'tf_gpt2', 'tf_hubert', 'tf_layoutlm', 'tf_led', 'tf_longformer', 'tf_lxmert', 'tf_marian', 'tf_mbart', 'tf_mobilebert', 'tf_mpnet', 'tf_mt5', 'tf_openai', 'tf_pegasus', 'tf_rag', 'tf_rembert', 'tf_roberta', 'tf_roformer', 'tf_t5', 'tf_transfo_xl', 'tf_wav2vec2', 'tf_xlm', 'tf_xlm_roberta', 'tf_xlnet', 'transfo_xl', 'visual_bert', 'vit', 'wav2vec2', 'xlm', 'xlm_prophetnet', 'xlm_roberta', 'xlm_roberta_fast', 'xlnet', 'xlnet_fast']\n"
     ]
    }
   ],
   "source": [
    "print(BLURR.get_architectures())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pretrained_model_name = 'distilroberta-base'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h4 id=\"BlurrUtil.get_hf_objects\" class=\"doc_header\"><code>BlurrUtil.get_hf_objects</code><a href=\"https://github.com/ohmeow/blurr/tree/master/blurr/utils.py#L147\" class=\"source_link\" style=\"float:right\">[source]</a></h4><blockquote><p><code>BlurrUtil.get_hf_objects</code>(<strong><code>pretrained_model_name_or_path</code></strong>:<code>Union</code>[<code>str</code>, <code>PathLike</code>, <code>NoneType</code>], <strong><code>model_cls</code></strong>:<code>PreTrainedModel</code>, <strong><code>config</code></strong>:<code>Union</code>[<code>PretrainedConfig</code>, <code>str</code>, <code>PathLike</code>]=<em><code>None</code></em>, <strong><code>tokenizer_cls</code></strong>:<code>PreTrainedTokenizerBase</code>=<em><code>None</code></em>, <strong><code>config_kwargs</code></strong>=<em><code>{}</code></em>, <strong><code>tokenizer_kwargs</code></strong>=<em><code>{}</code></em>, <strong><code>model_kwargs</code></strong>=<em><code>{}</code></em>, <strong><code>cache_dir</code></strong>:<code>Union</code>[<code>str</code>, <code>PathLike</code>]=<em><code>None</code></em>)</p>\n",
       "</blockquote>\n",
       "<p>Given at minimum a <code>pretrained_model_name_or_path</code> and <code>model_cls (such as</code>AutoModelForSequenceClassification\"), this method returns all the Hugging Face objects you need to train\n",
       "a model using Blurr</p>\n",
       "<p><a href=\"https://ohmeow.github.io/blurr/utils#BlurrUtil.get_hf_objects\" target=\"_blank\" rel=\"noreferrer noopener\">Show in docs</a></p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "doc(BLURR.get_hf_objects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_labels = 2\n",
    "hf_arch, hf_config, hf_tokenizer, hf_model = BLURR.get_hf_objects(pretrained_model_name, model_cls=model_cls, config_kwargs={'num_labels': num_labels})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "roberta\n",
      "<class 'transformers.models.roberta.configuration_roberta.RobertaConfig'>\n",
      "<class 'transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast'>\n",
      "<class 'transformers.models.roberta.modeling_roberta.RobertaForSequenceClassification'>\n"
     ]
    }
   ],
   "source": [
    "print(hf_arch)\n",
    "print(type(hf_config))\n",
    "print(type(hf_tokenizer))\n",
    "print(type(hf_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RobertaConfig {\n",
      "  \"_name_or_path\": \"distilroberta-base\",\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 6,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.10.0\",\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 50265\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(hf_config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DataBlock API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h2 id=\"HF_TextBlock\" class=\"doc_header\"><code>class</code> <code>HF_TextBlock</code><a href=\"https://github.com/ohmeow/blurr/tree/master/blurr/data/core.py#L159\" class=\"source_link\" style=\"float:right\">[source]</a></h2><blockquote><p><code>HF_TextBlock</code>(<strong><code>hf_arch</code></strong>:<code>str</code>=<em><code>None</code></em>, <strong><code>hf_config</code></strong>:<code>PretrainedConfig</code>=<em><code>None</code></em>, <strong><code>hf_tokenizer</code></strong>:<code>PreTrainedTokenizerBase</code>=<em><code>None</code></em>, <strong><code>hf_model</code></strong>:<code>PreTrainedModel</code>=<em><code>None</code></em>, <strong><code>before_batch_tfm</code></strong>:<code>HF_BeforeBatchTransform</code>=<em><code>None</code></em>, <strong><code>after_batch_tfm</code></strong>:<code>HF_AfterBatchTransform</code>=<em><code>None</code></em>, <strong><code>max_length</code></strong>:<code>int</code>=<em><code>None</code></em>, <strong><code>padding</code></strong>:<code>Union</code>[<code>bool</code>, <code>str</code>]=<em><code>True</code></em>, <strong><code>truncation</code></strong>:<code>Union</code>[<code>bool</code>, <code>str</code>]=<em><code>True</code></em>, <strong><code>is_split_into_words</code></strong>:<code>bool</code>=<em><code>False</code></em>, <strong><code>input_return_type</code></strong>=<em><code>HF_BaseInput</code></em>, <strong><code>dl_type</code></strong>:<code>DataLoader</code>=<em><code>None</code></em>, <strong><code>before_batch_kwargs</code></strong>=<em><code>{}</code></em>, <strong><code>after_batch_kwargs</code></strong>=<em><code>{}</code></em>, <strong><code>tok_kwargs</code></strong>=<em><code>{}</code></em>, <strong><code>text_gen_kwargs</code></strong>=<em><code>{}</code></em>, <strong>**<code>kwargs</code></strong>) :: <code>TransformBlock</code></p>\n",
       "</blockquote>\n",
       "<p>The core <code>TransformBlock</code> to prepare your data for training in Blurr with fastai's <code>DataBlock</code> API</p>\n",
       "<p><a href=\"https://ohmeow.github.io/blurr/data-core#HF_TextBlock\" target=\"_blank\" rel=\"noreferrer noopener\">Show in docs</a></p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "doc(HF_TextBlock)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "blocks = (HF_TextBlock(hf_arch, hf_config, hf_tokenizer, hf_model, ),\n",
    "          CategoryBlock)\n",
    "dblock = DataBlock(blocks=blocks, \n",
    "                   get_x=ColReader('text'), \n",
    "                   get_y=ColReader('label'), \n",
    "                   splitter=ColSplitter(col='is_valid')\n",
    "                   )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "HF_BeforeBatchTransforms gives for each sample input_ids (padding), attention_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting-up type transforms pipelines\n",
      "Collecting items from         label  ... is_valid\n",
      "0    negative  ...    False\n",
      "1    positive  ...    False\n",
      "2    negative  ...    False\n",
      "3    positive  ...    False\n",
      "4    negative  ...    False\n",
      "..        ...  ...      ...\n",
      "995  negative  ...     True\n",
      "996  positive  ...     True\n",
      "997  negative  ...     True\n",
      "998  negative  ...     True\n",
      "999  positive  ...     True\n",
      "\n",
      "[1000 rows x 3 columns]\n",
      "Found 1000 items\n",
      "2 datasets of sizes 800,200\n",
      "Setting up Pipeline: ColReader -- {'cols': 'text', 'pref': '', 'suff': '', 'label_delim': None}\n",
      "Setting up Pipeline: ColReader -- {'cols': 'label', 'pref': '', 'suff': '', 'label_delim': None} -> Categorize -- {'vocab': None, 'sort': True, 'add_na': False}\n",
      "\n",
      "Building one sample\n",
      "  Pipeline: ColReader -- {'cols': 'text', 'pref': '', 'suff': '', 'label_delim': None}\n",
      "    starting from\n",
      "      label                                                                                                                                                                                                                                                                                                                                                                                                                    negative\n",
      "text        Un-bleeping-believable! Meg Ryan doesn't even look her usual pert lovable self in this, which normally makes me forgive her shallow ticky acting schtick. Hard to believe she was the producer on this dog. Plus Kevin Kline: what kind of suicide trip has his career been on? Whoosh... Banzai!!! Finally this was directed by the guy who did Big Chill? Must be a replay of Jonestown - hollywood style. Wooofff!\n",
      "is_valid                                                                                                                                                                                                                                                                                                                                                                                                                    False\n",
      "Name: 0, dtype: object\n",
      "    applying ColReader -- {'cols': 'text', 'pref': '', 'suff': '', 'label_delim': None} gives\n",
      "      Un-bleeping-believable! Meg Ryan doesn't even look her usual pert lovable self in this, which normally makes me forgive her shallow ticky acting schtick. Hard to believe she was the producer on this dog. Plus Kevin Kline: what kind of suicide trip has his career been on? Whoosh... Banzai!!! Finally this was directed by the guy who did Big Chill? Must be a replay of Jonestown - hollywood style. Wooofff!\n",
      "  Pipeline: ColReader -- {'cols': 'label', 'pref': '', 'suff': '', 'label_delim': None} -> Categorize -- {'vocab': None, 'sort': True, 'add_na': False}\n",
      "    starting from\n",
      "      label                                                                                                                                                                                                                                                                                                                                                                                                                    negative\n",
      "text        Un-bleeping-believable! Meg Ryan doesn't even look her usual pert lovable self in this, which normally makes me forgive her shallow ticky acting schtick. Hard to believe she was the producer on this dog. Plus Kevin Kline: what kind of suicide trip has his career been on? Whoosh... Banzai!!! Finally this was directed by the guy who did Big Chill? Must be a replay of Jonestown - hollywood style. Wooofff!\n",
      "is_valid                                                                                                                                                                                                                                                                                                                                                                                                                    False\n",
      "Name: 0, dtype: object\n",
      "    applying ColReader -- {'cols': 'label', 'pref': '', 'suff': '', 'label_delim': None} gives\n",
      "      negative\n",
      "    applying Categorize -- {'vocab': None, 'sort': True, 'add_na': False} gives\n",
      "      TensorCategory(0)\n",
      "\n",
      "Final sample: (\"Un-bleeping-believable! Meg Ryan doesn't even look her usual pert lovable self in this, which normally makes me forgive her shallow ticky acting schtick. Hard to believe she was the producer on this dog. Plus Kevin Kline: what kind of suicide trip has his career been on? Whoosh... Banzai!!! Finally this was directed by the guy who did Big Chill? Must be a replay of Jonestown - hollywood style. Wooofff!\", TensorCategory(0))\n",
      "\n",
      "\n",
      "Collecting items from         label  ... is_valid\n",
      "0    negative  ...    False\n",
      "1    positive  ...    False\n",
      "2    negative  ...    False\n",
      "3    positive  ...    False\n",
      "4    negative  ...    False\n",
      "..        ...  ...      ...\n",
      "995  negative  ...     True\n",
      "996  positive  ...     True\n",
      "997  negative  ...     True\n",
      "998  negative  ...     True\n",
      "999  positive  ...     True\n",
      "\n",
      "[1000 rows x 3 columns]\n",
      "Found 1000 items\n",
      "2 datasets of sizes 800,200\n",
      "Setting up Pipeline: ColReader -- {'cols': 'text', 'pref': '', 'suff': '', 'label_delim': None}\n",
      "Setting up Pipeline: ColReader -- {'cols': 'label', 'pref': '', 'suff': '', 'label_delim': None} -> Categorize -- {'vocab': None, 'sort': True, 'add_na': False}\n",
      "Setting up after_item: Pipeline: ToTensor\n",
      "Setting up before_batch: Pipeline: HF_BeforeBatchTransform\n",
      "Setting up after_batch: Pipeline: HF_AfterBatchTransform\n",
      "\n",
      "Building one batch\n",
      "Applying item_tfms to the first sample:\n",
      "  Pipeline: ToTensor\n",
      "    starting from\n",
      "      (Un-bleeping-believable! Meg Ryan doesn't even look her usual pert lovable self in this, which normally makes me forgive her shallow ticky acting schtick. Hard to believe she was the producer on this dog. Plus Kevin Kline: what kind of suicide trip has his career been on? Whoosh... Banzai!!! Finally this was directed by the guy who did Big Chill? Must be a replay of Jonestown - hollywood style. Wooofff!, TensorCategory(0))\n",
      "    applying ToTensor gives\n",
      "      (Un-bleeping-believable! Meg Ryan doesn't even look her usual pert lovable self in this, which normally makes me forgive her shallow ticky acting schtick. Hard to believe she was the producer on this dog. Plus Kevin Kline: what kind of suicide trip has his career been on? Whoosh... Banzai!!! Finally this was directed by the guy who did Big Chill? Must be a replay of Jonestown - hollywood style. Wooofff!, TensorCategory(0))\n",
      "\n",
      "Adding the next 3 samples\n",
      "\n",
      "Applying before_batch to the list of samples\n",
      "  Pipeline: HF_BeforeBatchTransform\n",
      "    starting from\n",
      "      [(Un-bleeping-believable! Meg Ryan doesn't even look her usual pert lovable self in this, which normally makes me forgive her shallow ticky acting schtick. Hard to believe she was the producer on this dog. Plus Kevin Kline: what kind of suicide trip has his career been on? Whoosh... Banzai!!! Finally this was directed by the guy who did Big Chill? Must be a replay of Jonestown - hollywood style. Wooofff!, TensorCategory(0)), (This is a extremely well-made film. The acting, script and camera-work are all first-rate. The music is good, too, though it is mostly early in the film, when things are still relatively cheery. There are no really superstars in the cast, though several faces will be familiar. The entire cast does an excellent job with the script.<br /><br />But it is hard to watch, because there is no good end to a situation like the one presented. It is now fashionable to blame the British for setting Hindus and Muslims against each other, and then cruelly separating them into two countries. There is some merit in this view, but it's also true that no one forced Hindus and Muslims in the region to mistreat each other as they did around the time of partition. It seems more likely that the British simply saw the tensions between the religions and were clever enough to exploit them to their own ends.<br /><br />The result is that there is much cruelty and inhumanity in the situation and this is very unpleasant to remember and to see on the screen. But it is never painted as a black-and-white case. There is baseness and nobility on both sides, and also the hope for change in the younger generation.<br /><br />There is redemption of a sort, in the end, when Puro has to make a hard choice between a man who has ruined her life, but also truly loved her, and her family which has disowned her, then later come looking for her. But by that point, she has no option that is without great pain for her.<br /><br />This film carries the message that both Muslims and Hindus have their grave faults, and also that both can be dignified and caring people. The reality of partition makes that realisation all the more wrenching, since there can never be real reconciliation across the India/Pakistan border. In that sense, it is similar to \"Mr & Mrs Iyer\".<br /><br />In the end, we were glad to have seen the film, even though the resolution was heartbreaking. If the UK and US could deal with their own histories of racism with this kind of frankness, they would certainly be better off., TensorCategory(1)), (Every once in a long while a movie will come along that will be so awful that I feel compelled to warn people. If I labor all my days and I can save but one soul from watching this movie, how great will be my joy.<br /><br />Where to begin my discussion of pain. For starters, there was a musical montage every five minutes. There was no character development. Every character was a stereotype. We had swearing guy, fat guy who eats donuts, goofy foreign guy, etc. The script felt as if it were being written as the movie was being shot. The production value was so incredibly low that it felt like I was watching a junior high video presentation. Have the directors, producers, etc. ever even seen a movie before? Halestorm is getting worse and worse with every new entry. The concept for this movie sounded so funny. How could you go wrong with Gary Coleman and a handful of somewhat legitimate actors. But trust me when I say this, things went wrong, VERY WRONG., TensorCategory(0)), (Name just says it all. I watched this movie with my dad when it came out and having served in Korea he had great admiration for the man. The disappointing thing about this film is that it only concentrate on a short period of the man's life - interestingly enough the man's entire life would have made such an epic bio-pic that it is staggering to imagine the cost for production.<br /><br />Some posters elude to the flawed characteristics about the man, which are cheap shots. The theme of the movie \"Duty, Honor, Country\" are not just mere words blathered from the lips of a high-brassed officer - it is the deep declaration of one man's total devotion to his country.<br /><br />Ironically Peck being the liberal that he was garnered a better understanding of the man. He does a great job showing the fearless general tempered with the humane side of the man., TensorCategory(1))]\n",
      "    applying HF_BeforeBatchTransform gives\n",
      "      [({'input_ids': tensor([    0,  1890,    12,  5225, 24320,    12,  8494, 18421,   868,   328,\n",
      "        14938,  1774,   630,    75,   190,   356,    69,  4505, 32819,   784,\n",
      "        30289,  1403,    11,    42,     6,    61,  6329,   817,   162, 20184,\n",
      "           69, 16762, 10457,   219,  3501,  8447, 41791,     4,  6206,     7,\n",
      "          679,    79,    21,     5,  3436,    15,    42,  2335,     4,  4642,\n",
      "         2363,   229,  1902,    35,    99,   761,     9,  4260,  1805,    34,\n",
      "           39,   756,    57,    15,   116,  3394,  5212,   734,  5981, 23642,\n",
      "        16506,  3347,    42,    21,  3660,    30,     5,  2173,    54,   222,\n",
      "         1776, 25928,   116,  8495,    28,    10, 16462,     9,  4160,   990,\n",
      "         3355,   111,  1368,  9718,  2496,     4, 29935,  1529,   506,   328,\n",
      "            2,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0])}, TensorCategory(0)), ({'input_ids': tensor([    0,   152,    16,    10,  2778,   157,    12,  7078,   822,     4,\n",
      "           20,  3501,     6,  8543,     8,  2280,    12,  6014,    32,    70,\n",
      "           78,    12,  7954,     4,    20,   930,    16,   205,     6,   350,\n",
      "            6,   600,    24,    16,  2260,   419,    11,     5,   822,     6,\n",
      "           77,   383,    32,   202,  3487,  5851,  4270,     4,   345,    32,\n",
      "          117,   269, 19784,  2726,    11,     5,  2471,     6,   600,   484,\n",
      "         2419,    40,    28,  2950,     4,    20,  1445,  2471,   473,    41,\n",
      "         4206,   633,    19,     5,  8543, 49069,  3809,  1589, 49007,  3809,\n",
      "        48709,  1708,    24,    16,   543,     7,  1183,     6,   142,    89,\n",
      "           16,   117,   205,   253,     7,    10,  1068,   101,     5,    65,\n",
      "         2633,     4,    85,    16,   122, 27787,     7,  4887,     5,  1089,\n",
      "           13,  2749, 30618,     8,  6299,   136,   349,    97,     6,     8,\n",
      "          172,  3977,  1780, 13861, 18246,   106,    88,    80,   749,     4,\n",
      "          345,    16,   103, 13032,    11,    42,  1217,     6,    53,    24,\n",
      "           18,    67,  1528,    14,   117,    65,  1654, 30618,     8,  6299,\n",
      "           11,     5,   976,     7, 12241, 12353,   349,    97,    25,    51,\n",
      "          222,   198,     5,    86,     9, 34255,     4,    85,  1302,    55,\n",
      "          533,    14,     5,  1089,  1622,   794,     5,  5734,   227,     5,\n",
      "        24664,     8,    58, 13074,   615,     7, 14958,   106,     7,    49,\n",
      "          308,  3587, 49069,  3809,  1589, 49007,  3809, 48709,   133,   898,\n",
      "           16,    14,    89,    16,   203, 18744,     8, 37576,  1571,    11,\n",
      "            5,  1068,     8,    42,    16,   182, 26262,     7,  2145,     8,\n",
      "            7,   192,    15,     5,  2441,     4,   125,    24,    16,   393,\n",
      "        10122,    25,    10,   909,    12,   463,    12,  9830,   403,     4,\n",
      "          345,    16, 11909, 14186,     8, 45560,    15,   258,  2380,     6,\n",
      "            8,    67,     5,  1034,    13,   464,    11,     5,  3240,  2706,\n",
      "        49069,  3809,  1589, 49007,  3809, 48709,   970,    16, 21772,     9,\n",
      "           10,  2345,     6,    11,     5,   253,     6,    77,   221,  7367,\n",
      "           34,     7,   146,    10,   543,  2031,   227,    10,   313,    54,\n",
      "           34, 19750,    69,   301,     6,    53,    67,  3127,  2638,    69,\n",
      "            6,     8,    69,   284,    61,    34,  2982,  4447,    69,     6,\n",
      "          172,   423,   283,   546,    13,    69,     4,   125,    30,    14,\n",
      "          477,     6,    79,    34,   117,  1973,    14,    16,   396,   372,\n",
      "         2400,    13,    69, 49069,  3809,  1589, 49007,  3809, 48709,   713,\n",
      "          822,  5723,     5,  1579,    14,   258,  6299,     8, 30618,    33,\n",
      "           49,  9753, 28853,     6,     8,    67,    14,   258,    64,    28,\n",
      "        19210,  3786,     8, 10837,    82,     4,    20,  2015,     9, 34255,\n",
      "          817,    14,   588,  3258,    70,     5,    55, 34310,   154,     6,\n",
      "          187,    89,    64,   393,    28,   588, 13229,   420,     5,   666,\n",
      "           73, 20700,  1424,     4,    96,    14,  1472,     6,    24,    16,\n",
      "         1122,     7,    22, 10980,   359,  3801,    38,  6426,   845, 41552,\n",
      "         3809,  1589, 49007,  3809, 48709,  1121,     5,   253,     6,    52,\n",
      "           58,  7785,     7,    33,   450,     5,   822,     6,   190,   600,\n",
      "            5,  3547,    21, 17052,     4,   318,     5,   987,     8,   382,\n",
      "          115,   432,    19,    49,   308, 26271,     9,  8222,    19,    42,\n",
      "          761,     9, 26486,  1825,     6,    51,    74,  1819,    28,   357,\n",
      "          160,     4,     2]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1])}, TensorCategory(1)), ({'input_ids': tensor([    0,  4337,   683,    11,    10,   251,   150,    10,  1569,    40,\n",
      "          283,   552,    14,    40,    28,    98, 11522,    14,    38,   619,\n",
      "        19411,     7, 11345,    82,     4,   318,    38,  4178,    70,   127,\n",
      "          360,     8,    38,    64,  1871,    53,    65,  7047,    31,  2494,\n",
      "           42,  1569,     6,   141,   372,    40,    28,   127,  5823, 49069,\n",
      "         3809,  1589, 49007,  3809, 48709, 13841,     7,  1642,   127,  3221,\n",
      "            9,  2400,     4,   286,  9770,     6,    89,    21,    10,  4388,\n",
      "        27121,  1580,   358,   292,   728,     4,   345,    21,   117,  2048,\n",
      "          709,     4,  4337,  2048,    21,    10, 33006,     4,   166,    56,\n",
      "        21854,  2173,     6,  5886,  2173,    54, 24923,   218,  7046,     6,\n",
      "        36302,  1093,  2173,     6,  4753,     4,    20,  8543,  1299,    25,\n",
      "          114,    24,    58,   145,  1982,    25,     5,  1569,    21,   145,\n",
      "          738,     4,    20,   931,   923,    21,    98,  5784,   614,    14,\n",
      "           24,  1299,   101,    38,    21,  2494,    10,  3812,   239,   569,\n",
      "         5209,     4,  6319,     5,  5392,     6,  4426,     6,  4753,     4,\n",
      "          655,   190,   450,    10,  1569,   137,   116,  6579,   990,  8693,\n",
      "           16,   562,  3007,     8,  3007,    19,   358,    92,  3555,     4,\n",
      "           20,  4286,    13,    42,  1569, 12020,    98,  6269,     4,  1336,\n",
      "          115,    47,   213,  1593,    19,  4587, 10450,     8,    10,  6095,\n",
      "            9,  5568,  8134,  5552,     4,   125,  2416,   162,    77,    38,\n",
      "          224,    42,     6,   383,   439,  1593,     6, 39789,  8692,  9672,\n",
      "            4,     2,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0])}, TensorCategory(0)), ({'input_ids': tensor([    0, 10704,    95,   161,    24,    70,     4,    38,  3996,    42,\n",
      "         1569,    19,   127,  4252,    77,    24,   376,    66,     8,   519,\n",
      "         1665,    11,  1101,    37,    56,   372, 24287,    13,     5,   313,\n",
      "            4,    20,  6770,   631,    59,    42,   822,    16,    14,    24,\n",
      "          129, 14410,    15,    10,   765,   675,     9,     5,   313,    18,\n",
      "          301,   111, 40491,   615,     5,   313,    18,  1445,   301,    74,\n",
      "           33,   156,   215,    41,  9066, 10709,    12, 19017,    14,    24,\n",
      "           16, 16609,     7,  4744,     5,   701,    13,   931, 49069,  3809,\n",
      "         1589, 49007,  3809, 48709,  6323, 15736,  1615,  6343,     7,     5,\n",
      "        17107, 12720,    59,     5,   313,     6,    61,    32,  6162,  2347,\n",
      "            4,    20,  4782,     9,     5,  1569,    22,   495, 27237,     6,\n",
      "        11403,     6,  5093,   113,    32,    45,    95,  9198,  1617,  3089,\n",
      "        27694,    31,     5, 14638,     9,    10,   239,    12,  3809, 29591,\n",
      "         1036,   111,    24,    16,     5,  1844,  9394,     9,    65,   313,\n",
      "           18,   746, 27022,     7,    39,   247, 49069,  3809,  1589, 49007,\n",
      "         3809, 48709, 46805, 29550,   145,     5,  6176,    14,    37,    21,\n",
      "        15913,    10,   357,  2969,     9,     5,   313,     4,    91,   473,\n",
      "           10,   372,   633,  2018,     5, 29107,   937, 31380,    19,     5,\n",
      "        26899,   526,     9,     5,   313,     4,     2,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n",
      "            1,     1,     1]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0])}, TensorCategory(1))]\n",
      "\n",
      "Collating items in a batch\n",
      "\n",
      "Applying batch_tfms to the batch built\n",
      "  Pipeline: HF_AfterBatchTransform\n",
      "    starting from\n",
      "      ({'input_ids': tensor([[    0,  1890,    12,  ...,     1,     1,     1],\n",
      "        [    0,   152,    16,  ...,   160,     4,     2],\n",
      "        [    0,  4337,   683,  ...,     1,     1,     1],\n",
      "        [    0, 10704,    95,  ...,     1,     1,     1]], device='cuda:0'), 'attention_mask': tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 1, 1, 1],\n",
      "        [1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 0, 0, 0]], device='cuda:0')}, TensorCategory([0, 1, 0, 1], device='cuda:0'))\n",
      "    applying HF_AfterBatchTransform gives\n",
      "      ({'input_ids': tensor([[    0,  1890,    12,  ...,     1,     1,     1],\n",
      "        [    0,   152,    16,  ...,   160,     4,     2],\n",
      "        [    0,  4337,   683,  ...,     1,     1,     1],\n",
      "        [    0, 10704,    95,  ...,     1,     1,     1]], device='cuda:0'), 'attention_mask': tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 1, 1, 1],\n",
      "        [1, 1, 1,  ..., 0, 0, 0],\n",
      "        [1, 1, 1,  ..., 0, 0, 0]], device='cuda:0')}, TensorCategory([0, 1, 0, 1], device='cuda:0'))\n"
     ]
    }
   ],
   "source": [
    "dblock.summary(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting items from         label  ... is_valid\n",
      "0    negative  ...    False\n",
      "1    positive  ...    False\n",
      "2    negative  ...    False\n",
      "3    positive  ...    False\n",
      "4    negative  ...    False\n",
      "..        ...  ...      ...\n",
      "995  negative  ...     True\n",
      "996  positive  ...     True\n",
      "997  negative  ...     True\n",
      "998  negative  ...     True\n",
      "999  positive  ...     True\n",
      "\n",
      "[1000 rows x 3 columns]\n",
      "Found 1000 items\n",
      "2 datasets of sizes 800,200\n",
      "Setting up Pipeline: ColReader -- {'cols': 'text', 'pref': '', 'suff': '', 'label_delim': None}\n",
      "Setting up Pipeline: ColReader -- {'cols': 'label', 'pref': '', 'suff': '', 'label_delim': None} -> Categorize -- {'vocab': None, 'sort': True, 'add_na': False}\n",
      "Setting up after_item: Pipeline: ToTensor\n",
      "Setting up before_batch: Pipeline: HF_BeforeBatchTransform\n",
      "Setting up after_batch: Pipeline: HF_AfterBatchTransform\n"
     ]
    }
   ],
   "source": [
    "dls = dblock.dataloaders(source=df, verbose=True, bs=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = dls.one_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2,\n",
       " dict,\n",
       " fastai.torch_core.TensorCategory,\n",
       " {'attention_mask': tensor([[1, 1, 1,  ..., 1, 1, 1],\n",
       "          [1, 1, 1,  ..., 1, 1, 1],\n",
       "          [1, 1, 1,  ..., 1, 1, 1],\n",
       "          [1, 1, 1,  ..., 1, 1, 1]], device='cuda:0'),\n",
       "  'input_ids': tensor([[    0,  4833,  3009,  ...,  1916,     6,     2],\n",
       "          [    0,   978,    14,  ..., 38954, 18281,     2],\n",
       "          [    0,  2647,     6,  ...,     7,    69,     2],\n",
       "          [    0,    22, 15833,  ...,    15,    10,     2]], device='cuda:0')})"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(b), type(b[0]), type(b[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'attention_mask': tensor([[1, 1, 1,  ..., 1, 1, 1],\n",
       "         [1, 1, 1,  ..., 1, 1, 1],\n",
       "         [1, 1, 1,  ..., 1, 1, 1],\n",
       "         [1, 1, 1,  ..., 1, 1, 1]], device='cuda:0'),\n",
       " 'input_ids': tensor([[    0,  4833,  3009,  ...,  1916,     6,     2],\n",
       "         [    0,   978,    14,  ..., 38954, 18281,     2],\n",
       "         [    0,  2647,     6,  ...,     7,    69,     2],\n",
       "         [    0,    22, 15833,  ...,    15,    10,     2]], device='cuda:0')}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Raising Victor Vargas: A Review&lt;br /&gt;&lt;br /&gt;You know, Raising Victor Vargas is like sticking your hands into a big, steaming bowl of oatmeal. It's warm and gooey, but you're not sure if it feels right. Try as I might, no matter how warm and gooey Raising Victor Vargas became I was always aware that something didn't quite feel right. Victor Vargas suffers from a certain overconfidence on the director's part. Apparently, the director thought that the ethnic backdrop of a Latino family on the lower</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The year 2005 saw no fewer than 3 filmed productions of H. G. Wells' great novel, \"War of the Worlds\". This is perhaps the least well-known and very probably the best of them. No other version of WotW has ever attempted not only to present the story very much as Wells wrote it, but also to create the atmosphere of the time in which it was supposed to take place: the last year of the 19th Century, 1900  using Wells' original setting, in and near Woking, England.&lt;br /&gt;&lt;br /&gt;IMDb seems unfriendly</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dls.show_batch(dataloaders=dls, max_n=2, trunc_at=500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h2 id=\"HF_BaseModelWrapper\" class=\"doc_header\"><code>class</code> <code>HF_BaseModelWrapper</code><a href=\"https://github.com/ohmeow/blurr/tree/master/blurr/modeling/core.py#L45\" class=\"source_link\" style=\"float:right\">[source]</a></h2><blockquote><p><code>HF_BaseModelWrapper</code>(<strong><code>hf_model</code></strong>:<code>PreTrainedModel</code>, <strong><code>output_hidden_states</code></strong>:<code>bool</code>=<em><code>False</code></em>, <strong><code>output_attentions</code></strong>:<code>bool</code>=<em><code>False</code></em>, <strong><code>hf_model_kwargs</code></strong>=<em><code>{}</code></em>) :: <code>Module</code></p>\n",
       "</blockquote>\n",
       "<p>Same as <code>nn.Module</code>, but no need for subclasses to call <code>super().__init__</code></p>\n",
       "<p><a href=\"https://ohmeow.github.io/blurr/modeling-core#HF_BaseModelWrapper\" target=\"_blank\" rel=\"noreferrer noopener\">Show in docs</a></p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "doc(HF_BaseModelWrapper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = HF_BaseModelWrapper(hf_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h2 id=\"Learner\" class=\"doc_header\"><code>class</code> <code>Learner</code><a href=\"https://github.com/fastai/fastai/tree/master/fastai/learner.py#L83\" class=\"source_link\" style=\"float:right\">[source]</a></h2><blockquote><p><code>Learner</code>(<strong><code>dls</code></strong>, <strong><code>model</code></strong>, <strong><code>loss_func</code></strong>=<em><code>None</code></em>, <strong><code>opt_func</code></strong>=<em><code>Adam</code></em>, <strong><code>lr</code></strong>=<em><code>0.001</code></em>, <strong><code>splitter</code></strong>=<em><code>trainable_params</code></em>, <strong><code>cbs</code></strong>=<em><code>None</code></em>, <strong><code>metrics</code></strong>=<em><code>None</code></em>, <strong><code>path</code></strong>=<em><code>None</code></em>, <strong><code>model_dir</code></strong>=<em><code>'models'</code></em>, <strong><code>wd</code></strong>=<em><code>None</code></em>, <strong><code>wd_bn_bias</code></strong>=<em><code>False</code></em>, <strong><code>train_bn</code></strong>=<em><code>True</code></em>, <strong><code>moms</code></strong>=<em><code>(0.95, 0.85, 0.95)</code></em>) :: <code>GetAttr</code></p>\n",
       "</blockquote>\n",
       "<p>Group together a <code>model</code>, some <code>dls</code> and a <code>loss_func</code> to handle training</p>\n",
       "<p><a href=\"https://docs.fast.ai/learner#Learner\" target=\"_blank\" rel=\"noreferrer noopener\">Show in docs</a></p>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "doc(Learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#del learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(dls, model, \n",
    "                opt_func=partial(OptimWrapper, opt=torch.optim.Adam), #partial(Adam)\n",
    "                loss_func=CrossEntropyLossFlat(), \n",
    "                metrics=[accuracy], \n",
    "                cbs=[HF_BaseModelCallback], \n",
    "                splitter=hf_splitter).to_fp16()\n",
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#learn.summary() #Q: What is the purpose of HF_BaseModelWrapper, HF_BaseModelCallback, hf_splitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    }
   ],
   "source": [
    "print(len(learn.opt.param_groups))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SuggestedLRs(minimum=1.0964781722577755e-07, steep=0.033113110810518265, valley=0.0005754399462603033, slide=0.0020892962347716093)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEKCAYAAADw2zkCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXzU1b3/8dcn+x4IhDUoiyCrrC5oVZCqKKK01lIvveqtvbbXW7eqV/vQKrXa218vt3Vp1eqtcttSl6KtUtG6XFxBhSC7gGyWQCAJgSyTZCaTnN8fM8QASUhMZsnM+/l45MHM+X7n+/2cDJnPnHO+33PMOYeIiMS3hEgHICIikadkICIiSgYiIqJkICIiKBmIiAhKBiIiAiRFOoCO6t27txs8eHCkwxAR6VYKCwvLnHP5rW3vdslg8ODBrFq1KtJhiIh0K2b2eVvb1U0kIiJKBiIiomQgIiJ0wzGDltTX11NUVERdXV2kQ+n20tLSKCgoIDk5OdKhiEgYxUQyKCoqIjs7m8GDB2NmkQ6n23LOceDAAYqKihgyZEikwxGRMIqJbqK6ujp69eqlRNBJZkavXr3UwhKJQzGRDAAlgi6i36NIdHpj034+218VsuPHTDLoDl5++WV+/vOft7nP3r17+cY3vhGmiESku7h+USEvrN4TsuPHxJhBh617Ht66DyqKILcAZtwDp3wz5Ke99NJLufTSS9vcZ8CAASxevDjksYhI9+H1N1Df4MhKTQzZOeKvZbDueVhyI1TsBlzg3yU3Bso7YdeuXYwcOZJrrrmGESNGMG/ePN58803OOusshg8fzscff8zChQv5wQ9+AMA111zDjTfeyJlnnsnQoUObEsCuXbsYO3YsAAsXLmTOnDmcf/75DB48mF//+tf88pe/ZOLEiZxxxhmUl5cDMG3atKa7ssvKyjg8XUd7Xy8i0a3G2wBAZmrovr/HXzJ46z6orz2yrL42UN5J27Zt49Zbb2Xz5s1s3ryZP/3pT7z//vssWLCAn/3sZ8fsX1xczPvvv8/f/vY37rzzzhaPuWHDBl588UVWrlzJXXfdRUZGBp988glTp07l97///XFj6uzrRSTyqr1+ADJTlAy6TkVRx8o7YMiQIYwbN46EhATGjBnDjBkzMDPGjRvHrl27jtl/zpw5JCQkMHr0aPbv39/iMadPn052djb5+fnk5uYye/ZsgFaP2dWvF5HI8/iCyUAtgy6UW9Cx8g5ITU1tepyQkND0PCEhAb/f3+b+zrkvfcykpCQaGxsBjrkstKMxiUj08RxuGWjMoAvNuAeS048sS04PlHdTgwcPprCwEECDzyIxqDo4ZpCllkEXOuWbMPthyB0EWODf2Q+H5WqiULntttt47LHHmDhxImVlZZEOR0S62Bctg9AlA2uteyJaTZkyxR29nsGnn37KqFGjIhRR7NHvUyS6/HnVbm5fvI73/mM6g/IyvtQxzKzQOTelte3x1zIQEelmwtEyUDIQEYlyHl9gzCAjRQPIIiJxq9rrJynBSE0K3Ue2koGISJTzeP1kpiaFdCJJJQMRkShX7fWH9LJSUDIQEYl6Nd6GkN5wBiFMBmb2lJmVmNmGVrbPM7N1ZrbezJab2fhQxRIpDz74IDU1NZEOQ0S6OY/PH9IriSC0LYOFwMw2tu8EznXOjQN+CjwRwliO8MqOV7hg8QWc8r+ncMHiC3hlxyshOY+SgYh0hWqvP6ST1EEIk4Fz7l2g1TmSnXPLnXMHg08/BDo/OVA7vLLjFeYvn0+xpxiHo9hTzPzl8zudEDweD7NmzWL8+PGMHTuWn/zkJ+zdu5fp06czffp0AF5//XWmTp3KpEmTuOKKK6iurgagsLCQc889l8mTJ3PhhRdSXFwMBKamvummm5gwYQJjx47l448/7lzlRaRbCgwgd9Nuog66Fni1tY1mdp2ZrTKzVaWlpZ060UOrH6Ku4cjJ3Ooa6nho9UOdOu5rr73GgAEDWLt2LRs2bODmm29mwIABLFu2jGXLllFWVsb999/Pm2++yerVq5kyZQq//OUvqa+v54YbbmDx4sUUFhbyne98h7vuuqvpuDU1NaxZs4ZHH32U73znO52KUUS6J4+3IeTdRBFf6czMphNIBl9pbR/n3BMEu5GmTJnSqfkz9nn2dai8vcaNG8ett97KHXfcwSWXXMLZZ599xPYPP/yQTZs2cdZZZwHg8/mYOnUqW7ZsYcOGDZx//vkANDQ00L9//6bXXXnllQCcc845VFZWcujQIXr06NGpWEWkewnH1UQRTQZmdgrwP8BFzrkD4Thnv8x+FHuKWyzvjBEjRrB69WqWLl3K3XffzYwZM47Y7pzj/PPP55lnnjmifP369YwZM4YVK1a0eNyjryvWgvUi8aemmw8gt8nMTgBeBP7ZObc1XOe9adJNpCWmHVGWlpjGTZNu6tRx9+7dS0ZGBt/+9re5/fbbWb16NdnZ2VRVVQFwxhln8MEHH7Bt2zYgMMawdetWTj75ZEpLS5uSQX19PRs3bmw67nPPPQfA+++/T25uLrm5uZ2KU0S6ly/WP+6mLQMzewaYBvQ2syLgXiAZwDn3OHAP0At4NPht19/WjHpdZdbQWUBg7GCfZx/9Mvtx06Sbmsq/rPXr13P77beTkJBAcnIyjz32GCtWrGDmzJlNYwcLFy7kyiuvxOv1AnD//fczYsQIFi9ezI033khFRQV+v5+bb76ZMWPGAJCWlsbEiROpr6/nqaee6lzlRaTb8Rxe/ziE8xKBprCOatOmTWPBggVMmRLyHHmEWP19inRHu8trOPsXy/jFN07hm1MGfenjaAprEZFurDo4fXW37SaSznv77bcjHYKIRFg41jIAtQxERKLa4bUMsuLkpjMREWmBWgYiItI0ZtBt5yYSEZHOU8sghmVlZQGwa9cuxo4dG+FoRCSafZEMNGbQ5SqWLOGz82bw6ajRfHbeDCqWLIl0SCIiLar2NpCcaKQmKRl0qYolSyj+8T349+4F5/Dv3Uvxj+/pVEK48847+c1vftP0fP78+dx///3MmDGDSZMmMW7cOF566aU2j9HQ0MDtt9/OqaeeyimnnMJvf/tbAK666ir++te/Nu03b9684x5LRGJHOOYlgjhMBiW/ehBXd+QU1q6ujpJfPfiljzl37lyef/75pufPP/88V199NX/5y19YvXo1y5Yt49Zbb6Wtu71/97vfkZuby8qVK1m5ciVPPvkkO3fu5Nprr2XhwoUAVFRUsHz5cmbN6tzUGSLSfYRjYRuIw5vO/MXHzljaVnl7TJw4kZKSEvbu3UtpaSk9e/akX79+3HLLLbz77rskJCSwZ88e9u/fT79+Lc+O+vrrr7Nu3ToWL14MBD74P/vsMy644AKuv/56SktLeeGFF7j88stJSoq7t00kbnnCMH01xGEySOrfP9BF1EJ5Z1xxxRUsXryYffv2MXfuXBYtWkRpaSmFhYUkJyczePBg6o5qkTTnnOORRx7hwgsvPGbbVVddxR//+EeeffZZnn766U7FKSLdS2Bhm9COF0AcdhP1ueVmLO3IKawtLY0+t9zcqePOnTuXZ599lsWLF3PFFVdQUVFBnz59SE5OZtmyZXz++edtvv7CCy/kscceo76+HoCtW7fi8XgAuOaaa3jwwUA31ujRozsVp4h0L9Xe8IwZxF3LIHf2bCAwduAvLiapf3/63HJzU/mXNWbMGKqqqhg4cCD9+/dn3rx5zJ49m3HjxjFlyhRGjhzZ5uu/+93vsmvXLiZNmoRzjvz8/KaB4759+zJq1CjmzJnTqRhFpPvxeP30y0k7/o6dpCmsu4GamhrGjRvH6tWrw7K4Taz/PkW6kzP/8y2mDuvNf39zfKeOoymsu7k333yTUaNGccMNN2iVM5E45PE1hHySOojDbqLu5qtf/epxxxtEJDY55/CEacxALQMRkSjl9Tfib3RKBiIi8axpXqIQr38MSgYiIlHL4w0sbKOWgYhIHAvX+segZBBS06ZN4/BlsBdffDGHDh06Zp/58+ezYMGCcIcmIt1AjS88axlAnF5NtPWjfax4aTvV5V6y8lKZetkwRpze8pxBXWXp0qUhPb6IxJ7qMC1sA3HYMtj60T6WLdpMdbkXgOpyL8sWbWbrR/s6dVyPx8OsWbMYP348Y8eO5bnnnjti++DBgykrKwPggQceYMSIEXzlK19hy5YtTfts376dmTNnMnnyZM4++2w2b97cqZhEpHs7PGagbqIQWPHSdvy+xiPK/L5GVry0vVPHfe211xgwYABr165lw4YNzJw5s8X9CgsLefbZZ1mzZg1Lly5l5cqVTduuu+46HnnkEQoLC1mwYAHXX399p2ISke7t8NVEGd35aiIze8rMSsxsQyvbR5rZCjPzmtltoYrjaIdbBO0tb69x48bxxhtvcMcdd/Dee++1erfwe++9x9e+9jUyMjLIycnh0ksvDZy/uprly5dzxRVXMGHCBL73ve9R3IlptUWk+wvnAHIoz7AQ+DXw+1a2lwM3AmGdfS0rL7XFD/6svNROHXfEiBGsXr2apUuXcvfddzNjxowOvb6xsZEePXqwZs2aTsUhIrHDEwtjBs65dwl84Le2vcQ5txKoD1UMLZl62TCSUo6sdlJKAlMvG9ap4+7du5eMjAy+/e1vc/vtt7N69eoW9zvnnHP461//Sm1tLVVVVSwJLreZk5PDkCFD+POf/wwEbkNfu3Ztp2ISke7N42sgJTGBlKTQ9+jH3ZjBiNP7MX3eyKaWQFZeKtPnjez01UTr16/ntNNOY8KECfzkJz/h7rvvbnG/SZMmMXfuXMaPH89FF13Eqaee2rRt0aJF/O53v2P8+PGMGTNGax2LxLnAvEShHy+AEE9hbWaDgb8558a2sc98oNo51+rF9mZ2HXAdwAknnDD56InbNOVy19LvUyQ6/PC5NXy8q5z37ziv08eKiSmsnXNPOOemOOem5OfnRzocEZGwqA7T+sfQTZKBiEg88vj8YbmsFEJ4NZGZPQNMA3qbWRFwL5AM4Jx73Mz6AauAHKDRzG4GRjvnKkMVk4hId1LtbSAnLTwtg5CdxTl35XG27wMKuvB8mFlXHS5udbdlUEViWY3Xz4Dc0K9/DDHSTZSWlsaBAwf0QdZJzjkOHDhAWlp4/vOJSNvCtcoZxMhEdQUFBRQVFVFaWhrpULq9tLQ0Cgq6rMEmIp0QzgHkmEgGycnJDBkyJNJhiIh0GeccHl9D2O4ziIluIhGRWOP1N9LQ6MhI0aWlIiJxK5yT1IGSgYhIVKoJ4/rHoGQgIhKVvmgZaMxARCRuecK4/jEoGYiIRKVwrn8MSgYiIlHJowFkEREJ5/rHoGQgIhKVPMGridQyEBGJY+Fc/xiUDEREolK1z09KUgLJieH5mFYyEBGJQp4wTlIHSgYiIlHJ4w3fJHWgZCAiEpWqvX4ywzRJHSgZiIhEpRpf+Ba2ASUDEZGoVO1tUDIQEYl3gQFkjRmIiMQ1j8YMRESk2qsxAxGRuOacC7QM1E0kIhK/vP5GGl34pqIAJQMRkahTVReYlyhbyUBEJH4drPEB0DMzJWznDFkyMLOnzKzEzDa0st3M7GEz22Zm68xsUqhiERHpTg5UB5JBXkYMJANgITCzje0XAcODP9cBj4UwFhGRbuNwyyAvKwaSgXPuXaC8jV0uA37vAj4EephZ/1DFIyLSXRzwxFbL4HgGArubPS8Klh3DzK4zs1Vmtqq0tDQswYmIRMpBTwyNGXQl59wTzrkpzrkp+fn5kQ5HRCSkyj0+stOSwrawDUQ2GewBBjV7XhAsExGJa+UeH73C2CqAyCaDl4GrglcVnQFUOOeKIxiPiEhUKPf4wtpFBNCuOxrMLBOodc41mtkIYCTwqnOuvo3XPANMA3qbWRFwL5AM4Jx7HFgKXAxsA2qAf+lEPUREYka5x8eAHmlhPWd7b297FzjbzHoCrwMrgbnAvNZe4Jy7sq0DOucc8O/tPL+ISNwo9/gYMyAnrOdsbzeROedqgK8DjzrnrgDGhC4sEZH45JyjvMYX1nsMoAPJwMymEmgJvBIsC990eiIiccLja8DnbwzrPQbQ/mRwM/Aj4C/OuY1mNhRYFrqwRETi0+F7DPKicQDZOfcO8A6AmSUAZc65G0MZmIhIPDoQoWTQrpaBmf3JzHKCVxVtADaZ2e2hDU1EJP5EqmXQ3m6i0c65SmAO8CowBPjnkEUlIhKnorplACSbWTKBZPBy8P4CF7qwRETiU7S3DH4L7AIygXfN7ESgMlRBiYjEqwMeH8mJRlYYVzmD9g8gPww83KzoczObHpqQRETi10GPj7zMFMwsrOdt7wByrpn98vA00mb23wRaCSIi0oUOeHz0DPM9BtD+bqKngCrgm8GfSuDpUAUlIhKvDtb46BXmu4+h/XMTDXPOXd7s+U/MbE0oAhIRiWeRmJcI2t8yqDWzrxx+YmZnAbWhCUlEJH5FYi0DaH/L4PvA780sN/j8IHB1aEISEYlP9Q2NVNTWh30tA2j/1URrgfFmlhN8XmlmNwPrQhmciEg8OVQTWCImEi2DDq105pyrDN6JDPDDEMQjIhK3yoM3nEWiZdCZZS/DexGsiEiMK4/Q3cfQuWSg6ShERLpQJJNBm2MGZlZFyx/6BqSHJCIRkThVXhOlycA5lx2uQERE4l15dXDMIIrvQBYRkRA7WOMjJy2J5MTwfzQrGYiIRIkDwUnqIkHJQEQkShxUMhAREbUMREQkdlsGZjbTzLaY2TYzu7OF7Sea2Vtmts7M3jazglDGIyISrZxzlHt8Ebn7GEKYDMwsEfgNcBEwGrjSzEYftdsC4PfOuVOA+4D/DFU8IiLRzONrwNfQSF4ELiuF0LYMTgO2Oed2OOd8wLPAZUftMxr4v+DjZS1sFxGJC4fvMYjFbqKBwO5mz4uCZc2tBb4efPw1INvMeh19IDO77vCSm6WlpSEJVkQkkiJ59zFEfgD5NuBcM/sEOBfYAzQcvZNz7gnn3BTn3JT8/PxwxygiEnLlHi8QuWTQ3sVtvow9wKBmzwuCZU2cc3sJtgzMLAu43Dl3KIQxiYhEpXJPYC2DWGwZrASGm9kQM0sBvgW83HwHM+ttZodj+BHwVAjjERGJWpFuGYQsGTjn/MAPgL8DnwLPO+c2mtl9ZnZpcLdpwBYz2wr0BR4IVTwiItGs3FNPcqKRlRrKDpvWhfSszrmlwNKjyu5p9ngxsDiUMYiIdAflHi95mSmYRWbdsEgPIIuICIGWQSSmrj5MyUBEJAqUe7z0ylIyEBGJawdr1DIQEYl7B6q99IrQlUSgZCAiEnYNjY7l28qoqgvcW1Df0EhlnT9ik9RBiK8mEhGRY729pYRr/3cVKUkJTD85n3NGBGZWiGTLQMlARCTMdpfXAPD1iQN5a3MJf9+4H4C8zNSIxaRkICISZiVVXpISjJ99bRwO+GjnAVbtOsi0kyM395qSgYhImO2v9JKfnUpCQuAGszOH9ebMYb0jGpMGkEVEwqykqo4+OWmRDuMISgYiImFWUumlT3bkxgdaomQgIhJmJVV19M1RMhARiVtefwMHa+rpm61uIhGRuFVaFVi3oI9aBiIi8Wt/5eFkoJaBiEjcKqmsA9AAsohIPCsJdhP1VctARCR+7a+sIynByIvgdNUtUTIQEQmjkqoj7z6OFkoGIiJhtL8y+u4+BiUDEZGwKq2KvruPQclARCSs9ldG393HoGQgIhI2h+8+7hNldx+DkoGISNiUNl1WqpaBiEjcarr7ON5aBmY208y2mNk2M7uzhe0nmNkyM/vEzNaZ2cWhjEdEJJJKq4J3H8dTy8DMEoHfABcBo4ErzWz0UbvdDTzvnJsIfAt4NFTxiIhE2uGWQbTdfQyhbRmcBmxzzu1wzvmAZ4HLjtrHATnBx7nA3hDGIyISUSVV0Xn3MYR2DeSBwO5mz4uA04/aZz7wupndAGQCXw1hPCIiEXX02sfRJNIDyFcCC51zBcDFwB/M7JiYzOw6M1tlZqtKS0vDHqSISFcoidIbziC0yWAPMKjZ84JgWXPXAs8DOOdWAGlA76MP5Jx7wjk3xTk3JT8/P0ThioiEVkmUTkUBoU0GK4HhZjbEzFIIDBC/fNQ+/wBmAJjZKALJQF/9RSQmRevdxxDCZOCc8wM/AP4OfErgqqGNZnafmV0a3O1W4F/NbC3wDHCNc86FKiYRkUiJ5ruPIbQDyDjnlgJLjyq7p9njTcBZoYxBRCQaRPPdxxD5AWQRkbhweIWzaG0ZKBmIiIRB09rHahmIiMQvtQxERIT9lXUkJhi9MqPv7mNQMhARCYuSysANZ9F49zGE+GoiEZFYs3V/FQ+88im7DngYkJvOgB7pDOyZzuj+OZw3sg9v/ONVHlr9EPs8++iX2Y+bJt3ErKGz2B/Fdx+DkoGISLtU1tXz4Buf8b8rdpGVmsRXTurNvso6PthWxv6qOpyDHn3WQ68/04APgGJPMfOXzwegpDKbQXkZkavAcSgZiIgcx9L1xdzz0gYOeHxcedoJ3HbByeQ16/v3+Rv5YFsZt330X/iCieCwuoY6Hlr9EGVVtzP5xJ7hDr3dlAxERNqwr6KOm59dw8n9snn6mtMYV5B7zD4pSQlMH9mH+o/KWz6GZx+VHl9UrmNwmAaQRUTa8Pg722lwjkfnTWoxETTXL7Nfi+X56X0BonrMQMlARKQVJZV1PPPxP/j6xIHt6u+/adJNpCUe+e3fNSaTWxuYji2aWwbqJhIRacXj7+zA3+j4wXkntWv/WUNnARxxNVGe9zI+XD8YiN67j0HJQESkRSVVdSz66HPmTBjIib0y2/26WUNnNSUFCAwuf6d2JR9sL6N/bnooQu0SSgYiIi148t0d1Dc0trtV0JqUpASevGoKm/dVHnEFUrTRmIGIyFHKqr384cPPuWzCQIb0bn+roDXpKYlMPCF6LysFJQMRkWM8+d4OfP7Otwq6EyUDEZFmXtuwj6c/2MXs8QMYlp8V6XDCRslARCToqfd38m+LChkzIId7Z4+JdDhhpQFkEYl7jY2O+1/5lKc+2MmFY/ry0LcmkpacGOmwwkrJQETimsfr59bn1/Laxn38y1mDuXvWaBKjdJrpUFIyEJG4tWVfFdcvKmRnmYcfXzKaa78yJNIhRYySgYjEnIZGx86yatbvqWBdUQV7D9Vy+pBenD+6b9O0EosLi7j7r+vJSk3mj989nTOH9Y5w1JGlZCAi3Ya/oZHiijpSkhJanOfn8wMeHn9nB0vW7qXa6wcgLTmB/OxU/r5xP/f9bROj+ucwIDeNtzaXcMbQPB6+cmLUrkscTkoGEnbbSqopOljDuSPyMevavlnnHFv3V/PRzgP4/I1kpSaRlZZEdloyA3ukM6R3Zlz2BwNULFlCya8exF9cTFL//vS55WZyZ8+OdFgtamx07CjzsH7PIdYVVbCtpJrd5TUUHazF3+gAGN0/h6+O7sv5o/qSmGA89s52Xlm3l6TEBC4bP4DThuRxSkEPhuVnkpSYwK4yD29s2s/rm/bxwfYybjjvJG7+6oi4/f9wNHPORTqGDpkyZYpbtWpVpMOQL6Gx0fHUBzv5xd+34PM3Mm5gLj+8YATTOpEUPF4/W/ZXsWlvJR/tLGfF9jLKqn2t7p+WnMDIfjmMGZDD+IIeTB3WK6pXn+oqFUuWUPzje3B1dU1llpZG/5/eF9aEUFbt5an3d7LrgIfSKi8lVV7KqryYGWnJiWSkJJKWnMDeQ3VHfLMf3iebE3plcGJeBif2yuBgTT1vfbqfws8PEswNZKYk8u2pJ3LtWUPoc5zZQZ1zXf5FJNqZWaFzbkqr25UMJByKK2q57c9r+WDbAWaM7MOMUX159O1tFB2sZfKJPZk7ZRCl1V52lXn4/EANB2t8jB6Qw8RBPZhwQk9G9M2i6GAtW/dXsXVfFVv3V7N5XyWfl9dw+L9w35xUzhzWm6nDejF1aC9yM5KprvNT7fVTWVvPrgM1bNpbyca9FWwqrqSqLvBhM7BHOlOH9eK0IXlMHNSDoflZUftt0TlHRW09+yrr2F/pZX9lHSWVdZRUeSmp9FLu8ZGQAMmJCaQkJpCSlEBeZgqX/+LfST9YeuwB+/Zj8Btvkp4Smssot360jxUvbae63ItlJvFmkpd1ifWc0CuDPtmp9MlOo3dWYCbP2voGan1+ausb6JOdxikFuUd8s2/JgWovy7aUUllbz+WTCsjNSA5JPWJBRJOBmc0EHgISgf9xzv38qO2/AqYHn2YAfZxzPdo6ppJB91JcUctrG/bxqze2Ut/guGf2aL516iDMDJ+/kedX7eaR//uM/ZVeILD4x+DemWSnJrFhb0VTeXMJBif2ymRkv2xG9c9p+regZ3q7v+055/ispJoV2w+wYvsBPtx5gEM19UDgG+bYgbkMzc/CW9+Ax+enxtdAra/hiGM0OofX34jX30hdfQP1DY1kpibRIz2Znhkp5KYnU+31U1rtpbTKS1m1l6SEBHLTk+mREfjJTU8mKzXQjZWVmkRSglHnb6DW10idvwGP10+5x8fBGh8HPfWUVXvx+huPqU9OWhJ9ctLolZmCA+obGqlvaKSuvpFyj48/LLqxxTtMG4HZX1vAkN6ZjOqfw6j+OeSmJ1NZV09VXSCJ+hscKUkJpCYFkktOejJDemcyLD+TQXkZpCYlUuPzs+dgLUUHaymt9pKbnozbVc3O14porP8i3gaDCV8fwjnnx+9VO5ESsWRgZonAVuB8oAhYCVzpnNvUyv43ABOdc99p67jdORnsLPPw0po9JCcmkJWaREZKIvnZqZw9PD9qv4m2V31DI4dq6jlU46Pc4+PjneW88el+1hVVADD5xJ4suGJ8i5N+1dU3sLu8hoE908lIOXIYq7iilk/+cYjtJdUMystgeN8shuVndfkNQYE+6mrW7q5gbdEh1hZVUFReQ3pKIpkpSWSmJpKWnEjzXGMYqUkJpCYnkJaUSHJiAtVeP4dqfcHfRT3ZaUnkZ6eSn5VKr6wUGhrhUI2PQ7X1HKzxUVXnp6qunuo6P55gsklMMNKSEkhPSSQjJYmemSnkZSTTMzOFXpkp9M1Jo19uGn1z0uibnUafnNTj/j4+m34e/uLiY8p9vfvw1r1P8rT9ku8AAA1cSURBVGlxJZ8WV1J0sLZpW3KikZ2WTHJiIHF7/Y34/I1NffYQSMw56clNibS56ypSyXXHpqCsvFSu/tlZx31PpGsdLxmEcgD5NGCbc25HMJBngcuAFpMBcCVwbwjjOYK/oZE3Nu3n7BH5ZKV2/NfgnOMvn+xhZ5mH8QU9mHBCj6bmbkteXruXH72wrukPvrnxBbk88LVxjB3Y9pJ6odDY6Civ8dEjPbnVpjhAVV0920qq+aykmu2l1eyrqKOk0ktJVaCL4nCXS3MTT+jBf8w8mQtG92VYflar39rTkhMZ3je7xW39c9PpPy70c8AnJBgn9cnmpD7ZXD65IOTna0lDo6PROZLbeB++rD4/vKXFMYPBd9zGD88f0VRWUVuPt76B7LRk0pITWnzPKmrr2VXmYUdZNTtLPRzw+BjQI52CnukMysugd2YqlXX1vPPTwhZjqS4/trUnkRfKZDAQ2N3seRFweks7mtmJwBDg/1rZfh1wHcAJJ5zQJcE9/cEuHlj6KSfkZfCruROYfGL7p5f1+hu46y8bWFxYdET5oLx0Th2cxwWj+3LOiHwyUpKoq2/gp3/bxKKP/sHkE3vyyJUTyctMweMNdD2s+rycB17ZzKW/fp+rzxzMD88fQXZa6Po9SyrrWPX5wcC3392H2LCnkmqvHzPIy0ihd1YqPTKSqW9opLY+0P1RVeenrPqLP+CUxAT65abRJzuVk/tlc/bwfPIyU+iZkUyPjBR6ZCRzcr9sXa7XQYkJRiKhaSEeHiQ+3tVEuenJkN72/7/c9GTGD+rB+EFt9uhSmJfa4gd/Vl70rvYVz0LZTfQNYKZz7rvB5/8MnO6c+0EL+94BFDjnbjjecbuim6ikqo7zFrzD8L5ZlFZ5Ka6o4wfTT+KG805q89sxBD5Mv/fHQj75xyFumjGc684Zysa9lazZfZA1uw+xfHug7zktOYGzh+ez52Atm4or+d45Q7ntwpNb/NZXUVvPgr9v4Y8ffU7vrFTOHt6bk/tmc3KwL7yj66b6/I1Ue/1U1/mp8tazo9TDih0H+HDHAXaUeoDAB/qo/tmcUtCDIb0zqaitb+rbrqipD3R9JCeSHrzC44ReGQzvk83wPlkMysvo9t1aEnpbP9rHskWb8fu+GDNISklg+ryRjDi95YXjJXQi2U20BxjU7HlBsKwl3wL+PYSxHOH/vboFr7+B/75iPL2zU5n/0kYeeusz3v2slMfmTaZfbssfvmt3H+K6P6yiqs7PY/MmcdG4/gCcNiSP04bkAYG+85U7y/n7xn28vmk/Xn8jv7t6CjNG9W01ntz0ZH46ZyyXTy7g4bc+44NtZby4+otf1cl9s5l1Sn8uHtefk/o0m1J33fPw1n1QUQS5BXw6+hZu3TyCTcWVx5wjKzWJ04bk8a1TB3H6kF6M7J9NalJ8TcQl4XX4A//w1URZealMvWyYEkGUCmXLIInAAPIMAklgJfBPzrmNR+03EngNGOLaEUxnWwar/3GQrz+6nO+fO4w7LxrZVL5k7V7ufGEdJ/bK5M/fn0rmUeMI64sqmPvECvIyU3jyqimM6p9z3HM553Au0B/dUQc9Prbsr2LDngpe37iflZ+X4xyM7JfNuSfncwnvM7bwx5j/iwG/GpfCgpR/J/f0eeSmJ5EVvEKlf24aYwbkHLfVIyKxK9KXll4MPEjg0tKnnHMPmNl9wCrn3MvBfeYDac65O9tzzM4kg8ZGx5xHP2BfRR3/d9u0YwaOl20p4dqFK5kxqi+//fbkpg/x3eU1fO3R5aQmJfDi9Wd2uNumK+yrqOPVDcW8un4fn+w+yLLEGyhIKDtmP5dbgN2ysYUjiEg8001nzTy38h/c8cJ6Hpw7gTkTB7a4z9Mf7OQnSzbxvXOH8qOLRlHu8fGNx5ZzwOPjhX+bykl9Wr7qJZzq6htIfaAXRkvvncH8Q2GPSUSiWyTHDKJKRW09v3htC1NO7MllEwa0ut81Zw5me2k1v31nBwN7pPOXT/aw51Ati757elQkAghcikluAVTsPnZjbmQuixSR7i1uOpHf3LSfgzU+5l86ps27VM2Me2eP4ezhvbnnpY2s2X2Ih741kSmD88IYbTvMuAeSj7r+Pjk9UC4i0kFx0zK4fHIBk0/syeAW7oA9WnJiAr/+p0nc9OwnXDy2PzPHRuHVD6d8M/Bvs6uJmHHPF+UiIh0QV2MGIiLx6nhjBnHTTSQiIq1TMhARESUDERFRMhAREZQMREQEJQMREUHJQEREUDIQERG64U1nZlYKfN6sKBeoaOfj3sCxU322X/NjdnSflsqPLmvr+eHHzcs6U5/O1KW1be2Jv7XHem+OH2d799F7c+z70d3r0trjjtTnROdcfqtbA3Pud98f4In2PiYwdXaXnKuj+7RUfnRZW8+b1aF52ZeuT2fq8mXqo/dG702435tYqkuo6tP8Jxa6iZZ08HFXnauj+7RUfnRZW8+XtLLPl9WZurS2rT3xt/W4M/TetL0tHt+bWKpLW4+7RLfrJuoMM1vl2pibo7uJpfrEUl0gtuqjukSvrqxPLLQMOuKJSAfQxWKpPrFUF4it+qgu0avL6hNXLQMREWlZvLUMRESkBUoGIiKiZCAiIkoGTczsbDN73Mz+x8yWRzqezjCzBDN7wMweMbOrIx1PZ5nZNDN7L/j+TIt0PJ1lZplmtsrMLol0LJ1lZqOC78tiM/u3SMfTGWY2x8yeNLPnzOyCSMfTWWY21Mx+Z2aL27N/TCQDM3vKzErMbMNR5TPNbIuZbTOzO9s6hnPuPefc94G/Af8bynjb0hV1AS4DCoB6oChUsbZHF9XHAdVAGhGsTxfVBeAO4PnQRNl+XfR382nw7+abwFmhjLctXVSXvzrn/hX4PjA3lPEeTxfVZ4dz7tp2n7Sr7l6L5A9wDjAJ2NCsLBHYDgwFUoC1wGhgHIEP/OY/fZq97nkguzvXBbgT+F7wtYu7+3sDJARf1xdY1M3rcj7wLeAa4JLu/t4EX3Mp8CrwT929LsHX/TcwKRbem+Dr2vUZkEQMcM69a2aDjyo+DdjmnNsBYGbPApc55/4TaLF5bmYnABXOuaoQhtumrqiLmRUBvuDThtBFe3xd9d4EHQRSQxFne3TRezMNyCTwR1xrZkudc42hjLs1XfXeOOdeBl42s1eAP4Uu4tZ10XtjwM+BV51zq0Mbcdu6+O+mXWIiGbRiILC72fMi4PTjvOZa4OmQRfTldbQuLwKPmNnZwLuhDOxL6lB9zOzrwIVAD+DXoQ2twzpUF+fcXQBmdg1QFqlE0IaOvjfTgK8TSNJLQxpZx3X07+YG4KtArpmd5Jx7PJTBfQkdfW96AQ8AE83sR8Gk0apYTgYd5py7N9IxdAXnXA2BxBYTnHMvEkhwMcM5tzDSMXQF59zbwNsRDqNLOOceBh6OdBxdxTl3gMD4R7vExAByK/YAg5o9LwiWdUexVBeIrfrEUl0gtuoTS3WBENcnlpPBSmC4mQ0xsxQCg3YvRzimLyuW6gKxVZ9YqgvEVn1iqS4Q6vpEcsS8C0fenwGK+eJSymuD5RcDWwmMwN8V6TjjrS6xVp9Yqkus1SeW6hKp+miiOhERieluIhERaSclAxERUTIQERElAxERQclARERQMhAREZQMJEaYWXWYz9cla14E12qoMLM1ZrbZzBa04zVzzGx0V5xf5DAlA5EWmFmb83Y5587swtO955ybAEwELjGz460LMIfArKciXUbJQGKWmQ0zs9fMrNACK6WNDJbPNrOPzOwTM3vTzPoGy+eb2R/M7APgD8HnT5nZ22a2w8xubHbs6uC/04LbFwe/2S8KToWMmV0cLCs0s4fN7G9txeucqwXWEJidEjP7VzNbaWZrzewFM8swszMJrB/wX8HWxLDW6inSEUoGEsueAG5wzk0GbgMeDZa/D5zhnJsIPAv8R7PXjAa+6py7Mvh8JIHps08D7jWz5BbOMxG4OfjaocBZZpYG/Ba4KHj+/OMFa2Y9geF8Me34i865U51z44FPCUxJsJzAfDS3O+cmOOe2t1FPkXbTFNYSk8wsCzgT+HPwizp8sTBOAfCcmfUnsGLUzmYvfTn4Df2wV5xzXsBrZiUEVls7eunNj51zRcHzrgEGE1imc4dz7vCxnwGuayXcs81sLYFE8KBzbl+wfKyZ3U9gHYcs4O8drKdIuykZSKxKAA4F++KP9gjwS+fcy8HFWeY32+Y5al9vs8cNtPw305592vKec+4SMxsCfGhmzzvn1gALgTnOubXBxXCmtfDatuop0m7qJpKY5JyrBHaa2RUQWNLQzMYHN+fyxTzwV4cohC3A0GZLFx53gfVgK+LnwB3BomygONg1Na/ZrlXBbcerp0i7KRlIrMgws6JmPz8k8AF6bbALZiNwWXDf+QS6VQqBslAEE+xquh54LXieKqCiHS99HDgnmER+DHwEfABsbrbPs8DtwQHwYbReT5F20xTWIiFiZlnOuerg1UW/AT5zzv0q0nGJtEQtA5HQ+dfggPJGAl1Tv41wPCKtUstARETUMhARESUDERFByUBERFAyEBERlAxERAQlAxERAf4/HLXPtAk0CwUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find(suggest_funcs=[minimum, steep, valley, slide])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.706078</td>\n",
       "      <td>0.692356</td>\n",
       "      <td>0.490000</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.694096</td>\n",
       "      <td>0.687766</td>\n",
       "      <td>0.535000</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.687647</td>\n",
       "      <td>0.687146</td>\n",
       "      <td>0.535000</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(3, lr_max=slice(5e-4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "SuggestedLRs(minimum=0.03019951581954956, steep=0.25118863582611084, valley=1.737800812406931e-05, slide=0.2089296132326126)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZxU1bXo8d+qruq5aaBpZJZBUAREsAPiCNeAI0Jy4/RM1Bu95EbjEI1P39WoSUxePu/5jEMSjTcSYq5jcA7EOASjxgnoMCmICKgNDT1Bz11dw3p/1OlO0VTPdbqqutf386kPdfaZVh2gVp2999lbVBVjjDGmLU+iAzDGGJOcLEEYY4yJyRKEMcaYmCxBGGOMickShDHGmJgsQRhjjInJm+gA4mnYsGE6fvz4RIdhjDEpY/369RWqWhhrXb9KEOPHj2fdunWJDsMYY1KGiHze3jqrYjLGGBOTJQhjjDExWYIwxhgTU79qg4glEAhQUlJCU1NTokNJaZmZmYwZMwafz5foUIwxfcS1BCEiY4HHgCMABR5R1fvbbDMfeBHY5RQ9p6o/dtadBdwPpAG/VdWf9ySOkpIS8vLyGD9+PCLSo88y0KkqlZWVlJSUMGHChESHY4xxrNq5ivuL72df/T5G5Izg+tnXc+7Ec+N2fDfvIILATapaLCJ5wHoReU1VP26z3duqel50gYikAb8CFgIlwFoReSnGvp1qamqy5NBLIkJBQQHl5eWJDsUY41i1cxV3vXsXTaFI7UhpfSl3vXsXQNyShGttEKpaqqrFzvtaYCswuou7zwF2qOpOVW0GngKW9DQWSw69Z9fQmORyf/H9rcmhRVOoifuL729nj+7rk0ZqERkPzAI+iLF6nohsFJE/i8g0p2w08GXUNiV0PbmknJdeeomf/7zjGrS9e/fyjW98o48iMsYku331+7pV3hOuN1KLSC7wLHCDqta0WV0MHKmqdSJyDvACMLmbx18GLAMYN25c7wPe9Ay88WOoLoH8MXDGHXDchb0/bgfOP/98zj///A63GTVqFCtXrnQ1DmNM6hiRM4LS+tKY5fHi6h2EiPiIJIfHVfW5tutVtUZV65z3qwGfiAwD9gBjozYd45QdRlUfUdUiVS0qLIz5tHjXbXoGXr4Oqr8ENPLny9dFynto9+7dHHPMMVxxxRVMmTKFSy+9lNdff52TTz6ZyZMn8+GHH7JixQq+973vAXDFFVdw3XXXcdJJJzFx4sTWpLB7926mT58OwIoVK1i6dCkLFy5k/Pjx/PKXv+Tee+9l1qxZnHjiiVRVVQEwf/781ifLKyoqaBmGpKv7G2OS1/WzrycjLfOQssy0TK6ffX3czuFagpBIpfWjwFZVvbedbUY42yEic5x4KoG1wGQRmSAi6cDFwEtuxdrqjR9DoPHQskBjpLwXduzYwU033cS2bdvYtm0bTzzxBO+88w733HMPP/vZzw7bvrS0lHfeeYc//elP3HrrrTGPuWXLFp577jnWrl3LbbfdRnZ2Nv/4xz+YN28ejz32WKcx9XZ/Y0xinTvxXM4deS3h5sEIwsickdx10l0p04vpZOBbwGYR2eCU/ScwDkBVHwa+AXxXRIJAI3CxRibJDorI94C/EOnmulxVP3Ix1ojqku6Vd9GECROYMWMGANOmTeOMM85ARJgxYwa7d+8+bPulS5fi8Xg49thj2b9/f8xjLliwgLy8PPLy8sjPz2fx4sUAzJgxg02bNnUaU2/3N8YkXuOBmXj3/pB//HAhHk/8O5K4liBU9R2gw4hV9ZfAL9tZtxpY7UJo7csf41QvxSjvhYyMjNb3Ho+nddnj8RAMBjvcPpIve3ZMr9dLOBwGOOxBwe7GZIxJPh/uquIr44e6khzAhto41Bl3gC/r0DJfVqQ8BY0fP57169cDWAO3Mf3M/pomdlc2cOLEoa6dwxJEtOMuhMUPQP5YQCJ/Ln7A9V5MbvnBD37AQw89xKxZs6ioqEh0OMaYOHp/ZyUAcycUuHYOaa8KIxUVFRVp2/kgtm7dytSpUxMUUf9i19KY5HHb85t5ccNeNtyxEG9az3/ri8h6VS2Ktc7uIIwxJgV9sKuKovFDepUcOmMJwhhjUkxFnZ8dZXWuVi+BJQhjjEk5a3dFHmad62IDNViCMMaYlPPBriqyfGnMGJ3v6nn6/YRBxhjTX2z/YB/vvfgZw6r8XJWewa51ZUyZG7+xl9qyBGGMMSlg+wf7WPP4NoLNYQTIbFbWPL4NwLUkYVVMCXDffffR0NCQ6DCMMSnkvRc/I9gcPqQs2BzmvRc/c+2cliDaWLVzFYtWLuK43x/HopWLWLVzVdzPYQnCGNNddVX+bpXHgyWIKC1T+JXWl6Jo6xR+vUkS9fX1nHvuucycOZPp06fzox/9iL1797JgwQIWLFgAwKuvvsq8efOYPXs2F1xwAXV1dQCsX7+e008/nRNOOIEzzzyT0tLI2O/z58/n+uuv5/jjj2f69Ol8+OGHvf/wxpikljs0o1vl8WAJIoobU/i98sorjBo1io0bN7JlyxZuuOEGRo0axZo1a1izZg0VFRXcfffdvP766xQXF1NUVMS9995LIBDg2muvZeXKlaxfv55vf/vb3Hbbba3HbWhoYMOGDfz617/m29/+do/jM8akhnlLJuFNP/Qr25vuYd6SSa6d0xqpo7gxhd+MGTO46aabuOWWWzjvvPM49dRTD1n//vvv8/HHH3PyyScD0NzczLx58/jkk0/YsmULCxcuBCAUCjFy5MjW/S655BIATjvtNGpqajh48CCDBw/ucZzGmOTW0hD98uNbyWhW8oZmMG/JJOvF1FfcmMJvypQpFBcXs3r1am6//XbOOOOMQ9arKgsXLuTJJ588pHzz5s1MmzaN9957L+ZxnXmW2l02xvQ/U+aO4NV3t3NkQTaPXBZz+KS4siqmKNfPvp7MOE/ht3fvXrKzs/nmN7/JzTffTHFxMXl5edTW1gJw4okn8ve//50dO3YAkTaL7du3c/TRR1NeXt6aIAKBAB999M85k55++mkA3nnnHfLz88nPd/eBGWNMciiv81OY5167QzS7g4jSMlXf/cX3s69+HyNyRnD97Ot7NYXf5s2bufnmm/F4PPh8Ph566CHee+89zjrrrNa2iBUrVnDJJZfg90d6I9x9991MmTKFlStXct1111FdXU0wGOSGG25g2rRpAGRmZjJr1iwCgQDLly/v/Yc3xiS9QChMVX1znyUIG+47Bc2fP5977rmHoiL3bzGj9cdraUwq2V/TxNyfvcFPvzadS+ceGZdjJmS4bxEZKyJrRORjEflIRA6rpxGRS0Vkk4hsFpF3RWRm1LrdTvkGEVnXdl9jjBloymsjtQyFualfxRQEblLVYhHJA9aLyGuq+nHUNruA01X1gIicDTwCzI1av0BVbSq0Nt58881Eh2CMSYCWBDEs1dsgVLUUKHXe14rIVmA08HHUNu9G7fI+MMateIwxJtX19R1En/RiEpHxwCzggw42uxL4c9SyAq+KyHoRWdbBsZeJyDoRWVdeXh6PcI0xJimV1zkJItXvIFqISC7wLHCDqta0s80CIgnilKjiU1R1j4gMB14TkW2q+lbbfVX1ESJVUxQVFfWfFndjjGmjvNZPXqaXTF9an5zP1TsIEfERSQ6Pq+pz7WxzHPBbYImqVraUq+oe588y4HlgjpuxGmNMsuvLZyDA3V5MAjwKbFXVe9vZZhzwHPAtVd0eVZ7jNGwjIjnAImCLW7Emk9zcXAB2797N9OnTExyNMSaZlNf6+6z9AdytYjoZ+BawWUQ2OGX/CYwDUNWHgTuAAuDXzlARQac/7hHA806ZF3hCVV9xMdZW1S+/TNkv7iNYWop35EiGf/8G8hcv7otTG2NMhypq/UwdNajPzudmL6Z3gA4HCFLVq4CrYpTvBGYevoe7ql9+mdIf3oE2RUZ0De7dS+kP7wDocZK49dZbGTt2LNdccw0Ad911F16vlzVr1nDgwAECgQB33303S5YsafcYoVCIW2+9lTfffBO/388111zDd77zHS677DK+/vWvs3TpUgAuvfRSLrzwwg6PZYxJXeW1fk7rwzsIG4spStkv7mtNDi20qYmyX9zX42NedNFFPPPMM63LzzzzDJdffjnPP/88xcXFrFmzhptuuomOnmh/9NFHyc/PZ+3ataxdu5b/+q//YteuXVx55ZWsWLECgOrqat59913OPbfnw4IYY5JXUyBErT/Yp20QNhZTlGDp4SO5dlTeFbNmzaKsrIy9e/dSXl7OkCFDGDFiBN///vd566238Hg87Nmzh/379zNiROxRY1999VU2bdrEypUrgUgy+PTTT1m0aBFXX3015eXlPPvss/zrv/4rXq/9lRrTH7U+A2EJIjG8I0cS3Ls3ZnlvXHDBBaxcuZJ9+/Zx0UUX8fjjj1NeXs769evx+XyMHz+epjZ3LtFUlQcffJAzzzzzsHWXXXYZ//3f/81TTz3F7373u17FaYxJXn39DARYFdMhhn//BiTz0OG+JTOT4d+/oVfHveiii3jqqadYuXIlF1xwAdXV1QwfPhyfz8eaNWv4/PPPO9z/zDPP5KGHHiIQCACwfft26uvrAbjiiiu4775IFdixxx7bqziNMcmrr5+iBruDOERLQ3S8ezFNmzaN2tpaRo8ezciRI7n00ktZvHgxM2bMoKioiGOOOabD/a+66ip2797N7NmzUVUKCwt54YUXADjiiCOYOnVqa0O1MaZ/SkQVkw33neIaGhqYMWMGxcXFrk8a1N+vpTHJ7BevbeeBv37K9rvPxpcWv8qfhAz3bdz3+uuvM3XqVK699lqbUc6Yfq68zs/Q7PS4JofOWBVTCvvqV7/aafuFMaZ/qKjt22E2wO4gjDEmJfT1OExgCcIYY1JCX4/DBJYgjDEm6akq5bX+PptJroUlCGOMSXK1/iD+YNjuIAaC+fPn09Id95xzzuHgwYOHbXPXXXdxzz339HVoxpgklIhnIMB6MR1m+wf7eO/Fz6ir8pM7NIN5SyYxZW7sMZLiYfXq1a4d2xjTP1QkKEHYHUSU7R/sY83j26irivxl1FX5WfP4NrZ/sK/Hx6yvr+fcc89l5syZTJ8+naeffvqQ9ePHj6eiogKAn/70p0yZMoVTTjmFTz75pHWbzz77jLPOOosTTjiBU089lW3btvU4HmNM6knEOExgCeIQ7734GcHm8CFlweYw7734WY+P+corrzBq1Cg2btzIli1bOOuss2Jut379ep566ik2bNjA6tWrWbt2beu6ZcuW8eCDD7J+/Xruuecerr766h7HY4xJPYkYhwmsiukQLXcOXS3vihkzZnDTTTdxyy23cN5553HqqafG3O7tt9/ma1/7GtnZ2QCcf/75kXPX1fHuu+9ywQUXtG7r9/c8HmNM6imv9eP1CPlZvj49r2sJQkTGAo8RmT5UgUdU9f422whwP3AO0ABcoarFzrrLgdudTe9W1d+7FWuL3KEZMZNB7tCeZ+0pU6ZQXFzM6tWruf322znjjDO6tX84HGbw4MFs2LCh842NMf1Sea2fYbkZeDwdTtIZd25WMQWBm1T1WOBE4BoRaTse9dnAZOe1DHgIQESGAncCc4E5wJ0iMsTFWAGYt2QS3vRDL4k33cO8JZN6fMy9e/eSnZ3NN7/5TW6++WaKi4tjbnfaaafxwgsv0NjYSG1tLS+//DIAgwYNYsKECfzxj38EIv2hN27c2ON4jDGpJxFPUYOLCUJVS1vuBlS1FtgKjG6z2RLgMY14HxgsIiOBM4HXVLVKVQ8ArwGxK+/jaMrcESy49JjWO4bcoRksuPSYXvVi2rx5M3PmzOH444/nRz/6EbfffnvM7WbPns1FF13EzJkzOfvss/nKV77Suu7xxx/n0UcfZebMmUybNo0XX3yxx/EYY1JPRYISRJ+0QYjIeGAW8EGbVaOBL6OWS5yy9spdN2XuiLh2az3zzDMPmwnuzTffbH2/e/fu1ve33XYbt91222HHmDBhAq+88krcYjLGpJbyWj/TRvb9iM2u92ISkVzgWeAGVa1x4fjLRGSdiKwrLy+P9+GNMSahwmGloq65f1UxAYiIj0hyeFxVn4uxyR5gbNTyGKesvfLDqOojqlqkqkWFhYXxCdwYY5LEgYZmQmFlWG56n5/btQTh9FB6FNiqqve2s9lLwGUScSJQraqlwF+ARSIyxGmcXuSUGWPMgFLmPAPR1wP1gbttECcD3wI2i0hLH83/BMYBqOrDwGoiXVx3EOnm+m/OuioR+QnQ8rTYj1W1qqeBqCqRfGV6qj9NTWtMKtm+vxaASYW5fX5u1xKEqr4DdPitrJFvnWvaWbccWN7bODIzM6msrKSgoMCSRA+pKpWVlWRmZiY6FGMGnI1fVpPp8zB5eD9KEMlizJgxlJSUYA3YvZOZmcmYMWMSHYYxA87mPQeZNiofbx/ORd2i3ycIn8/HhAkTEh2GMcZ0WzAUZsueGi6eM7bzjV1gg/UZY0yS+qy8nsZAiOPG9P0zEGAJwhhjktbGkshkYseNGZyQ81uCMMaYJLW5pJq8DC8TCnIScn5LEMYYk6Q2lRxk+uj8Ph/FtYUlCGOMSULNwTBbS2sT1v4AliCMMSYpfbKvluZQOGHtD2AJwhhjktKmPS0N1HYHYYwxJsqmL6sZku1jzJCshMVgCcIYY5LQpj3VzBgzOKFDBFmCMMaYJNPYHGL7/lpmJrB6CSxBGGNM0vm4tIZQWJkx2hKEMcaYKJsS/AR1C0sQxhiTZDaXVDM8L4MR+YkdYt8ShDHGJJmNJQcT2r21hSUIY4xJInX+IDsr6pkxOrHVS2AJwhhjksqWPdWownFjE38H4dqEQSKyHDgPKFPV6THW3wxcGhXHVKDQmY96N1ALhICgqha5FacxxiSTzSXVAAnvwQTu3kGsAM5qb6Wq/l9VPV5Vjwf+F/A3Va2K2mSBs96SgzFmwNi0p5rRg7MYlpuR6FDcSxCq+hZQ1emGEZcAT7oVizHGpIrNJQeT4u4BkqANQkSyidxpPBtVrMCrIrJeRJZ1sv8yEVknIuvKy8vdDNUYY1xV3RBgd2UDM5KgBxMkQYIAFgN/b1O9dIqqzgbOBq4RkdPa21lVH1HVIlUtKiwsdDtWY4xxzZa9kfaHZOjiCsmRIC6mTfWSqu5x/iwDngfmJCAuY4zpU5uSqIEaEpwgRCQfOB14MaosR0TyWt4Di4AtiYnQGGP6zuY9Bxk3NJvB2emJDgVwt5vrk8B8YJiIlAB3Aj4AVX3Y2exrwKuqWh+16xHA884Qt17gCVV9xa04jTEmWWwqqWbm2MQ/INfCtQShqpd0YZsVRLrDRpftBGa6E5UxxiSnqvpmSg408q0Tj0x0KK2SoQ3CGGMGvM17nPaHJGmgBksQxhiTFDY7Q3xPT5IGarAEYYwxSWFTSTUTh+UwKNOX6FBaWYIwxpgksHlPdVJVL4ElCGOMSbiy2iZKq5uS5vmHFl1KEM6zCR7n/RQROV9Ekuc+yBhjUtiWPS1PUCdPF1fo+h3EW0CmiIwGXgW+RZvuqcYYY3pmU0k1IjBt1KBEh3KIriYIUdUG4OvAr1X1AmCae2EZY8zAsbmkmqMKc8nJcO3RtB7pcoIQkXlEJvhZ5ZSluROSMcYMLLsq65l8RG6iwzhMVxPEDUQm9XleVT8SkYnAGvfCMsaYgaOqvpmCnMRPENRWl+5nVPVvwN8AnMbqClW9zs3AjDFmIAiGwhxsCDA0JzkG6IvW1V5MT4jIIGd01S3Ax86c0sYYY3rhQEMAgILcFE0QwLGqWgMsBf4MTCDSk8kYY0wvHGhoBmBIkgzxHa2rCcLnPPewFHhJVQNEpgU1xhjTC5V1kQRRkKpVTMBvgN1ADvCWiBwJ1LgVlDHGDBRV9ZEEMTQJq5i62kj9APBAVNHnIrLAnZCMMWbgqKr3A6R0I3W+iNwrIuuc1/8jcjdhjDGmF6rqI43UqdwGsRyoBS50XjXA7zraQUSWi0iZiMScT1pE5otItYhscF53RK07S0Q+EZEdInJrF2M0xpiUU1XvZ1CmF19a8o2d2tXnuiep6r9GLf9IRDZ0ss8K4JfAYx1s87aqnhddICJpwK+AhUAJsFZEXlLVj7sYqzHGpIzK+mYKcpPvITno+h1Eo4ic0rIgIicDjR3toKpvAVU9iGkOsENVd6pqM/AUsKQHxzHGmKRXVd+clO0P0PU7iP8AHhORlsHKDwCXx+H880RkI7AX+IGqfgSMBr6M2qYEmNveAURkGbAMYNy4cXEIyRhj+k5VfTNjhmQnOoyYunQHoaobVXUmcBxwnKrOAv6ll+cuBo50jvsg8EJPDqKqj6hqkaoWFRYW9jIkY4zpW5FxmJLzDqJbrSKqWuM8UQ1wY29O7Byrznm/msjDeMOAPcDYqE3HOGXGGNOvqCoHGpqT8hkI6N2Uo9KbE4vICBER5/0cJ5ZKYC0wWUQmiEg6cDHwUm/OZYwxyaimKUggpEl7B9Gb2Sk6HGpDRJ4E5gPDRKQEuBPwAajqw8A3gO+KSJBIg/fFqqpAUES+B/yFyJwTy522CWOM6VcO1CfvOEzQSYIQkVpiJwIBsjraV1Uv6WT9L4l0g421bjWwuqP9jTEm1VUm8TAb0EmCUNW8vgrEGGMGmpZxmJK1iin5Ht0zxpgBIpnHYQJLEMYYkzAt4zBZgjDGGHOIqno/mT4P2em96S/kHksQxhiTIJX1zRTkJOc4TGAJwhhjEiaZx2ECSxDGGJMwB+qbGWIJwhhjTFuVSTwOE1iCMMaYhLEqJmOMMYdpCoRoaA5ZgjDGGHOolqeoLUEYY4w5hCUIY4wxMVUm+ThMYAnCGGMSItnHYQJLEMYYkxDJPg4TWIIwxpiEqKr3k+YRBmX6Eh1KuyxBGGNMAlTVNzMkOx2Pp1ezN7vKtQQhIstFpExEtrSz/lIR2SQim0XkXRGZGbVut1O+QUTWuRWjMcYkSmVdcj9FDe7eQawAzupg/S7gdFWdAfwEeKTN+gWqeryqFrkUnzHGJMyBhmaG5CRv9RK4mCBU9S2gqoP176rqAWfxfWCMW7EYY0yySfahviF52iCuBP4ctazAqyKyXkSWJSgmY4xxTbKPwwSQ8GmMRGQBkQRxSlTxKaq6R0SGA6+JyDbnjiTW/suAZQDjxo1zPV5jjOmtYCjMwYZA0ieIhN5BiMhxwG+BJapa2VKuqnucP8uA54E57R1DVR9R1SJVLSosLHQ7ZGOM6bWDjcn/DAQkMEGIyDjgOeBbqro9qjxHRPJa3gOLgJg9oYwxJhWlwjhM4GIVk4g8CcwHholICXAn4ANQ1YeBO4AC4NciAhB0eiwdATzvlHmBJ1T1FbfiNMaYvlZZl/zjMIGLCUJVL+lk/VXAVTHKdwIzD9/DGGP6h9Y7iNzkThDJ0ovJGGMGjKoGJ0FkW4IwxhgTpcqpYhoyUKuYjDHGRDQFQrz9aQX7qhspq/Xz+tYyBmV68aUl9290SxDGGOOigw3NXPn7daz/PDJwhEegIDeDxTNHJTiyzlmCMMYYl+w92Mhlyz/ki8oG7r1wJqccNYyC3AzSkngE12iWIIwxxgXb99dy2aMfUu8P8tiVczhxYkGiQ+o2SxDGGBNn+6qbuODh90j3enj6O/M4dtSgRIfUI5YgjDEmzt7Ytp/qxgCrrjslZZMDWDdXY4yJuw92VjE8L4NjR6ZucgBLEMYYE1eqyvs7K5k7sQBnyKCUZQnCGGPiaHdlA2W1fk6cODTRofSaJQhjjImj93dGZi6YOyH1ei21ZQnCGGPi6IOdlQzLzWBSYU6iQ+k1SxDGGBMnkfaHKk6cODTl2x/AEoQxxsTN55UN7KtpYm4KPhQXiyUIY4yJkw92Rdof5vWDBmqwBGGMMXHz/s4qhuWmM6kwN9GhxIUlCGOMiQNV5YOdlcydkPrPP7RwNUGIyHIRKRORLe2sFxF5QER2iMgmEZkdte5yEfnUeV3uZpzGGNNdew82UlHnb13+sqqRvdVNzO0n1Uvg/h3ECuCsDtafDUx2XsuAhwBEZChwJzAXmAPcKSJDXI3UGGO6SFW5+JH3+Zd73mT15lIA3nfaH1Jx1Nb2uDpYn6q+JSLjO9hkCfCYqirwvogMFpGRwHzgNVWtAhCR14gkmifdjNcYY7pia2ktX1Q1MDQnnasfL+aSOeOoaQwwNCedycP7R/sDJH4019HAl1HLJU5Ze+WHEZFlRO4+GDdunDtRGmNMlDe27gfgT9eewmPvfc7Df/sMgLOnj+g37Q/QDxqpVfURVS1S1aLCwsJEh2OMGQDe2FbGzLGDGTU4i1vPPoY/XDmHo4bnsnRWzN+xKSvRCWIPMDZqeYxT1l65McYkVHmtn40lB/nqMcNby06dXMjrN57OmdNGJDCy+Et0gngJuMzpzXQiUK2qpcBfgEUiMsRpnF7klBljTEKt2VaGKvzL1OGdb5ziXG2DEJEniTQ4DxOREiI9k3wAqvowsBo4B9gBNAD/5qyrEpGfAGudQ/24pcHaGGMS6fWt+xmVn5nykwF1hdu9mC7pZL0C17Szbjmw3I24jDGmJ5oCId7ZUcHXZ4/uV43R7Ul0FZMxxqSM93dW0tAc4oxjjkh0KH3CEoQxxnTRG1vLyPKlMW9S/3kYriOWIIwxpgtUlb9uK+OUycPI9KUlOpw+YQnCGGO6YNu+WvYcbOSrA6D3UgtLEMYY0wWvfxx5enrB0ZYgjDHGOMpr/Tz6913Mm1jA8EGZiQ6nz1iCMMaYTtz18kc0+EP8ZOm0RIfSpyxBGGNMB/7y0T5WbSrlujOO4qjheYkOp08lejRXY4zpM02BEAcamglrpFeSKuRmeMnP8uHxHP7gW3VjgB++sIWpIwfxndMnJSDixLIEYYzp95oCIX7/7m5+tWYHNU3Bw9aLwOAsH8NyM5g7cSgLjh7OvEkF/GzVVirrm1l+xVfwpQ28ChdLEMaYfisUVp4rLuEXr21nb3UT848uZNGxI0jz0DpURl1TkIMNzVQ1NLPnQCPPFe/hv9//gnSvh+ZgmO/On8T00fkJ/iSJYQnCGNMvhcLKd/6wjte3ljFzTD73XDiTkyYN63Q/fzDE2l0HWPNJGZV1fq4/Y3IfRKGTQ7UAABBnSURBVJucLEEYY/qle1/7hNe3lnHbOVO56tQJXR5cL8ObximTh3HK5M6TSX838CrVjDH93qpNpfxqzWdcMmdst5KDOZQlCGNMv7K1tIYf/HEjs8cN5q7zp1ly6AVLEMaYfqOizs+yP6wjL9PLw988gQzvwBhUzy3WBmGMSXk1TQEefXsXj76zi+ZgmKe+c+KAGhLDLW5POXoWcD+QBvxWVX/eZv0vgAXOYjYwXFUHO+tCwGZn3Reqer6bsRpjUk8wFOa37+zi4b99xsGGAGdNG8GNi6Yw5YiB9cSzW1xLECKSBvwKWAiUAGtF5CVV/bhlG1X9ftT21wKzog7RqKrHuxWfMSb13f/Gpzz41x0sOLqQGxcezYwxA/N5Bbe4eQcxB9ihqjsBROQpYAnwcTvbXwLc6WI8xph+ZP3nB/jVmh1844Qx3HPBzESH0y+52Ug9GvgyarnEKTuMiBwJTAD+GlWcKSLrROR9EVna3klEZJmz3bry8vJ4xG2MSXL1/iA3PrOBkflZ3Ln42ESH028lSy+mi4GVqhqKKjtSVYuA/wHcJyIxR8pS1UdUtUhViwoLC/siVmNMgt29aitfVDVw74Uzycv0JTqcfsvNKqY9wNio5TFOWSwXA9dEF6jqHufPnSLyJpH2ic/iH6YxJpHCYeVgYyAyumpLmSqhcOQVDkOGz0N+lo9MXxpvbN3Pkx9+wXdOn8jciQUJjb2/czNBrAUmi8gEIonhYiJ3A4cQkWOAIcB7UWVDgAZV9YvIMOBk4P+4GKsxphvKaprYXdlAKKwoCgqDs9OZNDynw2cPymqa2LK3mo/31vBpWR07yur4rLyOpkC4S+dN93pQVY4ZkceNC6fE6+OYdriWIFQ1KCLfA/5CpJvrclX9SER+DKxT1ZecTS8GnlJVjdp9KvAbEQkTqQb7eXTvJzPwVL/8MmW/uI9gaSnekSMZ/v0byF+8ONFh9Zg/GKKirpmymiYONDSTne5lSHY6Q7J9DMrykeH1dPgE8P6aJv647kueLd5DnT9IfpaP/CwfeZleAqEw9f4Qjc0hAqEwhXkZjMzPZER+Flm+NEoONPDlgQa+rGrEHwwzKMvLoMzIviJCczCEPxgmEAozJDudIwZlMmJQJlnpaXy8t4Z/fHGAvdVNMePyeoRJhbkcMzKP7HQvTYEQDc1B6vxBtu+vo7zW37rt6MFZHDU8lxMnFjBmSBZpznwMAiCC1yOkeYQ0ERoDIaobA9Q0BvAHw/zbyePtIbg+IId+L6e2oqIiXbduXaLDMHFW/fLLlP7wDrTpn19KkpnJyJ/8OCFJIhRW6puDNDWHaAqEaQyEaA5GfgG3fKcHw0q9P/LFWO8PUlrdxK6KenZV1LO7op7K+uZOz5PmiXxJ5mV6GTs0myOHZjOuIIetpTX8dVsZobBy8lEFjB2STXVjgOrGALVNQdK9HrLT08hOT8Pr8VBW20RpdRP7a5oIhJQjBmUwZkg2Y4dkkelLo7YpSE1ToHWehAyvhwyvB69HqGoIUFbTRFmtn1BYGTMki+PHDmbWuCFMHp6L1yMgIAgVdX627athW2kt2/bV4g+GyUr3kO3zkpWexqTCXKaPHsT00flMHTmI3Ax7TjcZiMh6p7338HWWINxTUednwxcHqWpopq4p8mVR5w/S0BykwR+ioTlEWJWC3HQKcjIoyE1ncLaP3AwfORlp5GX4yM30kpfpJTfDS6Yv8otJVQlr5IvKlyaH/NIMhMJUNwY42BBgf00TX1Q1tL4a/EHCGqnfjf5rF4mMjZ+XGfklmZ/lIzcjDRHBI4JI5FxNgciv0sZAKFK1EHWM3Ewvg7N8kfgzvTQFwq2f2R8Mk+nzkOVLIys9DV+aB2k5L0KGz8OgLB+DMn0MyvQSCCmNgWDkV3AgxKj/uBhv+f7Drm9g2HC++PWT5Ga0XCMf6V4PHqE17shn8XZrPJ5wWCmv8/Pp/jo+Lq3mo72RL73K+mbq/UEaA6HODxLD8LwMJgzLYcKwHEYNzqIwL4PheRkMzUmnsTlEVUMzBxoiv5Kbg2GC4TDBkFLTFODzygY+r2ygtLqRoTnpXFA0lou/MpYjC3K69bmCYSXd2/2+KaGw0tActAbhfqijBGEpPE6aAiF2lNWxtbSG4i8O8MGuKnaW1x+2XYbXQ06Gt/UXniAUf3GQAw3NhMIdJ2tfmqAa+XUaLT3N0/qfvs5/+GxZvjRh9OAs8rN8IILzow8RoeUHQiislBxooMb5JRoIHR6LCGT70sj0peFNa6kOEMIa+bVc3xz7i9OXJjGP11WrYiQHgLSKMr73xD863T/D62FYbgZDciJfbsGQOnXntCauTF8azcEwpdVNlFY3HhLvqPxMpo4cxOwjh5CbkUZOhpec9Miv4kxfGlm+tNbr33I90zxCboaXnIxIch+WlxGXX8zNwXCk2iXG9Jid8XiE9B7sB5HPY8lh4LEE0UY4rKzaXMreg43U+4PU+oPUNgU5UB+ZcepAfTONgRDZ6V6yfGnkZKRxoCHAror61i/4vEwvXxk/lAuLxlJ05BCOGJRJXmbky6K9aQtbenJUNwYi522944hUG7QsC+BNi9z+ewSaQ0pzMExzMIyiDM6K3IUMzvZRmJvBuIJsRuZndesLRVUJhJSw82UXVsUj0mm9eHMw7FRzBMhKT4t8QaZ78Xjkn3cggUi9uEbaNVFVmgLhSBWHU0XiSxOy0yNJNNOXhrw7Asr2HXY+zxEjeOWGU52qnBC1TQECoTDhcOTYoXAknsq6Zsrr/Byob0acuu2WBOd3qojq/EG8HmHWuMGcO3gkowZnMaEgh2NHDWJoTnqXr53bevLr35iesgQRJRRWbnl2EyvXlwCRX8y56V5yM70MzUlnaE46Y4dkk+VLoyEQotFpfBtfkMPZ00dwzIhBHD0ijwnDcrr9C8/jkdZzJJqIkO7t/i/NdK+HwrwMCvMyDluX5pHIL+8e/IquvvnG2G0QP7iR/BGDun08Y0zXWIJwhMLK/1y5iWeLS7j+jMksO21ipArIxpJPuJaG6P7Ui8mYVGAJgkhyuPmPG3nuH3u4ceEUrhvAc9Amq/zFiy0hGNPHBnyCCIWVm57ZwAsb9vKDRVP43r9YcjDGGLAEQZ0/yCf767j5zKO5ZsFRiQ7HGGOSxoBPEPlZPp6/+qTWZwyMMcZEWJ85sORgjDExWIIwxhgTkyUIY4wxMVmCMMYYE5MlCGOMMTFZgjDGGBOTJQhjjDExWYIwxhgTU7+aMEhEqoFPo4rygeouvh8GVPTgtNHH6s76WOVtyzqLubexdxRfZ+s7i7+9z9KX176jbTq61m2X7dp3PbaubBOvaw8D8/9tvK/9YFUtjHkUVe03L+CR9pY7e09knuxen7Or62OVdzf+3sbuZvztfZa+vPbdid+ufepdezfjT+b/t25e+7av/lbF9HIHy115H49zdnV9rPLuxt/b2LtyjJ7G395n6ctr39E2HV3rtst27bsWQ1e36c/Xvu2yG/G7ee0P0a+qmHpDRNZpO/OyJrtUjh1SO/5Ujh0s/kRKhdj72x1EbzyS6AB6IZVjh9SOP5VjB4s/kZI+druDMMYYE5PdQRhjjInJEoQxxpiYLEEYY4yJyRJEF4jIqSLysIj8VkTeTXQ83SEiHhH5qYg8KCKXJzqe7hKR+SLytnP95yc6nu4SkRwRWSci5yU6lu4SkanOdV8pIt9NdDzdISJLReS/RORpEVmU6Hi6S0QmisijIrIykXH0+wQhIstFpExEtrQpP0tEPhGRHSJya0fHUNW3VfU/gD8Bv3cz3mjxiB1YAowBAkCJW7HGEqf4FagDMunD+OMUO8AtwDPuRNm+OP273+r8u78QONnNeKPFKfYXVPXfgf8ALnIz3rbiFP9OVb3S3Ug71+97MYnIaUS+YB5T1elOWRqwHVhI5EtnLXAJkAb87zaH+Laqljn7PQNcqaq1qRK78zqgqr8RkZWq+o2+iD2O8VeoalhEjgDuVdVLUyj2mUABkeRWoap/6ovYnVjj8u9eRM4Hvgv8QVWfSKXYnf3+H/C4qhb3RezOOeMZf5/+n23Lm6gT9xVVfUtExrcpngPsUNWdACLyFLBEVf83ELMqQETGAdV9lRwgPrGLSAnQ7CyG3Iv2cPG69o4DQIYbccYSp2s/H8gBjgUaRWS1qobdjLtFvK69qr4EvCQiq4A+SRBxuvYC/Bz4c18mB4j7v/uE6vcJoh2jgS+jlkuAuZ3scyXwO9ci6rruxv4c8KCInAq85WZgXdSt+EXk68CZwGDgl+6G1qluxa6qtwGIyBU4d0KuRte57l77+cDXiSTm1a5G1rnu/ru/FvgqkC8iR6nqw24G1wXdvfYFwE+BWSLyv5xE0ucGaoLoNlW9M9Ex9ISqNhBJbilJVZ8jkuRSlqquSHQMPaGqbwJvJjiMHlHVB4AHEh1HT6lqJZH2k4Tq943U7dgDjI1aHuOUpYJUjh1SO/5Ujh1SO/5Ujh1SNP6BmiDWApNFZIKIpAMXAy8lOKauSuXYIbXjT+XYIbXjT+XYIVXj78l45Kn0Ap4ESvlnN88rnfJziPQq+Ay4LdFx9rfYUz3+VI491eNP5dj7Q/zRr37fzdUYY0zPDNQqJmOMMZ2wBGGMMSYmSxDGGGNisgRhjDEmJksQxhhjYrIEYYwxJiZLEKZfE5G6Pj5fXOYLkcg8GNUiskFEtonIPV3YZ6mIHBuP8xsDliCM6RYR6XD8MlU9KY6ne1tVjwdmAeeJSGdzMiwlMnKsMXFhCcIMOCIySUReEZH1Epmt7hinfLGIfCAi/xCR1505KBCRu0TkDyLyd+APzvJyEXlTRHaKyHVRx65z/pzvrF/p3AE87gxBjYic45StF5EHRKTDeSJUtRHYQGREUETk30VkrYhsFJFnRSRbRE4Czgf+r3PXMam9z2lMV1mCMAPRI8C1qnoC8APg1075O8CJqjoLeAr4n1H7HAt8VVUvcZaPITIM+RzgThHxxTjPLOAGZ9+JwMkikgn8BjjbOX9hZ8GKyBBgMv8crv05Vf2Kqs4EthIZyuFdImP73Kyqx6vqZx18TmO6xIb7NgOKiOQCJwF/dH7Qwz8nIhoDPC0iI4F0YFfUri85v+RbrFJVP+AXkTLgCA6fEvVDVS1xzrsBGE9kprGdqtpy7CeBZe2Ee6qIbCSSHO5T1X1O+XQRuZvIHBm5wF+6+TmN6RJLEGag8QAHnbr9th4kMq3pS85kOXdFratvs60/6n2I2P+XurJNR95W1fNEZALwvog8o6obgBXAUlXd6ExGND/Gvh19TmO6xKqYzICiqjXALhG5ACJTU4rITGd1Pv8co/9yl0L4BJgYNSXlRZ3t4Nxt/By4xSnKA0qdaq3oObprnXWdfU5jusQShOnvskWkJOp1I5Ev1Sud6puPgCXOtncRqZJZD1S4EYxTTXU18Ipznlqgugu7Pgyc5iSWHwIfAH8HtkVt8xRws9PIPon2P6cxXWLDfRvTx0QkV1XrnF5NvwI+VdVfJDouY9qyOwhj+t6/O43WHxGp1vpNguMxJia7gzDGGBOT3UEYY4yJyRKEMcaYmCxBGGOMickShDHGmJgsQRhjjInJEoQxxpiY/j8eHi1DsYzlvQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find(suggest_funcs=[minimum, steep, valley, slide])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.589672</td>\n",
       "      <td>0.487191</td>\n",
       "      <td>0.775000</td>\n",
       "      <td>01:36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.382374</td>\n",
       "      <td>0.431692</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>01:36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(2, lr_max=1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       ""
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The trouble with the book, \"Memoirs of a Geisha\" is that it had Japanese surfaces but underneath the surfaces it was all an American man's way of thinking. Reading the book is like watching a magnificent ballet with great music, sets, and costumes yet performed by barnyard animals dressed in those costumesso far from Japanese ways of thinking were the characters.&lt;br /&gt;&lt;br /&gt;The movie isn't about Japan or real geisha. It is a story about a few American men's mistaken ideas about Japan and geish</td>\n",
       "      <td>negative</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;br /&gt;&lt;br /&gt;I'm sure things didn't exactly go the same way in the real life of Homer Hickam as they did in the film adaptation of his book, Rocket Boys, but the movie \"October Sky\" (an anagram of the book's title) is good enough to stand alone. I have not read Hickam's memoirs, but I am still able to enjoy and understand their film adaptation. The film, directed by Joe Johnston and written by Lewis Colick, records the story of teenager Homer Hickam (Jake Gyllenhaal), beginning in October of 195</td>\n",
       "      <td>positive</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How viewers react to this new \"adaption\" of Shirley Jackson's book, which was promoted as NOT being a remake of the original 1963 movie (true enough), will be based, I suspect, on the following: those who were big fans of either the book or original movie are not going to think much of this one...and those who have never been exposed to either, and who are big fans of Hollywood's current trend towards \"special effects\" being the first and last word in how \"good\" a film is, are going to love it.</td>\n",
       "      <td>negative</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.show_results(learner=learn, max_n=3, trunc_at=500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('positive',), (#1) [tensor(1)], (#1) [tensor([0.0640, 0.9360])])]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.blurr_predict(\"excellent movie, loved the whole plot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('negative',), (#1) [tensor(0)], (#1) [tensor([0.9270, 0.0730])]),\n",
       " (('positive',), (#1) [tensor(1)], (#1) [tensor([0.0640, 0.9360])])]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.blurr_predict([\"the movie plot has lot of holes in it, hated the songs\", \"excellent movie, loved the whole plot\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}