Q1_Q2_Q3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  CS6910: Fundamentals of Deep Learning\n",
    "#### Assignment 3 - Team 3\n",
    "\n",
    "S Renganathan, CH16B058\t     \n",
    "S Nithya, CH16B113\t\t         \n",
    "Vasistha Singhal, CH16B119"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7_qTq010ijWD"
   },
   "source": [
    "### Run this cell only once (the first ever time you run) to process data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "v8PCDpQkAHwN"
   },
   "outputs": [],
   "source": [
    "# # from google.colab import drive\n",
    "# # drive.mount('/content/drive')\n",
    "\n",
    "# # For extracting\n",
    "# !pip install pyunpack\n",
    "# !pip install patool\n",
    "\n",
    "# from pyunpack import Archive\n",
    "# Archive('CUB_200_2011.tgz').extractall('Assignment3_Data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fjjzGKEwFmsN"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "0Z5GNnFsDPna",
    "outputId": "957eaf3d-9c94-4942-ef91-ccb0300004b6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import tarfile\n",
    "import pandas as pd\n",
    "import os\n",
    "import re\n",
    "from torch.utils.data import Dataset, DataLoader, ConcatDataset, random_split\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch.optim as optim\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.decomposition import PCA\n",
    "from io import StringIO\n",
    "from PIL import Image\n",
    "import re\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score\n",
    "import pickle\n",
    "import torchvision.models as models\n",
    "\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "J6JD3ycyPuYi",
    "outputId": "439532ab-b3d2-4b1a-e5bb-0382057d7dd6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DP8iACDGDSWD"
   },
   "outputs": [],
   "source": [
    "class DatasetClass(Dataset):\n",
    "    \n",
    "    def __init__(self, directory, img_size):\n",
    "        \n",
    "        self.directory = directory\n",
    "        self.classes = ['026.Bronzed_Cowbird',\t'084.Red_legged_Kittiwake',\t'131.Vesper_Sparrow',\t'085.Horned_Lark',\t'015.Lazuli_Bunting',\t'041.Scissor_tailed_Flycatcher',\t'114.Black_throated_Sparrow']\n",
    "        print('Number of Classes =', len(self.classes))\n",
    "        self.files = []\n",
    "        for class_name in self.classes:\n",
    "            images = os.listdir(directory + '/' + class_name)\n",
    "            images = [class_name + '/' + image for image in images]\n",
    "            self.files.extend(images)\n",
    "        \n",
    "        self.img_size = img_size\n",
    "        self.size = len(self.files)\n",
    "        \n",
    "    def __getitem__(self, idx):     \n",
    "        \n",
    "        image_name = self.files[idx]\n",
    "        y = self.classes.index(re.split('/', image_name)[0])\n",
    "        img = Image.open(self.directory + '/' + image_name).convert(mode='RGB').resize(self.img_size)\n",
    "        \n",
    "        trans = transforms.ToTensor()\n",
    "        # return trans(img), torch.Tensor(y, dtype=torch.long)\n",
    "        \n",
    "        return trans(img), y        # Multiplying by pixel value\n",
    "      \n",
    "    def __len__(self):\n",
    "        \n",
    "        return self.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TKVGwDDOKAsM"
   },
   "outputs": [],
   "source": [
    "def train_test_loader(directory, img_size, train_fraction=0.7, cv_fraction=0.2, num_workers=0, batch_size=32):\n",
    "\n",
    "    dataset = DatasetClass(directory, img_size)\n",
    "    \n",
    "    N = dataset.size\n",
    "    train_size = int(N*train_fraction)\n",
    "    cv_size = int(N*cv_fraction)\n",
    "    test_size = N - train_size - cv_size\n",
    "\n",
    "    train_data, cv_data, test_data = torch.utils.data.random_split(dataset, [train_size, cv_size, test_size])\n",
    "\n",
    "    trainloader = DataLoader(train_data, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n",
    "    cvloader = DataLoader(cv_data, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n",
    "    testloader = DataLoader(test_data, batch_size=batch_size, shuffle=True, num_workers=num_workers)\n",
    "    \n",
    "    return trainloader, cvloader, testloader, train_size, cv_size, test_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "fT66UkOpUDtv",
    "outputId": "462e6575-f7d3-4a0d-c6d6-9b2a5f15e0b8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Classes = 7\n"
     ]
    }
   ],
   "source": [
    "# trainloader, cvloader, testloader, train_size, cv_size, test_size = train_test_loader('D:/_SEM8/DL/Assignment 3/Assignment3_Data/CUB_200_2011/images/', (224, 224))\n",
    "trainloader, cvloader, testloader, train_size, cv_size, test_size = train_test_loader('/content/drive/My Drive/Assignment3_Data/CUB_200_2011/images', (224, 224), batch_size=32)\n",
    "\n",
    "# if __name__ == \"__main__\":\n",
    "  # trainloader, cvloader, testloader, train_size, cv_size, test_size = train_test_loader('Assignment3_Data/CUB_200_2011/images/', (224, 224))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "81_AjruYX-U6",
    "outputId": "2236bc5e-f3ff-42e6-8a9e-90c814091e40"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 / 9, 2 / 9, 3 / 9, 4 / 9, 5 / 9, 6 / 9, 7 / 9, 8 / 9, 9 / 9, "
     ]
    }
   ],
   "source": [
    "RGB_mean = torch.zeros(3)\n",
    "i = 0\n",
    "for X, y in trainloader:\n",
    "    i += 1\n",
    "    RGB_mean += (X.sum(0).sum(1).sum(1)/(X.shape[2]*X.shape[2]))/train_size\n",
    "    print(i, '/', len(trainloader), end=', ')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Vc8N6oxxiwx0"
   },
   "source": [
    "### Question 1. a\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xzGC9uLa9xum"
   },
   "outputs": [],
   "source": [
    "class VGGNet(nn.Module):\n",
    "    \n",
    "    def __init__(self, RGB_mean, num_classes):\n",
    "        super(VGGNet, self).__init__()\n",
    "        \n",
    "        self.RGB_mean = RGB_mean.to(device)\n",
    "        self.num_classes = num_classes\n",
    "\n",
    "        self.c11 = nn.Conv2d(3, 64, 3, stride=1, padding=1)\n",
    "        self.c12 = nn.Conv2d(64, 64, 3, stride=1, padding=1)\n",
    "        self.p1 = nn.MaxPool2d(2, stride=2)\n",
    "\n",
    "        self.c21 = nn.Conv2d(64, 128, 3, stride=1, padding=1)\n",
    "        self.c22 = nn.Conv2d(128, 128, 3, stride=1, padding=1)\n",
    "        self.p2 = nn.MaxPool2d(2, stride=2)\n",
    "\n",
    "        self.c31 = nn.Conv2d(128, 256, 3, stride=1, padding=1)\n",
    "        self.c32 = nn.Conv2d(256, 256, 3, stride=1, padding=1)\n",
    "        self.c33 = nn.Conv2d(256, 256, 3, stride=1, padding=1)\n",
    "        self.p3 = nn.MaxPool2d(2, stride=2)\n",
    "\n",
    "        self.c41 = nn.Conv2d(256, 512, 3, stride=1, padding=1)\n",
    "        self.c42 = nn.Conv2d(512, 512, 3, stride=1, padding=1)\n",
    "        self.c43 = nn.Conv2d(512, 512, 3, stride=1, padding=1)\n",
    "        self.p4 = nn.MaxPool2d(2, stride=2)\n",
    "\n",
    "        self.c51 = nn.Conv2d(512, 512, 3, stride=1, padding=1)\n",
    "        self.c52 = nn.Conv2d(512, 512, 3, stride=1, padding=1)\n",
    "        self.c53 = nn.Conv2d(512, 512, 3, stride=1, padding=1)\n",
    "        self.p5 = nn.MaxPool2d(2, stride=2)\n",
    "\n",
    "        self.flat = nn.Flatten(1, -1)\n",
    "        self.fc1 = nn.Linear(7*7*512, 4096)\n",
    "        self.fc2 = nn.Linear(4096, 4096)\n",
    "        self.out = nn.Linear(4096, self.num_classes)\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        x = x - self.RGB_mean[None, :, None, None]\n",
    "        x = self.p1(F.relu(self.c12(F.relu(self.c11(x)))))\n",
    "        x = self.p1(F.relu(self.c22(F.relu(self.c21(x)))))\n",
    "        x = self.p3(F.relu(self.c33(F.relu(self.c32(F.relu(self.c31(x)))))))\n",
    "        x = self.p4(F.relu(self.c43(F.relu(self.c42(F.relu(self.c41(x)))))))\n",
    "        x = self.p5(F.relu(self.c53(F.relu(self.c52(F.relu(self.c51(x)))))))\n",
    "        x = F.relu(self.fc2(F.relu(self.fc1(self.flat(x)))))\n",
    "        Z = self.out(x)\n",
    "\n",
    "\n",
    "        return Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EwuOkncsWXVy"
   },
   "outputs": [],
   "source": [
    "VGG_model = VGGNet(RGB_mean, 7)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(VGG_model.parameters(), lr=0.001, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sZHF1Sn-Dhl0"
   },
   "outputs": [],
   "source": [
    "# vgg16 = models.vgg16(pretrained=True)\n",
    "# pickle.dump(vgg16, open('/content/drive/My Drive/vgg_init.sav', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Bms2NosWGTC8"
   },
   "outputs": [],
   "source": [
    "vgg16 = pickle.load(open('/content/drive/My Drive/vgg_init.sav', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aX8N4KctEfc4"
   },
   "outputs": [],
   "source": [
    "params = list(vgg16.parameters())\n",
    "\n",
    "with torch.no_grad():\n",
    "    \n",
    "    VGG_model.c11.weight = params[0]\n",
    "    VGG_model.c11.bias = params[1]\n",
    "    VGG_model.c12.weight = params[2]\n",
    "    VGG_model.c12.bias = params[3]\n",
    "    \n",
    "    VGG_model.c21.weight = params[4]\n",
    "    VGG_model.c21.bias = params[5]\n",
    "    VGG_model.c22.weight = params[6]\n",
    "    VGG_model.c22.bias = params[7]\n",
    "\n",
    "    VGG_model.c31.weight = params[8]\n",
    "    VGG_model.c31.bias = params[9]\n",
    "    VGG_model.c32.weight = params[10]\n",
    "    VGG_model.c32.bias = params[11]\n",
    "    VGG_model.c33.weight = params[12]\n",
    "    VGG_model.c33.bias = params[13]\n",
    "\n",
    "    VGG_model.c41.weight = params[14]\n",
    "    VGG_model.c41.bias = params[15]\n",
    "    VGG_model.c42.weight = params[16]\n",
    "    VGG_model.c42.bias = params[17]\n",
    "    VGG_model.c43.weight = params[18]\n",
    "    VGG_model.c43.bias = params[19]\n",
    "\n",
    "    VGG_model.c51.weight = params[20]\n",
    "    VGG_model.c51.bias = params[21]\n",
    "    VGG_model.c52.weight = params[22]\n",
    "    VGG_model.c52.bias = params[23]\n",
    "    VGG_model.c53.weight = params[24]\n",
    "    VGG_model.c53.bias = params[25]\n",
    "\n",
    "    VGG_model.fc1.weight = params[26]\n",
    "    VGG_model.fc1.bias = params[27]\n",
    "\n",
    "    VGG_model.fc2.weight = params[28]\n",
    "    VGG_model.fc2.bias = params[29]\n",
    "\n",
    "VGG_model = VGG_model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "VUTJaWtqSqQj",
    "outputId": "a0d51b7e-e628-4667-d04c-a7749d160401"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 : Loss = 1.8460866445448338 inf\n",
      "Epoch 2 : Loss = 1.0807606449110583 0.70813644375139\n",
      "Epoch 3 : Loss = 0.6524373722824071 0.6564971456651195\n",
      "Epoch 4 : Loss = 0.4732703884304193 0.3785721402223944\n",
      "Epoch 5 : Loss = 0.38688288800392423 0.22329108654094523\n",
      "Epoch 6 : Loss = 0.32553923711544136 0.18843704197392783\n",
      "Epoch 7 : Loss = 0.2902372030846333 0.12163166422367284\n",
      "Epoch 8 : Loss = 0.26113165608681865 0.11145928239408057\n",
      "Epoch 9 : Loss = 0.24088090819141178 0.08406954310930682\n",
      "Epoch 10 : Loss = 0.2186906454558987 0.10146873310128844\n",
      "Epoch 11 : Loss = 0.20416518692770902 0.0711456186373873\n",
      "Epoch 12 : Loss = 0.19065284765348203 0.07087404904009678\n",
      "Epoch 13 : Loss = 0.17924669883392416 0.06363380131271432\n",
      "Epoch 14 : Loss = 0.16815741092797357 0.06594587681122446\n",
      "Epoch 15 : Loss = 0.15963291021174253 0.05340064717810287\n",
      "Epoch 16 : Loss = 0.150935411323446 0.057623978442396605\n",
      "Epoch 17 : Loss = 0.14390422232475014 0.04886019940977488\n",
      "Epoch 18 : Loss = 0.13576473467026023 0.05995288595568472\n",
      "Epoch 19 : Loss = 0.13111437348330893 0.03546797397878964\n",
      "Epoch 20 : Loss = 0.12481511732115563 0.05046869559834637\n",
      "Epoch 21 : Loss = 0.12068857107220626 0.03419169033396307\n",
      "Epoch 22 : Loss = 0.11453054061334723 0.053767583963900197\n",
      "Epoch 23 : Loss = 0.10944787412881851 0.046439152199032205\n",
      "Epoch 24 : Loss = 0.1047580350670665 0.04476829924071747\n",
      "Epoch 25 : Loss = 0.1018730155630394 0.028319761499961092\n",
      "Epoch 26 : Loss = 0.09735744597085261 0.046381348104989235\n",
      "Epoch 27 : Loss = 0.09501377973406988 0.024666593028319857\n",
      "Epoch 28 : Loss = 0.09170677777678293 0.03606060574210001\n",
      "Epoch 29 : Loss = 0.08798246116796021 0.04233021626563648\n",
      "Epoch 30 : Loss = 0.08541762031848422 0.03002706982368322\n",
      "Epoch 31 : Loss = 0.08222835934120604 0.038785414215117324\n",
      "Epoch 32 : Loss = 0.07978247544802856 0.03065690653796216\n",
      "Epoch 33 : Loss = 0.07776238979570542 0.025977669380149476\n",
      "Epoch 34 : Loss = 0.07549703074456923 0.03000593571422201\n",
      "Epoch 35 : Loss = 0.07324027630925595 0.030813024595704338\n",
      "Epoch 36 : Loss = 0.07152482140354993 0.0239840501806674\n",
      "Epoch 37 : Loss = 0.06925546142957352 0.03276795688213187\n",
      "Epoch 38 : Loss = 0.06776119784195664 0.02205190632996245\n",
      "Epoch 39 : Loss = 0.06567215021271323 0.031810251719746385\n",
      "Epoch 40 : Loss = 0.0639694510125117 0.026617380222139303\n",
      "Epoch 41 : Loss = 0.06277076733859989 0.019096208708837877\n",
      "Epoch 42 : Loss = 0.061094235992286264 0.027441727015381678\n",
      "Epoch 43 : Loss = 0.05988659718778076 0.020165427010635145\n",
      "Epoch 44 : Loss = 0.0582114722618658 0.028776542850168268\n",
      "Epoch 45 : Loss = 0.057171784211534245 0.01818533503318235\n",
      "Epoch 46 : Loss = 0.05600896389434562 0.020761325265401257\n",
      "Epoch 47 : Loss = 0.054567586474821544 0.02641453493987961\n",
      "Epoch 48 : Loss = 0.05360048113695835 0.018042848074293828\n",
      "Epoch 49 : Loss = 0.05233452714434485 0.024189651874026757\n",
      "Epoch 50 : Loss = 0.05155399158752754 0.015140157585899545\n",
      "Epoch 51 : Loss = 0.05021944300690179 0.026574340548586796\n",
      "Epoch 52 : Loss = 0.04925371824604709 0.01960714429782578\n",
      "Epoch 53 : Loss = 0.048683343721287596 0.011716009648492686\n",
      "Epoch 54 : Loss = 0.047545176051323426 0.023938657178081663\n",
      "Epoch 55 : Loss = 0.04658432125733704 0.02062614132936525\n",
      "Epoch 56 : Loss = 0.0456821639285478 0.019748568176418225\n",
      "Epoch 57 : Loss = 0.044909997320756685 0.017193646267135952\n",
      "Epoch 58 : Loss = 0.044255546485609296 0.014787995790769396\n",
      "Epoch 59 : Loss = 0.04355728190119673 0.016030949451723692\n",
      "Epoch 60 : Loss = 0.04270058528938775 0.02006287749929027\n",
      "Epoch 61 : Loss = 0.041986414844669945 0.017009560053171967\n",
      "Epoch 62 : Loss = 0.04130131683802355 0.016587800561740605\n",
      "Epoch 63 : Loss = 0.04073164322598471 0.013986020865355617\n",
      "Epoch 64 : Loss = 0.040063708060086814 0.016671825905284085\n",
      "Epoch 65 : Loss = 0.039332595111912554 0.018587966191756063\n",
      "Epoch 66 : Loss = 0.039036850000403904 0.007576049591747018\n",
      "Converged\n",
      "Finished Training\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5xcdX3/8dd7d2Z3k73kQi5AgiRc5KaAGOMF1HjDaCtYa1vQWrQqP620Wq0ttv7UYu/Wiv6qVX6VUvtTKGKpqUIBQcUbmgUh3IoECCYhIRtyv+xt9vP743wnOdnMJrvJzs5s5v18POYxc67zmc1m3nu+33POVxGBmZnZcE21LsDMzOqTA8LMzCpyQJiZWUUOCDMzq8gBYWZmFTkgzMysIgeEWQOQ9FZJt9bw/T8h6f/V6v3t0Dgg7JBIeoukbkk7JK2TdLOk82pdl1UWEV+NiPPL05JC0km1rGkkklZJenWt6zAHhB0CSR8ErgT+CpgLPAv4AnBhLevKk1SodQ1HKv9sG4cDwsZE0jTgCuB9EfEfEbEzIgYi4r8i4sNpnVZJV0p6Kj2ulNSali2RtEbShyRtSEcf70jLXihpvaTm3Pv9mqQV6XWTpMslPSbpGUnXS5qZli1IfxW/U9IvgTskNUv6tKSNkp6QdFlap1D+LJK+nGpYK+kvyu8t6e2Sfijp7yVtTtu/LlfXTEn/kj7fZkn/mVv2q5LulbRF0o8lnXmAn2dI+j1Jj0raLumTkk5M221Ln7Elt/67Ja2UtEnSMknHDtvXe9K+tkj6vCTlP096fWfa5L50BPhbo9z3+yQ9Cjxa4XOUf/6Xpp/JOkl/dIDPfYGkB1Od35N0Wpr/b2R/cPxXqu2PR9qHTYCI8MOPUT+ApcAgUDjAOlcAdwFzgNnAj4FPpmVL0vZXAEXg9cAuYEZa/hjwmty+vg5cnl6/P+13PtAKfAm4Ni1bAATwFaAdmAK8B3gorT8D+E5ap5C2uTHtoz3V+jPgf6VlbwcGgHcDzcB7gacApeXfBv497bcIvDzNfx6wAXhh2u4SYBXQOsLPKoBvAl3AGUAfcDtwAjAt1X9JWveVwEbgnPT5/w9w57B9fQuYTvYl2wMszX2eHw5b96Tc9Gj2fRswE5hS4XOUf/7Xpp/nc9P7vzot/wTw/9LrZwM7gdekn90fAyuBlrR8VXk7P2r8/73WBfgxuR7AW4H1B1nnMeD1uenXAqvS6yXAbnIBk75QX5Re/wVwdXrdmb5Ijk/TDwOvym13TPoSL+S+oE7ILb+D9IWfpl+d1imQNY315b/sgIuB76bXbwdW5pZNTdsend53iBRqwz77P5HCMDfvEVKAVFg/gHNz03cDf5Kb/jRwZXr9ZeDvcss60udfkNvXebnl17M3XN/OgQNiNPt+5QH+zcs//1Nz8/4O+HJ6nQ+I/w1cn1uvCVgLLEnTDog6ebiJycbqGWDWQdqhjwWezE0/mebt2UdEDOamd5F9IQF8DXhTapJ6E3BPRJT3dTxwY2qW2EIWGCWyL/uy1cPqWD3CsuPJ/npdl9vfl8iOJMrWl19ExK70sgM4DtgUEZsrfPbjgQ+V95n2e9ywzz/c07nXuytMl382+/xcI2IH2b/HvEo1s+/P9WBGs+/VwzeqIL/O8H/3kd5rKG03r8K6VkMOCBurn5D95f3GA6zzFNkXZdmz0ryDioiHyL48Xge8hSwwylYDr4uI6blHW0Ssze8i93odWfNS2XHD9tUHzMrtqysizhhFmauBmZKmj7DsL4fVODUirh3Ffg9mn5+rpHbgKLK/vidi36O59XP+ZzzSv/vw91LarvxevsV0nXBA2JhExFbgY8DnJb1R0lRJRUmvk/R3abVrgY9Kmi1pVlp/LOfAf42sv+FlZH0QZV8E/lLS8QBp/wc6c+p64P2S5qUv8z/JfY51wK3ApyV1pQ7wEyW9/GDFpW1vBr4gaUb6/C9Li/8v8J7U4S5J7ZJ+RVLn6D/+iK4F3iHp7HSE9VfATyNi1SHs62myfo7x3vf/Tr8TZwDvIOunGe564FckvUpSEfgQWVj/eITarEYcEDZmEfFp4IPAR8k6IlcDlwHlM3n+AugGVgD3A/ekeaN1LfBy4I6I2Jib/1lgGXCrpO1kHdYvPMB+/i9ZCKwAfg7cRNZBXkrLfwdoIesI3gzcQNa/MBpvI2uj/x+yPpQPAEREN1nH9j+mfa4ka/8/bBHxHbL2+2+QHR2dCFx0iLv7BPCvqRnsN8dx398n+8y3A38fEftdnBcRjwC/TdYRvhF4A/CGiOhPq/w12R8YWw50JpRVX/mMDLMjXjpN9YsRcfxBV7YxkbQAeAIoDutfsknMRxB2xJI0RdLrJRUkzQM+TnZqq5mNggPCjmQC/pysqefnZGc9faymFZlNIm5iMjOzinwEYWZmFR1RN92aNWtWLFiwoNZlmJlNGnfffffGiJhdadkRFRALFiygu7u71mWYmU0akp4caZmbmMzMrCIHhJmZVeSAMDOzihwQZmZWkQPCzMwqckCYmVlFDggzM6vIAQF87vZH+f4vempdhplZXXFAAFfd+Tjff8QBYWaW54AAOloL7OgbqHUZZmZ1xQEBdLYV2NHnMU7MzPIcEEBHW4HtvQ4IM7M8BwRZE5MDwsxsXw4IoKut6CYmM7NhHBCUjyDcSW1mlueAIOuD2OEmJjOzfVQtICRdLWmDpAdGWP5hSfemxwOSSpJmpmWrJN2fllV9BKDOtgI7+0uUhjw+t5lZWTWPIK4Blo60MCI+FRFnR8TZwEeA70fEptwqr0jLF1WxRiBrYgLY2e+jCDOzsqoFRETcCWw66IqZi4Frq1XLwXS2ZQHhM5nMzPaqeR+EpKlkRxrfyM0O4FZJd0u69CDbXyqpW1J3T8+h3S6js60I4H4IM7OcmgcE8AbgR8Oal86LiHOA1wHvk/SykTaOiKsiYlFELJo9e/YhFVBuYvLtNszM9qqHgLiIYc1LEbE2PW8AbgQWV7OAjtTEtM1HEGZme9Q0ICRNA14OfDM3r11SZ/k1cD5Q8Uyo8dJZPoJwQJiZ7VGo1o4lXQssAWZJWgN8HCgCRMQX02q/BtwaETtzm84FbpRUru9rEfHf1aoTcn0QvprazGyPqgVERFw8inWuITsdNj/vceCs6lRVWcees5jcB2FmVlYPfRA1N7XYjOQmJjOzPAcE0NSk7H5MbmIyM9vDAZF0+pbfZmb7cEAkvmGfmdm+HBBJp8eEMDPbhwMi8ZgQZmb7ckAkHW3upDYzy3NAJF3ugzAz24cDIunwWUxmZvtwQCQdrUV2D5QYLA3VuhQzs7rggEjKgwbt7CvVuBIzs/rggEj23vLbZzKZmYEDYo89t/z2mUxmZoADYo/yEYQDwsws44BIPC61mdm+HBBJeVxq90GYmWUcEEmnm5jMzPbhgEj2BISbmMzMgCoGhKSrJW2Q9MAIy5dI2irp3vT4WG7ZUkmPSFop6fJq1Zg3pdhMk/DV1GZmSTWPIK4Blh5knR9ExNnpcQWApGbg88DrgNOBiyWdXsU6Se9LR2vBTUxmZknVAiIi7gQ2HcKmi4GVEfF4RPQD1wEXjmtxI+hsK/oIwswsqXUfxIsl3SfpZklnpHnzgNW5ddakeRVJulRSt6Tunp6ewyqms81jQpiZldUyIO4Bjo+Is4D/A/znoewkIq6KiEURsWj27NmHVZCbmMzM9qpZQETEtojYkV7fBBQlzQLWAsflVp2f5lVdZ5sDwsysrGYBIeloSUqvF6dangGWAydLWiipBbgIWDYRNXW4D8LMbI9CtXYs6VpgCTBL0hrg40ARICK+CLwZeK+kQWA3cFFEBDAo6TLgFqAZuDoiHqxWnXkeNMjMbK+qBUREXHyQ5f8I/OMIy24CbqpGXQeSNTG5k9rMDGp/FlNd6Wwt0DswxIBHlTMzc0Dkdfh2G2Zmezggcjo8aJCZ2R4OiJzymBC+5beZmQNiH76jq5nZXg6IHDcxmZnt5YDI8aBBZmZ7OSByymcxbXMTk5mZAyKvszXrpHYfhJmZA2IfbcUmCk3y1dRmZjgg9iGJjjbfj8nMDBwQ++loLbiJycwMB8R+OtuKbPdZTGZmDojhOls97KiZGTgg9tPhUeXMzAAHxH7cB2FmlnFADNPps5jMzAAHxH462grupDYzo4oBIelqSRskPTDC8rdKWiHpfkk/lnRWbtmqNP9eSd3VqrGSztYC/YND9A2WJvJtzczqTjWPIK4Blh5g+RPAyyPiucAngauGLX9FRJwdEYuqVF9F5TEh3A9hZo2uagEREXcCmw6w/McRsTlN3gXMr1YtY+FbfpuZZeqlD+KdwM256QBulXS3pEsPtKGkSyV1S+ru6ek57ELKd3R1R7WZNbpCrQuQ9AqygDgvN/u8iFgraQ5wm6T/SUck+4mIq0jNU4sWLYrDrafTAWFmBtT4CELSmcA/AxdGxDPl+RGxNj1vAG4EFk9UTXtu+e0mJjNrcDULCEnPAv4DeFtE/CI3v11SZ/k1cD5Q8UyoaujYM6qcb7dhZo2tak1Mkq4FlgCzJK0BPg4UASLii8DHgKOAL0gCGExnLM0FbkzzCsDXIuK/q1XncHuGHXUTk5k1uKoFRERcfJDl7wLeVWH+48BZ+28xMcpnMXnYUTNrdPVyFlPdaC00UWyW+yDMrOE5IIaR5Bv2mZnhgKios63oMSHMrOE5ICroaPWYEGZmDogKOnzLbzMzB0QlXQ4IMzMHRCVuYjIzc0BU5HGpzcwcEBWVz2KKOOx7/5mZTVoOiAo6WgsMlIK+waFal2JmVjMOiAr23I/JzUxm1sAcEBV4TAgzMwdERR1pTAhfTW1mjcwBUcHM9iwgNu3sr3ElZma144CoYE5nGwAbtvXVuBIzs9pxQFQwu7MVgA3be2tciZlZ7TggKmgrNjNtSpGnfQRhZg3MATGCuV2tPoIws4bmgBjBnM42H0GYWUOrakBIulrSBkkPjLBckj4naaWkFZLOyS27RNKj6XFJNeusZE5XKz3bHRBm1rhGFRCS2iU1pdfPlnSBpOIoNr0GWHqA5a8DTk6PS4F/Su8xE/g48EJgMfBxSTNGU+t4mdvVxobtvQwN+X5MZtaYRnsEcSfQJmkecCvwNrIv/wOKiDuBTQdY5ULgK5G5C5gu6RjgtcBtEbEpIjYDt3HgoBl3czpbGSgFm3f5Wggza0yjDQhFxC7gTcAXIuI3gDPG4f3nAatz02vSvJHm71+YdKmkbkndPT0941BSZm5XuhbCzUxm1qBGHRCSXgy8Ffh2mtdcnZLGJiKuiohFEbFo9uzZ47bfOelaiKe3+UwmM2tMow2IDwAfAW6MiAclnQB8dxzefy1wXG56fpo30vwJ4yMIM2t0owqIiPh+RFwQEX+bOqs3RsQfjMP7LwN+J53N9CJga0SsA24Bzpc0I3VOn5/mTZg9V1P7CMLMGlRhNCtJ+hrwHqAELAe6JH02Ij51kO2uBZYAsyStITszqQgQEV8EbgJeD6wEdgHvSMs2Sfpkei+AKyLiQJ3d485XU5tZoxtVQACnR8Q2SW8FbgYuB+4GDhgQEXHxQZYH8L4Rll0NXD3K+qrCV1ObWSMbbR9EMV338EZgWUQMAEf8BQK+mtrMGtloA+JLwCqgHbhT0vHAtmoVVS98NbWZNbLRdlJ/LiLmRcTr00VtTwKvqHJtNTenM7uaOmsJMzNrLKO91cY0Sf9QviBN0qfJjiaOaHO7yldTe+hRM2s8o21iuhrYDvxmemwD/qVaRdWL8rUQvljOzBrRaM9iOjEifj03/eeS7q1GQfUkfzX1acd01bgaM7OJNdojiN2SzitPSDoX2F2dkuqHr6Y2s0Y22iOI9wBfkTQtTW8GJnyMhonmq6nNrJGNKiAi4j7gLEldaXqbpA8AK6pZXK2Vr6b2EYSZNaIxjSgXEdsionz9wwerUE/dmdvV6k5qM2tIhzPkqMatijrmq6nNrFEdTkA0xNVjvprazBrVAfsgJG2nchAImFKViupM/mpqqSEOmszMgIMERER0TlQh9Sp/NfXM9pZal2NmNmEOp4mpIczp9NXUZtaYHBAHMbcrXQvhfggzazAOiIPw/ZjMrFE5IA7CV1ObWaOqakBIWirpEUkrJV1eYflnJN2bHr+QtCW3rJRbtqyadR6Ir6Y2s0Y12nsxjZmkZuDzwGuANcByScsi4qHyOhHxh7n1fx94Xm4XuyPi7GrVNxZzOn01tZk1nmoeQSwGVkbE4xHRD1wHXHiA9S8Grq1iPYdsblebjyDMrOFUMyDmAatz02vSvP2kMa4XAnfkZrel0evukvTGkd5E0qXlke56enrGo+79zOlsZYNvt2FmDaZeOqkvAm6IiFJu3vERsQh4C3ClpBMrbRgRV0XEoohYNHv27KoUN6fLY1ObWeOpZkCsBY7LTc9P8yq5iGHNSxGxNj0/DnyPffsnJpTHpjazRlTNgFgOnCxpoaQWshDY72wkSacCM4Cf5ObNkNSaXs8CzgUeGr7tRPHV1GbWiKoWEBExCFwG3AI8DFwfEQ9KukLSBblVLwKui33bb04DuiXdB3wX+Jv82U8TzVdTm1kjqtpprgARcRNw07B5Hxs2/YkK2/0YeG41axsLH0GYWSOql07qujYnHUF4XAgzayQOiFEoX03tIwgzayQOiFHy1dRm1mgcEKPkq6nNrNE4IEZpTlcr67f6CMLMGocDYpROPbqTdVt72bDdIWFmjcEBMUovWDATgOVPbK5xJWZmE8MBMUrPmTeNKcVmlq/aVOtSzMwmhANilIrNTTzvWdP52RMOCDNrDA6IMVi8cCYPr9/Gtl7ftM/MjnwOiDFYvGAmEXD3k+6HMLMjnwNiDJ73rBkUmsRyNzOZWQNwQIzBlJZmnjNvmvshzKwhOCDGaPHCmaxYs5XegdLBVzYzm8QcEGP0ggUz6S8Ncd/qLbUuxcysqhwQY7To+BkAvh7CzI54DogxmtHewilzO/nZKp/JZGZHNgfEIXjBwhnc8+RmBktDtS7FzKxqqhoQkpZKekTSSkmXV1j+dkk9ku5Nj3flll0i6dH0uKSadY7VCxbMZEffIA+v217rUszMqqZqY1JLagY+D7wGWAMsl7QsIh4atuq/R8Rlw7adCXwcWAQEcHfati7adRYvzG7c97NVm3ju/Gk1rsbMrDqqeQSxGFgZEY9HRD9wHXDhKLd9LXBbRGxKoXAbsLRKdY7ZMdOmMH/GFF8wZ2ZHtGoGxDxgdW56TZo33K9LWiHpBknHjXFbJF0qqVtSd09Pz3jUPSqLF8xk+apNRMSEvaeZ2USqdSf1fwELIuJMsqOEfx3rDiLiqohYFBGLZs+ePe4FjuQFC2fyzM5+Ht+4c8Le08xsIlUzINYCx+Wm56d5e0TEMxFRHuj5n4Hnj3bbWisPIOTbbpjZkaqaAbEcOFnSQkktwEXAsvwKko7JTV4APJxe3wKcL2mGpBnA+Wle3ThxdjtzOlu56f51tS7FzKwqqhYQETEIXEb2xf4wcH1EPCjpCkkXpNX+QNKDku4D/gB4e9p2E/BJspBZDlyR5tUNSbzj3IX84NGN/PyXdXFylZnZuNKR1Mm6aNGi6O7unrD329k3yHl/ewdnHTeda96xeMLe18xsvEi6OyIWVVpW607qSa29tcC7XnoC33ukh3t98z4zO8I4IA7TJS9ZwPSpRT53+6O1LsXMbFw5IA5TR2uBd523kDv+ZwMr1vgowsyOHA6IcXDJSxYwbYqPIszsyOKAGAedbUXeed5CvvPwBh5Yu7XW5ZiZjQsHxDi55CUL6Gwr8FkfRZjZEcIBMU6mTSnyu+cu5LaHnvZ1EWZ2RHBAjKN3vnQhx0xr40Nfv4/d/aVal2NmdlgcEOOoq63I3//GWTzes5O/vvnhg29gZlbHHBDj7NyTZvG75y7kKz95ku89sqHW5ZiZHTIHRBX88dJTePbcDj58wwo27+yvdTlmZofEAVEFbcVmPvNbZ7NlVz9/euP9HlTIzCYlB0SVnHHsND74mlO4+YH1/Mc9dTWUhZnZqDggqujSl53A4gUz+bP/vJ9vrXiq1uWYmY2JA6KKmpvEF377HM44dhqXfe3n/MOtjzA05OYmM5scHBBVNqujla+9+4X8xvPn87k7VvJ7X72HnX2DtS7LzOygHBAToLXQzN+9+Uw++iuncetD6/n1f/oxqzftqnVZZmYH5ICYIJJ410tP4Oq3v4C1W3bz+s/+gK93r/YZTmZWtxwQE2zJKXP49u+/lNOO6eLDN6zg3V/pZsP23lqXZWa2n6oGhKSlkh6RtFLS5RWWf1DSQ5JWSLpd0vG5ZSVJ96bHsmrWOdGeddRUrrv0RXz0V07jzkc3cv5n7uRbK57y0YSZ1RVV60tJUjPwC+A1wBpgOXBxRDyUW+cVwE8jYpek9wJLIuK30rIdEdExlvdctGhRdHd3j9tnmAgrN2znQ9ffx31rtvL842fw3pefyKtOm4OkWpdmZg1A0t0RsajSsmoeQSwGVkbE4xHRD1wHXJhfISK+GxHl3tq7gPlVrKcunTSnk2+89yV88sIzWL+1l3d9pZulV/6Ab967lsHSUK3LM7MGVs2AmAeszk2vSfNG8k7g5tx0m6RuSXdJeuNIG0m6NK3X3dPTc3gV10ihuYm3vXgB3/vwEv7hN89iKIL3X3cvr/z09/nG3Wso+doJM6uBuuiklvTbwCLgU7nZx6fDnrcAV0o6sdK2EXFVRCyKiEWzZ8+egGqrp9jcxJvOmc8tH3gZV73t+XS2FfjQ1+/jtVfeyU33r/NFdmY2oaoZEGuB43LT89O8fUh6NfBnwAUR0VeeHxFr0/PjwPeA51Wx1rrS1CTOP+No/uuy8/jCW88B4Pe+eg9v+Mcfsuy+p9jV7wvtzKz6qtlJXSDrpH4VWTAsB94SEQ/m1nkecAOwNCIezc2fAeyKiD5Js4CfABfmO7grmYyd1KNRGgq+ee9arvzOo/xy0y6mFJt55WlzeMOZx7DklDm0FZtrXaKZTVIH6qQuVOtNI2JQ0mXALUAzcHVEPCjpCqA7IpaRNSl1AF9PZ+38MiIuAE4DviRpiOwo528OFg5HsuYm8aZz5nPh2fNYvmoT31rxFDffv55vr1hHe0szS06dw2vPOJpXnDKbzrZircs1syNE1Y4gauFIPYKoZLA0xF2Pb+Lb9z/FbQ89zcYd/bQ0N/GSk47iNafP5WUnz+a4mVNrXaaZ1bkDHUE4II4ApaHgnl9u5pYH1nPLQ+tZvWk3AM+aOZVzT5rFeSfN4iUnHsWM9pYaV2pm9cYB0UAigsd6dvDDRzfyw5XPcNfjz7CjbxAJTju6i3NPOoqXnDiLFyycSUdr1VoYzWyScEA0sIHSECvWbOFHK5/hx49t5J4nt9BfGqK5STx7bifPndfFc+dN4znzpnHaMV3u8DZrMA4I26N3oMTdT27mJ489w31rtvDA2q1s3jUAQCGFxpnzp/Hc+dM4c950nn10B60Fh4bZkcoBYSOKCNZu2c0Da7eyYs1W7l+bPbak0GhuEscfNZWT53Rw8pxOTp7bwalHd3Hi7HYKzXVxnaWZHYaanOZqk4Mk5s+YyvwZU1n6nGOALDTWbN7NfWu28Mj67Tz69A4e3bCd7zy8Yc9tP1oLTZx6dCenH9vFqUd3MW/6FI6e1sYx09qY2d7imw2aHQF8BGGj1j84xBMbd/LQuq089NQ2HkyPrbsH9lmvpbmJeTOmsOCoqSyc1cHCWdnzMdPbmNPZSkdrwQFiVid8BGHjoqXQxClHd3LK0Z38WrrxSUTQs72PdVt7Wbe1l/Vbd7Nuay9rNu/m8Y07uevxTeweKO2zn7ZiE3M625jb1cqzZrazcNZUFsxqZ8FR7Rx/1FRf7GdWJxwQdlgkMaerjTldbZx13P7LI4Knt/XxxMadPL2tlw3be9mwrY8N2/tYv62XH67s4Rv39O2zTWdbgXnTpzBv+hSOnT6FY6a3cey0vU1Yc7vafLaV2QRwQFhVSeLoaW0cPa1txHV29Q+yauMunnxmJ09u2sVTW3bz1JbdrN3Sy/JVm9jWu//NCduKTUyf0sK0KUWmTSkys72FuV2tzJ3WxtFd2WNGewsdrYXs0Vag6E51szFxQFjNTW0pcPqxXZx+bFfF5Tv7Blm/rZf1qRnr6W29bN09wJZd/WzdPcDW3QM81rODHz22ke0VwqSstdDEjKktzGhvYWZ7kRlTW5jZ3sJR7a3M6kzPHS1Mn9pCV1uBzrYibcUm95dYw3JAWN1rby1w4uwOTpx98BFod/UPsn5rL+u39bJt9wDbewfZ3jvIjr5BtvcOsHlXFiybdvbz1JZtPLOjr+IRSlmxWXS2FZk+pcj0qVmoTJ/awoypRTrbinS0FehsK9DVVqCjtcjU1mamtjTT3lJgakszHW0FX0dik5YDwo4oU1sKnDC7gxNGESZl/YNDbNrZz8YdfWzc0cfWFCzbetPz7gG27B5g885+ntray0PrtrFl18B+ne8jKTZrTzNXe0sWKO3lpq/WAlNbCkxpaWJqS4G2YjNTis20tzZn67Vk23W0ZsvaCs20FptoLfjIxqrPAWENr6XQdNB+kkoGS0PpyCR77OwfZGffILv6S+zsy17v7C+xo2+QHb3Z9PY0f9POfn75zC629w2yu7/Erv5BxjpgYDlI2lPIdLQ2M6WlwJRiE1OKzUxpaaat2ExLoYnW5qbsOQVMOYjK67UW9i7Pnpv2WeYwakwOCLNDVGhuYnpqcjpcEUF/aYje/iF2DQyysy8Llp0pgHb1D9I7METvQInewRK9A0Ps7h9kR18WLjv7sma0rbsHeHprid0D2aO3v0RfaYj+waHDqm9KMQuWluYmiilsis1Kz9n8crC0FrNQyR/xZOuIQm771uamPftsLTZRaGqi0JStU2gWxaYm2op7Q60cYoWm7L0dWtXngDCrA5KyL8JCM9MY/+tAIoKBUtCXwqV3oERvCpFd/SX6B4foG8yCpL9Uoi+ts3tgKAuagRK7+0sMDg3RP5iFWf9giYFSpG2G2N47yMbBIfoGs+3L79WX1htvxWbtCZWmJtGcHoUm7QmhltxzsZDNzx7Z60JTEy2Fva8Lzdn25aAq7wGZRQYAAAhrSURBVG/Pc3PTPtOFNF1+z0KqqdicrVMcto/8dNPwZ+1dr17CzwFh1gAk0VLI/uLvHFtL2riICAaHgoHSEAODQV9p31DqGxyiNDTEQCkYLEUKoix4+gay596BUrZ9CqXyOqUhGIpsm9JQ1vQ3UBpKIZbte6A0RO/AEDt6B+kvZXUMpn0NlIay2gaz58FURy3lw65S0DRJNAmaJCQ4qr2V69/z4nGvwwFhZlUnac9f7bQAVThKGk8RQWkoC7WhFG6lUjAwNMTQEAwODaUgiz2v9wRNKZsub18Ovvz04FDsWWco8sv2PrJ1hvbML0+XhoIgC8WhyJ47qzS2S1UDQtJS4LNkY1L/c0T8zbDlrcBXgOcDzwC/FRGr0rKPAO8ESsAfRMQt1azVzKxMUtZc1OBnKFft0lJJzcDngdcBpwMXSzp92GrvBDZHxEnAZ4C/TdueDlwEnAEsBb6Q9mdmZhOkmvceWAysjIjHI6IfuA64cNg6FwL/ml7fALxKWe/MhcB1EdEXEU8AK9P+zMxsglQzIOYBq3PTa9K8iutExCCwFThqlNsCIOlSSd2Sunt6esapdDMzm/R3L4uIqyJiUUQsmj17dq3LMTM7YlQzINYC+RtAz0/zKq4jqQBMI+usHs22ZmZWRdUMiOXAyZIWSmoh63ReNmydZcAl6fWbgTsiG+JuGXCRpFZJC4GTgZ9VsVYzMxumaqe5RsSgpMuAW8hOc706Ih6UdAXQHRHLgC8D/yZpJbCJLERI610PPAQMAu+LiNHdGc3MzMaFx6Q2M2tgBxqT+ogKCEk9wJOHuPksYOM4ljORJmvtk7VucO214trH3/ERUfEMnyMqIA6HpO6RUrTeTdbaJ2vd4NprxbVPrEl/mquZmVWHA8LMzCpyQOx1Va0LOAyTtfbJWje49lpx7RPIfRBmZlaRjyDMzKwiB4SZmVXU8AEhaamkRyStlHR5res5EElXS9og6YHcvJmSbpP0aHqeUcsaRyLpOEnflfSQpAclvT/Nr/v6JbVJ+pmk+1Ltf57mL5T00/S78+/pljJ1R1KzpJ9L+laanix1r5J0v6R7JXWneXX/+wIgabqkGyT9j6SHJb14stSe19ABMcpBjerJNWQDKOVdDtweEScDt6fpejQIfCgiTgdeBLwv/awnQ/19wCsj4izgbGCppBeRDXD1mTTg1WayAbDq0fuBh3PTk6VugFdExNm56wcmw+8LZCNp/ndEnAqcRfbznyy17xURDfsAXgzckpv+CPCRWtd1kJoXAA/kph8BjkmvjwEeqXWNo/wc3wReM9nqB6YC9wAvJLsqtlDpd6leHmR3Qr4deCXwLUCToe5U2ypg1rB5df/7QnZX6idIJwFNptqHPxr6CIIxDExUx+ZGxLr0ej0wt5bFjIakBcDzgJ8ySepPzTT3AhuA24DHgC2RDXQF9fu7cyXwx8BQmj6KyVE3QAC3Srpb0qVp3mT4fVkI9AD/kpr2/llSO5Oj9n00ekAcUSL706Suz1uW1AF8A/hARGzLL6vn+iOiFBFnk/1Fvhg4tcYlHZSkXwU2RMTdta7lEJ0XEeeQNQG/T9LL8gvr+PelAJwD/FNEPA/YybDmpDqufR+NHhBHwsBET0s6BiA9b6hxPSOSVCQLh69GxH+k2ZOmfoCI2AJ8l6xpZnoa6Arq83fnXOACSavIxoR/JVnbeL3XDUBErE3PG4AbyYJ5Mvy+rAHWRMRP0/QNZIExGWrfR6MHxGgGNap3+UGXLiFr2687kkQ2/sfDEfEPuUV1X7+k2ZKmp9dTyPpOHiYLijen1equ9oj4SETMj4gFZL/bd0TEW6nzugEktUvqLL8GzgceYBL8vkTEemC1pFPSrFeRjW1T97Xvp9adILV+AK8HfkHWpvxnta7nILVeC6wDBsj+SnknWZvy7cCjwHeAmbWuc4TazyM7pF4B3Jser58M9QNnAj9PtT8AfCzNP4FspMOVwNeB1lrXeoDPsAT41mSpO9V4X3o8WP6/ORl+X1KdZwPd6XfmP4EZk6X2/MO32jAzs4oavYnJzMxG4IAwM7OKHBBmZlaRA8LMzCpyQJiZWUUOCGs4knak5wWS3jIO+1sl6Ru56TdLuuZw95v29QlJfzQe+zIbKweENbIFwJgCIncF8nDPr7c7ASvj/+N2yPzLY43sb4CXpvEG/jDdkO9TkpZLWiHpfwFIWiLpB5KWkV0RW8mngT8bPnP4EYCkB9KRy4I0VsA1kn4h6auSXi3pR2m8gMW53Zwl6Sdp/rtz+/pwrtbyGBULlI1v8hWyi/ryt5IxG5OR/hoyawSXA38UEb8KkO4YujUiXiCpFfiRpFvTuucAz4mIJ0bY1/XA70k6aQzvfxLwG8Dvkt325S1kV5xfAPwp8Ma03plkY2i0Az+X9G3gOcDJZPcnErAs3czul2n+JRFx1xhqMduPA8Jsr/OBMyWV71M0jezLth/42QHCAaAEfIpsTJGbR/l+T0TE/QCSHiQbTCYk3U/W/FX2zYjYDeyW9F2yUDgv1fvztE5HqvWXwJMOBxsPDgizvQT8fkTcss9MaQnZLZsP5t/IAuKB3LxB9m3Kbcu97su9HspND7Hv/83h98OJVOtfR8SXhtW6YJS1mh2U+yCskW0HOnPTtwDvTbclR9Kz051ERyUiBoDPAH+Ym72KrHkKSeeQDSYzVhcqGxf7KLKb7i1Ptf5uGl8DSfMkzTmEfZuNyEcQ1shWACVJ95GN9/1Zsqade9LtyXvY2w8wWl8GPpqb/gbwO6kJ6adkdw4+lDq/C8wCPhkRTwFPSToN+ElWKjuA3yZr6jIbF76bq5mZVeQmJjMzq8gBYWZmFTkgzMysIgeEmZlV5IAwM7OKHBBmZlaRA8LMzCr6/7ysf9SFfjjhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "old_loss = np.inf\n",
    "\n",
    "max_epoch = 100\n",
    "losses = []\n",
    "for epoch in range(max_epoch):\n",
    "\n",
    "    running_loss = 0.0\n",
    "    \n",
    "    for data in trainloader:\n",
    "        \n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Forward\n",
    "        y_hat = VGG_model(X)\n",
    "        \n",
    "        # Calculate Loss (Cross Entropy)\n",
    "        loss = criterion(y_hat, y)\n",
    "        \n",
    "        # Backpropagation\n",
    "        loss.backward()\n",
    "        \n",
    "        # Update Parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "        running_loss += loss.item()*len(X)/train_size\n",
    "    \n",
    "    print('Epoch', epoch+1, ': Loss =', running_loss, abs(running_loss-old_loss)/running_loss)\n",
    "    losses.append(running_loss)\n",
    "\n",
    "    if abs(running_loss-old_loss)/running_loss < 1e-2 and running_loss<0.05:\n",
    "        print('Converged')\n",
    "        break\n",
    "    \n",
    "    old_loss = running_loss\n",
    "\n",
    "print('Finished Training')\n",
    "plt.plot(losses)\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Iter Number')\n",
    "plt.title('Convergence monitor plot')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 283
    },
    "colab_type": "code",
    "id": "LuguuhIInaRW",
    "outputId": "a1909fab-8721-47a8-ac70-ba7291903fc3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Loss = 0.03823063522577286\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4   5   6\n",
       "0  43   0   0   0   0   0   0\n",
       "1   0  36   0   0   0   0   0\n",
       "2   0   0  46   0   0   0   0\n",
       "3   0   0   0  37   0   0   0\n",
       "4   0   0   0   0  40   0   0\n",
       "5   0   0   0   0   0  42   0\n",
       "6   0   0   0   0   0   0  43"
      ]
     },
     "execution_count": 16,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    train_loss = 0.0\n",
    "    y_train = []\n",
    "    y_train_pred = []\n",
    "\n",
    "    for data in trainloader:\n",
    "\n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        y_hat = VGG_model(X)\n",
    "        train_loss += criterion(y_hat, y)*len(X)/train_size\n",
    "        \n",
    "        y_train.extend(list(y.detach().cpu().numpy()))\n",
    "        y_train_pred.extend(list(torch.argmax(y_hat, axis=1).detach().cpu().numpy()))\n",
    "\n",
    "print('Train Loss =', train_loss.item())\n",
    "pd.DataFrame(confusion_matrix(y_train, y_train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "rHtBQ8zvtwUa",
    "outputId": "b90f3559-c9ef-4f46-9c86-ae944d34cca9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy = 1.0 Train Precision = 1.0 Train F1 = 1.0\n"
     ]
    }
   ],
   "source": [
    "acc_tr = accuracy_score(y_train, y_train_pred)\n",
    "prec_tr = precision_score(y_train, y_train_pred, average='weighted')\n",
    "f1_tr = f1_score(y_train, y_train_pred, average='weighted')\n",
    "\n",
    "print('Train Accuracy =', acc_tr, 'Train Precision =', prec_tr, 'Train F1 =', f1_tr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 283
    },
    "colab_type": "code",
    "id": "Ok3viz8oX4nv",
    "outputId": "b346bedf-acf7-4585-f24b-5b43b13fa75b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss = 0.27743086218833923\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3  4  5  6\n",
       "0  8  0  0  0  0  0  0\n",
       "1  0  8  0  0  0  0  0\n",
       "2  0  0  2  1  0  0  0\n",
       "3  0  0  1  8  0  0  1\n",
       "4  0  0  0  1  4  0  0\n",
       "5  0  0  0  0  0  4  0\n",
       "6  0  0  0  0  0  0  4"
      ]
     },
     "execution_count": 18,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    test_loss = 0.0\n",
    "    y_test = []\n",
    "    y_test_pred = []\n",
    "\n",
    "    for data in testloader:\n",
    "\n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        y_hat = VGG_model(X)      \n",
    "        test_loss += criterion(y_hat, y)*len(X)/test_size\n",
    "        \n",
    "        y_test.extend(list(y.detach().cpu().numpy()))\n",
    "        y_test_pred.extend(list(torch.argmax(y_hat, axis=1).detach().cpu().numpy()))\n",
    "\n",
    "print('Test Loss =', test_loss.item())\n",
    "pd.DataFrame(confusion_matrix(y_test, y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "mbtJA8AXtybh",
    "outputId": "af6312a8-721e-43f4-d25a-f827a7001f87"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy = 0.9047619047619048 Test Precision = 0.8952380952380953 Test F1 = 0.8920634920634922\n"
     ]
    }
   ],
   "source": [
    "acc = accuracy_score(y_test, y_test_pred)\n",
    "prec = precision_score(y_test, y_test_pred, average='macro')\n",
    "f1 = f1_score(y_test, y_test_pred, average='macro')\n",
    "\n",
    "print('Test Accuracy =', acc, 'Test Precision =', prec, 'Test F1 =', f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5Nt359MkcYxa"
   },
   "outputs": [],
   "source": [
    "torch.save(VGG_model, '/content/drive/My Drive/A3Q1a_model.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "r_MY7HHT7GYv"
   },
   "source": [
    "### Question 1. b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZEvZrYhn7JuX"
   },
   "outputs": [],
   "source": [
    "class GoogLeNet(nn.Module):\n",
    "    \n",
    "    def __init__(self, n_classes, RGB_mean):\n",
    "    \n",
    "        super(GoogLeNet, self).__init__()\n",
    "        self.RGB_mean = RGB_mean.to(device)\n",
    "        self.n_classes = n_classes\n",
    "        # Convolution\n",
    "        # 3x224x224\n",
    "        self.c1 = nn.Conv2d(3, 64, 7, stride=2, padding=3)\n",
    "        # 64x112x112\n",
    "        self.mp1 = nn.MaxPool2d(3, stride=2, padding=1)\n",
    "\n",
    "        # Deep Convolution\n",
    "        # 64x56x56\n",
    "        self.c21 = nn.Conv2d(64, 64, 1, stride=1, padding=0)\n",
    "        # 64x56x56\n",
    "        self.c22 = nn.Conv2d(64, 192, 3, stride=1, padding=1)\n",
    "        # 192x56x56\n",
    "        self.mp2 = nn.MaxPool2d(3, stride=2)\n",
    "\n",
    "        # Inception 3a\n",
    "        # 192x28x28\n",
    "        # P1\n",
    "        self.c3a1 = nn.Conv2d(192, 64, 1, stride=1, padding=0)\n",
    "        # P2\n",
    "        self.c3a21 = nn.Conv2d(192, 96, 1, stride=1, padding=0)\n",
    "        self.c3a22 = nn.Conv2d(96, 128, 3, stride=1, padding=1)\n",
    "        # P3\n",
    "        self.c3a31 = nn.Conv2d(192, 16, 1, stride=1, padding=0)\n",
    "        self.c3a32 = nn.Conv2d(16, 32, 3, stride=1, padding=1)\n",
    "        # P4\n",
    "        self.mp3a4 = nn.MaxPool2d(3, stride=1, padding=1)\n",
    "        self.c3a4 = nn.Conv2d(192, 32, 1, stride=1, padding=0)\n",
    "\n",
    "        # Inception 3b\n",
    "        # 256x28x28\n",
    "        # P1\n",
    "        self.c3b1 = nn.Conv2d(256, 128, 1, stride=1, padding=0)\n",
    "        # P2\n",
    "        self.c3b21 = nn.Conv2d(256, 128, 1, stride=1, padding=0)\n",
    "        self.c3b22 = nn.Conv2d(128, 192, 3, stride=1, padding=1)\n",
    "        # P3\n",
    "        self.c3b31 = nn.Conv2d(256, 32, 1, stride=1, padding=0)\n",
    "        self.c3b32 = nn.Conv2d(32, 96, 3, stride=1, padding=1)\n",
    "        # P4\n",
    "        self.mp3b4 = nn.MaxPool2d(3, stride=1, padding=1)\n",
    "        self.c3b4 = nn.Conv2d(256, 64, 1, stride=1, padding=0)\n",
    "\n",
    "        # 480x28x28\n",
    "        # MP\n",
    "        self.mp3 = nn.MaxPool2d(3, stride=2, padding=1)\n",
    "\n",
    "        # Inception 4a\n",
    "        # 480x14x14\n",
    "        # P1\n",
    "        self.c4a1 = nn.Conv2d(480, 192, 1, stride=1, padding=0)\n",
    "        # P2\n",
    "        self.c4a21 = nn.Conv2d(480, 96, 1, stride=1, padding=0)\n",
    "        self.c4a22 = nn.Conv2d(96, 208, 3, stride=1, padding=1)\n",
    "        # P3\n",
    "        self.c4a31 = nn.Conv2d(480, 16, 1, stride=1, padding=0)\n",
    "        self.c4a32 = nn.Conv2d(16, 48, 3, stride=1, padding=1)\n",
    "        # P4\n",
    "        self.mp4a4 = nn.MaxPool2d(3, stride=1, padding=1)\n",
    "        self.c4a4 = nn.Conv2d(480, 64, 1, stride=1, padding=0)\n",
    "\n",
    "        # Auxiliary 1\n",
    "        # 512x14x14\n",
    "        self.apa1 = nn.AvgPool2d(5, stride=3, padding=0)\n",
    "        # 512x4x4\n",
    "        self.ca1 = nn.Conv2d(512, 128, 1, stride=1)\n",
    "        # 128x4x4\n",
    "        self.flat1 = nn.Flatten(1, -1)\n",
    "        # (128x4x4)x1\n",
    "        self.fca1 = nn.Linear(2048, 1024)\n",
    "        self.a1drop = nn.Dropout(0.7)\n",
    "        self.a1out = nn.Linear(1024, self.n_classes)\n",
    "\n",
    "        # Inception 4b\n",
    "        # 512x14x14\n",
    "        # P1\n",
    "        self.c4b1 = nn.Conv2d(512, 160, 1, stride=1, padding=0)\n",
    "        # P2\n",
    "        self.c4b21 = nn.Conv2d(512, 112, 1, stride=1, padding=0)\n",
    "        self.c4b22 = nn.Conv2d(112, 224, 3, stride=1, padding=1)\n",
    "        # P3\n",
    "        self.c4b31 = nn.Conv2d(512, 24, 1, stride=1, padding=0)\n",
    "        self.c4b32 = nn.Conv2d(24, 64, 3, stride=1, padding=1)\n",
    "        # P4\n",
    "        self.mp4b4 = nn.MaxPool2d(3, stride=1, padding=1)\n",
    "        self.c4b4 = nn.Conv2d(512, 64, 1, stride=1, padding=0)\n",
    "\n",
    "        # Inception 4c\n",
    "        # 512x14x14\n",
    "        # P1\n",
    "        self.c4c1 = nn.Conv2d(512, 128, 1, stride=1, padding=0)\n",
    "        # P2\n",
    "        self.c4c21 = nn.Conv2d(512, 128, 1, stride=1, padding=0)\n",
    "        self.c4c22 = nn.Conv2d(128, 256, 3, stride=1, padding=1)\n",
    "        # P3\n",
    "        self.c4c31 = nn.Conv2d(512, 24, 1, stride=1, padding=0)\n",
    "        self.c4c32 = nn.Conv2d(24, 64, 3, stride=1, padding=1)\n",
    "        # P4\n",
    "        self.mp4c4 = nn.MaxPool2d(3, stride=1, padding=1)\n",
    "        self.c4c4 = nn.Conv2d(512, 64, 1, stride=1, padding=0)\n",
    "\n",
    "        # Inception 4d\n",
    "        # 512x14x14\n",
    "        # P1\n",
    "        self.c4d1 = nn.Conv2d(512, 112, 1, stride=1, padding=0)\n",
    "        # P2\n",
    "        self.c4d21 = nn.Conv2d(512, 144, 1, stride=1, padding=0)\n",
    "        self.c4d22 = nn.Conv2d(144, 288, 3, stride=1, padding=1)\n",
    "        # P3\n",
    "        self.c4d31 = nn.Conv2d(512, 32, 1, stride=1, padding=0)\n",
    "        self.c4d32 = nn.Conv2d(32, 64, 3, stride=1, padding=1)\n",
    "        # P4\n",
    "        self.mp4d4 = nn.MaxPool2d(3, stride=1, padding=1)\n",
    "        self.c4d4 = nn.Conv2d(512, 64, 1, stride=1, padding=0)\n",
    "\n",
    "        # Auxiliary 2\n",
    "        # 528x14x14\n",
    "        self.apa2 = nn.AvgPool2d(5, stride=3, padding=0)\n",
    "        # 528x4x4\n",
    "        self.ca2 = nn.Conv2d(528, 128, 1, stride=1)\n",
    "        # 128x4x4\n",
    "        self.flat1 = nn.Flatten(1, -1)\n",
    "        # (128x4x4)x1\n",
    "        self.fca2 = nn.Linear(2048, 1024)\n",
    "        self.a2drop = nn.Dropout(0.7)\n",
    "        self.a2out = nn.Linear(1024, self.n_classes)\n",
    "\n",
    "        # Inception 4e\n",
    "        # 528x14x14\n",
    "        # P1\n",
    "        self.c4e1 = nn.Conv2d(528, 256, 1, stride=1, padding=0)\n",
    "        # P2\n",
    "        self.c4e21 = nn.Conv2d(528, 160, 1, stride=1, padding=0)\n",
    "        self.c4e22 = nn.Conv2d(160, 320, 3, stride=1, padding=1)\n",
    "        # P3\n",
    "        self.c4e31 = nn.Conv2d(528, 32, 1, stride=1, padding=0)\n",
    "        self.c4e32 = nn.Conv2d(32, 128, 3, stride=1, padding=1)\n",
    "        # P4\n",
    "        self.mp4e4 = nn.MaxPool2d(3, stride=1, padding=1)\n",
    "        self.c4e4 = nn.Conv2d(528, 128, 1, stride=1, padding=0)\n",
    "\n",
    "        # 832x14x14\n",
    "        # MP\n",
    "        self.mp4 = nn.MaxPool2d(3, stride=2, padding=1)\n",
    "\n",
    "        # Inception 5a\n",
    "        # 832x7x7\n",
    "        # P1\n",
    "        self.c5a1 = nn.Conv2d(832, 256, 1, stride=1, padding=0)\n",
    "        # P2\n",
    "        self.c5a21 = nn.Conv2d(832, 160, 1, stride=1, padding=0)\n",
    "        self.c5a22 = nn.Conv2d(160, 320, 3, stride=1, padding=1)\n",
    "        # P3\n",
    "        self.c5a31 = nn.Conv2d(832, 32, 1, stride=1, padding=0)\n",
    "        self.c5a32 = nn.Conv2d(32, 128, 3, stride=1, padding=1)\n",
    "        # P4\n",
    "        self.mp5a4 = nn.MaxPool2d(3, stride=1, padding=1)\n",
    "        self.c5a4 = nn.Conv2d(832, 128, 1, stride=1, padding=0)\n",
    "\n",
    "        # Inception 5b\n",
    "        # 832x7x7\n",
    "        # P1\n",
    "        self.c5b1 = nn.Conv2d(832, 384, 1, stride=1, padding=0)\n",
    "        # P2\n",
    "        self.c5b21 = nn.Conv2d(832, 192, 1, stride=1, padding=0)\n",
    "        self.c5b22 = nn.Conv2d(192, 384, 3, stride=1, padding=1)\n",
    "        # P3\n",
    "        self.c5b31 = nn.Conv2d(832, 48, 1, stride=1, padding=0)\n",
    "        self.c5b32 = nn.Conv2d(48, 128, 3, stride=1, padding=1)\n",
    "        # P4\n",
    "        self.mp5b4 = nn.MaxPool2d(3, stride=1, padding=1)\n",
    "        self.c5b4 = nn.Conv2d(832, 128, 1, stride=1, padding=0)\n",
    "\n",
    "        # 1024x7x7\n",
    "        self.ap = nn.AvgPool2d(7, stride=1)\n",
    "        # 1024x1x1\n",
    "        self.drop = nn.Dropout(0.4)\n",
    "        self.flat = nn.Flatten(1, -1)\n",
    "        self.out = nn.Linear(1024, self.n_classes)\n",
    "\n",
    "    def forward(self, x, auxiliary=True):\n",
    "\n",
    "        x = x - self.RGB_mean[None, :, None, None]\n",
    "\n",
    "        # Layer 1\n",
    "        x = self.mp1(F.relu(self.c1(x)))\n",
    "\n",
    "        # Layer 2\n",
    "        x = self.mp2(F.relu(self.c22(F.relu(self.c21(x)))))\n",
    "\n",
    "        # Layer 3a\n",
    "        x1 = F.relu(self.c3a1(x))\n",
    "        x2 = F.relu(self.c3a22(F.relu(self.c3a21(x))))\n",
    "        x3 = F.relu(self.c3a32(F.relu(self.c3a31(x))))\n",
    "        x4 = F.relu(self.c3a4(self.mp3a4(x)))\n",
    "        x = torch.cat((x1, x2, x3, x4), 1)\n",
    "\n",
    "        # Layer 3b\n",
    "        x1 = F.relu(self.c3b1(x))\n",
    "        x2 = F.relu(self.c3b22(F.relu(self.c3b21(x))))\n",
    "        x3 = F.relu(self.c3b32(F.relu(self.c3b31(x))))\n",
    "        x4 = F.relu(self.c3b4(self.mp3b4(x)))\n",
    "        x = self.mp3(torch.cat((x1, x2, x3, x4), 1))\n",
    "\n",
    "        # Layer 4a\n",
    "        x1 = F.relu(self.c4a1(x))\n",
    "        x2 = F.relu(self.c4a22(F.relu(self.c4a21(x))))\n",
    "        x3 = F.relu(self.c4a32(F.relu(self.c4a31(x))))\n",
    "        x4 = F.relu(self.c4a4(self.mp4a4(x)))\n",
    "        x = torch.cat((x1, x2, x3, x4), 1)\n",
    "\n",
    "        # Auxiliary 1\n",
    "        if auxiliary == True:\n",
    "            z1 = self.flat1(F.relu(self.ca1(self.apa1(x))))\n",
    "            z1 = self.a1out(self.a1drop(F.relu(self.fca1(z1))))\n",
    "        else:\n",
    "            z1 = None\n",
    "\n",
    "        # Layer 4b\n",
    "        x1 = F.relu(self.c4b1(x))\n",
    "        x2 = F.relu(self.c4b22(F.relu(self.c4b21(x))))\n",
    "        x3 = F.relu(self.c4b32(F.relu(self.c4b31(x))))\n",
    "        x4 = F.relu(self.c4b4(self.mp4b4(x)))\n",
    "        x = torch.cat((x1, x2, x3, x4), 1)\n",
    "\n",
    "        # Layer 4c\n",
    "        x1 = F.relu(self.c4c1(x))\n",
    "        x2 = F.relu(self.c4c22(F.relu(self.c4c21(x))))\n",
    "        x3 = F.relu(self.c4c32(F.relu(self.c4c31(x))))\n",
    "        x4 = F.relu(self.c4c4(self.mp4c4(x)))\n",
    "        x = torch.cat((x1, x2, x3, x4), 1)\n",
    "\n",
    "        # Layer 4d\n",
    "        x1 = F.relu(self.c4d1(x))\n",
    "        x2 = F.relu(self.c4d22(F.relu(self.c4d21(x))))\n",
    "        x3 = F.relu(self.c4d32(F.relu(self.c4d31(x))))\n",
    "        x4 = F.relu(self.c4d4(self.mp4d4(x)))\n",
    "        x = torch.cat((x1, x2, x3, x4), 1)\n",
    "\n",
    "        # Auxiliary 2\n",
    "        if auxiliary == True:\n",
    "            z2 = self.flat(F.relu(self.ca2(self.apa2(x))))\n",
    "            z2 = self.a2out(self.a2drop(F.relu(self.fca2(z2))))\n",
    "        else:\n",
    "            z2 = None\n",
    "\n",
    "        # Layer 4e\n",
    "        x1 = F.relu(self.c4e1(x))\n",
    "        x2 = F.relu(self.c4e22(F.relu(self.c4e21(x))))\n",
    "        x3 = F.relu(self.c4e32(F.relu(self.c4e31(x))))\n",
    "        x4 = F.relu(self.c4e4(self.mp4e4(x)))\n",
    "        x = self.mp4(torch.cat((x1, x2, x3, x4), 1))\n",
    "\n",
    "        # Layer 5a\n",
    "        x1 = F.relu(self.c5a1(x))\n",
    "        x2 = F.relu(self.c5a22(F.relu(self.c5a21(x))))\n",
    "        x3 = F.relu(self.c5a32(F.relu(self.c5a31(x))))\n",
    "        x4 = F.relu(self.c5a4(self.mp5a4(x)))\n",
    "        x = torch.cat((x1, x2, x3, x4), 1)\n",
    "\n",
    "        # Layer 5b\n",
    "        x1 = F.relu(self.c5b1(x))\n",
    "        x2 = F.relu(self.c5b22(F.relu(self.c5b21(x))))\n",
    "        x3 = F.relu(self.c5b32(F.relu(self.c5b31(x))))\n",
    "        x4 = F.relu(self.c5b4(self.mp5b4(x)))\n",
    "        x = torch.cat((x1, x2, x3, x4), 1)\n",
    "\n",
    "        # Final Output\n",
    "        x = self.out(self.flat(self.drop(self.ap(x))))\n",
    "\n",
    "        return x, z1, z2\n",
    "\n",
    "    \n",
    "    def predict(self, x):\n",
    "\n",
    "        y_hat = self.forward(x)\n",
    "        y_hat = torch.argmax(y_hat, axis=1)\n",
    "        return y_hat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aOTtnofE32ZR"
   },
   "outputs": [],
   "source": [
    "# googlenet = models.googlenet(pretrained=True)\n",
    "# pickle.dump(googlenet, open('/content/drive/My Drive/google_init.sav', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CnTY_tSQ5Zt3"
   },
   "outputs": [],
   "source": [
    "googlenet = pickle.load(open('/content/drive/My Drive/google_init.sav', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "k-5ltA0Q2kdX"
   },
   "outputs": [],
   "source": [
    "classifier = GoogLeNet(7, RGB_mean)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(classifier.parameters(), lr=0.01, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Fo6APIXm53j9"
   },
   "outputs": [],
   "source": [
    "params = list(googlenet.parameters())\n",
    "\n",
    "with torch.no_grad():\n",
    "    \n",
    "    classifier.c1.weight = params[0]\n",
    "    # classifier.c1.bias = params[1]\n",
    "\n",
    "    classifier.c21.weight = params[3]\n",
    "    # classifier.c21.bias = params[4]\n",
    "    classifier.c22.weight = params[6]\n",
    "    # classifier.c22.bias = params[7]\n",
    "\n",
    "    classifier.c3a1.weight = params[9]\n",
    "    # classifier.c3a1.bias = params[10]\n",
    "    classifier.c3a21.weight = params[12]\n",
    "    # classifier.c3a21.bias = params[13]\n",
    "    classifier.c3a22.weight = params[15]\n",
    "    # classifier.c3a22.bias = params[16]\n",
    "    classifier.c3a31.weight = params[18]\n",
    "    # classifier.c3a31.bias = params[19]\n",
    "    classifier.c3a32.weight = params[21]\n",
    "    # classifier.c3a32.bias = params[22]\n",
    "    classifier.c3a4.weight = params[24]\n",
    "    # classifier.c3a4.bias = params[25]\n",
    "    \n",
    "    classifier.c3b1.weight = params[27]\n",
    "    # classifier.c3b1.bias = params[28]\n",
    "    classifier.c3b21.weight = params[30]\n",
    "    # classifier.c3b21.bias = params[31]\n",
    "    classifier.c3b22.weight = params[33]\n",
    "    # classifier.c3b22.bias = params[34]\n",
    "    classifier.c3b31.weight = params[36]\n",
    "    # classifier.c3b31.bias = params[37]\n",
    "    classifier.c3b32.weight = params[39]\n",
    "    # classifier.c3b32.bias = params[40]\n",
    "    classifier.c3b4.weight = params[42]\n",
    "    # classifier.c3b4.bias = params[43]\n",
    "    \n",
    "    classifier.c4a1.weight = params[45]\n",
    "    # classifier.c4a1.bias = params[46]\n",
    "    classifier.c4a21.weight = params[48]\n",
    "    # classifier.c4a21.bias = params[49]\n",
    "    classifier.c4a22.weight = params[51]\n",
    "    # classifier.c4a22.bias = params[52]\n",
    "    classifier.c4a31.weight = params[54]\n",
    "    # classifier.c4a31.bias = params[55]\n",
    "    classifier.c4a32.weight = params[57]\n",
    "    # classifier.c4a32.bias = params[58]\n",
    "    classifier.c4a4.weight = params[60]\n",
    "    # classifier.c4a4.bias = params[61]\n",
    "\n",
    "    classifier.c4b1.weight = params[63]\n",
    "    # classifier.c4b1.bias = params[64]\n",
    "    classifier.c4b21.weight = params[66]\n",
    "    # classifier.c4b21.bias = params[67]\n",
    "    classifier.c4b22.weight = params[69]\n",
    "    # classifier.c4b22.bias = params[70]\n",
    "    classifier.c4b31.weight = params[72]\n",
    "    # classifier.c4b31.bias = params[73]\n",
    "    classifier.c4b32.weight = params[75]\n",
    "    # classifier.c4b32.bias = params[76]\n",
    "    classifier.c4b4.weight = params[78]\n",
    "    # classifier.c4b4.bias = params[79]\n",
    "\n",
    "    classifier.c4c1.weight = params[81]\n",
    "    # classifier.c4c1.bias = params[82]\n",
    "    classifier.c4c21.weight = params[84]\n",
    "    # classifier.c4c21.bias = params[85]\n",
    "    classifier.c4c22.weight = params[87]\n",
    "    # classifier.c4c22.bias = params[88]\n",
    "    classifier.c4c31.weight = params[90]\n",
    "    # classifier.c4c31.bias = params[91]\n",
    "    classifier.c4c32.weight = params[93]\n",
    "    # classifier.c4c32.bias = params[94]\n",
    "    classifier.c4c4.weight = params[96]\n",
    "    # classifier.c4c4.bias = params[97]\n",
    "\n",
    "    classifier.c4d1.weight = params[99]\n",
    "    # classifier.c4d1.bias = params[100]\n",
    "    classifier.c4d21.weight = params[102]\n",
    "    # classifier.c4d21.bias = params[103]\n",
    "    classifier.c4d22.weight = params[105]\n",
    "    # classifier.c4d22.bias = params[106]\n",
    "    classifier.c4d31.weight = params[108]\n",
    "    # classifier.c4d31.bias = params[109]\n",
    "    classifier.c4d32.weight = params[111]\n",
    "    # classifier.c4d32.bias = params[112]\n",
    "    classifier.c4d4.weight = params[114]\n",
    "    # classifier.c4d4.bias = params[115]\n",
    "\n",
    "    classifier.c4e1.weight = params[117]\n",
    "    # classifier.c4e1.bias = params[118]\n",
    "    classifier.c4e21.weight = params[120]\n",
    "    # classifier.c4e21.bias = params[121]\n",
    "    classifier.c4e22.weight = params[123]\n",
    "    # classifier.c4e22.bias = params[124]\n",
    "    classifier.c4e31.weight = params[126]\n",
    "    # classifier.c4e31.bias = params[127]\n",
    "    classifier.c4e32.weight = params[129]\n",
    "    # classifier.c4e32.bias = params[130]\n",
    "    classifier.c4e4.weight = params[132]\n",
    "    # classifier.c4e4.bias = params[133]\n",
    "\n",
    "    classifier.c5a1.weight = params[135]\n",
    "    # classifier.c5a1.bias = params[136]\n",
    "    classifier.c5a21.weight = params[138]\n",
    "    # classifier.c5a21.bias = params[139]\n",
    "    classifier.c5a22.weight = params[141]\n",
    "    # classifier.c5a22.bias = params[142]\n",
    "    classifier.c5a31.weight = params[144]\n",
    "    # classifier.c5a31.bias = params[145]\n",
    "    classifier.c5a32.weight = params[147]\n",
    "    # classifier.c5a32.bias = params[148]\n",
    "    classifier.c5a4.weight = params[150]\n",
    "    # classifier.c5a4.bias = params[151]\n",
    "\n",
    "    classifier.c5b1.weight = params[153]\n",
    "    # classifier.c5b1.bias = params[154]\n",
    "    classifier.c5b21.weight = params[156]\n",
    "    # classifier.c5b21.bias = params[157]\n",
    "    classifier.c5b22.weight = params[159]\n",
    "    # classifier.c5b22.bias = params[160]\n",
    "    classifier.c5b31.weight = params[162]\n",
    "    # classifier.c5b31.bias = params[163]\n",
    "    classifier.c5b32.weight = params[165]\n",
    "    # classifier.c5b32.bias = params[166]\n",
    "    classifier.c5b4.weight = params[168]\n",
    "    # classifier.c5b4.bias = params[169]\n",
    "\n",
    "classifier = classifier.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "QOH2pEviO6nE",
    "outputId": "ec13719c-1e1b-4912-f609-6cb3fed2d7f9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 : Loss = 3.1121552248034323\n",
      "Epoch 2 : Loss = 3.1037406331570723\n",
      "Epoch 3 : Loss = 3.100610318499575\n",
      "Epoch 4 : Loss = 3.098109191303054\n",
      "Epoch 5 : Loss = 3.083987043294342\n",
      "Epoch 6 : Loss = 3.075826416447603\n",
      "Epoch 7 : Loss = 3.066208663302432\n",
      "Epoch 8 : Loss = 3.0496589456285745\n",
      "Epoch 9 : Loss = 3.026289643310919\n",
      "Epoch 10 : Loss = 3.018028483573568\n",
      "Epoch 11 : Loss = 2.9945229412371273\n",
      "Epoch 12 : Loss = 2.990751808113337\n",
      "Epoch 13 : Loss = 2.969864595227125\n",
      "Epoch 14 : Loss = 2.9476608922672605\n",
      "Epoch 15 : Loss = 2.9458387761996603\n",
      "Epoch 16 : Loss = 2.932173090113995\n",
      "Epoch 17 : Loss = 2.9030247108446185\n",
      "Epoch 18 : Loss = 2.876336592830432\n",
      "Epoch 19 : Loss = 2.8782254935141642\n",
      "Epoch 20 : Loss = 2.8447000814231846\n",
      "Epoch 21 : Loss = 2.8092686689689184\n",
      "Epoch 22 : Loss = 2.8065642562892794\n",
      "Epoch 23 : Loss = 2.768598222566399\n",
      "Epoch 24 : Loss = 2.7850758929701223\n",
      "Epoch 25 : Loss = 2.75470313128694\n",
      "Epoch 26 : Loss = 2.7355639012433093\n",
      "Epoch 27 : Loss = 2.712163524760602\n",
      "Epoch 28 : Loss = 2.6756305511820195\n",
      "Epoch 29 : Loss = 2.6337790713493003\n",
      "Epoch 30 : Loss = 2.633684078575427\n",
      "Epoch 31 : Loss = 2.6222823206140604\n",
      "Epoch 32 : Loss = 2.6058054742912797\n",
      "Epoch 33 : Loss = 2.586942533167397\n",
      "Epoch 34 : Loss = 2.5493121928038915\n",
      "Epoch 35 : Loss = 2.5347473197697763\n",
      "Epoch 36 : Loss = 2.50716773142798\n",
      "Epoch 37 : Loss = 2.477218887947162\n",
      "Epoch 38 : Loss = 2.448247921176073\n",
      "Epoch 39 : Loss = 2.442304324605324\n",
      "Epoch 40 : Loss = 2.398467775005912\n",
      "Epoch 41 : Loss = 2.3707925542306403\n",
      "Epoch 42 : Loss = 2.368723256247384\n",
      "Epoch 43 : Loss = 2.340957535268538\n",
      "Epoch 44 : Loss = 2.2707061127918524\n",
      "Epoch 45 : Loss = 2.2697044374220052\n",
      "Epoch 46 : Loss = 2.2480283414860636\n",
      "Epoch 47 : Loss = 2.2264451689836453\n",
      "Epoch 48 : Loss = 2.2168386414491343\n",
      "Epoch 49 : Loss = 2.200134618772447\n",
      "Epoch 50 : Loss = 2.1562271500298373\n",
      "Epoch 51 : Loss = 2.1287369927462803\n",
      "Epoch 52 : Loss = 2.0961228978758486\n",
      "Epoch 53 : Loss = 2.0792939031581015\n",
      "Epoch 54 : Loss = 2.0587069971636196\n",
      "Epoch 55 : Loss = 2.0461498752288287\n",
      "Epoch 56 : Loss = 2.038017343560993\n",
      "Epoch 57 : Loss = 2.0264013004635273\n",
      "Epoch 58 : Loss = 2.012854382966869\n",
      "Epoch 59 : Loss = 1.9848498716586975\n",
      "Epoch 60 : Loss = 1.9901892399538685\n",
      "Epoch 61 : Loss = 1.9931232285416498\n",
      "Epoch 62 : Loss = 1.934301640513882\n",
      "Epoch 63 : Loss = 1.8592513446608487\n",
      "Epoch 64 : Loss = 1.857108891217966\n",
      "Epoch 65 : Loss = 1.8254925718706245\n",
      "Epoch 66 : Loss = 1.7840780526503455\n",
      "Epoch 67 : Loss = 1.7958979868307348\n",
      "Epoch 68 : Loss = 1.7788980887741992\n",
      "Epoch 69 : Loss = 1.7366729804447718\n",
      "Epoch 70 : Loss = 1.72011195657976\n",
      "Epoch 71 : Loss = 1.7082747632202784\n",
      "Epoch 72 : Loss = 1.6746741030689734\n",
      "Epoch 73 : Loss = 1.6420188714402895\n",
      "Epoch 74 : Loss = 1.6200998041272578\n",
      "Epoch 75 : Loss = 1.6732176106150556\n",
      "Epoch 76 : Loss = 1.5914378984464586\n",
      "Epoch 77 : Loss = 1.5769651675473524\n",
      "Epoch 78 : Loss = 1.5831792599648131\n",
      "Epoch 79 : Loss = 1.5728993557056068\n",
      "Epoch 80 : Loss = 1.6017766027915767\n",
      "Epoch 81 : Loss = 1.5486310271851278\n",
      "Epoch 82 : Loss = 1.4866077862550156\n",
      "Epoch 83 : Loss = 1.4577310724956234\n",
      "Epoch 84 : Loss = 1.4557836803409698\n",
      "Epoch 85 : Loss = 1.4694392406981995\n",
      "Epoch 86 : Loss = 1.4261156262421024\n",
      "Epoch 87 : Loss = 1.4346332587431532\n",
      "Epoch 88 : Loss = 1.4538957673913513\n",
      "Epoch 89 : Loss = 1.434916499184399\n",
      "Epoch 90 : Loss = 1.4110821979801826\n",
      "Epoch 91 : Loss = 1.474852840244147\n",
      "Epoch 92 : Loss = 1.3852422540611506\n",
      "Epoch 93 : Loss = 1.3707525094627087\n",
      "Epoch 94 : Loss = 1.316497345834659\n",
      "Epoch 95 : Loss = 1.3389848117628993\n",
      "Epoch 96 : Loss = 1.3000081677885422\n",
      "Epoch 97 : Loss = 1.2884438939210847\n",
      "Epoch 98 : Loss = 1.3094463045173403\n",
      "Epoch 99 : Loss = 1.2782295925691984\n",
      "Epoch 100 : Loss = 1.2617640362384013\n",
      "Epoch 101 : Loss = 1.2234129486183671\n",
      "Epoch 102 : Loss = 1.2417133585501214\n",
      "Epoch 103 : Loss = 1.2115277338526391\n",
      "Epoch 104 : Loss = 1.1780003003957795\n",
      "Epoch 105 : Loss = 1.1883815531115913\n",
      "Epoch 106 : Loss = 1.1446373400372496\n",
      "Epoch 107 : Loss = 1.1601628443089926\n",
      "Epoch 108 : Loss = 1.1653832691471753\n",
      "Epoch 109 : Loss = 1.115526293212944\n",
      "Epoch 110 : Loss = 1.1088987347971686\n",
      "Epoch 111 : Loss = 1.132129934606652\n",
      "Epoch 112 : Loss = 1.1103400542345612\n",
      "Epoch 113 : Loss = 1.1458134169362566\n",
      "Epoch 114 : Loss = 1.062436119812291\n",
      "Epoch 115 : Loss = 1.0417015797585145\n",
      "Epoch 116 : Loss = 1.098059244687549\n",
      "Epoch 117 : Loss = 1.0742569821221488\n",
      "Epoch 118 : Loss = 1.0666840483502644\n",
      "Epoch 119 : Loss = 1.0201193289474328\n",
      "Epoch 120 : Loss = 1.0267893894208846\n",
      "Epoch 121 : Loss = 1.0537227329892147\n",
      "Epoch 122 : Loss = 0.9847439814112328\n",
      "Epoch 123 : Loss = 0.9633538268169045\n",
      "Epoch 124 : Loss = 0.9405163615838161\n",
      "Epoch 125 : Loss = 0.9700623538851323\n",
      "Epoch 126 : Loss = 0.9171978046254414\n",
      "Epoch 127 : Loss = 0.9301379599222324\n",
      "Epoch 128 : Loss = 0.9266236698585936\n",
      "Epoch 129 : Loss = 0.905419751949842\n",
      "Epoch 130 : Loss = 0.9266656878102532\n",
      "Epoch 131 : Loss = 0.8708270314678497\n",
      "Epoch 132 : Loss = 0.8777364641947197\n",
      "Epoch 133 : Loss = 0.8901138756333329\n",
      "Epoch 134 : Loss = 0.8534176760849637\n",
      "Epoch 135 : Loss = 0.8690690998416329\n",
      "Epoch 136 : Loss = 0.8261989724760688\n",
      "Epoch 137 : Loss = 0.7794799758997528\n",
      "Epoch 138 : Loss = 0.7775393541681641\n",
      "Epoch 139 : Loss = 0.7962042843007876\n",
      "Epoch 140 : Loss = 0.7862572520452094\n",
      "Epoch 141 : Loss = 0.7701943358893178\n",
      "Epoch 142 : Loss = 0.8358120762512659\n",
      "Epoch 143 : Loss = 0.8318916444994431\n",
      "Epoch 144 : Loss = 0.7792670506633532\n",
      "Epoch 145 : Loss = 0.8023060499998751\n",
      "Epoch 146 : Loss = 0.7521303831907931\n",
      "Epoch 147 : Loss = 0.7721348550261521\n",
      "Epoch 148 : Loss = 0.7210632622865019\n",
      "Epoch 149 : Loss = 0.7247114763027284\n",
      "Epoch 150 : Loss = 0.7006377802077901\n",
      "Epoch 151 : Loss = 0.6737057886472563\n",
      "Epoch 152 : Loss = 0.6709913952010019\n",
      "Epoch 153 : Loss = 0.6653204389980862\n",
      "Epoch 154 : Loss = 0.6655975517910948\n",
      "Epoch 155 : Loss = 0.6497646142381409\n",
      "Epoch 156 : Loss = 0.6218487275602095\n",
      "Epoch 157 : Loss = 0.6533539895396616\n",
      "Epoch 158 : Loss = 0.6400266574234913\n",
      "Epoch 159 : Loss = 0.6424777030114097\n",
      "Epoch 160 : Loss = 0.6324668327275054\n",
      "Epoch 161 : Loss = 0.6461077610790107\n",
      "Epoch 162 : Loss = 0.6310064375192861\n",
      "Epoch 163 : Loss = 0.6467010453602993\n",
      "Epoch 164 : Loss = 0.6757563950708103\n",
      "Epoch 165 : Loss = 0.7206065548836976\n",
      "Epoch 166 : Loss = 0.6730202496674834\n",
      "Epoch 167 : Loss = 0.6145505095192779\n",
      "Epoch 168 : Loss = 0.5611164443168906\n",
      "Epoch 169 : Loss = 0.5730815349140234\n",
      "Epoch 170 : Loss = 0.5508918970927128\n",
      "Epoch 171 : Loss = 0.524984878114707\n",
      "Epoch 172 : Loss = 0.5394636567046003\n",
      "Epoch 173 : Loss = 0.561384179450909\n",
      "Epoch 174 : Loss = 0.6164477008560393\n",
      "Epoch 175 : Loss = 0.5695830238820784\n",
      "Epoch 176 : Loss = 0.5774058122252753\n",
      "Epoch 177 : Loss = 0.5921399755344988\n",
      "Epoch 178 : Loss = 0.5515237714355415\n",
      "Epoch 179 : Loss = 0.5363255004018856\n",
      "Epoch 180 : Loss = 0.5560798796627163\n",
      "Epoch 181 : Loss = 0.5580255027847423\n",
      "Epoch 182 : Loss = 0.5121155096678784\n",
      "Epoch 183 : Loss = 0.585112270162496\n",
      "Epoch 184 : Loss = 0.524904015172234\n",
      "Epoch 185 : Loss = 0.47772428382979865\n",
      "Epoch 186 : Loss = 0.4870477515022929\n",
      "Epoch 187 : Loss = 0.4546845237552496\n",
      "Epoch 188 : Loss = 0.45525841388968225\n",
      "Epoch 189 : Loss = 0.43647172309795745\n",
      "Epoch 190 : Loss = 0.44853516115128783\n",
      "Epoch 191 : Loss = 0.4406364629700623\n",
      "Epoch 192 : Loss = 0.4409895951531905\n",
      "Epoch 193 : Loss = 0.49327708079839827\n",
      "Epoch 194 : Loss = 0.46386974198477615\n",
      "Epoch 195 : Loss = 0.41758434967712244\n",
      "Epoch 196 : Loss = 0.4067636684673589\n",
      "Epoch 197 : Loss = 0.3984455441348644\n",
      "Epoch 198 : Loss = 0.4462163608248641\n",
      "Epoch 199 : Loss = 0.5342376356756231\n",
      "Epoch 200 : Loss = 0.5656480072682742\n",
      "Epoch 201 : Loss = 0.49196814753452656\n",
      "Epoch 202 : Loss = 0.4121818513405032\n",
      "Epoch 203 : Loss = 0.3481754286779344\n",
      "Epoch 204 : Loss = 0.39318106616830995\n",
      "Epoch 205 : Loss = 0.37802443520948026\n",
      "Epoch 206 : Loss = 0.39723992981146433\n",
      "Epoch 207 : Loss = 0.38343144106947996\n",
      "Epoch 208 : Loss = 0.381474522348065\n",
      "Epoch 209 : Loss = 0.3680926913376054\n",
      "Epoch 210 : Loss = 0.3574323609523241\n",
      "Epoch 211 : Loss = 0.3542941251075227\n",
      "Epoch 212 : Loss = 0.3428978511888391\n",
      "Epoch 213 : Loss = 0.37470655013459897\n",
      "Epoch 214 : Loss = 0.35412977308761784\n",
      "Epoch 215 : Loss = 0.32301931244155674\n",
      "Epoch 216 : Loss = 0.33893253828174974\n",
      "Epoch 217 : Loss = 0.3376712610081929\n",
      "Epoch 218 : Loss = 0.31992285135315685\n",
      "Epoch 219 : Loss = 0.36691268146661105\n",
      "Epoch 220 : Loss = 0.3828273904655868\n",
      "Epoch 221 : Loss = 0.3889967973639325\n",
      "Epoch 222 : Loss = 0.3705592670507132\n",
      "Epoch 223 : Loss = 0.3138750781165598\n",
      "Epoch 224 : Loss = 0.3134893597937627\n",
      "Epoch 225 : Loss = 0.3362697391975217\n",
      "Epoch 226 : Loss = 0.34803404947191163\n",
      "Epoch 227 : Loss = 0.38011479294673906\n",
      "Epoch 228 : Loss = 0.27786328651348474\n",
      "Epoch 229 : Loss = 0.28770830458880303\n",
      "Epoch 230 : Loss = 0.2935521029429153\n",
      "Epoch 231 : Loss = 0.2574968367088132\n",
      "Epoch 232 : Loss = 0.2742531519733655\n",
      "Epoch 233 : Loss = 0.2625041665308151\n",
      "Epoch 234 : Loss = 0.26554493715123434\n",
      "Epoch 235 : Loss = 0.2761579138268993\n",
      "Epoch 236 : Loss = 0.2848512371450351\n",
      "Epoch 237 : Loss = 0.28697922441602175\n",
      "Epoch 238 : Loss = 0.2947665245275464\n",
      "Epoch 239 : Loss = 0.32169125325173037\n",
      "Epoch 240 : Loss = 0.27452775597156964\n",
      "Epoch 241 : Loss = 0.3094736889575832\n",
      "Epoch 242 : Loss = 0.24506594681989027\n",
      "Epoch 243 : Loss = 0.23732272652383468\n",
      "Epoch 244 : Loss = 0.26565309767108347\n",
      "Epoch 245 : Loss = 0.28505842792655534\n",
      "Epoch 246 : Loss = 0.27092456423031747\n",
      "Epoch 247 : Loss = 0.2519514661425082\n",
      "Epoch 248 : Loss = 0.23957005516992627\n",
      "Epoch 249 : Loss = 0.20967576229613832\n",
      "Epoch 250 : Loss = 0.22780451027981494\n",
      "Epoch 251 : Loss = 0.2536542607724459\n",
      "Epoch 252 : Loss = 0.2326733878576797\n",
      "Epoch 253 : Loss = 0.23587278248125668\n",
      "Epoch 254 : Loss = 0.2647039481156379\n",
      "Epoch 255 : Loss = 0.26078819532843\n",
      "Epoch 256 : Loss = 0.24239191049482767\n",
      "Epoch 257 : Loss = 0.23386196683092814\n",
      "Epoch 258 : Loss = 0.27742392935819327\n",
      "Epoch 259 : Loss = 0.23496392265429483\n",
      "Epoch 260 : Loss = 0.24200768234007033\n",
      "Epoch 261 : Loss = 0.21049597921687138\n",
      "Epoch 262 : Loss = 0.18555736084848332\n",
      "Epoch 263 : Loss = 0.20742466178712946\n",
      "Epoch 264 : Loss = 0.22712489993730076\n",
      "Epoch 265 : Loss = 0.2385926716522888\n",
      "Epoch 266 : Loss = 0.28254979059671276\n",
      "Epoch 267 : Loss = 0.38997310784220274\n",
      "Epoch 268 : Loss = 0.33694518515872623\n",
      "Epoch 269 : Loss = 0.21523781183081636\n",
      "Epoch 270 : Loss = 0.22683066918874867\n",
      "Epoch 271 : Loss = 0.2090144905063749\n",
      "Epoch 272 : Loss = 0.1866205075580484\n",
      "Epoch 273 : Loss = 0.18890988494461006\n",
      "Converged\n",
      "Finished Training\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3gVVfrA8e+b3khIhZCQBAi9QxAEVMSCWMBesK8dbGtb3d2fbd11bWvDvqCiiIuKihUUEaQIhNBDCy0hCUkoIQmk5/z+uAOGkEAIuZnc3PfzPPdhypm577kT7nvPnJkzYoxBKaWU+/KwOwCllFL20kSglFJuThOBUkq5OU0ESinl5jQRKKWUm9NEoJRSbk4TgVIthIhcKyKzbXz/J0XkY7veXzWcJgJVJxEZJyLJIlIkItki8oOIDLc7LlU7Y8xUY8y5h+ZFxIhIop0x1UVEtovI2XbHoRw0EahaicgDwCvAv4A2QBzwJjDWzriqExEvu2NoqfSzdS+aCNRRRCQEeBqYYIyZYYw5YIwpN8Z8Y4x52CrjKyKviEiW9XpFRHytdSNEZKeIPCgiuVZr4mZr3WAR2SUintXe7xIRWW1Ne4jIoyKyRUT2iMh0EQmz1iVYv3JvEZF04BcR8RSRl0Rkt4hsE5G7rTJeh+oiIpOsGDJF5JlD7y0iN4nIAhF5UUT2WduPrhZXmIi8b9Vvn4h8VW3dhSKyUkTyRWSRiPQ5xudpRGS8iGwWkUIR+YeIdLK2K7Dq6FOt/G0ikiYie0Vkpoi0q7GvO6195YvIGyIi1etjTc+3Nlllteiuque+J4jIZmBzLfU49Pnfbn0m2SLy0DHqPUZE1llx/ioi3a3lH+H4YfGNFdsjde1DNRFjjL70dcQLOA+oALyOUeZp4HcgCogEFgH/sNaNsLZ/GvAGzgcOAqHW+i3AOdX29RnwqDV9n7XfWMAXeAeYZq1LAAwwBQgE/IE7gVSrfCjws1XGy9rmS2sfgVasS4E7rHU3AeXAbYAncBeQBYi1/jvgf9Z+vYEzrOX9gVxgsLXdjcB2wLeOz8oAXwPBQE+gFJgDdARCrPhvtMqOBHYDA6z6vw7Mr7Gvb4HWOL5M84DzqtVnQY2yidXm67Pvn4AwwL+Wehz6/KdZn2dv6/3PttY/CXxsTXcBDgDnWJ/dI0Aa4GOt335oO33Z/7I9AH01vxdwLbDrOGW2AOdXmx8FbLemRwDFVEsk1hfnEGv6GWCyNd3K+sKIt+bXA2dV2y7a+rL2qvZF1LHa+l+wvtit+bOtMl44TmmVVv9SA64B5lrTNwFp1dYFWNu2td63Cit51aj7W1hJr9qyjViJopbyBhhWbX458Jdq8y8Br1jTk4Dnq60LsuqfUG1fw6utn84fSfQmjp0I6rPvkcc45oc+/27Vlj0PTLKmqyeC/wOmVyvnAWQCI6x5TQTN6KWnhlRt9gARxzlP3A7YUW1+h7Xs8D6MMRXV5g/i+OIB+AS41DqVdCmQYow5tK944EvrdEI+jsRQieNL/ZCMGnFk1LEuHsev0exq+3sHR8vgkF2HJowxB63JIKA9sNcYs6+WuscDDx7ap7Xf9jXqX1NOteniWuYPfTZHfK7GmCIcxyOmtpg58nM9nvrsO6PmRrWoXqbmca/rvaqs7WJqKatspolA1WYxjl/SFx+jTBaOL8RD4qxlx2WMScXxJTEaGIcjMRySAYw2xrSu9vIzxmRW30W16Wwcp4UOaV9jX6VARLV9BRtjetYjzAwgTERa17HunzViDDDGTKvHfo/niM9VRAKBcBy/ppti3/UZjrj6Z1zXca/5XmJtd+i9dNjjZkQTgTqKMWY/8DjwhohcLCIBIuItIqNF5Hmr2DTg7yISKSIRVvkTuYb8Exz9Aafj6CM45G3gnyISD2Dt/1hXKk0H7hORGOtL+y/V6pENzAZeEpFgqyO6k4iccbzgrG1/AN4UkVCr/qdbq98D7rQ6vkVEAkXkAhFpVf/q12kacLOI9LNaTP8ClhhjtjdgXzk4+iEae9//Z/1N9ARuxtGPUtN04AIROUtEvIEHcSTlRXXEpmykiUDVyhjzEvAA8HccHYIZwN3AoStnngGSgdXAGiDFWlZf04AzgF+MMburLX8VmAnMFpFCHB3Hg4+xn/dwfNmvBlYA3+PoqK601t8A+ODokN0HfI7j/H99XI/jHPoGHH0c9wMYY5JxdDBPtPaZhuP8/EkzxvyM4/z6FzhaO52Aqxu4uyeBD63TV1c24r7n4ajzHOBFY8xRN7EZYzYC1+HokN4NXARcZIwps4o8i+OHRP6xrjxSTePQ1RFKtQjW5Z9vG2Pij1tYnRARSQC2Ad41+n+Ui9MWgXJpIuIvIueLiJeIxABP4LhkVClVT5oIlKsT4Ckcp2hW4LjK6HFbI1LKxeipIaWUcnPaIlBKKTfncgNLRUREmISEBLvDUEopl7J8+fLdxpjI2ta5XCJISEggOTnZ7jCUUsqliMiOutbpqSGllHJzmgiUUsrNaSJQSik3p4lAKaXcnCYCpZRyc5oIlFLKzWkiUEopN+c2iSCvsJRnvk0lt7DE7lCUUqpZcbkbyhpq8dY9vL9oO1N+30H3tq3o0qYVN5yaQO/YELtDU0opW7lNi2BM33b8/MAZ3DAkniA/L2at28Xlby/ikyXpVFRW2R2eUkrZxuVGH01KSjKNMcTEnqJSxk9NYcm2vSRGBfH3C7ozomvU8TdUSikXJCLLjTFJta1zmxZBTeFBvnx6+xDeunYAVVWGmz9YxpcrdtodllJKNTm3TQQAIsLo3tF8d+9pnNoxnIc+W82K9H12h6WUUk3KaYlARPxEZKmIrBKRdSLyVC1lfEXkfyKSJiJLrGeiNjl/H0/eum4gbYP9uPfTFSzbvteOMJRSyhbObBGUAiONMX2BfsB5IjKkRplbgH3GmETgZeA5J8ZzTCH+3rx2TX+Kyyq54u3FfLc6265QlFKqSTktERiHImvW23rV7JkeC3xoTX8OnCUi4qyYjmdgfCi/PTKSfu1b89cv1zA9OYPiskq7wlFKqSbh1D4CEfEUkZVALvCTMWZJjSIxQAaAMaYC2A+E17Kf20UkWUSS8/LynBky/j6evHJVPwJ8PHnk89XcNXU5VVWudWWVUkqdCKcmAmNMpTGmHxALnCIivRq4n3eNMUnGmKTIyFqftNaoEiICWfiXkTx5UQ9+3ZjHG3PTnP6eSilllya5s9gYky8ic4HzgLXVVmUC7YGdIuIFhAB7miKm4/HwEG4cmsDKjHz+8/Mmgv29GdktivZhAXaHppRSjcqZVw1Fikhra9ofOAfYUKPYTOBGa/py4BfTjO5wExH+dWlvurUN5omZ6zjn5Xl8v0Y7kZVSLYszTw1FA3NFZDWwDEcfwbci8rSIjLHKTALCRSQNeAB41InxNEiAjxdfTxjGd/cOp0d0MPdOW0FabqHdYSmlVKNx2yEmGmJ3USlnvvgrfWNb89Etp2DjBU5KKXVCdIiJRhIR5MuD53RhQdpuZq3bZXc4SinVKDQRnKDrhsTTrW0r/u/rdUz4JIXM/GK7Q1JKqZOiieAEeXl68M9LeuHj6cGstbt47efNdoeklFInRRNBAwyMD2PhoyMZNziOGSt2kqWtAqWUC9NEcBLuOKMTxsC787faHYpSSjWYJoKTENPan0v6xzBtaTp5haV2h6OUUg2iieAk3TWiE+WVVZz54q9MXrDN7nCUUuqEaSI4SR0jg5h00yC6tAnipdkb2birUB9uo5RyKZoIGsGZXaN4akwvDpRVMvrV+Vz1zu9k7D1od1hKKVUvmggaSe/YEIZ2Cic8yBcPD3hh1ka7Q1JKqXppktFH3cWkGwcB8MbcNCbOTeP0LpFcPjDW5qiUUurYtEXQiPx9PPH38eTeszoztFM4j81YzaqMfLvDUkqpY9JE4AQ+Xh68de1AIoJ8uf9/KzlYVmF3SEopVSdNBE4SEuDNS1f0ZdvuA3yyJN3ucJRSqk6aCJxoaGIEgzuE8f7C7ZRXVtkdjlJK1UoTgZPdfnpHMvOL+WGtDlutlGqeNBE42Zldo2gf5s80PT2klGqmNBE4mYeHcPWgOBZv3cP8TXmUlFfaHZJSSh1BE0ETuHxgLF4ewg2Tl3LOy/NYvkOHoFBKNR+aCJpAm2A/vpowjFeu6ocgXPvf31mYttvusJRSCtBE0GR6xYRwcf8Yvhw/lITwQG6bksyanfvtDksppTQRNLXwIF+m3HIKoQE+3DplmfYZKKVsp4nABlGt/Hjhij7kFJTy9cpMu8NRSrk5TQQ2ObVjON2jg5m0YBsVerOZUspGTksEItJeROaKSKqIrBOR+2opM0JE9ovISuv1uLPiaW5EhPEjOrEpp4jbpiTrKSKllG2cOQx1BfCgMSZFRFoBy0XkJ2NMao1yvxljLnRiHM3WRX3bsb+4nL9/tZZJC7Yx4cxEu0NSSrkhp7UIjDHZxpgUa7oQWA/EOOv9XNV1Q+I5t0cb3pibRk5Bid3hKKXcUJP0EYhIAtAfWFLL6lNFZJWI/CAiPZsinubmbxd0p6LS8PyP+lQzpVTTc3oiEJEg4AvgfmNMQY3VKUC8MaYv8DrwVR37uF1EkkUkOS8vz7kB2yA+PJA/De/AFyk7WakPslFKNTGnJgIR8caRBKYaY2bUXG+MKTDGFFnT3wPeIhJRS7l3jTFJxpikyMhIZ4Zsm7tHJhIR5MvT36zDGGN3OEopN+LMq4YEmASsN8b8p44yba1yiMgpVjx7nBVTcxbk68Ujo7qSkp7Psz9sYN+BMrtDUkq5CWdeNTQMuB5YIyIrrWV/BeIAjDFvA5cDd4lIBVAMXG3c+Ofw5QNjmbMhh3fnbyU1q4CPbx1sd0hKKTfgtERgjFkAyHHKTAQmOisGV+PhIbxzfRKv/ryZl3/eRPqeg8SFB9gdllKqhdM7i5uhy5NiEYHPU3baHYpSyg1oImiGYlr7Mzwxgk+W7KCgpNzucJRSLZwmgmbqkVHd2HOgjL99uZbUrJpX3SqlVOPRRNBM9Y4N4dbhHfhmVRYXvP4bq/T+AqWUk2giaMb+en535j98JuGBvjw+cx1VVW57QZVSyok0ETRjIkJceAAPj+rCqox8VmirQCnlBJoIXMBZ3dsAsGz7XpsjUUq1RJoIXEBEkC8dIwJJ1kSglHICTQQuYlBCGMu279N+AqVUo9NE4CKSEkLZX1zOQ5+vIn3PQbvDUUq1IJoIXMSIrlH0jQ3h+zXZ3DplGQfLKuwOSSnVQmgicBGRrXz5+u7hvHdDEptzi/jbl2t1uGqlVKPQROBiTuscyf1ndeHLFZm8/PNmyiqq7A5JKeXiNBG4oHtGJjKmbztem7OZK95exP5iHY9IKdVwmghckIeH8No1/Xlj3ABSswu46+PldoeklHJhmghc2AV9onlkVDcWbdnD+mwdmE4p1TCaCFzcZQNj8fYUPl+uzy5QSjWMJgIXFxbow9nd2/DlikyKSvWSUqXUidNE0ALcdnpH9h4o4/U5m+0ORSnlgjQRtAAD4kK5MimWSQu2sX33AbvDUUq5GE0ELcRDo7ri5Sm89ou2CpRSJ0YTQQsR1cqP64fE89WKTB2lVCl1QjQRtCDjRyQSHx7ITe8vY23mfrvDUUq5CE0ELUhooA/TbhtCsJ8Xt09JJq+w1O6QlFIuQBNBC9M2xI93b0hi78Eyxk5cwLxNeTo4nVLqmJyWCESkvYjMFZFUEVknIvfVUkZE5DURSROR1SIywFnxuJNeMSF8dsdQvDw9uHHyUm6YvJTdRdo6UErVzpktggrgQWNMD2AIMEFEetQoMxrobL1uB95yYjxupXdsCD89cDpPXNSDpdv2csFrv+kzj5VStXJaIjDGZBtjUqzpQmA9EFOj2FhginH4HWgtItHOisnd+Hp5cvOwDnw5fhj+3p5cP2kJmfnFdoellGpmmqSPQEQSgP7AkhqrYoCMavM7OTpZICK3i0iyiCTn5eU5K8wWq0e7YD6+dTAA46emcPW7izn9+bk6UJ1SCmiCRCAiQcAXwP3GmAZ98xhj3jXGJBljkiIjIxs3QDcRGxrAPSM7syojn8KSCrLyi/lmVZbdYSmlmgGnJgIR8caRBKYaY2bUUiQTaF9tPtZappxg/IhOrHr8XL679zQGxIcyb5O2rpRSzr1qSIBJwHpjzH/qKDYTuMG6emgIsN8Yk+2smNydiBAS4A3AGV0iWZdVQG5hic1RKaXs5swWwTDgemCkiKy0XueLyJ0icqdV5ntgK5AGvAeMd2I8qpoRXR2n2H7btNvmSJRSdvNy1o6NMQsAOU4ZA0xwVgyqbt3bBhPs50Xyjn1cNjDW7nCUUjbSO4vdlIeH0C8ulBXp++wORSllM00Ebqx/+9ZsyinUJ5sp5eY0EbixAfGhVBlYlZFvdyhKKRtpInBj/WJbA/DYjDXM1HsKlHJbmgjcWEiAN4+N7oa3p/DXGWvYd6DM7pCUUjbQRODm7jijE29dN5ADZRW8PW+L3eEopWygiUDRpU0rLuzTjk+WpHOwTDuOlXI3mggUANcPiaewtILnf9zIG3PTqKissjskpVQTcdoNZcq1DEoIJTEqiA8WbQdg465CXrmqHx4ex7wnUCnVAmgiUIBjHKInLurBoi178PIQXv8ljcsGxnJGFx3tVamWrl6nhkQkUEQ8rOkuIjLGGllUtSCndY7kL+d14+6RiYQGeDN9WcbxN1JKubz69hHMB/xEJAaYjWMwuQ+cFZSyl6+XJ5cOiGV26i527dfRSZVq6eqbCMQYcxC4FHjTGHMF0NN5YSm7XTckHi8PD66ftITs/fp4S6VasnonAhE5FbgW+M5a5umckFRz0CEikPdvHkRmfjHnvfIby3fog++VaqnqmwjuBx4DvjTGrBORjsBc54WlmoMhHcP59p7hBPl68cTMdThGDVdKtTT1SgTGmHnGmDHGmOesTuPdxph7nRybagY6RgZx39mdWZtZwJz1uQAcKK0gLbfQ5siUUo2lvlcNfSIiwSISCKwFUkXkYeeGppqLS/rHEBcWwCtzNmGM4YVZGxkzcSGlFZV2h6aUagT1PTXUwxhTAFwM/AB0wHHlkHID3p4e3D0ykbWZBfyUmsOPa3dxsKySjbu0VaBUS1DfROBt3TdwMTDTGFMO6AljN3KoVfDojDXsKnBcUrp6536bo1JKNYb6JoJ3gO1AIDBfROKBAmcFpZofb08Pnr+8D/kHy/D0EFr5erFGE4FSLUK9hpgwxrwGvFZt0Q4ROdM5IanmakjHcJ65uDe79hezIiOf1ZlHJoKfUnPoHBVEQkSgTREqpRqivp3FISLyHxFJtl4v4WgdKDczbnAcD5zblT6xIWzKKSR9z0G25hWxKiOf2z9K5tkf1tsdolLqBNV30LnJOK4WutKavx54H8edxsoNje0Xw0eLd3DGi3MxBnw8PTAG5m/aTUl5JX7eer+hUq6ivn0EnYwxTxhjtlqvp4COzgxMNW9d2rRixvih3DAknodHdSUm1J8rk2IpLq9kwebddoenlDoB9W0RFIvIcGPMAgARGQboADRuLjGqFU+N7QXAhDMTKauo4oc1u/gpNYeze7SxOTqlVH3VNxHcCUwRkRBrfh9w47E2EJHJwIVArjGmVy3rRwBfA9usRTOMMU/XMx7VDPl4eTC8cwS/bc7DGIOIPtRGKVdQ3yEmVhlj+gJ9gD7GmP7AyONs9gFw3nHK/GaM6We9NAm0AMM7R5C1v4Stuw/YHYpSqp5O6JnFxpgC6w5jgAeOU3Y+oENWupnTEh1PNFuYpv0ESrmKk3l4fWO0+08VkVUi8oOI1Pl8AxG5/dClq3l5eY3wtspZ4sIDaB/mz/xNepyUchUnkwhOdoiJFCDeOuX0OvBVnW9kzLvGmCRjTFJkpD5Dt7m7sE875mzIZW3mfrLyi5myeDsVlVVHlftw0XYemL6y6QNUSh3hmJ3FIlJI7V/4AvifzBtXO8WEMeZ7EXlTRCKMMXpOwcXdNaIT05dlMH5qCgfLKthdVEZEkC+je7Xlm9XZ9I0NIT48kE+WpJOWV8S/Lumt9x0oZaNjJgJjTCtnvbGItAVyjDFGRE7B0TrZ46z3U00n2M+bF6/oy6tzNtM22A8POcD05Ayy8ot55rv1RIf48d4NSWzMcYxempZbRK+YkOPsVSnlLPW9fPSEicg0YAQQISI7gScAbwBjzNvA5cBdIlKB456Eq40+AqvFOLNbFGd2iwLgpdkbef2XNH7dmMewxHBWpudzzXu/Hy6bmlWgiUApGzktERhjrjnO+onARGe9v2o+rjkljoVpuzm7Rxv+NKwDs9bt4r5PV9LKz4vKKkNqtg5kq5SdnJYIlDqkXWt/Zowfdnh+TN92pOzYR4CvF79v3aOJQCmbncxVQ0o1iIjw1Nhe/OW8bnSPDiY1q4DMfB2xRCm7aCJQthp3ShwicOmbCyksKbc7HKXckiYCZateMSG8fd1AcgpK+XWj3oSmlB00ESjbDekYTnigD1+k7OSxGWvYsUfHKVKqKWlnsbKdp4cwslsUny3fCUBZRRUvXdnX5qiUch/aIlDNwnm92gIQFxbAN6uz2FNUytvztjA9OcPmyJRq+bRFoJqFkd2i+P7e0/DxEs7+z3yufvd3NucW4evlwWmdI4gOOakRTZRSx6AtAtUsiAg92gWTGNWK5y7rzc59xQxKCKXKGF6YtRG96Vwp59EWgWp2rhoUx3k9own09eTlnzfxxtwttAvx56FRXe0OTakWSVsEqlkKCfDGy9ODB8/pyuUDY3nj1zS27z5Agd5roFSj00SgmjUPD+GRUV3x8hBumLyUAU//xLLt+uA7pRqTJgLV7EUF+3FB72jS9x6k0hie/3GD9hko1Yi0j0C5hMfO707/OEfn8VPfpDJvUx4jukbZHZZSLYImAuUS2gT7cePQBMoqqpi0YBsvzt7Itt0HOK1zBIlRTnt+klJuQU8NKZfi4+XBfWd1Zm1mAU99k8oz3623OySlXJ62CJTLuaR/DJn5xWzILmR26i4m/rKZGSmZhAR4M+OuoYiI3SEq5VK0RaBcjpenB/ef3YVHR3ejysCLszdRWlHFivR81mX98ZAbYwzP/biBNTv32xitUs2ftgiUy0qICOS6IXH4enly14hODP7XHL5bk02Ivzcvzt5Iv/ateevXLeQfLOfZ2N52h6tUs6WJQLm0Zy7+4wt+aKdwvl6RyaIte1iVkc/XK7MAWJupLQKljkVPDakW49bTOpJfXM6qjHyuHRyHh0BMa3827iqkrKKKzTmFLErbbXeYSjU72iJQLcYZXSJZ8tez2Lb7AH1iW/PQuV1ZkLabe6atYFNOIc98l8ryHfv4cvwwesWE2B2uUs2GtghUi9LKz5s+sa0BCA30obf1hb94yx6Wbd9HeaXhzo+X8+PabDvDVKpZ0USgWrT48ACC/bx4Z/5WKqsMj43uhq+XB3d+nMLGXYUYY/jWehCOUu5KE4Fq0USEe8/qzO6iUloHeHPraR353x2n4ukhfLUykx/X7uLuT1YweeE2u0NVyjZO6yMQkcnAhUCuMaZXLesFeBU4HzgI3GSMSXFWPMp93TK8A5n5xUS28sXTQ4gI8mV4YgQzUnYyI8XxnOSl23REU+W+nNki+AA47xjrRwOdrdftwFtOjEW5MRHhiYt6Mn5E4uFllw6IIaeglMoqw2mdI1iVsZ+S8kobo1TKPk5rERhj5otIwjGKjAWmGMd4wr+LSGsRiTbGaC+ecroxfdsRFxZAz3YhzNuUx2+bd7MqI5/BHcPtDk2pJmdnH0EMkFFtfqe17CgicruIJItIcl5eXpMEp1o2EaF/XCg+Xh4MSggFYP5m/dtS7sklOouNMe8aY5KMMUmRkZF2h6NamNYBPozq2Yb35m9j9c58vlyxk8dmrLY7LKWajJ2JIBNoX20+1lqmVJP796V9iAjyYcInKTz+9TqmLc1gx54DdoelVJOwMxHMBG4QhyHAfu0fUHYJDfTh9XEDyM4v4UBpBQCz1+VQWFKuj8VULZ44649cRKYBI4AIIAd4AvAGMMa8bV0+OhHHlUUHgZuNMcnH229SUpJJTj5uMaUa5Ps12RSVVPDBou1s33OAg2WVtA3247nL+/Du/C2MOyWeC/pE2x2mUidMRJYbY5JqW+fMq4auOc56A0xw1vsr1RDn93Z8yecVlfLCrI1cNySOXzfmcePkpQAYgyYC1eLooHNK1eLOMzoxpm872ocFkJpVwA2TlxIW6M3SbXspKCknr7CUwpIK+rVvbXeoSp00p50achY9NaTsYIwhecc+rnh7MRf0iebn1Bw8RFj82EhaB/jYHZ5Sx3WsU0MucfmoUnYTEfq3b01YoA/frc6mb/vWFJdXcvMHyxj54q/s1kHrlAvTU0NK1ZOXpwdTbx2MMdCjXTDXT1rCb5sdD7r5akUmt57W0eYIlWoYbREodQK6RwfTo10wAP+8uDf/vKQXvWNCmJFS+y0w8zblccdHyXyWnFHreqWaA00ESjVQXHgA1w6O57IBMaRmF7A+u+CoMh8s3MasdTk8/PlqNuw6er1SzYEmAqVO0ph+MXh7Cl8sdwxpXV5ZhTGGisoqlm3fx7k92iAC367KZtrSdIrLdJRT1bxoH4FSJyks0Iczu0bx1cosrj81nmve/Z3wIF+uGtSeotIKxvRrx54DZbz5axpVBsoqqrhxaILdYSt1mLYIlGoElw2MZXdRKee98huFpRXkFJTw96/WAjC4QzijerahyrpS+/s12fy2OU8fj6maDW0RKNUIzuoWxX1ndSZj30GuGxJPbGt/rv3vEgJ9vYhs5ctVg+KorIL84jLembeVJZOWMrZfO169ur/doSulN5Qp5SyVVYbSikoCfP74vbVxVyGjXpmPp4cQ4O1J8v+dja+Xp41RKnehN5QpZQNPDzkiCQB0bduKL+46lVev7kdhaQWfJe8kr1BPESl7aSJQqokNjA/jnB5taOXnxd+/WsvFbyykyBr6uqEmTE3h3z9saKQIlbvRRKCUDXy9PHnz2gE8PKorWfuLeXHWxgbva9vuA3y3JpuPf99BSblemqpOnCYCpWxyWudIJpyZyI2nJvDh4u0kb9/boP18meK4f6/YrkoAABNqSURBVKGotIK5G3IbMULlLjQRKGWzh0d1pV2IP3dNTeGxGauZtW4Xy3fspaKy6rjbllZU8kVKJkM7hRMR5MvXK7OaIGLV0mgiUMpmgb5eTBzXn85RQXy7Kps7PlrOZW8t5s6PUygpryQ1q4B1Wftr3XbKoh1k5hdz++kdOa9XG+ZtytPTQ+qE6X0ESjUD/eNC+eS2IZRWVLJ6536St+/juR838Pev1vLLhlwKiss5v3c0rQO86RARyEeLd3DdkHhem7OZM7tGMqJrFMbAx7+ns2TbXs7oEml3lZQL0USgVDPi6+XJoIQwBiWEkb2/mCmLdwAwPDGCRVt2U1BSQVlFFf7enjz9bSoxrf15emwvAE7tFI6ftwe/rM/RRKBOiCYCpZqpB8/pyg9rd3FKhzDeGDcAgB17DrBk217O7dGGj3/fwWUDY4kO8QfAz9uT4YkRzNmQy5NjDCJy1D7/8W0qlVWGJ8f0bNK6qOZNE4FSzVRIgDdzHjyDAO8/7jyODw8kPjwQgLtHdj5qm5Hd2vDz+lw25xbRpU2rI9aVlFcydckOKqsMfz67C34+HmTsLSYxKsi5FVHNnnYWK9WMBft54+VZ//+mI7tFAfBLLZeRLtqym5LyKsorDR8v2cH1/13KOS/Pq7Uj+pMl6bw0u+H3NijXoolAqRakbYgfPdsF8868LQz658+szfzjS/7n9bkE+niSEB7AC7M2sjIjH29PD6YuST9qPx8s2sY787Y2yrMT1mbu57fNeSe9H+U8mgiUamHO7dGWfQfLOVBawYRPUigsKScrv5hvV2VxepdIHju/O9cNieP7+4Yztm87vlqRyda8Iq58ZzE9H/+R537cwKacIsoqq1iybc9JxVJWUcVdU5dz77QVuNoAl+5E+wiUamHuGtGJ83q1paCknCvfWcxT36SyObeIKgOPnNeNDhGBjOrZFoCbhiXw9cosznnZMSJq+1B/3p635fC+5m/azYiuUQ2O5dNl6WTsLQZg6+4DdIrU/ojmyKktAhE5T0Q2ikiaiDxay/qbRCRPRFZar1udGY9S7sDHy4OubVsxKCGMm4Ym8PnynazL3M+LV/SlQ0TgEWV7tgth8k2DaNfaj5eu6Mufz+mCMeAhMDA+lPkneUpn2tIM2oX4AbB8x76T2pdyHqe1CETEE3gDOAfYCSwTkZnGmNQaRf9njLnbWXEo5c4eOrcrB0srGduvHUMTI2otM7xzBL89MhJwXFnUyteLuPAALuoTzZPfpLIpp/CoK5Dqo7Siks05hdx6Wkc+XZbO8u37uDKp/UnVRzmHM1sEpwBpxpitxpgy4FNgrBPfTylVQ6CvF89d3qfOJFCTn7cnz1/eh0dHd+OCPu3wEPh6ZSY5BSV1brM5p5C3ft1yVB/A5pwiKqoMvWKCGRAXyvJ0bRE0V85MBDFARrX5ndaymi4TkdUi8rmI1PpzQURuF5FkEUnOy9OrD5RyptG9ozmtcySRrXwZlhjBm79uYcizc1iytfaO44lz03juxw38sHbXEctTswsA6BEdzMD4UNJyi8g/WOb0+NWJs/uqoW+ABGNMH+An4MPaChlj3jXGJBljkiIj9dZ5pZrKn4Z3oEtUK8ICfHhp9qajBrQrq6g6fM/Csz+sp7Tij/WpWQUE+HiSEB7IwPhQAFK0VdAsOTMRZALVf+HHWssOM8bsMcYcek7ff4GBToxHKXWCzuwaxaw/n859Z3dm6fa99Hj8R16fs5lPl6YzfVkGc9bnUFhSwTWnxJGxt5i5G/5osadmF9A9OhgPD6FvbGu8PITk7ZoImiNnXj66DOgsIh1wJICrgXHVC4hItDEm25odA6x3YjxKqQa65pQ4KioNi7bs4aWfNh2xLtDHk79f0J2fUnfx9cpMzuvVlgOlFazemc/Vg+IA8PfxpGe7YL1yqJlyWiIwxlSIyN3ALMATmGyMWSciTwPJxpiZwL0iMgaoAPYCNzkrHqVUw3l7evCn4R24/tR4nv1+Az3bBRMd4sfGnEJ6RAcT6OvFRX3bMfX3dHILSli8dQ8l5VWc3zv68D4GxofxydIdlFdW4X0Cw2Yo5xNXu9svKSnJJCcn2x2GUqqGjbsKuej1BQT5eREW6MOB0goW/mUkHh6OUVC/W53NhE9S+HrCMPq2bw1AcVklft4etY6UqhqXiCw3xiTVtk7TslKqUXRt24pv7x1O56gg0nKLGNOv3eEkAJCU4Ogwnp26i+snLeHDRdsZ+u85XPXO76TlFtbr0Zw1rUjfx39mb2TX/rovb3Ulv23O46sVmccv2Mi0RaCUalTGGDbmFJIQHohftSG0AYb9+xfyCksps770QwO8OVhWSWlFFR0jAnnn+oF0tm5eK6+sYtKCbVzQO5r2YQFHvcdLszcxcW4aAJ2jgvj8zqGEBHgfLjNzVRYTf9nMzLuHHxVHc3XlO4tZl7mflMfPwdercWPWFoFSqsmICN3aBtf65ZuUEEpZZRW9YoK5/fSOfHTLYH68/3T+dUlvCkoquGjiAl79eTOrd+bz+fKd/PuHDdz0/lIKS8qP2M/XK7OYODeNq5LaM+nGJLbuPsB7v209vL6kvJJnv1/PppwituQVOb3OjWVr3gEOlFWyaMvJDfZ3ojQRKKWazKH7Ca4aFMdfz+9Or5gQOkQEMm5wHN/eM5zhiRG8/PMmxkxcyJMz15EQHsD2PQd59ocNHCyrIC23kJLySj5bnkFCeADPXtqbs7q3YWB86OH7GSoqq3j+x41kW6eL0nJdIxHsLy5nd5HjavrZ63Ka9L119FGlVJO5oHc0W3KLuKT/0YMMtA3x4783DiJ7fzGfLs3g/YXbeP7yvny3OouPl6Tzy/pcdhWUEB3ix66CEu4Z2flwH8SZXaN47scN7NpfwtPfruP7Nbu4bEAsX63MdJlEsNVqubQO8OaXDTlA7yZ7b20RKKWaTHiQL0+N7UWQb92/QaND/PnzOV1Y9cS5nNIhjLtHdsbXy4Oyyir+MbYnB0orMAbG9G13eJtDT2YbP3U536/ZxcOjuvLSlX2JDws4oUSwp6iUYf/+hWlL0ykoKaeotKLhlT1BW/IOADCqR1tyCkqPOh3mTNoiUEo1S4cuKY1s5csXdw2ldYA30SH+DIwPY01m/hHPWu7SJog+sSGkpOdzQZ9o7jqjEwCdrCuYAApKymnl63XMS1X/9f0GMvOL+WjxDqYs3kF0iB+TbxrkxFr+YUteEV4ewrDOEfwvOYMdew7SKyakSd5bE4FSqtnrHh18eLpHu2B6tAs+Yr2IMPPu4VRWGTyrXbKaGBXET6k5XPLmQlak5/OPi3tx/ZD4I7bdnFPItKUZpKTvY2VGPgnhAYcHzNuaV0RJeSV+3p7sPVDG58szuGpQHCH+3jS2rXlFxIcH0NF6ZkT6Xk0ESil1wqonAYBubR2Xom7OKaJDRCBv/7qF1Rn5hPh7c83gOD5avIOPft+Bl4eQGBXEkxf14NyebRn+3C94egilFVWsSM+nf1xrbpuSzPId+5ievJOPbxlMW+uBO7XJyi8mspXv4TuoJ0xNAYE3xg2otXxllWFFej5JCaHEhzsuld2x52BjfCT1oolAKdVind87Gh9PD0Z0jWJh2m5unZLMZ8t3AvDfBdvwEBg3OI4HzulKWKDP4e0ev7AHbUP8GT91OYu37uHXTbks37GPu89M5INF27np/aVMv/NUgv0cLYPs/cWk7znI4I7h/Jyaw61Tkgn08eRfl/YmKSGM79c6hlTbue8gsaEBR8U5f3MeuYWlXNSnHa38vAkL9CF9ryYCpZQ6ad6eHoy2xjsa2S2Kqwe1JykhjPjwALbkFnFKhzA61vIc5ZuGdQCgd2xrvli+k9zCEq4YGMtDo7oyuGMYN7+/jLETF/LeDQOJCwvkxslL2ZxbxMRrBvD8rA10jAgkNNCHB6evYmhiBMaACPzfV2sZGB/KhDMTD/dVGGOYviyD0ABvzureBoC4sADS9x5ook9J7yxWSqk6zduUx18+X01xeSU/PXA6Ua0cp4OWbN3D+KkpxIb6c3qXSF7/JY22wY7LWgE+/NMp9I9rza0fJLN0+16GdgrH29ODeZscw3S/f9MgzuwWRXllFQ9OX8XMVVnccUZHHhvdHYB7p61g5qosxo/oxJ0jOh1ueZyMY91ZrIlAKaWOoayiiuKyyiOGrwD4fPlOHvpsFQCXDYjlwXO7MGvdLvrHhdLPGlTPGMPirXuIDw/E18uDnfuKue/TFQT4ePHdPcP5x3epvL9wO38+uwt3j0w83MfxwqwNvDF3CwCDEkL56JbBJz1MhiYCpZRqZJVVhnHv/U6wvzdvjBuAj1f9bsuauSqLe6et4KxuUczZkMvNwxJ44qKeR5RJ33OQT5am0z7Mn799uZZrTmnP4xf2pLyqqsGtA00ESinlBMaYEx5C2xjDnz5YxtyNeQzpGMZHtww+5vMZnvtxA2/96mgd3DMykQfP7dqgWI+VCLSzWCmlGqghz1EQEZ67rA+TFmzj9tM7HvchPQ+c04UQf2/KK6o4tVN4Q0M9dkzaIlBKqZZPh6FWSilVJ00ESinl5jQRKKWUm9NEoJRSbk4TgVJKuTlNBEop5eY0ESillJvTRKCUUm7O5W4oE5E8YEcDN48AdjdiOM2N1s+1af1cW3OvX7wxJrK2FS6XCE6GiCTXdWddS6D1c21aP9fmyvXTU0NKKeXmNBEopZSbc7dE8K7dATiZ1s+1af1cm8vWz636CJRSSh3N3VoESimlatBEoJRSbs5tEoGInCciG0UkTUQetTuexiAi20VkjYisFJFka1mYiPwkIputf0PtjrO+RGSyiOSKyNpqy2qtjzi8Zh3P1SIywL7I66eO+j0pIpnWMVwpIudXW/eYVb+NIjLKnqjrT0Tai8hcEUkVkXUicp+13OWP4THq1jKOnzGmxb8AT2AL0BHwAVYBPeyOqxHqtR2IqLHseeBRa/pR4Dm74zyB+pwODADWHq8+wPnAD4AAQ4AldsffwPo9CTxUS9ke1t+pL9DB+vv1tLsOx6lfNDDAmm4FbLLq4fLH8Bh1axHHz11aBKcAacaYrcaYMuBTYKzNMTnLWOBDa/pD4GIbYzkhxpj5wN4ai+uqz1hginH4HWgtItFNE2nD1FG/uowFPjXGlBpjtgFpOP6Omy1jTLYxJsWaLgTWAzG0gGN4jLrVxaWOn7skghggo9r8To59EF2FAWaLyHIRud1a1sYYk21N7wLa2BNao6mrPi3pmN5tnRqZXO1UnkvXT0QSgP7AElrYMaxRN2gBx89dEkFLNdwYMwAYDUwQkdOrrzSONmqLuT64pdXH8hbQCegHZAMv2RvOyRORIOAL4H5jTEH1da5+DGupW4s4fu6SCDKB9tXmY61lLs0Yk2n9mwt8iaPpmXOoeW39m2tfhI2irvq0iGNqjMkxxlQaY6qA9/jj9IFL1k9EvHF8UU41xsywFreIY1hb3VrK8XOXRLAM6CwiHUTEB7gamGlzTCdFRAJFpNWhaeBcYC2Oet1oFbsR+NqeCBtNXfWZCdxgXXkyBNhf7fSDy6hxTvwSHMcQHPW7WkR8RaQD0BlY2tTxnQgREWASsN4Y859qq1z+GNZVtxZz/OzurW6qF44rFDbh6L3/m93xNEJ9OuK4KmEVsO5QnYBwYA6wGfgZCLM71hOo0zQczetyHOdUb6mrPjiuNHnDOp5rgCS7429g/T6y4l+N48sjulr5v1n12wiMtjv+etRvOI7TPquBldbr/JZwDI9RtxZx/HSICaWUcnPucmpIKaVUHTQRKKWUm9NEoJRSbk4TgVJKuTlNBEop5eY0EagWS0SKrH8TRGRcI+xvu4h8UW3+chH54GT3a+3rSRF5qDH2pdSJ0kSg3EECcEKJQES86lg1UER6nHREjci6IUv/L6sG0z8e5Q7+DZxmjRf/ZxHxFJEXRGSZNVjYHQAiMkJEfhORmUBqHft6CceNQkeo+YteRNZaLZEEEdkgIh+IyCYRmSoiZ4vIQmt8/uojUvYVkcXW8tuq7evharE+ZS1LsMa5n4LjbtbqwxkodULq+tWjVEvyKI4x4y8EsEZq3W+MGSQivsBCEZltlR0A9DKOoYNrMx0YLyKJJ/D+icAVwJ9wDHcyDsedqmOAv/LHsMx9cIzLHwisEJHvgF44hic4BceduDOtwQXTreU3GscQzko1mCYC5Y7OBfqIyOXWfAiOL9UyYOkxkgBAJfAC8BiOh6rUxzZjzBoAEVkHzDHGGBFZg+O01SFfG2OKgWIRmYvjy3+4Fe8Kq0yQFWs6sEOTgGoMmgiUOxLgHmPMrCMWiowADtRj+49wJIK11ZZVcOSpVr9q06XVpquqzVdx5P/BmuO9GCvWZ40x79SINaGesSp1XNpHoNxBIY7HCx4yC7jLGlYYEelijeBaL8aYcuBl4M/VFm/HcVoJ69m7HRoQ51gR8RORcGAEjtNIs4A/WePgIyIxIhLVgH0rVSdtESh3sBqoFJFVwAfAqzhOyaRYwwvnceKP9JwE/L3a/Bc4hlReh+PJVZsaGOdcIAL4hzEmC8gSke7AYkeoFAHX4ThFpVSj0NFHlVLKzempIaWUcnOaCJRSys1pIlBKKTeniUAppdycJgKllHJzmgiUUsrNaSJQSik39/8XkagMG2CVFAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "old_loss = np.inf\n",
    "\n",
    "max_epoch = 500\n",
    "losses = []\n",
    "for epoch in range(max_epoch):\n",
    "\n",
    "    running_loss = 0.0\n",
    "    \n",
    "    for data in trainloader:\n",
    "        \n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Forward\n",
    "        y_hat, y_hat1, y_hat2 = classifier(X)\n",
    "        \n",
    "        # Calculate Loss (Cross Entropy)\n",
    "        loss_main = criterion(y_hat, y)\n",
    "        loss1 = criterion(y_hat1, y)\n",
    "        loss2 = criterion(y_hat2, y)\n",
    "\n",
    "        # Weighted Loss\n",
    "        loss = loss_main + 0.3*loss1 + 0.3*loss2\n",
    "\n",
    "        # Backpropagation\n",
    "        loss.backward()\n",
    "        \n",
    "        # Update Parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "        running_loss += loss.item()*len(X)/train_size\n",
    "    \n",
    "    print('Epoch', epoch+1, ': Loss =', running_loss)\n",
    "    losses.append(running_loss)\n",
    "    if abs(running_loss-old_loss)/running_loss < 0.05 and running_loss<0.2:\n",
    "        print('Converged')\n",
    "        break\n",
    "    \n",
    "    old_loss = running_loss\n",
    "\n",
    "print('Finished Training')\n",
    "plt.plot(losses)\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Iter Number')\n",
    "plt.title('Convergence monitor plot')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 283
    },
    "colab_type": "code",
    "id": "fVvSW-o33fn7",
    "outputId": "ec659f6b-33aa-4551-fecc-b25737763f5d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Loss = 0.1609659045934677\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>43</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4   5   6\n",
       "0  42   0   0   0   0   0   1\n",
       "1   0  36   0   0   0   0   0\n",
       "2   0   0  43   1   2   0   0\n",
       "3   0   0   0  36   0   0   1\n",
       "4   0   0   0   0  40   0   0\n",
       "5   0   0   0   0   2  40   0\n",
       "6   0   0   0   0   0   0  43"
      ]
     },
     "execution_count": 105,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    train_loss = 0.0\n",
    "    y_train = []\n",
    "    y_train_pred = []\n",
    "\n",
    "    for data in trainloader:\n",
    "\n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        y_hat, _, _ = classifier(X)      \n",
    "        train_loss += criterion(y_hat, y)*len(X)/train_size\n",
    "        \n",
    "        y_train.extend(list(y.detach().cpu().numpy()))\n",
    "        y_train_pred.extend(list(torch.argmax(y_hat, axis=1).detach().cpu().numpy()))\n",
    "\n",
    "print('Train Loss =', train_loss.item())\n",
    "pd.DataFrame(confusion_matrix(y_train, y_train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "ylNoy_9_3fA4",
    "outputId": "9cd81662-4cff-47a4-ab2d-83d90020c5bd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy = 0.975609756097561 Train Precision = 0.9768027768027768 Train F1 = 0.9761091196676174\n"
     ]
    }
   ],
   "source": [
    "acc = accuracy_score(y_train, y_train_pred)\n",
    "prec = precision_score(y_train, y_train_pred, average='macro')\n",
    "f1 = f1_score(y_train, y_train_pred, average='macro')\n",
    "\n",
    "print('Train Accuracy =', acc, 'Train Precision =', prec, 'Train F1 =', f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 283
    },
    "colab_type": "code",
    "id": "ATw2OTqQiJCq",
    "outputId": "c83c8253-6335-4892-d7e6-2dfb8892aa51"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss = 1.312967300415039\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3  4  5  6\n",
       "0  4  0  0  0  1  1  2\n",
       "1  0  4  0  1  0  1  2\n",
       "2  1  0  2  0  0  0  0\n",
       "3  0  0  1  8  0  1  0\n",
       "4  0  0  0  0  4  0  1\n",
       "5  0  0  0  0  1  2  1\n",
       "6  0  0  1  0  0  0  3"
      ]
     },
     "execution_count": 107,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    test_loss = 0.0\n",
    "    y_test = []\n",
    "    y_test_pred = []\n",
    "\n",
    "    for data in testloader:\n",
    "\n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        y_hat, _, _ = classifier(X)      \n",
    "        test_loss += criterion(y_hat, y)*len(X)/test_size\n",
    "        \n",
    "        y_test.extend(list(y.detach().cpu().numpy()))\n",
    "        y_test_pred.extend(list(torch.argmax(y_hat, axis=1).detach().cpu().numpy()))\n",
    "\n",
    "print('Test Loss =', test_loss.item())\n",
    "pd.DataFrame(confusion_matrix(y_test, y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "rgq5cyyZ4m15",
    "outputId": "63ce4e90-4bf3-4dc1-ff28-f1eb84ac414e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy = 0.6428571428571429 Test Precision = 0.6555555555555556 Test F1 = 0.6184058214133402\n"
     ]
    }
   ],
   "source": [
    "acc = accuracy_score(y_test, y_test_pred)\n",
    "prec = precision_score(y_test, y_test_pred, average='macro')\n",
    "f1 = f1_score(y_test, y_test_pred, average='macro')\n",
    "\n",
    "print('Test Accuracy =', acc, 'Test Precision =', prec, 'Test F1 =', f1)\n",
    "torch.save(classifier, '/content/drive/My Drive/A3Q1b_model.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "s1RPoXsreDy4"
   },
   "source": [
    "### Question 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NuqwHPPlQavd"
   },
   "outputs": [],
   "source": [
    "class CNN2(nn.Module):\n",
    "    \n",
    "    def __init__(self, n_classes):\n",
    "        super(CNN2, self).__init__()\n",
    "\n",
    "        self.n_classes = n_classes\n",
    "        # 3x224x224\n",
    "        self.cl1 = nn.Conv2d(3, 4, 3, stride=1, padding=1)\n",
    "        # 4x224x224\n",
    "        self.pl1 = nn.AvgPool2d(2, stride=2)\n",
    "        # 4x112x112\n",
    "        self.cl2 = nn.Conv2d(4, 16, 3, stride=1, padding=1)\n",
    "        # 16x112x112\n",
    "        self.pl2 = nn.AvgPool2d(2, stride=2)\n",
    "\n",
    "        # 16x56x56\n",
    "        self.flat = nn.Flatten(1, -1)\n",
    "        # (16x56x56)x1\n",
    "        self.out = nn.Linear(16*56*56, self.n_classes)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        \n",
    "        x = self.pl1(F.relu(self.cl1(x)))\n",
    "        x = self.pl2(F.relu(self.cl2(x)))\n",
    "        x = self.out(self.flat(x))\n",
    "\n",
    "        return x\n",
    "\n",
    "\n",
    "    def predict(self, x):\n",
    "\n",
    "        y_hat = self.forward(x)\n",
    "        y_hat = torch.argmax(y_hat, axis=1)\n",
    "        return y_hat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y4dEykLIhv7s"
   },
   "outputs": [],
   "source": [
    "classifier = CNN2(7)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(classifier.parameters(), lr=0.00005, momentum=0.9)\n",
    "classifier = classifier.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "_3RX6TtEhMFp",
    "outputId": "449f89a5-c6fb-4d6d-9080-f8259f1440d8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 : Loss = 1.9484334253683324\n",
      "Epoch 2 : Loss = 1.947373752809983\n",
      "Epoch 3 : Loss = 1.9462925332763885\n",
      "Epoch 4 : Loss = 1.9449584060429694\n",
      "Epoch 5 : Loss = 1.9439449260459127\n",
      "Epoch 6 : Loss = 1.9427635395568421\n",
      "Epoch 7 : Loss = 1.9418055026788745\n",
      "Epoch 8 : Loss = 1.940997310631782\n",
      "Epoch 9 : Loss = 1.9404067469806208\n",
      "Epoch 10 : Loss = 1.9397513779198252\n",
      "Epoch 11 : Loss = 1.9390972383346292\n",
      "Epoch 12 : Loss = 1.938540056607449\n",
      "Epoch 13 : Loss = 1.9379960842664232\n",
      "Epoch 14 : Loss = 1.9372729299791183\n",
      "Epoch 15 : Loss = 1.9368470446157953\n",
      "Epoch 16 : Loss = 1.9365565810054022\n",
      "Epoch 17 : Loss = 1.9359457754507299\n",
      "Epoch 18 : Loss = 1.935442887532171\n",
      "Epoch 19 : Loss = 1.9347893161640763\n",
      "Epoch 20 : Loss = 1.93448837168956\n",
      "Epoch 21 : Loss = 1.9339445463871707\n",
      "Epoch 22 : Loss = 1.9335623850805834\n",
      "Epoch 23 : Loss = 1.9328854491071004\n",
      "Epoch 24 : Loss = 1.932512533373949\n",
      "Epoch 25 : Loss = 1.9318959422227813\n",
      "Epoch 26 : Loss = 1.9314164147559776\n",
      "Epoch 27 : Loss = 1.931144365450231\n",
      "Epoch 28 : Loss = 1.9304854313255604\n",
      "Epoch 29 : Loss = 1.929869052840442\n",
      "Epoch 30 : Loss = 1.929652035028677\n",
      "Epoch 31 : Loss = 1.9288988561995768\n",
      "Epoch 32 : Loss = 1.9284731153827095\n",
      "Epoch 33 : Loss = 1.927922998986593\n",
      "Epoch 34 : Loss = 1.927448688899183\n",
      "Epoch 35 : Loss = 1.9267448344712472\n",
      "Epoch 36 : Loss = 1.9265186043151163\n",
      "Epoch 37 : Loss = 1.9256972332033961\n",
      "Epoch 38 : Loss = 1.9251209194236516\n",
      "Epoch 39 : Loss = 1.9244895193219598\n",
      "Epoch 40 : Loss = 1.9239521574890988\n",
      "Epoch 41 : Loss = 1.9232810831236093\n",
      "Epoch 42 : Loss = 1.9228265750698927\n",
      "Epoch 43 : Loss = 1.9221636925009484\n",
      "Epoch 44 : Loss = 1.9213716900722486\n",
      "Epoch 45 : Loss = 1.9207152688960165\n",
      "Epoch 46 : Loss = 1.9202305917540492\n",
      "Epoch 47 : Loss = 1.9194150290838101\n",
      "Epoch 48 : Loss = 1.9187590162097783\n",
      "Epoch 49 : Loss = 1.9180176631914196\n",
      "Epoch 50 : Loss = 1.917649082605847\n",
      "Epoch 51 : Loss = 1.916613891980374\n",
      "Epoch 52 : Loss = 1.9160641348735796\n",
      "Epoch 53 : Loss = 1.915375442039676\n",
      "Epoch 54 : Loss = 1.9145038331427224\n",
      "Epoch 55 : Loss = 1.9139695674284825\n",
      "Epoch 56 : Loss = 1.9129308881659957\n",
      "Epoch 57 : Loss = 1.9120563268661501\n",
      "Epoch 58 : Loss = 1.9113547266152677\n",
      "Epoch 59 : Loss = 1.9103483634543337\n",
      "Epoch 60 : Loss = 1.909640578858113\n",
      "Epoch 61 : Loss = 1.9090816310058487\n",
      "Epoch 62 : Loss = 1.9079128894241013\n",
      "Epoch 63 : Loss = 1.9069307623009233\n",
      "Epoch 64 : Loss = 1.9059749077421448\n",
      "Epoch 65 : Loss = 1.9053806602331822\n",
      "Epoch 66 : Loss = 1.904361585706784\n",
      "Epoch 67 : Loss = 1.9033612154086708\n",
      "Epoch 68 : Loss = 1.9022744885720442\n",
      "Epoch 69 : Loss = 1.9014995779309953\n",
      "Epoch 70 : Loss = 1.9002110916563029\n",
      "Epoch 71 : Loss = 1.8994283493387574\n",
      "Epoch 72 : Loss = 1.8981249560877835\n",
      "Epoch 73 : Loss = 1.8969933293006975\n",
      "Epoch 74 : Loss = 1.8957710718859362\n",
      "Epoch 75 : Loss = 1.894872873917689\n",
      "Epoch 76 : Loss = 1.8935659916143384\n",
      "Epoch 77 : Loss = 1.8924650842719788\n",
      "Epoch 78 : Loss = 1.8913409406715154\n",
      "Epoch 79 : Loss = 1.8901387357545647\n",
      "Epoch 80 : Loss = 1.8892231328146798\n",
      "Epoch 81 : Loss = 1.8874746199684276\n",
      "Epoch 82 : Loss = 1.8858791490049729\n",
      "Epoch 83 : Loss = 1.884718432659056\n",
      "Epoch 84 : Loss = 1.883460458978128\n",
      "Epoch 85 : Loss = 1.881971483861943\n",
      "Epoch 86 : Loss = 1.8804235549753967\n",
      "Epoch 87 : Loss = 1.8789370919769237\n",
      "Epoch 88 : Loss = 1.877619432655361\n",
      "Epoch 89 : Loss = 1.8765157361479172\n",
      "Epoch 90 : Loss = 1.8741211293051054\n",
      "Epoch 91 : Loss = 1.8730310288871204\n",
      "Epoch 92 : Loss = 1.8714256078939406\n",
      "Epoch 93 : Loss = 1.8694362532386384\n",
      "Epoch 94 : Loss = 1.8680261751500573\n",
      "Epoch 95 : Loss = 1.8661254199955106\n",
      "Epoch 96 : Loss = 1.8644354966459376\n",
      "Epoch 97 : Loss = 1.8624358114880553\n",
      "Epoch 98 : Loss = 1.8602975025409605\n",
      "Epoch 99 : Loss = 1.858435933182879\n",
      "Epoch 100 : Loss = 1.8568378105396177\n",
      "Epoch 101 : Loss = 1.8547404302537234\n",
      "Epoch 102 : Loss = 1.8521631025686498\n",
      "Epoch 103 : Loss = 1.8509790324167925\n",
      "Epoch 104 : Loss = 1.8487435903283362\n",
      "Epoch 105 : Loss = 1.8459609763547518\n",
      "Epoch 106 : Loss = 1.8438414210641842\n",
      "Epoch 107 : Loss = 1.8414342959998793\n",
      "Epoch 108 : Loss = 1.8391415985619148\n",
      "Epoch 109 : Loss = 1.8370874036064546\n",
      "Epoch 110 : Loss = 1.8344429860131664\n",
      "Epoch 111 : Loss = 1.8317405051885998\n",
      "Epoch 112 : Loss = 1.828751387496443\n",
      "Epoch 113 : Loss = 1.8262698924499938\n",
      "Epoch 114 : Loss = 1.8237245332073253\n",
      "Epoch 115 : Loss = 1.8209417929632739\n",
      "Epoch 116 : Loss = 1.8177170383805596\n",
      "Epoch 117 : Loss = 1.814727600858602\n",
      "Epoch 118 : Loss = 1.8126589022447006\n",
      "Epoch 119 : Loss = 1.8095792822721526\n",
      "Epoch 120 : Loss = 1.8059389740747858\n",
      "Epoch 121 : Loss = 1.803112215696727\n",
      "Epoch 122 : Loss = 1.8001132746606754\n",
      "Epoch 123 : Loss = 1.7969251671734585\n",
      "Epoch 124 : Loss = 1.7932925220150566\n",
      "Epoch 125 : Loss = 1.7893641094712844\n",
      "Epoch 126 : Loss = 1.7862413423817334\n",
      "Epoch 127 : Loss = 1.7823951182049742\n",
      "Epoch 128 : Loss = 1.7784557774507213\n",
      "Epoch 129 : Loss = 1.7749311861676205\n",
      "Epoch 130 : Loss = 1.7709770568156489\n",
      "Epoch 131 : Loss = 1.7669505300422164\n",
      "Epoch 132 : Loss = 1.7645847917849178\n",
      "Epoch 133 : Loss = 1.760190910993968\n",
      "Epoch 134 : Loss = 1.7560418823454855\n",
      "Epoch 135 : Loss = 1.751572580287681\n",
      "Epoch 136 : Loss = 1.7465850829247396\n",
      "Epoch 137 : Loss = 1.7445571015520793\n",
      "Epoch 138 : Loss = 1.740903426130474\n",
      "Epoch 139 : Loss = 1.7352485249682172\n",
      "Epoch 140 : Loss = 1.7308010215958651\n",
      "Epoch 141 : Loss = 1.7274742292610197\n",
      "Epoch 142 : Loss = 1.7203743287495206\n",
      "Epoch 143 : Loss = 1.7178441676528613\n",
      "Epoch 144 : Loss = 1.7151629866623295\n",
      "Epoch 145 : Loss = 1.707442556939474\n",
      "Epoch 146 : Loss = 1.7042793301339765\n",
      "Epoch 147 : Loss = 1.6975591942823725\n",
      "Epoch 148 : Loss = 1.6928629721498658\n",
      "Epoch 149 : Loss = 1.687849085920779\n",
      "Epoch 150 : Loss = 1.6833556250828068\n",
      "Epoch 151 : Loss = 1.679483095112578\n",
      "Epoch 152 : Loss = 1.6738587239893472\n",
      "Epoch 153 : Loss = 1.6674563170310097\n",
      "Epoch 154 : Loss = 1.6634196404380663\n",
      "Epoch 155 : Loss = 1.6624023241448485\n",
      "Epoch 156 : Loss = 1.65202645225392\n",
      "Epoch 157 : Loss = 1.649968918192262\n",
      "Epoch 158 : Loss = 1.6402907076613\n",
      "Epoch 159 : Loss = 1.6372030259009438\n",
      "Epoch 160 : Loss = 1.6337670025509825\n",
      "Epoch 161 : Loss = 1.6284710156377598\n",
      "Epoch 162 : Loss = 1.6208588192271856\n",
      "Epoch 163 : Loss = 1.6210817484905498\n",
      "Epoch 164 : Loss = 1.613494554878527\n",
      "Epoch 165 : Loss = 1.6075170177200526\n",
      "Epoch 166 : Loss = 1.6014824823220015\n",
      "Epoch 167 : Loss = 1.5992170131995702\n",
      "Epoch 168 : Loss = 1.5899677363837639\n",
      "Epoch 169 : Loss = 1.584151668831031\n",
      "Epoch 170 : Loss = 1.579569311092124\n",
      "Epoch 171 : Loss = 1.5774732537385894\n",
      "Epoch 172 : Loss = 1.5700023224129496\n",
      "Epoch 173 : Loss = 1.5664709577992404\n",
      "Epoch 174 : Loss = 1.5606142174491482\n",
      "Epoch 175 : Loss = 1.5581509493784622\n",
      "Epoch 176 : Loss = 1.551198738377269\n",
      "Epoch 177 : Loss = 1.5409033859232992\n",
      "Epoch 178 : Loss = 1.53887057636673\n",
      "Epoch 179 : Loss = 1.5323508820051932\n",
      "Epoch 180 : Loss = 1.529034399820122\n",
      "Epoch 181 : Loss = 1.5246286122225716\n",
      "Epoch 182 : Loss = 1.5159315128359645\n",
      "Epoch 183 : Loss = 1.5199778727953444\n",
      "Epoch 184 : Loss = 1.506733967452099\n",
      "Epoch 185 : Loss = 1.5007071137843646\n",
      "Epoch 186 : Loss = 1.4956993537497436\n",
      "Epoch 187 : Loss = 1.4915352619483497\n",
      "Epoch 188 : Loss = 1.4879117988127866\n",
      "Epoch 189 : Loss = 1.4813738886902972\n",
      "Epoch 190 : Loss = 1.47741304415859\n",
      "Epoch 191 : Loss = 1.4707226960916553\n",
      "Epoch 192 : Loss = 1.463084547361846\n",
      "Epoch 193 : Loss = 1.467499514074691\n",
      "Epoch 194 : Loss = 1.4528035025148027\n",
      "Epoch 195 : Loss = 1.452091575499611\n",
      "Epoch 196 : Loss = 1.4418052235546845\n",
      "Epoch 197 : Loss = 1.4408822013941376\n",
      "Epoch 198 : Loss = 1.4342955948999119\n",
      "Epoch 199 : Loss = 1.4263192917827117\n",
      "Epoch 200 : Loss = 1.4210830777364325\n",
      "Epoch 201 : Loss = 1.4218337319869196\n",
      "Epoch 202 : Loss = 1.4083437014124534\n",
      "Epoch 203 : Loss = 1.4105852269129469\n",
      "Epoch 204 : Loss = 1.413748165040897\n",
      "Epoch 205 : Loss = 1.3995864345636932\n",
      "Epoch 206 : Loss = 1.400226497484001\n",
      "Epoch 207 : Loss = 1.3787788356222757\n",
      "Epoch 208 : Loss = 1.3803562312176003\n",
      "Epoch 209 : Loss = 1.3714518364298218\n",
      "Epoch 210 : Loss = 1.3692589130135777\n",
      "Epoch 211 : Loss = 1.35675529818917\n",
      "Epoch 212 : Loss = 1.3605072303100747\n",
      "Epoch 213 : Loss = 1.351082197049769\n",
      "Epoch 214 : Loss = 1.3394809059980437\n",
      "Epoch 215 : Loss = 1.3351450755621082\n",
      "Epoch 216 : Loss = 1.3436840510949855\n",
      "Epoch 217 : Loss = 1.3296017592792313\n",
      "Epoch 218 : Loss = 1.3194917589945245\n",
      "Epoch 219 : Loss = 1.323899618424605\n",
      "Epoch 220 : Loss = 1.3122343356601038\n",
      "Epoch 221 : Loss = 1.3041860355317385\n",
      "Epoch 222 : Loss = 1.3037693928343077\n",
      "Epoch 223 : Loss = 1.323867414886528\n",
      "Epoch 224 : Loss = 1.293704618975676\n",
      "Epoch 225 : Loss = 1.288493911563727\n",
      "Epoch 226 : Loss = 1.2761711812600858\n",
      "Epoch 227 : Loss = 1.2742859536347073\n",
      "Epoch 228 : Loss = 1.2722517771172606\n",
      "Epoch 229 : Loss = 1.2540094644765822\n",
      "Epoch 230 : Loss = 1.2506482783925659\n",
      "Epoch 231 : Loss = 1.2496351776222734\n",
      "Epoch 232 : Loss = 1.2347703553242966\n",
      "Epoch 233 : Loss = 1.2524244664853459\n",
      "Epoch 234 : Loss = 1.2360219033337636\n",
      "Epoch 235 : Loss = 1.2356645944641855\n",
      "Epoch 236 : Loss = 1.2173446711762856\n",
      "Epoch 237 : Loss = 1.214133893571249\n",
      "Epoch 238 : Loss = 1.2179407421304789\n",
      "Epoch 239 : Loss = 1.2121365871994338\n",
      "Epoch 240 : Loss = 1.1920861097163022\n",
      "Epoch 241 : Loss = 1.1859466448062805\n",
      "Epoch 242 : Loss = 1.1820607771026133\n",
      "Epoch 243 : Loss = 1.1983451290828426\n",
      "Epoch 244 : Loss = 1.182583916478041\n",
      "Epoch 245 : Loss = 1.1919575647194627\n",
      "Epoch 246 : Loss = 1.163543256317697\n",
      "Epoch 247 : Loss = 1.1702897062700384\n",
      "Epoch 248 : Loss = 1.1535261773896965\n",
      "Epoch 249 : Loss = 1.1369711090048014\n",
      "Epoch 250 : Loss = 1.1482565747736224\n",
      "Epoch 251 : Loss = 1.1637464765887644\n",
      "Epoch 252 : Loss = 1.1322903487740492\n",
      "Epoch 253 : Loss = 1.1341912036158066\n",
      "Epoch 254 : Loss = 1.115450781812236\n",
      "Epoch 255 : Loss = 1.0965962007070669\n",
      "Epoch 256 : Loss = 1.089672771895804\n",
      "Epoch 257 : Loss = 1.0895297793145795\n",
      "Epoch 258 : Loss = 1.0886816787387437\n",
      "Epoch 259 : Loss = 1.0972745065489713\n",
      "Epoch 260 : Loss = 1.0639859184570843\n",
      "Epoch 261 : Loss = 1.0752269795962743\n",
      "Epoch 262 : Loss = 1.0599984836079932\n",
      "Epoch 263 : Loss = 1.0382687318615798\n",
      "Epoch 264 : Loss = 1.080560523458474\n",
      "Epoch 265 : Loss = 1.0572939874403153\n",
      "Epoch 266 : Loss = 1.0446396837666476\n",
      "Epoch 267 : Loss = 1.0223810975975276\n",
      "Epoch 268 : Loss = 1.0269975882374036\n",
      "Epoch 269 : Loss = 1.0016940398083332\n",
      "Epoch 270 : Loss = 1.043285569662832\n",
      "Epoch 271 : Loss = 1.0379214361569606\n",
      "Epoch 272 : Loss = 1.0245605935618438\n",
      "Epoch 273 : Loss = 1.0093155918636387\n",
      "Epoch 274 : Loss = 0.9779924474111417\n",
      "Epoch 275 : Loss = 0.9939035530289706\n",
      "Epoch 276 : Loss = 0.9579423396014171\n",
      "Epoch 277 : Loss = 0.9865340642397412\n",
      "Epoch 278 : Loss = 0.988103288391326\n",
      "Epoch 279 : Loss = 0.975360847100979\n",
      "Epoch 280 : Loss = 0.9510958443950693\n",
      "Epoch 281 : Loss = 0.927291606569124\n",
      "Epoch 282 : Loss = 0.937792461923606\n",
      "Epoch 283 : Loss = 0.9317210233585344\n",
      "Epoch 284 : Loss = 0.9419554825028477\n",
      "Epoch 285 : Loss = 0.944180411536519\n",
      "Epoch 286 : Loss = 0.9394850429757547\n",
      "Epoch 287 : Loss = 0.9673254608277246\n",
      "Epoch 288 : Loss = 0.896248078304716\n",
      "Epoch 289 : Loss = 0.9005924126827758\n",
      "Epoch 290 : Loss = 0.8695522077407571\n",
      "Epoch 291 : Loss = 0.8656025119774848\n",
      "Epoch 292 : Loss = 0.8793085404389411\n",
      "Epoch 293 : Loss = 0.8886011568926768\n",
      "Epoch 294 : Loss = 0.8885934280186164\n",
      "Epoch 295 : Loss = 0.8454554807849048\n",
      "Epoch 296 : Loss = 0.8321388945346926\n",
      "Epoch 297 : Loss = 0.8536291041440665\n",
      "Epoch 298 : Loss = 0.8055006074573104\n",
      "Epoch 299 : Loss = 0.8263254105421723\n",
      "Epoch 300 : Loss = 0.8085758613377082\n",
      "Epoch 301 : Loss = 0.8303367601454466\n",
      "Epoch 302 : Loss = 0.8164430912778767\n",
      "Epoch 303 : Loss = 0.7852524317515438\n",
      "Epoch 304 : Loss = 0.7849185927819707\n",
      "Epoch 305 : Loss = 0.7934113630849725\n",
      "Epoch 306 : Loss = 0.7977287918848444\n",
      "Epoch 307 : Loss = 0.7804159156536806\n",
      "Epoch 308 : Loss = 0.7919045028370847\n",
      "Epoch 309 : Loss = 0.7878510376717571\n",
      "Epoch 310 : Loss = 0.7546081277136187\n",
      "Epoch 311 : Loss = 0.7537897615482584\n",
      "Epoch 312 : Loss = 0.7335736599948763\n",
      "Epoch 313 : Loss = 0.7479814357458507\n",
      "Epoch 314 : Loss = 0.7742005106879444\n",
      "Epoch 315 : Loss = 0.7460213976454652\n",
      "Epoch 316 : Loss = 0.70029343399852\n",
      "Epoch 317 : Loss = 0.7099444722880054\n",
      "Epoch 318 : Loss = 0.6836463568102608\n",
      "Epoch 319 : Loss = 0.7010550613187332\n",
      "Epoch 320 : Loss = 0.7010968036767913\n",
      "Epoch 321 : Loss = 0.692012071609497\n",
      "Epoch 322 : Loss = 0.6949641181201469\n",
      "Epoch 323 : Loss = 0.681998949848401\n",
      "Epoch 324 : Loss = 0.6425717738862652\n",
      "Epoch 325 : Loss = 0.6294385794978524\n",
      "Epoch 326 : Loss = 0.6511840105887489\n",
      "Epoch 327 : Loss = 0.643166295327376\n",
      "Epoch 328 : Loss = 0.6438309388293622\n",
      "Epoch 329 : Loss = 0.610070807177846\n",
      "Epoch 330 : Loss = 0.6942905544819318\n",
      "Epoch 331 : Loss = 0.6428474426684894\n",
      "Epoch 332 : Loss = 0.6557813469541198\n",
      "Epoch 333 : Loss = 0.6049705853860968\n",
      "Epoch 334 : Loss = 0.5762568905378468\n",
      "Epoch 335 : Loss = 0.5972974917199138\n",
      "Epoch 336 : Loss = 0.6029168408922201\n",
      "Epoch 337 : Loss = 0.656944196606347\n",
      "Epoch 338 : Loss = 0.6402796835018782\n",
      "Epoch 339 : Loss = 0.6179257045639517\n",
      "Epoch 340 : Loss = 0.5583508952361781\n",
      "Epoch 341 : Loss = 0.5440889636398608\n",
      "Epoch 342 : Loss = 0.5843890012348986\n",
      "Epoch 343 : Loss = 0.5574868020280312\n",
      "Epoch 344 : Loss = 0.5915941963627778\n",
      "Epoch 345 : Loss = 0.5430683811367181\n",
      "Epoch 346 : Loss = 0.5296311901836861\n",
      "Epoch 347 : Loss = 0.5433248275248431\n",
      "Epoch 348 : Loss = 0.527530153557814\n",
      "Epoch 349 : Loss = 0.5622191836194294\n",
      "Epoch 350 : Loss = 0.5660555161250178\n",
      "Epoch 351 : Loss = 0.5491992386791349\n",
      "Epoch 352 : Loss = 0.5229843680451556\n",
      "Epoch 353 : Loss = 0.5311014126817524\n",
      "Epoch 354 : Loss = 0.5154975681770139\n",
      "Epoch 355 : Loss = 0.51721343989987\n",
      "Epoch 356 : Loss = 0.5023463245883636\n",
      "Epoch 357 : Loss = 0.4667678236546001\n",
      "Epoch 358 : Loss = 0.5041804645950371\n",
      "Epoch 359 : Loss = 0.54444850550296\n",
      "Epoch 360 : Loss = 0.539512231997912\n",
      "Epoch 361 : Loss = 0.5274121926428964\n",
      "Epoch 362 : Loss = 0.49500950340194566\n",
      "Epoch 363 : Loss = 0.5119300686108526\n",
      "Epoch 364 : Loss = 0.4888258364557805\n",
      "Epoch 365 : Loss = 0.5225499907852466\n",
      "Epoch 366 : Loss = 0.47094366016703615\n",
      "Epoch 367 : Loss = 0.46579672515599985\n",
      "Epoch 368 : Loss = 0.4564843358478479\n",
      "Epoch 369 : Loss = 0.40679480410619057\n",
      "Epoch 370 : Loss = 0.4257308736910803\n",
      "Epoch 371 : Loss = 0.4299790113645148\n",
      "Epoch 372 : Loss = 0.42303662640707834\n",
      "Epoch 373 : Loss = 0.4255626843366057\n",
      "Epoch 374 : Loss = 0.394255845596566\n",
      "Epoch 375 : Loss = 0.40860047842982755\n",
      "Epoch 376 : Loss = 0.4185934183074207\n",
      "Epoch 377 : Loss = 0.39280291342984514\n",
      "Epoch 378 : Loss = 0.3916854461726411\n",
      "Epoch 379 : Loss = 0.38245227361805345\n",
      "Epoch 380 : Loss = 0.37348780160581607\n",
      "Epoch 381 : Loss = 0.39231201653281156\n",
      "Epoch 382 : Loss = 0.38131260269610306\n",
      "Epoch 383 : Loss = 0.35048954225167994\n",
      "Epoch 384 : Loss = 0.3568873592370063\n",
      "Epoch 385 : Loss = 0.32898384914165585\n",
      "Epoch 386 : Loss = 0.3657814585165695\n",
      "Epoch 387 : Loss = 0.3511504485216706\n",
      "Epoch 388 : Loss = 0.3248704857734853\n",
      "Epoch 389 : Loss = 0.32436372895274007\n",
      "Epoch 390 : Loss = 0.3398112377430919\n",
      "Epoch 391 : Loss = 0.3564128431293606\n",
      "Epoch 392 : Loss = 0.3508485325122128\n",
      "Epoch 393 : Loss = 0.3529411669599885\n",
      "Epoch 394 : Loss = 0.3396880260120285\n",
      "Epoch 395 : Loss = 0.39141056076574826\n",
      "Epoch 396 : Loss = 0.40443235261930405\n",
      "Epoch 397 : Loss = 0.33379200962777755\n",
      "Epoch 398 : Loss = 0.3776924361958321\n",
      "Epoch 399 : Loss = 0.389659152006023\n",
      "Epoch 400 : Loss = 0.3385411231982999\n",
      "Epoch 401 : Loss = 0.3316994316694213\n",
      "Epoch 402 : Loss = 0.34486013391292053\n",
      "Epoch 403 : Loss = 0.3953414156461842\n",
      "Epoch 404 : Loss = 0.35068372390411456\n",
      "Epoch 405 : Loss = 0.33114017499448534\n",
      "Epoch 406 : Loss = 0.31555366973013\n",
      "Epoch 407 : Loss = 0.32544991104029614\n",
      "Epoch 408 : Loss = 0.27686542417944926\n",
      "Converged\n",
      "Finished Training\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXgV1fnA8e+bHZKQhYQACRB2BNlTQEFEBAS0Yt2XWm1tcbe12rr1p1W7WK1bWzdUtG5Yq1VxXxAFZA37vgcIWxISSEJCQpL398dMwiUkIWBuJsv7eZ77ZOacmbnvncB9M3POnCOqijHGGFNZgNcBGGOMaZgsQRhjjKmSJQhjjDFVsgRhjDGmSpYgjDHGVMkShDHGmCpZgjCmGRCRq0TkSw/f/48i8oZX729OjiUIc1JE5EoRSRWRfBHZLSKficgIr+MyVVPVN1V1XPm6iKiIdPMypuqISJqIjPE6DmMJwpwEEfkt8BTwFyAB6Ag8C0zyMi5fIhLkdQxNlZ3b5sMShDkhIhIFPATcrKr/U9WDqnpYVT9S1d+524SKyFMisst9PSUioW7dKBFJF5E7RCTDvfr4uVs3VET2iEigz/v9RERWuMsBInK3iGwWkX0i8o6IxLp1ye5fxdeJyHbgGxEJFJHHRSRLRLaKyC3uNkHln0VEXnZj2Ckifyp/bxG5VkTmiMjfRSTH3X+CT1yxIvKK+/lyROQDn7rzRGSZiOwXkbki0q+G86kicpOIbBSRPBF5WES6uvvlup8xxGf7X4nIJhHJFpHpItK+0rFucI+1X0SeERHx/Tzu8ix3l+XuFeBltTz2zSKyEdhYxecoP/+T3XOyW0TurOFzny8iq904vxWRU9zy13H+4PjIje331R3D1ANVtZe9av0CxgMlQFAN2zwEzAfaAPHAXOBht26Uu/9DQDAwESgAYtz6zcBYn2P9F7jbXf61e9wkIBR4AZjm1iUDCrwGhAMtgBuANe72McDX7jZB7j7vu8cId2NdCFzv1l0LHAZ+BQQCNwK7AHHrPwH+4x43GDjTLR8IZABD3f2uAdKA0GrOlQIfAq2APkARMAPoAkS58V/jbjsayAIGuZ//n8CsSsf6GIjG+ZLNBMb7fJ45lbbt5rNem2N/BcQCLar4HOXnf5p7Pvu67z/Grf8j8Ia73AM4CIx1z93vgU1AiFufVr6fvTz+/+51APZqXC/gKmDPcbbZDEz0WT8HSHOXRwGF+CQY9wt1mLv8J2CquxzpfpF0ctfXAmf77NfO/RIP8vmC6uJT/w3uF767PsbdJgjn1liR75cdcAUw012+FtjkU9fS3bet+75luEmt0md/DjcZ+pStx00gVWyvwHCf9cXAXT7rjwNPucsvA4/61EW4nz/Z51gjfOrf4UhyvZaaE0Rtjj26ht95+fnv5VP2KPCyu+ybIP4PeMdnuwBgJzDKXbcE0UBedovJnKh9QNxx7kO3B7b5rG9zyyqOoaolPusFOF9IAG8BF7q3pC4Elqhq+bE6Ae+7tyX24ySMUpwv+3I7KsWxo5q6Tjh/ve72Od4LOFcS5faUL6hqgbsYAXQAslU1p4rP3gm4o/yY7nE7VPr8le31WS6sYr383Bx1XlU1H+f3kVhVzBx9Xo+nNsfeUXmnKvhuU/n3Xt17lbn7JVaxrfGQJQhzoubh/OV9QQ3b7ML5oizX0S07LlVdg/PlMQG4EidhlNsBTFDVaJ9XmKru9D2Ez/JunNtL5TpUOlYREOdzrFaq2qcWYe4AYkUkupq6P1eKsaWqTqvFcY/nqPMqIuFAa5y/vuvj2LUZ+tn3HFf3e6/8XuLuV/5eNsR0A2EJwpwQVT0A3A88IyIXiEhLEQkWkQki8qi72TTgDyISLyJx7vYn0gf+LZz2hpE4bRDlngf+LCKdANzj19Rz6h3g1yKS6H6Z3+XzOXYDXwKPi0grtwG8q4icebzg3H0/A54VkRj38490q18EbnAb3EVEwkXkXBGJrP3Hr9Y04OciMsC9wvoLsEBV007iWHtx2jnq+tj/5/6b6AP8HKedprJ3gHNF5GwRCQbuwEnWc6uJzXjEEoQ5Yar6OPBb4A84DZE7gFuA8p48fwJSgRXASmCJW1Zb04AzgW9UNcun/GlgOvCliOThNFgPreE4L+IkgRXAUuBTnAbyUrf+Z0AITkNwDvAuTvtCbVyNc49+HU4bym8AVDUVp2H7X+4xN+Hc///BVPVrnPv37+FcHXUFLj/Jw/0R+Ld7G+zSOjz2dzifeQbwd1U95uE8VV0P/BSnITwL+DHwY1Utdjf5K84fGPtr6gll/K+8R4YxTZ7bTfV5Ve103I3NCRGRZGArEFypfck0YnYFYZosEWkhIhNFJEhEEoEHcLq2GmNqwRKEacoEeBDnVs9SnF5P93sakTGNiN1iMsYYUyW7gjDGGFOlJjXoVlxcnCYnJ3sdhjHGNBqLFy/OUtX4qur8liBEpAPOuDgJOA++TFHVpyttIzhdF8vH47lWVZe4ddfgdKME+JOq/vt475mcnExqamrdfQhjjGniRGRbdXX+vIIoAe5Q1SXuQ0KLReQr90nZchOA7u5rKM44NkPFGaHzASAFJ7ksFpHp1QxtYIwxxg/81gahqrvLrwZUNQ+nB0nlsVYmAa+pYz4QLSLtcAZ3+0pVy8e7+QpnFFFjjDH1pF4aqd2HaAYCCypVJXL04F7pbll15VUde7I4M5ulZmZm1lXIxhjT7Pk9QYhIBM7j+79R1dy6Pr6qTlHVFFVNiY+vsp3FGGPMSfBrgnAH4noPeFNV/1fFJjs5evTHJLesunJjjDH1xG8Jwu2h9DKwVlWfqGaz6cDP3FEvhwEH3JEyvwDGuSNlxgDj3DJjjDH1xJ+9mIbjjHi5UkSWuWX34owRj6o+jzO65kSc0R8LcIYHRlWzReRhYJG730Oqmu3HWI0xxlTitwShqnNwxsKpaRsFbq6mbiow1Q+hHeMfMzYSEx5C17hwusRHkNAqFHeud2OMabaa1JPUJ6OktIwXZ28h79CREYrDQwLpHB9Ol7gIOseF06ZVKHERocRFhNA6PJS4yFDCQwItiRhjmrRmnyCCAgNY8cA49uQeYkvmQbZk5rM58yBbsg6yeFsOH63YRVXjGYYGBRAXEUpUi2BiwoOJbhFCdMtgWoeHEBMeQqz7imkZUpFcggJt6CtjTOPR7BMEgIjQLqoF7aJaMLxb3FF1xSVlZB8sJiu/iKz8IvblO8v73LIDBYfZX3iYtQdy2V9wmJyC4ioTigi0DneSRZtWYbSJDCU+MtTn55Gy8FD7tRhjvGffRMcREhRA26gw2kaF1Wr70jLlQOFhsg8Wu68isvKLycgrIjPvEJl5RWTkFbFxbx6ZeUWUlB2bTVqFBZEU05LEmBYkxbQgKaal+9NZjmoRXNcf0xhjjmEJoo4FBkjF7aXjKStTcgqKycwvIiO3qCJ57D5QSHpOIdv2HeT7TVkUFJcetV9kWBCJ0Ucnjm5tIuiREEm7qDBrGzHG1AlLEB4KCBBaR4TSOiKUXm2r3kZV2V9wmPScQtJzCo76uSO7gHmbszjok0AiQ4Po0TaSHglOwuiZEEn3hEjiIkIscRhjTogliAZORIhxG777JkUdU6+q5BQcZlNGPuv35rFhTx4b9ubx2ao9TFt4ZDir2PAQeiZE0q9DFAOSounfIdquNowxNbIE0ciJOLe0hnSOZUjn2IpyVSUzv4iNe/NZ7yaNtbtzeWVOGsWlZQDER4bSPymaYV1iOa1ra05p24qAAEsYxhiHJYgmSkTcnlFhR/XMKiopZe3uPJbv2M/yHftZsj2Hr9fuBSC6ZTCndWnN6V1bc1rXOLrGh9sVhjHNmCWIZiY0KJABHaIZ0CG6omz3gULmbd7HvM37mLt5H5+t2gM4Vxind3USxlm92tAmsnY9uYwxTYNoVZ32G6mUlBS1KUd/GFVlR3YhczdnMddNGFn5RYjAjzrFMv7UtoztnUBSTAu7ujCmCRCRxaqaUmWdJQhTE1Vl3Z48Pl+1h89X7WH93jwAOsS24MKBSVw0KImOrVt6HKUx5mRZgjB1ZktmPrM2ZDJjXQZzNmWhCkM6x3Lx4CQm9m1HhD0FbkyjYgnC+MWu/YW8v3Qn7y1OZ0vWQVqGBHLJ4CSuHd6ZznHhXodnjKkFSxDGr1SVJdv38+aCbXy8fDfFpWWM7tWGXwzvzPBura2twpgGzBKEqTcZeYd4c/523lywjaz8YnokRPCL4Z25YGAiYcGBXodnjKnEEoSpd4cOl/LR8l1M/T6NtbtziYsI5aZRXblyaEdLFMY0IJYgjGdUlXlb9vGPGRuZvyWbdlFh3Dq6O5ekJBFs82MY47maEoT9DzV+JSKc3jWOtyefxlu/HEq7qDDufX8lE56ezeyNmV6HZ4ypgSUIU29O7xbHezeezos/S6GktIyrX17I5NdS2b6vwOvQjDFVsARh6pWIMLZ3Al/cPpLfj+/JnE1ZjHniOx79fB35RSXHP4Axpt74LUGIyFQRyRCRVdXU/05ElrmvVSJSKiKxbl2aiKx066xRoQkKDQrkplHd+OaOUZzXrx3PfruZ0X//lvcWp9OU2sWMacz81kgtIiOBfOA1VT31ONv+GLhdVUe762lAiqpmnch7WiN147Vkew4PfrSG5Tv2c0b3OB69uB/tolp4HZYxTZ4njdSqOgvIruXmVwDT/BWLafgGdYzh/RtP5+ELTiU1LYeJT8/mle+3UlBst52M8YrnbRAi0hIYD7znU6zAlyKyWEQmH2f/ySKSKiKpmZnWK6YxCwgQrh7WiU9uG0HH1uE8+NEaxj4xi+U79nsdmjHNkucJAvgx8L2q+l5tjFDVQcAE4Gb3dlWVVHWKqqaoakp8fLy/YzX1oEt8BB/ePJx3bzgNgEtemMez326ixJ0JzxhTPxpCgricSreXVHWn+zMDeB8Y4kFcxmMpybF8dOsIzuwRz6Ofr+fO/y4nI/eQ12EZ02x4miBEJAo4E/jQpyxcRCLLl4FxQJU9oUzTFxsewos/S+E3Y7rzwbJdnP7IN7wxf5vXYRnTLPht8H4RmQaMAuJEJB14AAgGUNXn3c1+Anypqgd9dk0A3ndHAA0C3lLVz/0Vp2kcfjOmBxcMSOSB6au5/8NVbM06yL0TTyEwwEaKNcZfbCwm06gcLCrh4Y/X8PaiHZzXrx1/u6gf4TZJkTEnraZurvY/yzQq4aFBPHJRP5Ljwvnb5+tYuzuXZ68aTM+2kV6HZkyT0xAaqY05YTec2ZU3rxvKgcISJj0zh/8s2m5PYBtTxyxBmEbr9G5xfPrrEQzuFMNd763kT5+spazMkoQxdcUShGnU2kSG8dovhnLt6cm8PGcrE56ezfTlu7wOy5gmwRKEafQCA4QHftybRy/qR+6hw9z53+U214QxdcAShGkSRIRLf9SB928aTkRoEFe/vJDPVu72OixjGjVLEKZJaRsVxsw7R3FKu1bc+d/lPP31Rhvwz5iTZAnCNDlRLYJ58WeDGdE9jie/3sDFz80j+2Cx12EZ0+hYgjBNUlJMS164OoVXrv0RmzLzGfXYTF6avcXrsIxpVCxBmCbtrF5tePbKQcRFhPKnT9by77lpXodkTKNhCcI0eWPcObDH9k7ggemrefzL9eQdOux1WMY0eJYgTLMQHBjAs1cN4oIB7fnnN5u46qUFHLb5JYypkSUI02wEBwbw5GUDeOqyAaxIP8Cv315KYXGp12EZ02BZgjDNiohwwcBE7p3Yi89W7WHC07PseQljqmEJwjRLk0d25dWfDyEsOJBbpy3ljfnbbLA/YyqxBGGarTN7xPPODaeRkhzDHz5Yxa3TlrIju8DrsIxpMCxBmGatVVgw0341jNvH9ODL1Xu56Lm57Nxf6HVYxjQIliBMsyci/HpMdz6+bQQHi0q4451lNjyHMViCMKZCj4RIfndOT+Zvyebsx79jc2a+1yEZ4ylLEMb4uOb0ZF6/bgiHDpcy4anZ/OubjdZ4bZotvyUIEZkqIhkisqqa+lEickBElrmv+33qxovIehHZJCJ3+ytGYyoTEc7oHs8nt53BuD4J/P3LDXS+51Nen5fmdWjG1Dt/XkG8Cow/zjazVXWA+3oIQEQCgWeACUBv4AoR6e3HOI05RvvoFvzzioGM650AwAPTV3OgwIbnMM2L3xKEqs4Csk9i1yHAJlXdoqrFwNvApDoNzphaEBGe++lg3rvxdBSY/HqqJQnTrHjdBnGaiCwXkc9EpI9blgjs8Nkm3S2rkohMFpFUEUnNzLRpJk3dCgwQBneK4anLBrBkew7nPzOHORuzvA7LmHrhZYJYAnRS1f7AP4EPTuYgqjpFVVNUNSU+Pr5OAzSm3KQBibz5y2EA/PTlBfxzhjVem6bPswShqrmqmu8ufwoEi0gcsBPo4LNpkltmjKeGdI7li9+M5CcDE3n8qw3c+MYSGxHWNGmeJQgRaSsi4i4PcWPZBywCuotIZxEJAS4HpnsVpzG+woID+fsl/fn9+J58vnoP5/5jNk98tYHiEksUpukJ8teBRWQaMAqIE5F04AEgGEBVnwcuBm4UkRKgELhcnWv2EhG5BfgCCASmqupqf8VpzIkKDBBuGtWNosNlPD1jIxv2bqRLXDgXDKy2qcyYRkma0n3UlJQUTU1N9ToM00yoKt9uyOR3/11BVn4Rj17cj0tTOhx/R2MaEBFZrKopVdV53YvJmEZLRDirZxteuiaF5NYtued/K5kyazOlZU3njy7TvFmCMOYHGtAhmrcnn0Zy65b85dN13P3eCuvhZJoESxDG1IG2UWHMuGMUt47uxn8Xp/PXz9bZYH+m0bMEYUwdun1MD34yMJEps7Yw5onvWLXzAKt2HqDMbjuZRsgShDF1KCBAeOLS/vzryoGowk+e/Z7z/jmH1+aleR2aMSfMb91cjWmuRITz+rUnI7eImeszyMgt4o8frQHg2uGdPY7OmNqzKwhj/OQXIzrz+nVDuf/HzmDEf/50LYXFpR5HZUztWYIwxs+Gd4vjjeuGcrhU+dvn62x4DtNoWIIwph4M6RxLRGgQr85N48GPVlNiScI0ApYgjKkHIUEBvHP9aZzRPY435m/nyhcX2LMSpsGzBGFMPendvhWv/nwIt53dnYVp2XS+51O+WbfX67CMqZYlCGPqUWCAcNvobgzrEgvA799dSe4hm6XONEyWIIypZ0GBAbw9+TQ+umUEWflF/OrfqYx6bCaL0k5mhl5j/McShDEe6ZsUxaUpSSzYmk3avgL+/sV6r0My5iiWIIzx0F8v7MffL+nPhYMSWbA1m0ufn0eqeyUxdc5Wlu3Y73GEpjmzBGGMhwIDhIsHJ/GXn/QlMboFC9OyueWtpWzfV8BDH6/hpy8t8DpE04xZgjCmAQgLDuQ/1w/jngm92JN7iJGPzQQgv6jE48hMc2YJwpgGIimmJdef2ZWrh3U6qnxffpFHEZnmzhKEMQ3MQ5P68Nmvz+C5qwYB8Px3mz2OyDRXliCMaWBEhFPateKcPm25NCWJF2dvZe6mLK/DMs2QJQhjGqiAAOGhSafStlUYt0xbyv0frqKg2NokTP3xW4IQkakikiEiq6qpv0pEVojIShGZKyL9ferS3PJlIpLqrxiNaejCggN54erBDOsSy2vztvHWgu1eh2SaEX9eQbwKjK+hfitwpqr2BR4GplSqP0tVB6hqip/iM6ZR6N8hmmevGkzfxCg+WLaTktIySsuUjLxDXodmmji/zSinqrNEJLmG+rk+q/OBJH/FYkxTcGlKEv/34WoG/+lrWrUIYkd2IQvvO5s2kWFeh2aaqIbSBnEd8JnPugJfishiEZlc044iMllEUkUkNTMz069BGuOlnw7rxAtXDyY4UNiRXQjAo5+vp6jEZqkz/uF5ghCRs3ASxF0+xSNUdRAwAbhZREZWt7+qTlHVFFVNiY+P93O0xnhHRDinT1vunnBKRdm7i9N57HMbw8n4h6cJQkT6AS8Bk1R1X3m5qu50f2YA7wNDvInQmIbnokGJvPLzH5H6hzH0ahvJh8t3cehwqV1JmDrnWYIQkY7A/4CrVXWDT3m4iESWLwPjgCp7QhnTHIkIZ/VsQ1xEKLeO7k5mXhG9/u9zxj4xix3ZBV6HZ5oQf3ZznQbMA3qKSLqIXCciN4jIDe4m9wOtgWcrdWdNAOaIyHJgIfCJqn7urziNaczGn9qWW0d3o19SFNuzC/jHjI1eh2SaEGlK8+KmpKRoaqo9NmGap9+8vZTvNmSy8L4x3PXeCn7cvz1ndo8nIEC8Ds00YCKyuLrHCTxvpDbG1I1z+rQlp+Aw17++mP8t2cnPX1nE6Y98Q3FJmdehmUbKEoQxTcTY3gmMOSWBb9ZlVJTtyT3E2t25HkZlGrNaJQi34TjAXe4hIueLSLB/QzPGnIigwACe/+kg/vKTvjx12QCuPT0ZgL98utaeujYnpbZXELOAMBFJBL4ErsYZSsMY04AEBQZw5dCOXDAwkT+e3weABVuzufu9lagqf/10rU1jamqttglCVLUAuBB4VlUvAfr4LyxjTF24b6LzUN2cjVl8sXovL8zawt3vrfA4KtNY1DpBiMhpwFXAJ25ZoH9CMsbUlV+N7MJHt4yguLSMG95YDMDhUmu0NrVT2wTxG+Ae4H1VXS0iXYCZ/gvLGFNXTk1sRUjQkf/qmzMPsn2fPVBnjq9WCUJVv1PV81X1b25jdZaq3ubn2IwxdUBEeOTCvvRNjOKJS51pV0Y+NpPcQ4c9jsw0dLXtxfSWiLRyh75YBawRkd/5NzRjTF25cFASH906gvP6tef8/u0BmLF2r8dRmYautreYeqtqLnABzrDcnXF6MhljGpGQoACeumwA7aPC+Hj5bq/DMQ1cbRNEsPvcwwXAdFU9jDNngzGmkQkIECb2bcesjZkcKLTbTKZ6tU0QLwBpQDgwS0Q6AfZ4pjGN1Ln92nG4VPlg6U7yDh3m568sZNYGm3DLHK1WU46q6j+Af/gUbXMn+jHGNEIDOkQzpHMsD360mie/3sD+gsMEiDCyh026ZY6obSN1lIg8UT61p4g8jnM1YYxphESEF69O4eazunF2rwQAduQUkO6+Lnl+LntzbXiO5q62t5imAnnApe4rF3jFX0EZY/wvqmUwd4zryeOX9uf6kV3YsDefEX+byYuztrAoLYcXZ23xOkTjsVrdYgK6qupFPusPisgyfwRkjKl/XeMjKpbfWLAdgK1ZB70KxzQQtb2CKBSREeUrIjIcKPRPSMaY+nZuv3bcOa4HvdpGUlrmdFCcv2Wf9XJq5mqbIG4AnhGRNBFJA/4FXO+3qIwx9So8NIhbRnfngoGJAAQHCgeLS3n1+zQAVu08wKqdBzyM0HihtkNtLFfV/kA/oJ+qDgRG+zUyY0y9O6tnGwD6JkZxRvc4Pli2E4DLp8znvH/O4dv1GTXtbpqYE5pRTlVz3SeqAX7rh3iMMR7qkRDBvRN78filAzi7Vxu2Zh3kj9NXk19UAsCTX2+kKc1jb2r2Q6YcPe5M6CIyVUQyRGRVNfUiIv8QkU0iskJEBvnUXSMiG93XNT8gTmNMLYkIk0d2pXNcOKPcq4lX56YB0C8piuU79rNke46HEZr69EMSRG3+jHgVGF9D/QSgu/uaDDwHICKxwAPAUGAI8ICIxPyAWI0xJyg5LpynLhtQsf67c3rSKiyIl+ds9TAqU59qTBAikiciuVW88oD2xzu4qs4CsmvYZBLwmjrmA9Ei0g44B/hKVbNVNQf4ipoTjTHGD8obrQFObR/FRYOT+HL1Xg4dLvUwKlNfakwQqhqpqq2qeEWqam2foahJIrDDZz3dLauu/BgiMrn8Ce/MTBtLxpi69vCkPqR0iiEmPIQhybGUlCm3TVvKtn32nERTVxdf8p5S1SnAFICUlBRrPTOmjl19WjJXn5YMQN+kKAC+XLOXGesy6BTbkjG9E7jXnfvaNC0/pA2iLuwEOvisJ7ll1ZUbYzyUGN2iYrm0TNmSdZAXZ28hzZ66bpK8ThDTgZ+5vZmGAQdUdTfwBTBORGLcxulxbpkxxkMiwv3n9a5YP6N7HAK8tyT9mG0/WbGbAwX2JHZj5tcEISLTgHlATxFJF5HrROQGEbnB3eRTYAuwCXgRuAlAVbOBh4FF7usht8wY47FfjOjM5T9yLvCHd4ujX1I032/KOmqb1bsOcPNbS3jwo9VehGjqiF/bIFT1iuPUK3BzNXVTcUaRNcY0MOf0acvbi3ZwRvc48g+V8Nx3m8k7dJjIsGCAimE58twH7Ezj5PUtJmNMI3RWrzase3g8fdpHcXq31pSWKQu2ZJNfVMKfP1nDXe+tBKCVmzBM49ToezEZY7wRFhwIwKCOMYQEBvDL11KP2WbfwaL6DsvUIbuCMMb8IGHBgUSGHflb8/qRXZhz11n07xBNVr4liMbMriCMMT/Y45f2Z/bGLH55RmcSIsMICBC6t4k4pvHaNC6WIIwxP9ionm0qBvcrFxcRyr78YlQVkeOO7WkaILvFZIzxi7iIEIpLy3h9/javQzEnyRKEMcYv2kU5T13f/+FqikpKKS1TNuzNo6jEBvprLCxBGGP84pw+CVzojga7bnceN76xmHFPzuL+D+zhucbCEoQxxi+CAgO4fWwPAP72+Tq+XLMXgDW7c2vazTQgliCMMX6TFNOCmJbBzN28jy5x4Vw9rBNbsw7atKWNhCUIY4zfiAj3TjyFyLAgfj2mO13jw8kvKiErv9jr0EwtWDdXY4xfXZLSgYsHJyEifLs+A4CrXppPcUkZ3/7uLI+jMzWxBGGM8bvy5yC6xkcAsGFvPgCHS8sIDrQbGQ2V/WaMMfUmKaYFLUMCK9Z37S/kQOFh0nMKPIzKVMcShDGm3ogIz141qGJ9wtOzOe2vMxjxt5nk29DgDY4lCGNMvRrVsw1z7x4NQEFxKQXFzoNz7yza4WVYpgqWIIwx9S6hVdgxZd+sy/AgElMTa6Q2xtS7wAChRXAgQYHCiG5xhAQFMHNdBqrK/C3ZJMW0oENsS6/DbPYsQRhjPLH0/rEEiBASFMC0hdv5cNkubn5rCZ+u3EOrsCBG92rDjaO60bNtpNehNlt2i8kY44mw4EBCgpyvoL6JUQB8unIPAPJbSC8AABUPSURBVLmHSvhg2S4mv37sLHWm/liCMMZ4rk/7Vtw1vhd3jO3BjDvOBKBDbAu27Ssg+6Dz1PVbC7azcGu2l2E2O369xSQi44GngUDgJVV9pFL9k0D5o5QtgTaqGu3WlQIr3brtqnq+P2M1xnhHRLhxVNeK9TUPncPS7fu56qUFrNmVS0pyDPe+73wdpD1yrldhNjt+SxAiEgg8A4wF0oFFIjJdVdeUb6Oqt/tsfysw0OcQhao6wF/xGWMarpYhQfRu1wqAtxdtx3dCurIyJSDAZqirD/68ghgCbFLVLQAi8jYwCVhTzfZXAA/4MR5jTCMSEx5CcuuWfLxiN0u3768o35SZT48Ea7iuD/5sg0gEfJ98SXfLjiEinYDOwDc+xWEikioi80XkgureREQmu9ulZmZm1kXcxpgG4oObhxMRGsTO/YVEhjl/z65IP+BxVM1HQ2mkvhx4V1V95yLspKopwJXAUyLStaodVXWKqqaoakp8fHx9xGqMqSfRLUM4o3scAPdNPIXAAGHbvoMeR9V8+DNB7AQ6+KwnuWVVuRyY5lugqjvdn1uAbzm6fcIY00zcd+4pXH9mFy4clET76DDS9tnAfvXFnwliEdBdRDqLSAhOEpheeSMR6QXEAPN8ymJEJNRdjgOGU33bhTGmCUuKack9E04hJCiA5NbhbNtnM9LVF78lCFUtAW4BvgDWAu+o6moReUhEfLusXg68rUf/xk8BUkVkOTATeMS395MxpnnqGNuSFekHuGzKfK9DaRakKWXilJQUTU21Jy+Naar+/sV6/jVzU8V621Zh/PXCvqQkxxAZFnzUtgcKDxMWHEBoUGDlwxgfIrLYbe89RkNppDbGmOO6+rROnNXzSGeUPbmH+Pmri/jtO8uP2bb/g19y3av2B+MPYQnCGNNoJLQK47mfDq5Yf3vyMOIiQvhqzV7W7cmtKC8tc+6MzNmUVe8xNiWWIIwxjUpYcCDPXDmIj28dwbAurfnktjMAmLd5X8U2uYWHvQqvSbHhvo0xjc65/dpVLLeJDKVVWBCbMvIryrILiiuWVRURG5rjZNgVhDGmURMRurWJYFNGPsUlZagq+30SxIMfranoFvvu4nSe8WnkNjWzBGGMafTKE8TYJ7/j0hfmHTV206tz00jPKQTgzv8u57Ev1pOVX+RVqI2KJQhjTKPXvU0k+w4Ws21fAYvScvjTJ2uPqi9PEOU+WFrdoA7GlyUIY0yjN9GnTeLqYZ0qln8/vicA6TkFlJYpge4w4V+v3Vu/ATZSliCMMY1eYnQLrh/ZhYl923LnuJ4V5deN6IwI7NxfSEbeIUrLlFZhQaSm5ZBfVOJhxI2DJQhjTJNwz8RTePaqwUS1PPJEdWhQIAmRYaTnFLJrv3Ob6fIhHSkpUxal2fSlx2MJwhjT5ESGHunBnxTTgh3ZBezcfwiAEd2c4cMrt0uYY9lzEMaYJuf7e0ZTXFIGQJf4cD5YuouMvCLCQwIZ2DGaAIG9Bw55HGXDZ1cQxpgmp1VYMHERoQAM6BBDcWkZW7MO8q+rBhEZFkybyDD25B4i52AxL87aQvLdn3DocOlxjtr82BWEMaZJG9QpumL5zO7OQH8JUWHsOXCIgQ9/VVGXnlNItzYR9R5fQ2ZXEMaYJq17m0gAhnSOJcDt5tq2VSjLd+w/arsdOTZTXWV2BWGMadICA4S5d48m2qd3U9tWYeRV6ua6I9tJEN+uz+D3767g+jO7ct2IzvUaa0NjVxDGmCavfXQLWoYEHbVeWXmCePKrDWTkFfHS7C2UlTWdCdVOhiUIY0yzc06fthXLj1/Sn46xLdmeXcDczVms3ZNHXEQouw8cInVbzlH7lZUpHy7bSe6h5jGcuCUIY0yzkxwXzmldWnN619ZcNDiJXm0j+WL1Xq58cQHFJWXcNKorIkfPMQEwe1MWv357Gaf/9Ztm0evJEoQxpll661dDefOXQwG4YmjHo+qGdomlZ0IkT369gb98emTgvw178gDILyrhjfnb6i9Yj/g1QYjIeBFZLyKbROTuKuqvFZFMEVnmvn7pU3eNiGx0X9f4M05jTPMjIhUTCY3qEc+VQzsyqmc8cREhdGsTQc+2Tu+nKbO2MGPtXlSVjRl5xEWEkNIphncXpwPw6OfrePKrDZ59Dn+S8ok06vzAIoHABmAskA4sAq5Q1TU+21wLpKjqLZX2jQVSgRRAgcXAYFU9+oZgJSkpKZqaapOUG2N+uB3ZBbw2L40XZ28F4NmrBvHS7C2EBAUwqGMMz367metGdOblOU592iPnehjtyRORxaqaUlWdP68ghgCbVHWLqhYDbwOTarnvOcBXqprtJoWvgPF+itMYY47RIbYl953bm6nXOt+dL8/ZysaMfLq3ieTUxKiKsnL++mPbS/5MEInADp/1dLessotEZIWIvCsiHU5wX2OM8avRvRL43Tk9Wbwth7xDJXRrE8Gp7aOO2S6noOn1bPK6kfojIFlV++FcJfz7RA8gIpNFJFVEUjMzM+s8QGOM6Zt4JCF0bxNBh9gWjOudcNQ26U3wSWx/JoidQAef9SS3rIKq7lPV8slhXwIG13Zfn2NMUdUUVU2Jj4+vk8CNMcZXr3aRFcvdEiIQEab8LIUZd5zJFUOcr6qdTXD4cH8miEVAdxHpLCIhwOXAdN8NRKSdz+r5QHl/si+AcSISIyIxwDi3zBhj6l28OzJs5eWu8RHcPf4UAJZsr7EPTaPkt7GYVLVERG7B+WIPBKaq6moReQhIVdXpwG0icj5QAmQD17r7ZovIwzhJBuAhVbXpn4wxnhARYloGU3i4tKJrbLlWLYIY1zuBF2dv5dTEKCYNaDrNpX7r5uoF6+ZqjPGX/KISVJXIsOBj6kpKy7j0hXlszTrI4j+MrRg1tjHwqpurMcY0GRGhQVUmB4CgwAB+MiiJnILDZOQVVblNY2QJwhhj6kCHGGeE2KbUm8nmgzDGmDqQFNMSgL99vo6s/GJ+3K8d5w9IbNSz1NkVhDHG1IEk9wpiUVoOW7MO8o9vNnHHO8tYuzuX1+aleRrbybIEYYwxdSAsOLBi+X83nU7X+HCWpx9gwtOzuf/D1WTlN762CUsQxhhTR4IDnd5LgzrGcNvZ3Y+qW7j1xHvqT52zletf965nprVBGGNMHfn+rtHg9nDtlxRdUd4yJJCnv97I6V1bszEjn+DAAAZ0iK7mKI78ohIe+tgZ/Lq0TMktPExMeAhbsw7ywPTVPHPlwGp7VdUVu4Iwxpg60qZVGG0iwwDoFNuSSwYn8fbkYZzbtx3r9+Zx4xtLuOT5eVzwzPfHnbb0u/VHxpZ79PN1DHz4K7btO8hjX6xj1oZMZqzN8OtnAbuCMMYYvwgIEB67pD8Aw7q0ZkDHaO57f1VF/X8W7uBXI7tUu/8On+6yL8zaAsDGvfkEuE9y5xeV+CPso9gVhDHG1IMrh3RkbO8EIkKDiIsIrRi7admO/RQWHzu/dVWD/+3cX0hpmTP6xY5s/z9vYQnCGGPqgYjwrysH8sXtIxnWJZYFW7NZtfMAFzzzPb94dVHFduUP2qXnFBAWfPRX9M79heza7ySOrVkH/R6z3WIyxph6EhoUSGJ0C/omRvHxit2c9885AMzbso/8ohJmrsvg1mlLadsqjD25hxja2UkkALHhIaTnFJDuXlls22dXEMYY0+SMqTTZEMD3m7J4fd42APbkHgKcRu9yfdq3Ys2uXPYdLCYwQNiUmU/OwWK/xmkJwhhj6lnX+AgW3Ht2xXpIUABvL9zOwrRs7hrfi69uHwlA73atKrbp1Lolae5Vw0OT+lBapny9dq9f47RbTMYY44EEn6uDgR2imbk+k6AA4eLBScRHhjL37tHER4ayZncu7aPCuGpoJ6Yv28WQzrFcOaQjz87czNTv08jML+L6kV0J9MMQ45YgjDHGI+/ecBphwYEs2Z7Dgq3ZnD+gPfGRzox17aOdsZ3+ecXAiu0X3DuGgACnwfvSlA48+fUG1u7OZXjXOPof58G7k2EJwhhjPJKSHAvAqYlRXDGkI0HHuQpoEXJkvKfLfuQkCHCemfBHgrA2CGOMaQCCAwOOmc60Jm2jwlj6f2MB2O6nZyIsQRhjTCMVEx5CbHgIO7KPfaiuLliCMMaYRqxDbEu/zWJnCcIYYxqxDjEt/Dbshl8ThIiMF5H1IrJJRO6uov63IrJGRFaIyAwR6eRTVyoiy9zXdH/GaYwxjdXwbnEM69LaL8cWVfXPgUUCgQ3AWCAdWARcoaprfLY5C1igqgUiciMwSlUvc+vyVfWEJnNNSUnR1FTvJtcwxpjGRkQWq2pKVXX+vIIYAmxS1S2qWgy8DUzy3UBVZ6pq+bXRfCDJj/EYY4w5Af5MEInADp/1dLesOtcBn/msh4lIqojMF5ELqttJRCa726VmZmZWt5kxxpgT1CAelBORnwIpwJk+xZ1UdaeIdAG+EZGVqrq58r6qOgWYAs4tpnoJ2BhjmgF/XkHsBDr4rCe5ZUcRkTHAfcD5qlpUXq6qO92fW4BvgYGV9zXGGOM//kwQi4DuItJZREKAy4GjeiOJyEDgBZzkkOFTHiMioe5yHDAcWIMxxph647dbTKpaIiK3AF8AgcBUVV0tIg8Bqao6HXgMiAD+6z5ivl1VzwdOAV4QkTKcJPaIb+8nY4wx/ue3bq5esG6uxhhzYrzq5mqMMaYRa1JXECKSCWw7yd3jgKw6DKcuNMSYoGHGZTHVXkOMqyHGBA0zrrqOqZOqxldV0aQSxA8hIqnVXWZ5pSHGBA0zLoup9hpiXA0xJmiYcdVnTHaLyRhjTJUsQRhjjKmSJYgjpngdQBUaYkzQMOOymGqvIcbVEGOChhlXvcVkbRDGGGOqZFcQxhhjqmQJwhhjTJWafYI43qx39RxLmoisdGfRS3XLYkXkKxHZ6P6M8XMMU0UkQ0RW+ZRVGYM4/uGeuxUiMqie4/qjiOz0mXlwok/dPW5c60XkHD/F1EFEZrqzIq4WkV+75Z6drxpi8uxciUiYiCwUkeVuTA+65Z1FZIH73v9xx2xDRELd9U1ufXJdx3ScuF4Vka0+52qAW16f/94DRWSpiHzsrntzrlS12b5wxojaDHQBQoDlQG8P40kD4iqVPQrc7S7fDfzNzzGMBAYBq44XAzARZw4PAYbhzA5Yn3H9Ebizim17u7/LUKCz+zsO9ENM7YBB7nIkzgyKvb08XzXE5Nm5cj9vhLscDCxwP/87wOVu+fPAje7yTcDz7vLlwH/89G+qurheBS6uYvv6/Pf+W+At4GN33ZNz1dyvII47610DMAn4t7v8b6DayZPqgqrOArJrGcMk4DV1zAeiRaRdPcZVnUnA26papKpbgU04v+u6jmm3qi5xl/OAtTiTYnl2vmqIqTp+P1fu5813V4PdlwKjgXfd8srnqfz8vQucLeKM5llPcVWnXv69i0gScC7wkrsueHSumnuCONFZ7/xNgS9FZLGITHbLElR1t7u8B0jwIK7qYmgI5+8W93J/qs/tt3qPy720H4jzV2iDOF+VYgIPz5V7y2QZkAF8hXOlsl9VS6p434qY3PoDQOu6jqmquFS1/Fz92T1XT4o79QD19/t7Cvg9UOaut8ajc9XcE0RDM0JVBwETgJtFZKRvpTrXkZ72S24IMfh4DugKDAB2A497EYSIRADvAb9R1VzfOq/OVxUxeXquVLVUVQfgTBw2BOhVn+9fncpxicipwD048f0IiAXuqq94ROQ8IENVF9fXe9akuSeIWs16V1/0yCx6GcD7OP+R9pZfxro/M6o/gt9UF4On509V97r/wcuAFzlya6Te4hKRYJwv4jdV9X9usafnq6qYGsK5cuPYD8wETsO5RVM+J43v+1bE5NZHAfv8FVOluMa7t+lUnRkuX6F+z9Vw4HwRScO55T0aeBqPzlVzTxDHnfWuvohIuIhEli8D44BVbjzXuJtdA3zoQXjVxTAd+Jnbu2MYcMDn1orfVbr/+xOc81Ue1+VuD4/OQHdgoR/eX4CXgbWq+oRPlWfnq7qYvDxXIhIvItHucgtgLE7byEzgYnezyuep/PxdDHzjXonVqWriWueT3AXnXr/vufLr709V71HVJFVNxvk++kZVr8Krc1WXLd6N8YXTM2EDzj3R+zyMowtOb5LlwOryWHDuJ84ANgJfA7F+jmMazi2Iwzj3Oq+rLgac3hzPuOduJZBSz3G97r7vCvc/Sjuf7e9z41oPTPBTTCNwbh+tAJa5r4lenq8aYvLsXAH9gKXue68C7vf5N78Qp2H8v0CoWx7mrm9y67v46fdXXVzfuOdqFfAGR3o61du/d/f9RnGkF5Mn58qG2jDGGFOl5n6LyRhjTDUsQRhjjKmSJQhjjDFVsgRhjDGmSpYgjDHGVMkShGl2RCTf/ZksIlfWwfHSROQ9n/WLReTVH3pc91h/FJE76+JYxpwoSxCmOUsGTihB+DzNWtlgEen9gyOqQ+4DXfZ/3Jw0+8djmrNHgDPcMf9vdwdue0xEFrkDtV0PICKjRGS2iEwH1lRzrMdxHjg7SuUrABFZ5V65JIvIOnHmHtggIm+KyBgR+V6ceSR8R1TtLyLz3PJf+Rzrdz6xls9lkCzOvA6v4Tzo5Ts0hDEnpLq/hoxpDu7GmSPhPAB3BN0DqvojdwTP70XkS3fbQcCp6gyJXZV3gJtEpNsJvH834BLgFzjDvlyJ8yT0+cC9HBnSuR/O/APhwFIR+QQ4FWdYjCE4T/hOdwd33O6WX6POkNTGnDRLEMYcMQ7oJyLlY95E4XzZFgMLa0gOAKXAYzgjgX5Wy/fbqqorAURkNTBDVVVEVuLc/ir3oaoWAoUiMhMnKYxw413qbhPhxrod2GbJwdQFSxDGHCHArar6xVGFIqOAg7XY/3WcBLHKp6yEo2/lhvksF/ksl/msl3H0/83K4+GoG+tfVfWFSrEm1zJWY47L2iBMc5aHMy1nuS+AG93hshGRHu7IurWiqoeBJ4HbfYrTcG5PIc4cxp1PIs5J4syf3BpnALdFbqy/cOd9QEQSRaTNSRzbmGrZFYRpzlYApSKyHGce4qdxbu0scYd6zuTEp3h9GfiDz/p7OENEr8aZ2W3DScY5E4gDHlbVXcAuETkFmOeESj7wU5xbXcbUCRvN1RhjTJXsFpMxxpgqWYIwxhhTJUsQxhhjqmQJwhhjTJUsQRhjjKmSJQhjjDFVsgRhjDGmSv8PnyjlYl2FKMMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "old_loss = np.inf\n",
    "losses = []\n",
    "max_epoch = 500\n",
    "for epoch in range(max_epoch):\n",
    "\n",
    "    running_loss = 0.0\n",
    "    \n",
    "    for data in trainloader:\n",
    "            \n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Forward\n",
    "        y_hat = classifier(X)\n",
    "        \n",
    "        # Calculate Loss (Cross Entropy)\n",
    "        loss = criterion(y_hat, y)\n",
    "\n",
    "        # Backpropagation\n",
    "        loss.backward()\n",
    "        \n",
    "        # Update Parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "        running_loss += loss.item()*len(X)/train_size\n",
    "\n",
    "    print('Epoch', epoch+1, ': Loss =', running_loss)\n",
    "    losses.append(running_loss)\n",
    "\n",
    "    if (abs(running_loss-old_loss)/running_loss < 0.2) and running_loss<0.3:\n",
    "        print('Converged')\n",
    "        break\n",
    "    \n",
    "    old_loss = running_loss\n",
    "\n",
    "print('Finished Training')\n",
    "plt.plot(losses)\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Iter Number')\n",
    "plt.title('Convergence monitor plot')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 283
    },
    "colab_type": "code",
    "id": "RskMmPeG5RAU",
    "outputId": "5f7f7804-27e8-420c-9b2a-d7d3bad8fe78"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Loss = 0.2829172909259796\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3   4   5   6\n",
       "0  43   0   0   0   0   0   0\n",
       "1   0  30   1   0   0   2   3\n",
       "2   0   0  45   0   0   0   1\n",
       "3   0   0   1  32   0   0   4\n",
       "4   0   0   1   0  36   0   3\n",
       "5   0   0   3   0   0  38   1\n",
       "6   0   0   0   0   0   0  43"
      ]
     },
     "execution_count": 91,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    train_loss = 0.0\n",
    "    y_train = []\n",
    "    y_train_pred = []\n",
    "\n",
    "    for data in trainloader:\n",
    "\n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        y_hat = classifier(X)      \n",
    "        train_loss += criterion(y_hat, y)*len(X)/train_size\n",
    "        \n",
    "        y_train.extend(list(y.detach().cpu().numpy()))\n",
    "        y_train_pred.extend(list(torch.argmax(y_hat, axis=1).detach().cpu().numpy()))\n",
    "\n",
    "print('Train Loss =', train_loss.item())\n",
    "pd.DataFrame(confusion_matrix(y_train, y_train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "s1_WEeod6Bg2",
    "outputId": "146feb9e-b24a-4ea7-d577-27731e9924ca"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy = 0.9303135888501742 Train Precision = 0.941137341854702 Train F1 = 0.9312989047598114\n"
     ]
    }
   ],
   "source": [
    "acc_tr = accuracy_score(y_train, y_train_pred)\n",
    "prec_tr = precision_score(y_train, y_train_pred, average='weighted')\n",
    "f1_tr = f1_score(y_train, y_train_pred, average='weighted')\n",
    "\n",
    "print('Train Accuracy =', acc_tr, 'Train Precision =', prec_tr, 'Train F1 =', f1_tr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 283
    },
    "colab_type": "code",
    "id": "4eAKVd07iASJ",
    "outputId": "fe4c8159-aeca-4f55-81ad-721493b15a02"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss = 3.4638450145721436\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3  4  5  6\n",
       "0  4  0  0  0  0  2  2\n",
       "1  0  0  1  0  1  5  1\n",
       "2  1  0  1  0  0  0  1\n",
       "3  0  0  4  1  0  0  5\n",
       "4  0  1  3  0  0  1  0\n",
       "5  1  0  1  0  0  2  0\n",
       "6  0  0  0  0  0  1  3"
      ]
     },
     "execution_count": 109,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    test_loss = 0.0\n",
    "    y_test = []\n",
    "    y_test_pred = []\n",
    "\n",
    "    for data in testloader:\n",
    "\n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        y_hat = classifier(X)      \n",
    "        test_loss += criterion(y_hat, y)*len(X)/test_size\n",
    "        \n",
    "        y_test.extend(list(y.detach().cpu().numpy()))\n",
    "        y_test_pred.extend(list(torch.argmax(y_hat, axis=1).detach().cpu().numpy()))\n",
    "\n",
    "print('Test Loss =', test_loss.item())\n",
    "pd.DataFrame(confusion_matrix(y_test, y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "qJK9h5xu4DjA",
    "outputId": "9ab1cb05-06d2-4531-bc7d-1efd439cbc96"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy = 0.2619047619047619 Test Precision = 0.31406926406926405 Test F1 = 0.2212513676799391\n"
     ]
    }
   ],
   "source": [
    "acc = accuracy_score(y_test, y_test_pred)\n",
    "prec = precision_score(y_test, y_test_pred, average='macro')\n",
    "f1 = f1_score(y_test, y_test_pred, average='macro')\n",
    "\n",
    "print('Test Accuracy =', acc, 'Test Precision =', prec, 'Test F1 =', f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GgHWYeQ5boqa"
   },
   "outputs": [],
   "source": [
    "pickle.dump(list(classifier.parameters()), open('/content/drive/My Drive/A3Q2_params.sav', 'wb'))\n",
    "torch.save(classifier, '/content/drive/My Drive/A3Q2_model.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_vIZ1K6eE33v"
   },
   "source": [
    "## Question 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_3-K0hqPk_Sy"
   },
   "outputs": [],
   "source": [
    "params = pickle.load(open('/content/drive/My Drive/A3Q2_params.sav', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jVC5hAh8E33x"
   },
   "outputs": [],
   "source": [
    "class CNN3(nn.Module):\n",
    "    \n",
    "    def __init__(self, n_classes, k=4):\n",
    "        super(CNN3, self).__init__()\n",
    "\n",
    "        self.n_classes = n_classes\n",
    "        # 3x224x224\n",
    "        self.cl1 = nn.Conv2d(3, 4, 3, stride=1, padding=1)\n",
    "        # 4x224x224\n",
    "        self.pl1 = nn.AvgPool2d(2, stride=2)\n",
    "        # 4x112x112\n",
    "        self.cl2 = nn.Conv2d(4, 16, 3, stride=1, padding=1)\n",
    "        # 16x112x112\n",
    "        self.pl2 = nn.AvgPool2d(2, stride=2)\n",
    "\n",
    "        # NetVLAD\n",
    "        self.K = k\n",
    "        self.nv_conv = nn.Conv2d(16, self.K, 1)\n",
    "        self.nv_soft_ass = nn.Softmax2d()\n",
    "\n",
    "        # NetVLAD Parameter\n",
    "        self.c = nn.Parameter(torch.rand(self.K, 16))\n",
    "        \n",
    "        # Flatten to get h\n",
    "        self.flat = nn.Flatten(1, -1)\n",
    "\n",
    "        # Output layer\n",
    "        self.out = nn.Linear(self.K*16, self.n_classes)\n",
    "        \n",
    "\n",
    "    def forward(self, x):\n",
    "        \n",
    "        # print(x.shape)\n",
    "        x = self.pl1(F.relu(self.cl1(x)))\n",
    "        # print(x.shape)\n",
    "        x = self.pl2(F.relu(self.cl2(x)))\n",
    "        # print(x.shape)\n",
    "        \n",
    "        # NetVLAD Step 1\n",
    "        a = self.nv_soft_ass(self.nv_conv(x))\n",
    "\n",
    "        # NetVLAD Step 2\n",
    "        for k in range(self.K):\n",
    "            a_k = a[:, k, :, :]\n",
    "            c_k = self.c[k, :]\n",
    "            temp = (x - c_k.reshape(1, -1, 1, 1))*a_k.unsqueeze(1)\n",
    "            z_k = torch.sum(temp, axis=(2, 3))\n",
    "            if k==0:\n",
    "                Z = z_k.unsqueeze(1)\n",
    "            else:\n",
    "                Z = torch.cat((Z, z_k.unsqueeze(1)), 1)\n",
    "        \n",
    "        # Flatten\n",
    "        Z = self.flat(Z)\n",
    "        # print('Z shape', Z.shape)\n",
    "        Z = self.out(Z)\n",
    "\n",
    "        return Z\n",
    "\n",
    "\n",
    "    def predict(self, x):\n",
    "\n",
    "        y_hat = self.forward(x)\n",
    "        y_hat = torch.argmax(y_hat, axis=1)\n",
    "        return y_hat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2NO7TpyJE330"
   },
   "outputs": [],
   "source": [
    "classifier = CNN3(7)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(classifier.parameters(), lr=0.0001)\n",
    "classifier = classifier.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RLMoFCgKmfJa"
   },
   "outputs": [],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    classifier.cl1.weight = params[0]\n",
    "    classifier.cl1.bias = params[1]\n",
    "\n",
    "    classifier.cl2.weight = params[2]\n",
    "    classifier.cl2.bias = params[3]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "Tmxv4-HM1did",
    "outputId": "f46bdd2c-8f80-4074-91a2-2d11addbd605"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 : Loss = 893.5829023301394\n",
      "Epoch 2 : Loss = 831.1176611072925\n",
      "Epoch 3 : Loss = 770.8134737978414\n",
      "Epoch 4 : Loss = 710.1151720638475\n",
      "Epoch 5 : Loss = 655.7416118130036\n",
      "Epoch 6 : Loss = 611.2973830591926\n",
      "Epoch 7 : Loss = 572.7526660879314\n",
      "Epoch 8 : Loss = 541.3858713821252\n",
      "Epoch 9 : Loss = 509.4077240947232\n",
      "Epoch 10 : Loss = 478.02689353597293\n",
      "Epoch 11 : Loss = 446.2870749230999\n",
      "Epoch 12 : Loss = 414.2360910023546\n",
      "Epoch 13 : Loss = 382.3424581600814\n",
      "Epoch 14 : Loss = 350.97451569560513\n",
      "Epoch 15 : Loss = 324.4152578426986\n",
      "Epoch 16 : Loss = 303.46812263515346\n",
      "Epoch 17 : Loss = 285.8235572522525\n",
      "Epoch 18 : Loss = 267.80292215912186\n",
      "Epoch 19 : Loss = 250.90637600463438\n",
      "Epoch 20 : Loss = 234.28303261999469\n",
      "Epoch 21 : Loss = 220.06721454048824\n",
      "Epoch 22 : Loss = 205.6271059787232\n",
      "Epoch 23 : Loss = 192.2070586307539\n",
      "Epoch 24 : Loss = 180.8892187989132\n",
      "Epoch 25 : Loss = 171.59733145577565\n",
      "Epoch 26 : Loss = 162.9088388369889\n",
      "Epoch 27 : Loss = 157.39718101581215\n",
      "Epoch 28 : Loss = 152.62755986123966\n",
      "Epoch 29 : Loss = 148.56565405018242\n",
      "Epoch 30 : Loss = 145.0803792867096\n",
      "Epoch 31 : Loss = 141.8209965403487\n",
      "Epoch 32 : Loss = 138.93856843423345\n",
      "Epoch 33 : Loss = 135.79087438815975\n",
      "Epoch 34 : Loss = 133.16799777891578\n",
      "Epoch 35 : Loss = 130.21303773673984\n",
      "Epoch 36 : Loss = 127.56941074278295\n",
      "Epoch 37 : Loss = 124.66825731124611\n",
      "Epoch 38 : Loss = 121.80397586623135\n",
      "Epoch 39 : Loss = 119.11185856729433\n",
      "Epoch 40 : Loss = 116.21352724081963\n",
      "Epoch 41 : Loss = 113.55369655595841\n",
      "Epoch 42 : Loss = 110.80071665268741\n",
      "Epoch 43 : Loss = 108.04091460862642\n",
      "Epoch 44 : Loss = 105.33876694204085\n",
      "Epoch 45 : Loss = 102.58334576543614\n",
      "Epoch 46 : Loss = 99.89179378602562\n",
      "Epoch 47 : Loss = 96.99082667653154\n",
      "Epoch 48 : Loss = 94.40527660090748\n",
      "Epoch 49 : Loss = 91.80952801853938\n",
      "Epoch 50 : Loss = 89.11887928500825\n",
      "Epoch 51 : Loss = 86.1397461043833\n",
      "Epoch 52 : Loss = 83.90558616292601\n",
      "Epoch 53 : Loss = 81.01661631620719\n",
      "Epoch 54 : Loss = 78.58055295512236\n",
      "Epoch 55 : Loss = 76.20815936364365\n",
      "Epoch 56 : Loss = 73.5407916344832\n",
      "Epoch 57 : Loss = 71.10263555972001\n",
      "Epoch 58 : Loss = 68.6936218580718\n",
      "Epoch 59 : Loss = 66.26021061541728\n",
      "Epoch 60 : Loss = 63.959585502172594\n",
      "Epoch 61 : Loss = 61.682234308860856\n",
      "Epoch 62 : Loss = 59.938573245802814\n",
      "Epoch 63 : Loss = 57.78235882749125\n",
      "Epoch 64 : Loss = 56.5003913321146\n",
      "Epoch 65 : Loss = 54.18130687221833\n",
      "Epoch 66 : Loss = 52.3802164855319\n",
      "Epoch 67 : Loss = 50.95039184251313\n",
      "Epoch 68 : Loss = 49.225822315814185\n",
      "Epoch 69 : Loss = 47.7011901775719\n",
      "Epoch 70 : Loss = 46.16192119212931\n",
      "Epoch 71 : Loss = 44.94175558056981\n",
      "Epoch 72 : Loss = 43.817784246251975\n",
      "Epoch 73 : Loss = 42.75641133644024\n",
      "Epoch 74 : Loss = 41.62557583320432\n",
      "Epoch 75 : Loss = 40.673309153380714\n",
      "Epoch 76 : Loss = 39.78599469919238\n",
      "Epoch 77 : Loss = 38.97202928390237\n",
      "Epoch 78 : Loss = 38.31000831152088\n",
      "Epoch 79 : Loss = 37.72474480339875\n",
      "Epoch 80 : Loss = 36.99438334342079\n",
      "Epoch 81 : Loss = 35.99675672311816\n",
      "Epoch 82 : Loss = 35.78844043851314\n",
      "Epoch 83 : Loss = 35.3974884644618\n",
      "Epoch 84 : Loss = 34.72830654768994\n",
      "Epoch 85 : Loss = 33.7084681414561\n",
      "Epoch 86 : Loss = 33.10584902348003\n",
      "Epoch 87 : Loss = 32.72345717028043\n",
      "Epoch 88 : Loss = 32.4091339975284\n",
      "Epoch 89 : Loss = 31.88886871072058\n",
      "Epoch 90 : Loss = 31.515057400959293\n",
      "Epoch 91 : Loss = 30.652980272778233\n",
      "Epoch 92 : Loss = 30.15656858038819\n",
      "Epoch 93 : Loss = 29.84890065209791\n",
      "Epoch 94 : Loss = 29.491259910503747\n",
      "Epoch 95 : Loss = 28.858973626060354\n",
      "Epoch 96 : Loss = 28.486855025075453\n",
      "Epoch 97 : Loss = 28.236096412047274\n",
      "Epoch 98 : Loss = 27.599948377974776\n",
      "Epoch 99 : Loss = 27.52642252054779\n",
      "Epoch 100 : Loss = 27.253317862852942\n",
      "Epoch 101 : Loss = 26.862120611742398\n",
      "Epoch 102 : Loss = 26.34187766151561\n",
      "Epoch 103 : Loss = 26.005221430017556\n",
      "Epoch 104 : Loss = 26.24210670019276\n",
      "Epoch 105 : Loss = 25.49227981035718\n",
      "Epoch 106 : Loss = 24.96475143964282\n",
      "Epoch 107 : Loss = 24.717447500195654\n",
      "Epoch 108 : Loss = 24.288106253753554\n",
      "Epoch 109 : Loss = 23.84821322238404\n",
      "Epoch 110 : Loss = 23.692150295403774\n",
      "Epoch 111 : Loss = 23.513482655382326\n",
      "Epoch 112 : Loss = 23.677117942517643\n",
      "Epoch 113 : Loss = 22.809745203742583\n",
      "Epoch 114 : Loss = 22.724086429183906\n",
      "Epoch 115 : Loss = 22.304551041500076\n",
      "Epoch 116 : Loss = 22.05870716081679\n",
      "Epoch 117 : Loss = 21.445925470013236\n",
      "Epoch 118 : Loss = 21.130149947641616\n",
      "Epoch 119 : Loss = 20.93855963517565\n",
      "Epoch 120 : Loss = 20.555990863760172\n",
      "Epoch 121 : Loss = 20.754394404979536\n",
      "Epoch 122 : Loss = 20.16948250694142\n",
      "Epoch 123 : Loss = 20.009896447849606\n",
      "Epoch 124 : Loss = 19.649501022976864\n",
      "Epoch 125 : Loss = 19.605448074872488\n",
      "Epoch 126 : Loss = 19.30944813917738\n",
      "Epoch 127 : Loss = 19.17670662644018\n",
      "Epoch 128 : Loss = 18.747016421593855\n",
      "Epoch 129 : Loss = 18.960299880662447\n",
      "Epoch 130 : Loss = 18.453339128128746\n",
      "Epoch 131 : Loss = 18.473062259394947\n",
      "Epoch 132 : Loss = 18.43939353863121\n",
      "Epoch 133 : Loss = 17.930991309029718\n",
      "Epoch 134 : Loss = 17.71161227276101\n",
      "Epoch 135 : Loss = 17.75849092463583\n",
      "Epoch 136 : Loss = 17.65578862100528\n",
      "Epoch 137 : Loss = 17.561470636507362\n",
      "Epoch 138 : Loss = 17.02393025388286\n",
      "Epoch 139 : Loss = 17.174420705655724\n",
      "Epoch 140 : Loss = 16.761629051447745\n",
      "Epoch 141 : Loss = 16.526002232621355\n",
      "Epoch 142 : Loss = 16.48668804235159\n",
      "Epoch 143 : Loss = 16.369133437552104\n",
      "Epoch 144 : Loss = 16.13819084433313\n",
      "Epoch 145 : Loss = 15.873721169262398\n",
      "Epoch 146 : Loss = 15.525629026964568\n",
      "Epoch 147 : Loss = 15.385203491104606\n",
      "Epoch 148 : Loss = 15.37371409479334\n",
      "Epoch 149 : Loss = 15.420997227525879\n",
      "Epoch 150 : Loss = 15.578307058753037\n",
      "Epoch 151 : Loss = 15.257868364712918\n",
      "Epoch 152 : Loss = 14.71926277107478\n",
      "Epoch 153 : Loss = 14.545044393904949\n",
      "Epoch 154 : Loss = 14.3567238850876\n",
      "Epoch 155 : Loss = 14.179192313749201\n",
      "Epoch 156 : Loss = 14.260190189507782\n",
      "Epoch 157 : Loss = 13.915091880107173\n",
      "Epoch 158 : Loss = 13.877999375506146\n",
      "Epoch 159 : Loss = 13.81831583494924\n",
      "Epoch 160 : Loss = 13.526670023954702\n",
      "Epoch 161 : Loss = 13.394327874798392\n",
      "Epoch 162 : Loss = 13.51905839700732\n",
      "Epoch 163 : Loss = 13.181174806601495\n",
      "Epoch 164 : Loss = 12.841292015766847\n",
      "Epoch 165 : Loss = 12.926962476989534\n",
      "Epoch 166 : Loss = 12.979952084478187\n",
      "Epoch 167 : Loss = 12.96993749731509\n",
      "Epoch 168 : Loss = 12.607707030266418\n",
      "Epoch 169 : Loss = 12.394733718047988\n",
      "Epoch 170 : Loss = 12.275666655563723\n",
      "Epoch 171 : Loss = 12.097094884732874\n",
      "Epoch 172 : Loss = 11.929104622232789\n",
      "Epoch 173 : Loss = 11.743828866539932\n",
      "Epoch 174 : Loss = 11.738984526657475\n",
      "Epoch 175 : Loss = 11.72886217097372\n",
      "Epoch 176 : Loss = 11.482496301471564\n",
      "Epoch 177 : Loss = 11.330987714308897\n",
      "Epoch 178 : Loss = 11.185051552510012\n",
      "Epoch 179 : Loss = 11.038976679280246\n",
      "Epoch 180 : Loss = 10.914152813290057\n",
      "Epoch 181 : Loss = 11.145599026297859\n",
      "Epoch 182 : Loss = 11.17552820398417\n",
      "Epoch 183 : Loss = 10.853266815690628\n",
      "Epoch 184 : Loss = 11.048186740808784\n",
      "Epoch 185 : Loss = 10.698562974298458\n",
      "Epoch 186 : Loss = 10.701208215972688\n",
      "Epoch 187 : Loss = 10.699838236233916\n",
      "Epoch 188 : Loss = 10.362039210488986\n",
      "Epoch 189 : Loss = 10.374597772073248\n",
      "Epoch 190 : Loss = 10.565135025396579\n",
      "Epoch 191 : Loss = 10.195740839330163\n",
      "Epoch 192 : Loss = 10.325540000968694\n",
      "Epoch 193 : Loss = 10.619165217834897\n",
      "Epoch 194 : Loss = 10.590715248825656\n",
      "Epoch 195 : Loss = 10.336734279938277\n",
      "Epoch 196 : Loss = 10.209793453017177\n",
      "Epoch 197 : Loss = 10.058941262939666\n",
      "Epoch 198 : Loss = 10.201476849745376\n",
      "Epoch 199 : Loss = 9.884003479721652\n",
      "Epoch 200 : Loss = 9.944987503078341\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "old_loss = np.inf\n",
    "from IPython.display import clear_output\n",
    "losses = []\n",
    "max_epoch = 200\n",
    "for epoch in range(max_epoch):\n",
    "\n",
    "    running_loss = 0.0\n",
    "    \n",
    "    for data in trainloader:\n",
    "            \n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Forward\n",
    "        y_hat = classifier(X)\n",
    "        \n",
    "        # Calculate Loss (Cross Entropy)\n",
    "        loss = criterion(y_hat, y)\n",
    "\n",
    "        # Backpropagation\n",
    "        loss.backward()\n",
    "        \n",
    "        # Update Parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "        running_loss += loss.item()*len(X)/train_size\n",
    "\n",
    "    print('Epoch', epoch+1, ': Loss =', running_loss)\n",
    "    losses.append(running_loss)\n",
    "\n",
    "    \n",
    "    if (abs(running_loss-old_loss)/running_loss < 0.0001): #if (abs(running_loss-old_loss)/running_loss < 0.2) and epoch>=10 and running_loss<0.01:\n",
    "        print('Converged')\n",
    "        break\n",
    "    \n",
    "    old_loss = running_loss\n",
    "\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "colab_type": "code",
    "id": "6_ku1dTVp9sI",
    "outputId": "588cd094-735d-4fc8-b0d5-7e2c55ddd0f0"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3icdZ338fd3ZnJumvSQtmmSNi3ng6XFymFhFQFdQBB0QfGwosu1rKvsqugqPruruEd1lxXdR1EUV1jlJMpDVQQVUVaRQqGlB8qhtKVpm7TpKT0kTTKZ7/PH/Us6DUmatpnck8zndV1zzX2ae75zJ5lPfr/7ZO6OiIgIQCLuAkREJH8oFEREpI9CQURE+igURESkj0JBRET6KBRERKSPQkFkHDKz95nZL2J8/5vM7Ptxvb8cOYWCDIuZvdfMlpjZXjNrNrOfm9m5cdclA3P3H7j7W3vHzczN7Ng4axqMma03swvjrkMiCgU5JDO7AbgF+FdgOjAL+AZweZx1ZTOzVNw1jFfatoVFoSBDMrMq4B+Bj7r7j919n7t3u/tP3P1vwzIlZnaLmW0Oj1vMrCTMO8/MNprZJ81sa2hlfCjMO9PMWswsmfV+7zCz5WE4YWY3mtkrZrbdzO4zs8lhXmP47/daM9sA/NrMkmZ2s5ltM7N1ZnZ9WCbV+1nM7PZQwyYz++fe9zazD5rZ78zsP8xsZ3j9xVl1TTaz/w6fb6eZ/b+seZea2TIz22VmT5jZvCG2p5vZR8zsZTPbY2b/ZGbHhNftDp+xOGv5vzCzNWa2w8wWmdnMfuv6cFjXLjP7uplZ9ucJw4+HlzwXWnrvHua6P2pmLwMvD/A5erf/dWGbNJvZp4b43G83s1Whzt+Y2Ulh+v8Q/ZPxk1Dbpwdbh4wSd9dDj0EfwEVAGkgNscw/Ak8C04Aa4Angn8K888Lr/xEoAi4B2oFJYf4rwFuy1vVD4MYw/LGw3nqgBPgWcHeY1wg4cCdQAZQBHwaeD8tPAn4VlkmF1zwQ1lERan0K+Msw74NAN/AXQBL4K2AzYGH+z4B7w3qLgDeF6QuArcCZ4XXXAOuBkkG2lQMPAhOBU4BO4FFgLlAV6r8mLHs+sA04PXz+/wIe77eunwLVRF+srcBFWZ/nd/2WPTZrfDjr/iUwGSgb4HP0bv+7w/Z8XXj/C8P8m4Dvh+HjgX3AW8K2+zSwBigO89f3vk6P+B+xF6BHfj+A9wEth1jmFeCSrPE/AdaH4fOADrJCJXyJnhWG/xn4bhiuDF8es8P4auCCrNfVhi/uVNaX0tys+b8mfMmH8QvDMimibq/O7C844D3AY2H4g8CarHnl4bUzwvtmCEHW77PfSgjArGkvEkJjgOUdOCdr/BngM1njNwO3hOHbgS9nzZsQPn9j1rrOzZp/HwcC9YMMHQrDWff5Q/zMe7f/iVnTvgzcHoazQ+EfgPuylksAm4DzwrhCIY8e6j6SQ9kOTD1Ev/JM4NWs8VfDtL51uHs6a7yd6EsI4C7gnaG76Z3As+7eu67ZwAOhy2EXUUj0EH3B92rqV0fTIPNmE/2X2py1vm8RtRh6tfQOuHt7GJwANAA73H3nAJ99NvDJ3nWG9Tb0+/z9bcka7hhgvHfbHLRd3X0v0c+jbqCaOXi7Hspw1t3U/0UDyF6m/899sPfKhNfVDbCsxEyhIIfyB6L/sK8YYpnNRF+OvWaFaYfk7s8TfWFcDLyXKCR6NQEXu3t11qPU3TdlryJruJmo66hXQ791dQJTs9Y10d1PGUaZTcBkM6seZN6/9Kux3N3vHsZ6D+Wg7WpmFcAUov+yR2Pdw7mEcvY2Huzn3v+9LLyu9710qeY8olCQIbl7G/A54OtmdoWZlZtZkZldbGZfDovdDfy9mdWY2dSw/OEco34X0f6DNxLtU+j1TeBfzGw2QFj/UEc83Qd8zMzqwhf4Z7I+RzPwC+BmM5sYdmIfY2ZvOlRx4bU/B75hZpPC539jmP1t4MNhp7mZWYWZvc3MKof/8Qd1N/AhM5sfWlL/Cix29/VHsK4tRPstRnrd/xB+J04BPkS036W/+4C3mdkFZlYEfJIooJ8YpDaJkUJBDsndbwZuAP6eaGdiE3A90HsEzj8DS4DlwArg2TBtuO4G3gT82t23ZU3/KrAI+IWZ7SHa6XzmEOv5NtEX/3JgKfAQ0U7unjD/A0Ax0c7cncD9RPsLhuPPiPrcXyDaJ/JxAHdfQrRz+v+Gda4h6s8/au7+K6L++B8RtYKOAa4+wtXdBNwRurjeNYLr/i3RZ34U+A93f80Jc+7+IvB+op3Z24DLgMvcvSss8m9E/1TsGuoIJhkdvUdWiIw74ZDSb7r77EMuLIfFzBqBdUBRv/1FMsappSDjhpmVmdklZpYyszrg80SHoYrIMCkUZDwx4AtE3ThLiY5W+lysFYmMMeo+EhGRPmopiIhInzF9oaupU6d6Y2Nj3GWIiIwpzzzzzDZ3rxlo3pgOhcbGRpYsWRJ3GSIiY4qZvTrYPHUfiYhIH4WCiIj0USiIiEgfhYKIiPRRKIiISB+FgoiI9FEoiIhIn4IMhafX7+BLD7+ALvEhInKwggyF5RvbuPU3r7CzvTvuUkRE8kpBhsLMqlIAmts6Yq5ERCS/FGQo1FaXAdC8a3/MlYiI5JfCDAW1FEREBlSQoTB1QgmphNHcppaCiEi2ggyFZMKYPrFUoSAi0k9BhgJEXUjqPhIROVjhhkJ1mVoKIiL9FG4oVEXdRzqBTUTkgIIOha50hh37uuIuRUQkbxRwKIRzFdSFJCLSp4BDofdcBYWCiEivwg2Fap3AJiLSX8GGwtSKEoqSxmZd6kJEpE/BhkIinMDWopaCiEifgg0FgJlVZWzWPgURkT4FHQozqkppUSiIiPQp6FCorY5CIZPRCWwiIpDjUDCzT5jZKjNbaWZ3m1mpmc0xs8VmtsbM7jWz4rBsSRhfE+Y35rI2iLqPunoybNcJbCIiQA5DwczqgL8BFrr7qUASuBr4EvAVdz8W2AlcG15yLbAzTP9KWC6nZoRzFdSFJCISyXX3UQooM7MUUA40A+cD94f5dwBXhOHLwzhh/gVmZrksbmY4q3mzjkASEQFyGAruvgn4D2ADURi0Ac8Au9w9HRbbCNSF4TqgKbw2HZaf0n+9ZnadmS0xsyWtra1HVaNaCiIiB8tl99Ekov/+5wAzgQrgoqNdr7vf5u4L3X1hTU3NUa1rSkUxxcmEWgoiIkEuu48uBNa5e6u7dwM/Bs4BqkN3EkA9sCkMbwIaAML8KmB7DusjkTBmVJXSrLOaRUSA3IbCBuAsMysP+wYuAJ4HHgOuDMtcAzwYhheFccL8X/so3OxA5yqIiByQy30Ki4l2GD8LrAjvdRvwGeAGM1tDtM/g9vCS24EpYfoNwI25qi3bzKpSdR+JiASpQy9y5Nz988Dn+01eC5wxwLL7gatyWc9AaqvL2LKimUzGSSRyerCTiEjeK+gzmiG6r0J3j7NtX2fcpYiIxE6h0HsHNu1sFhFRKOgObCIiBygUqnQHNhGRXgUfCpMriilOJdRSEBFBoYCZUVtVqlAQEUGhAERdSM271H0kIqJQILpaqloKIiIKBSC61MWW3fvp0R3YRKTAKRSIzmpOZ5xte3UCm4gUNoUCUDtR5yqIiIBCAYDa6hAK2tksIgVOocCB23KqpSAihU6hAFSXF1GSSuisZhEpeAoFohPY6qrL2KyL4olIgVMoBHWTyti4sz3uMkREYqVQCOonlbNxp7qPRKSwKRSC+kllbN/XRXtXOu5SRERio1AI6idFRyBtUmtBRAqYQiGon1QOoC4kESloCoWgYXLUUmjSzmYRKWAKhaBmQgklqYRaCiJS0BQKgZnpsFQRKXgKhSw6LFVECp1CIUv9pDKFgogUNIVCloZJ5ezY18W+Tp2rICKFSaGQpfdchQ07tF9BRAqTQiHL3JoKANZt2xdzJSIi8VAoZJkzNQqFV7bujbkSEZF4KBSylBenmFlVylq1FESkQCkU+jlm2gReaVVLQUQKk0Khn7lTK1jbug93j7sUEZFRp1DoZ27NBPZ2pmnd0xl3KSIio06h0M8xNRMAWKMuJBEpQAqFfnoPS13bqp3NIlJ4FAr9zJhYSllRUjubRaQg5TQUzKzazO43sxfMbLWZnW1mk83sl2b2cnieFJY1M/uama0xs+VmdnouaxtMImHMralgjc5VEJEClOuWwleBh939ROA0YDVwI/Coux8HPBrGAS4GjguP64Bbc1zboE6YUcmLLXviensRkdjkLBTMrAp4I3A7gLt3ufsu4HLgjrDYHcAVYfhy4E6PPAlUm1ltruobykkzJrJ1Tyc79nXF8fYiIrHJZUthDtAK/LeZLTWz75hZBTDd3ZvDMi3A9DBcBzRlvX5jmHYQM7vOzJaY2ZLW1tacFH7CjEoAXmjZnZP1i4jkq1yGQgo4HbjV3RcA+zjQVQSAR2eIHdZZYu5+m7svdPeFNTU1I1ZsthNro1BQF5KIFJpchsJGYKO7Lw7j9xOFxJbebqHwvDXM3wQ0ZL2+PkwbdTUTSphcUcwLzQoFESksOQsFd28BmszshDDpAuB5YBFwTZh2DfBgGF4EfCAchXQW0JbVzTSqzIwTZ1TywhaFgogUllSO1//XwA/MrBhYC3yIKIjuM7NrgVeBd4VlHwIuAdYA7WHZ2Jwwo5J7nmoik3ESCYuzFBGRUZPTUHD3ZcDCAWZdMMCyDnw0l/UcjpNmTKSju4dXd7T33WdBRGS80xnNgzh55kQAVmxqi7kSEZHRo1AYxAkzKiktSrBsw664SxERGTUKhUEUJRO8rq6KpU074y5FRGTUKBSGML+hmlWbd9OVzsRdiojIqFAoDGF+wyS60hlWN+vMZhEpDAqFISyYVQ3AsibtVxCRwqBQGEJtVSnTKktYukH7FUSkMCgUhmBmzG+oVktBRAqGQuEQ5s+qZv32dnbqMtoiUgAUCocwvyHsV9io1oKIjH8KhUOYV19NwtBJbCJSEBQKhzChJMXx0ytZqv0KIlIAFArDML+hmueadhFds09EZPxSKAzD/IZq2jq6WbdtX9yliIjklEJhGBbMmgToJDYRGf8UCsNw7LQJVBQnWaqdzSIyzikUhiGZMObV6yQ2ERn/FArDNH9WNaubd7O/uyfuUkREckahMEwLGqpJZ5xVm3UnNhEZv4YVCmZWYWaJMHy8mb3dzIpyW1p+mR+umKr9CiIyng23pfA4UGpmdcAvgD8DvperovLRtMpS6qrLdBKbiIxrww0Fc/d24J3AN9z9KuCU3JWVn+Y3VOtyFyIyrg07FMzsbOB9wM/CtGRuSspfC2ZVs2lXB1v37I+7FBGRnBhuKHwc+CzwgLuvMrO5wGO5Kys/9V0xVa0FERmnUsNZyN1/C/wWIOxw3ubuf5PLwvLRqXVVpBLGsqZdvPWUGXGXIyIy4oZ79NFdZjbRzCqAlcDzZva3uS0t/5QWJTmxtlInsYnIuDXc7qOT3X03cAXwc2AO0RFIBWd+QzXLN7bRk9EVU0Vk/BluKBSF8xKuABa5ezdQkN+KCxomsbczzZqte+MuRURkxA03FL4FrAcqgMfNbDawO1dF5bPek9iWNe2MuRIRkZE3rFBw96+5e527X+KRV4E357i2vDRnSgUTS1ParyAi49JwdzRXmdl/mtmS8LiZqNVQcBIJ47SGal3uQkTGpeF2H30X2AO8Kzx2A/+dq6Ly3YJZk3hpyx72dabjLkVEZEQNNxSOcffPu/va8PgCMDeXheWzBQ3VZByWb9QVU0VkfBluKHSY2bm9I2Z2DtCRm5Ly32m9ZzZrv4KIjDPDOqMZ+DBwp5lVhfGdwDW5KSn/Ta4oZvaUch2BJCLjznCPPnrO3U8D5gHz3H0BcP5wXmtmSTNbamY/DeNzzGyxma0xs3vNrDhMLwnja8L8xiP6RKNkQdjZ7F6Qp2uIyDh1WHdec/fd4cxmgBuG+bKPAauzxr8EfMXdjyVqcVwbpl8L7AzTvxKWy1vzG6rZuqeT5jZdMVVExo+juR2nHXIBs3rgbcB3wrgRtTDuD4vcQXSWNMDlYZww/4KwfF6aP2sSoP0KIjK+HE0oDKff5Bbg00AmjE8Bdrl777GcG4G6MFwHNAGE+W1h+bx0Um0lxcmEQkFExpUhdzSb2R4G/vI3oOwQr70U2Oruz5jZeUdc4WvXex1wHcCsWbNGarWHrSSV5JS6ibq3goiMK0O2FNy90t0nDvCodPdDHbl0DvB2M1sP3EPUbfRVoNrMel9bD2wKw5uABoAwvwrYPkBNt7n7QndfWFNTM8yPmRvzG6pZvmkX3T2ZQy8sIjIGHE330ZDc/bPuXu/ujcDVwK/d/X1Ed2y7Mix2DfBgGF7EgcNcrwzL5/WhPfMbqtnfneHFlj1xlyIiMiJyFgpD+Axwg5mtIdpncHuYfjswJUy/AbgxhtoOy4IG7WwWkfFluCevHRV3/w3wmzC8FjhjgGX2A1eNRj0jpWFyGVMqilnWtIv3nzU77nJERI5aHC2FccPMmN9QzdINOrNZRMYHhcJRmt9QzSut+2jr6I67FBGRo6ZQOEq9d2JbvlH7FURk7FMoHKV59eGKqTpfQUTGAYXCUaoqK+LYaRNYqiOQRGQcUCiMgN6dzZlMXp9WISJySAqFEXDmnMnsbO/mpa06iU1ExjaFwgg4+5joun1PrHnNVTlERMYUhcIIqJ9Uzuwp5TzxikJBRMY2hcII+aNjprB47XbSujieiIxhCoURcvYxU9nTmWbV5t2HXlhEJE8pFEbI2XOj/Qr/+3JrzJWIiBw5hcIIqaksYX5DNQ+vaom7FBGRI6ZQGEGXvG4GKzftZsP29rhLERE5IgqFEXTxqbUAPLyqOeZKRESOjEJhBDVMLufUuok8tEJdSCIyNikURtglr6tlWdMu1m3bF3cpIiKHTaEwwv709HoSBvctaYq7FBGRw6ZQGGHTJ5Zy/onTuP+ZjXTrRDYRGWMUCjnw7jfMonVPJ4+9sDXuUkREDotCIQfefEIN0ypLuPdpdSGJyNiiUMiBVDLBla+v57EXt9LStj/uckREhk2hkCPvWthAxuH+Z9RaEJGxQ6GQI41TKzh77hTuXdKkO7KJyJihUMihq89ooGlHB39Yq/ssiMjYoFDIoT85ZQZVZUXa4SwiY4ZCIYdKi5K8Y0EdD69sYee+rrjLERE5JIVCjr37DQ109WR4YOmmuEsRETkkhUKOnVQ7kdMaqrnrqQ24a4eziOQ3hcIoeP+Zs1izdS9Prt0RdykiIkNSKIyCy06bSVVZEd9f/GrcpYiIDEmhMApKi5Jc9fp6HlnZQnNbR9zliIgMSqEwSq75o0YAvvXbtfEWIiIyBIXCKGmYXM47FtRx91Mb2LpH10MSkfykUBhFH3nzsXT3ZPjO/66LuxQRkQEpFEbRnKkVXHbaTL7/5Kvs0MlsIpKHchYKZtZgZo+Z2fNmtsrMPhamTzazX5rZy+F5UphuZvY1M1tjZsvN7PRc1Ran6998LB3dPdz+O+1bEJH8k8uWQhr4pLufDJwFfNTMTgZuBB519+OAR8M4wMXAceFxHXBrDmuLzXHTK7n41Bnc8cSrtLV3x12OiMhBchYK7t7s7s+G4T3AaqAOuBy4Iyx2B3BFGL4cuNMjTwLVZlabq/ridP2bj2NvZ5rvPbE+7lJERA4yKvsUzKwRWAAsBqa7e3OY1QJMD8N1QPblRDeGaf3XdZ2ZLTGzJa2trTmrOZdOnjmRC0+axnd/v449+9VaEJH8kfNQMLMJwI+Aj7v77ux5Hl0M6LAuCOTut7n7QndfWFNTM4KVjq6/Pv842jq6ufMPOstZRPJHTkPBzIqIAuEH7v7jMHlLb7dQeN4apm8CGrJeXh+mjUunNVRz/onTuPU3r7B1t85bEJH8kMujjwy4HVjt7v+ZNWsRcE0YvgZ4MGv6B8JRSGcBbVndTOPS5y49ma50hn95aHXcpYiIALltKZwD/BlwvpktC49LgC8CbzGzl4ELwzjAQ8BaYA3wbeAjOawtLzROreDDb5rLg8s28+jqLXGXIyKCjeVr/C9cuNCXLFkSdxlHZX93D+/8xhNs2tXBz/7mXOonlcddkoiMc2b2jLsvHGiezmiOWWlRklvffzqZjPPRu5bSlc7EXZKIFDCFQh6YPaWCf79qHs817eJftX9BRGKkUMgTF51ay7XnzuF7T6znll+9RCYzdrv1RGTsSsVdgBxw48UnsrO9i1t+9TIrN7XxxT+dx9QJJXGXJSIFRC2FPFKUTHDzVadx02Un8/jL27jolse5/5mNajWIyKhRKOQZM+OD58zhJ9efS111GZ/64XNc+l+/43cvb4u7NBEpAAqFPHXCjEoe+Mg5fPXq+eze3837b1/Mlbc+wSOrWuhRy0FEckTnKYwBneke7l68ge/8bh0bd3YwZ2oF1547hytfX09pUTLu8kRkjBnqPAWFwhiS7snw8KoWvv34Wp7b2EZ1eRFXnl7Pe86cxTE1E+IuT0TGCIXCOOPuPLVuB3f+4VUeWdVCOuOcNXcy7zljFhedOoOSlFoPIjK4oUJBh6SOQWbGmXOncObcKbTu6eSHzzRxz1NNfOyeZUyuKOaqhfW874zZzJqiS2aIyOFRS2GcyGSc37+yje8/+Sq/Wr2VjDtvPK6G9581m/NPnEYyYXGXKCJ5Qt1HBaalbT93P7WBe57ewJbdncysKuU9Z8zi3Wc0MK2yNO7yRCRmCoUC1d2T4dHVW/j+kxv43ZptpBLGW06eznvPnMU5x0wlodaDSEHSPoUCVZRMcNGptVx0ai3rtu3jrsWvcv8zG/n5yhZmTS7n6jMauOr1DdRU6lIaIhJRS6HAdKZ7eHhlC3ct3sDidTtIJYy3njKd954xmz86ZopaDyIFQN1HMqBXWvdy9+IN3P/sRna1dzN7SjlXv2EWVy2s14X4RMYxhYIMaX93D4+sOtB6KEoabz15Bu89cxZnz1XrQWS8USjIsK3Zupe7n9rAj0LroXFKOVefMYsrX6/Wg8h4oVCQw7a/+8C+h6fWR62HNx0/jbfNm8FFp9RSVqyzpkXGKoWCHJU1W/dwz1NN/GxFM81t+6ksSfH2+TN518IG5tVXYabuJZGxRKEgIyKTcZ5av4P7no4CojOdoXFKOZfOm8mlp9VywvRKBYTIGKBQkBHX1tHNz1c089PlzTzxyjYyDsdNm9AXELpqq0j+UihITrXu6eThlc38ZHkzT6/fgTucXDuRS0+r5bJ5M2mYrAvzieQThYKMmpa2/fxsRTM/Xb6ZpRt2AXBafRWXnTaTS15Xy8zqspgrFBGFgsSiaUd7X0Cs3LQbgPkN1bzp+BpOravijDmTqSorirlKkcKjUJDYrdu2j4dWNPPIqhZWbGrDHVIJ44w5k7ngpOmcOWcyx0+vpDil24aL5JpCQfLKvs40Kze18ZuXWnl09RZe2rIXgOJkghNmVPK6+ipeVxc9FBQiI0+hIHlt4852nmtqY/mmXazc1MaKjW3s3p8GoqA4qbaSU+uqmFdfxSkzq5hbU0F5sS7wK3KkFAoyprg7G3a0syIExIpN0WNPCAqA6RNLaJxSwZypFTRMLqe2qpTaqjJqq0qZUVVKaZHOuBYZjO6nIGOKmTF7SgWzp1Rw6byZQHTi3IYd7azavJv12/exblv0+OXzW9i+r+s165hcUcyMiaXUVpVSWZqirDhFRXGSSRXFTKkoZuqEEqZMiJ6nTijRZTtEAoWCjAmJhNE4tYLGqRWvmdfelaalbT8tbfvZ3LaflrYOmtv29z1e3pqmvauH9q7oeSDlxUkmVxQzqbyY6vIiqsuLqS4rYlJ5EVXlxUwqLzpoenV5MVVlRbr3tYw7CgUZ88qLU8ytmcDcYZxFvb+7h+37uti+t5NtezvZtreL7Xu72La3k+17O9nV0c2u9m6adrSzq6Obto5uhuphLS9OUl6coqIkep5QcmC8ojjFxLIiJpYWUVqUoCudoaIkxdTKEmomlDCxLEV5cYry4iRlxUnKi5KkktqpLvFSKEhBKS1KUlddRt0wT6LLZJzd+6Og2NXRzc72Ltrau9nV3sXO9m7au9Ls6+qhvTPN3s6oNbKro5vNuzrY25lmz/40ezvTh36joDiZiAKiNyiKk5QVJSkrTlFedGD6hJIocKrLi5hQkqIklaA4laA4mYyeUwmKk9FzSb/xomSCoqTpOlUyIIWCyBASCYu6jMqLj3gd6Z4M+9MZipMJ2rvStO7pZOueTvbs7w7dWj109D5399ARurnau3unp2nr6KalraNv2b2daTrTmSOuySy6h3dJCIr+QdIbHiX9phX3W74kGS030Dqyg6ooaQde0y+8kmYkEpBKRPN0U6d4KRREciyVTDAhdAsVp6KAOW565VGvd393D20d3ezZn6YrnaGrJxM9pzN092To7DetK91z8HiPh+eerGnRc2cY39uZpvugdUTLdGYNj+QBjGZQEbrfSlJJUkmjOJkglbSohZPIGg7PqWSCooSRShrJRIJUwkgmjKIwHj1bmB7Nz15HKpGgKNW7jqz1hvED7x+1rhJmGETPRngYCQMjmpYw6wvIVPLAa3qXzWd5FQpmdhHwVSAJfMfdvxhzSSJ5q7QoSWlRkukT46vB3Uln/KDgeE0YHWI4nXEymWg9HV1RN9zezu4o3DJOd1imuyfT9z77OtN09zjpTIbuEG49GafHnZ6wbE/GSfc43ZmRDa6RYBad0V9RkiKVMNzBgYx7NBwKToaAS1j/Z/jEW47n8vl1I15b3oSCmSWBrwNvATYCT5vZInd/Pt7KRGQwZtb3n/VR9LDlXCYERm9I9ITn7h4n3RM9d/dk+uYfHERhmYzj4Us70+/Z6R2PhjMZ72uJdYfWVO8yzoEw3deZJp3xg1seHGhN9AZdJuMHDWccplTk5va4eRMKwBnAGndfC2Bm9wCXAwoFETkqiYSRwChKQhk6J2Uo+XT8Wx3QlDW+MUw7iJldZ2ZLzGxJa2vrqBUnIlII8ikUhsXdb3P3he6+sKamJu5yRETGlXwKhU1AQ9Z4fZgmIiKjJJ9C4WngODObY2bFwNXAophrEhEpKHmzo9nd02Z2PfAI0dbmGOcAAAc1SURBVCGp33X3VTGXJSJSUPImFADc/SHgobjrEBEpVPnUfSQiIjFTKIiISJ8xfec1M2sFXj3Cl08Fto1gOSMpX2tTXYdHdR2+fK1tvNU1290HPKZ/TIfC0TCzJYPdji5u+Vqb6jo8quvw5WtthVSXuo9ERKSPQkFERPoUcijcFncBQ8jX2lTX4VFdhy9fayuYugp2n4KIiLxWIbcURESkH4WCiIj0KchQMLOLzOxFM1tjZjfGWEeDmT1mZs+b2Soz+1iYfpOZbTKzZeFxSQy1rTezFeH9l4Rpk83sl2b2cnieNMo1nZC1TZaZ2W4z+3hc28vMvmtmW81sZda0AbeRRb4WfueWm9npo1zXv5vZC+G9HzCz6jC90cw6srbdN0e5rkF/dmb22bC9XjSzP8lVXUPUdm9WXevNbFmYPirbbIjvh9z+jkW3lyucB9HF9l4B5gLFwHPAyTHVUgucHoYrgZeAk4GbgE/FvJ3WA1P7TfsycGMYvhH4Usw/xxZgdlzbC3gjcDqw8lDbCLgE+DnR3RbPAhaPcl1vBVJh+EtZdTVmLxfD9hrwZxf+Dp4DSoA54W82OZq19Zt/M/C50dxmQ3w/5PR3rBBbCn23/XT3LqD3tp+jzt2b3f3ZMLwHWM0Ad5vLI5cDd4ThO4ArYqzlAuAVdz/SM9qPmrs/DuzoN3mwbXQ5cKdHngSqzax2tOpy91+4ezqMPkl0v5JRNcj2GszlwD3u3unu64A1RH+7o16bRTdMfhdwd67ef5CaBvt+yOnvWCGGwrBu+znazKwRWAAsDpOuD03A7452N03gwC/M7Bkzuy5Mm+7uzWG4BZgeQ129rubgP9K4t1evwbZRPv3e/TnRf5S95pjZUjP7rZn9cQz1DPSzy6ft9cfAFnd/OWvaqG6zft8POf0dK8RQyDtmNgH4EfBxd98N3AocA8wHmomarqPtXHc/HbgY+KiZvTF7pkft1ViOZ7boJkxvB34YJuXD9nqNOLfRYMzs74A08IMwqRmY5e4LgBuAu8xs4iiWlJc/u37ew8H/gIzqNhvg+6FPLn7HCjEU8uq2n2ZWRPQD/4G7/xjA3be4e4+7Z4Bvk8Nm82DcfVN43go8EGrY0tscDc9bR7uu4GLgWXffEmqMfXtlGWwbxf57Z2YfBC4F3he+TAjdM9vD8DNEfffHj1ZNQ/zsYt9eAGaWAt4J3Ns7bTS32UDfD+T4d6wQQyFvbvsZ+ipvB1a7+39mTc/uB3wHsLL/a3NcV4WZVfYOE+2kXEm0na4Ji10DPDiadWU56D+3uLdXP4Nto0XAB8IRImcBbVldADlnZhcBnwbe7u7tWdNrzCwZhucCxwFrR7GuwX52i4CrzazEzOaEup4arbqyXAi84O4beyeM1jYb7PuBXP+O5XoPej4+iPbSv0SU8H8XYx3nEjX9lgPLwuMS4H+AFWH6IqB2lOuaS3Tkx3PAqt5tBEwBHgVeBn4FTI5hm1UA24GqrGmxbC+iYGoGuon6b68dbBsRHRHy9fA7twJYOMp1rSHqb+79PftmWPZPw894GfAscNko1zXozw74u7C9XgQuHu2fZZj+PeDD/ZYdlW02xPdDTn/HdJkLERHpU4jdRyIiMgiFgoiI9FEoiIhIH4WCiIj0USiIiEgfhYKMe2a2Nzw3mtl7R2B9683sR1njV5rZ9452vWFdN5nZp0ZiXSJHQqEghaQROKxQCGe0DuT1ZnbyUVc0gsJJS/qblqOiXyApJF8E/jhcA/8TZpa06D4DT4cLsv0lgJmdZ2b/a2aLgOcHWdfNRCdXHaT/f/pmtjK0UBotup/B98zsJTP7gZldaGa/D9fFz740x2lm9ocw/S+y1vW3WbV+IUxrtOh+A3cSnQ2cfZkDkcM22H9BIuPRjUTX7r8UIFz9tc3d32BmJcDvzewXYdnTgVM9umzzQO4DPmJmxx7G+x8LXEV0ldKniVot5xJd3O//cOASyPOIrodfASw1s58BpxJdTuEMojNXF4WLFG4I06/x6HLJIkdFoSCF7K3APDO7MoxXEX3BdgFPDREIAD3AvwOf5eDLUA9lnbuvADCzVcCj7u5mtoKoa6vXg+7eAXSY2WNEQXBuqHdpWGZCqHUD8KoCQUaKQkEKmQF/7e6PHDTR7Dxg3zBe/z9EoZB9Ab40B3fLlmYNd2YNZ7LGMxz8t9j/2jMeav03d/9Wv1obh1mryLBon4IUkj1EtzXs9QjwV+HyxJjZ8eGqsMPi7t3AV4BPZE1eT9T1hEX3yJ1zBHVebmalZjYFOI+oq+kR4M/DtfUxszozm3YE6xYZkloKUkiWAz1m9hzR1S+/StRt82y4THErh3+L0duBv88a/xHR5YtXEd0l66UjrPMxYCrwT+6+GdhsZicBf4hKZS/wfqJuLJERo6ukiohIH3UfiYhIH4WCiIj0USiIiEgfhYKIiPRRKIiISB+FgoiI9FEoiIhIn/8P/CBPZDElavQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses)\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Iter Number')\n",
    "plt.title('Convergence monitor plot')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 271
    },
    "colab_type": "code",
    "id": "BAJMMT7Z0_Ri",
    "outputId": "f9396c0f-7106-4a73-ed7e-7aadd27b5afe"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Loss = 9.868721008300781\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1   2   3  4  5  6\n",
       "0  14   5   5   9  3  2  2\n",
       "1   4  21   1   7  3  2  1\n",
       "2  12   5  10  16  0  0  2\n",
       "3  11   3   6  19  2  1  0\n",
       "4   5   8   7  13  2  2  6\n",
       "5   7   8   6  12  1  1  2\n",
       "6   8   5   7  15  0  2  4"
      ]
     },
     "execution_count": 20,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    train_loss = 0.0\n",
    "    y_train = []\n",
    "    y_train_pred = []\n",
    "\n",
    "    for data in trainloader:\n",
    "\n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        y_hat = classifier(X)      \n",
    "        train_loss += criterion(y_hat, y)*len(X)/train_size\n",
    "        \n",
    "        y_train.extend(list(y.detach().cpu().numpy()))\n",
    "        y_train_pred.extend(list(torch.argmax(y_hat, axis=1).detach().cpu().numpy()))\n",
    "\n",
    "\n",
    "print('Train Loss =', train_loss.item())\n",
    "pd.DataFrame(confusion_matrix(y_train, y_train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "KsXfxUyu1OJm",
    "outputId": "1375e691-f26b-427c-b24f-4cfc534636a3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy = 0.24738675958188153 Train Precision = 0.22550520587104275 Train F1 = 0.21349928256472567\n"
     ]
    }
   ],
   "source": [
    "acc_tr = accuracy_score(y_train, y_train_pred)\n",
    "prec_tr = precision_score(y_train, y_train_pred, average='weighted')\n",
    "f1_tr = f1_score(y_train, y_train_pred, average='weighted')\n",
    "\n",
    "print('Train Accuracy =', acc_tr, 'Train Precision =', prec_tr, 'Train F1 =', f1_tr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 271
    },
    "colab_type": "code",
    "id": "lcw20TIlE334",
    "outputId": "611e70dc-555c-4a21-84da-054c413a9fdb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss = 21.973230361938477\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   0  1  2  3  4  5  6\n",
       "0  3  2  0  2  0  1  0\n",
       "1  0  2  0  0  2  0  0\n",
       "2  1  0  3  2  0  1  0\n",
       "3  2  1  0  1  0  0  0\n",
       "4  0  0  2  4  0  0  1\n",
       "5  0  0  1  3  0  1  0\n",
       "6  2  0  1  1  1  0  2"
      ]
     },
     "execution_count": 21,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    \n",
    "    test_loss = 0.0\n",
    "    y_test = []\n",
    "    y_test_pred = []\n",
    "\n",
    "    for data in testloader:\n",
    "\n",
    "        X, y = data[0].to(device), data[1].to(device)\n",
    "        y_hat = classifier(X)      \n",
    "        test_loss += criterion(y_hat, y)\n",
    "        \n",
    "        y_test.extend(list(y.detach().cpu().numpy()))\n",
    "        y_test_pred.extend(list(torch.argmax(y_hat, axis=1).detach().cpu().numpy()))\n",
    "\n",
    "print('Test Loss =', test_loss.item())\n",
    "pd.DataFrame(confusion_matrix(y_test, y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "FckLra-d1Ulv",
    "outputId": "0c8fd20d-1040-4f7d-bfd3-4c0ed2ffceaa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy = 0.2857142857142857 Test Precision = 0.3257849293563579 Test F1 = 0.2879518474056289\n"
     ]
    }
   ],
   "source": [
    "acc = accuracy_score(y_test, y_test_pred)\n",
    "prec = precision_score(y_test, y_test_pred, average='macro')\n",
    "f1 = f1_score(y_test, y_test_pred, average='macro')\n",
    "\n",
    "print('Test Accuracy =', acc, 'Test Precision =', prec, 'Test F1 =', f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "J_Lz-dJ9p4DI"
   },
   "outputs": [],
   "source": [
    "torch.save(classifier, '/content/drive/My Drive/A3Q3_model.pt')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Q1_Q2_Q3.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}