Pytorch revision.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PyTorch quick revision from the official tutorials for tensorflow people"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.0000, 0.0000, 0.0000],\n",
      "        [0.0000, 0.0000, 0.0000],\n",
      "        [0.0000, 0.0000, 0.0000],\n",
      "        [0.0000, 0.0000, 0.0000],\n",
      "        [0.0000, 0.0000, 0.0000]])\n"
     ]
    }
   ],
   "source": [
    "## We do things like we would have done in numpy i.e lot of manual calculation\n",
    "x = torch.empty(5,3)\n",
    "print(x)\n",
    "\n",
    "## yay an empty 5 x 3 matrix or a tensor as they call it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.3880, 0.8199, 0.2898],\n",
      "        [0.6449, 0.5578, 0.1688],\n",
      "        [0.2335, 0.7963, 0.8927],\n",
      "        [0.2032, 0.4299, 0.2409],\n",
      "        [0.6283, 0.1152, 0.4861]])\n"
     ]
    }
   ],
   "source": [
    "## Getting random variables range 0 , 1\n",
    "x = torch.rand(5,3)\n",
    "print(x)\n",
    "\n",
    "## and many more exist like this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([4, 5, 6])\n",
      "torch.Size([3])\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([4,5,6]) ## This is a tensor\n",
    "print(x)\n",
    "print(x.size()) ## this is a tupple so supports tupple operation\n",
    "## TO BE NOTED ALL THE NORMAL NUMPY OPPS ARE SUPPORTED HERE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.6338, 0.9851, 0.0262, 0.1332, 0.0523],\n",
      "        [0.7835, 0.9359, 0.4455, 0.3878, 0.5530],\n",
      "        [0.0714, 0.0435, 0.9916, 0.5912, 0.4978],\n",
      "        [0.3708, 0.5061, 0.1259, 0.2301, 0.2187],\n",
      "        [0.1493, 0.2586, 0.6372, 0.3214, 0.7729],\n",
      "        [0.5047, 0.0031, 0.7603, 0.5704, 0.4037],\n",
      "        [0.9949, 0.7993, 0.1529, 0.6301, 0.2469],\n",
      "        [0.6789, 0.4863, 0.9660, 0.5266, 0.2509],\n",
      "        [0.6879, 0.4490, 0.4139, 0.5726, 0.8976],\n",
      "        [0.2292, 0.6735, 0.6236, 0.9390, 0.0746],\n",
      "        [0.2812, 0.8519, 0.9704, 0.0922, 0.1541],\n",
      "        [0.4538, 0.0789, 0.3859, 0.7943, 0.2468]])\n"
     ]
    }
   ],
   "source": [
    "x = torch.rand((3,4,5))\n",
    "print(x.view((12,5))) ## This is how reshape tensors by view opp cool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Lets check whether CUDA IS AVAILABLE OR NOT\n",
    "## In my case it is. It's a mx150 mobile gpu\n",
    "if torch.cuda.is_available():\n",
    "    device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_cuda = torch.rand((5,3) , device = device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_cuda = x.to(device) ## or we can move variable to the device and get a refference for it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[0.6338, 0.9851, 0.0262, 0.1332, 0.0523],\n",
      "         [0.7835, 0.9359, 0.4455, 0.3878, 0.5530],\n",
      "         [0.0714, 0.0435, 0.9916, 0.5912, 0.4978],\n",
      "         [0.3708, 0.5061, 0.1259, 0.2301, 0.2187]],\n",
      "\n",
      "        [[0.1493, 0.2586, 0.6372, 0.3214, 0.7729],\n",
      "         [0.5047, 0.0031, 0.7603, 0.5704, 0.4037],\n",
      "         [0.9949, 0.7993, 0.1529, 0.6301, 0.2469],\n",
      "         [0.6789, 0.4863, 0.9660, 0.5266, 0.2509]],\n",
      "\n",
      "        [[0.6879, 0.4490, 0.4139, 0.5726, 0.8976],\n",
      "         [0.2292, 0.6735, 0.6236, 0.9390, 0.0746],\n",
      "         [0.2812, 0.8519, 0.9704, 0.0922, 0.1541],\n",
      "         [0.4538, 0.0789, 0.3859, 0.7943, 0.2468]]]) tensor([[[0.6338, 0.9851, 0.0262, 0.1332, 0.0523],\n",
      "         [0.7835, 0.9359, 0.4455, 0.3878, 0.5530],\n",
      "         [0.0714, 0.0435, 0.9916, 0.5912, 0.4978],\n",
      "         [0.3708, 0.5061, 0.1259, 0.2301, 0.2187]],\n",
      "\n",
      "        [[0.1493, 0.2586, 0.6372, 0.3214, 0.7729],\n",
      "         [0.5047, 0.0031, 0.7603, 0.5704, 0.4037],\n",
      "         [0.9949, 0.7993, 0.1529, 0.6301, 0.2469],\n",
      "         [0.6789, 0.4863, 0.9660, 0.5266, 0.2509]],\n",
      "\n",
      "        [[0.6879, 0.4490, 0.4139, 0.5726, 0.8976],\n",
      "         [0.2292, 0.6735, 0.6236, 0.9390, 0.0746],\n",
      "         [0.2812, 0.8519, 0.9704, 0.0922, 0.1541],\n",
      "         [0.4538, 0.0789, 0.3859, 0.7943, 0.2468]]], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "print(x , x_cuda)\n",
    "## The output is pretty self evident duh ???"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "But this is not why we use pytorch we need it for its Automatic Differentiation capability for opperations on tensor i.e AutoGrad.\n",
    "\n",
    "This is an excerpt from the official docs\n",
    "\n",
    "torch.Tensor is the central class of the package. If you set its attribute .requires_grad as True, it starts to track all operations on it. When you finish your computation you can call .backward() and have all the gradients computed automatically. The gradient for this tensor will be accumulated into .grad attribute.\n",
    "\n",
    "To stop a tensor from tracking history, you can call .detach() to detach it from the computation history, and to prevent future computation from being tracked.\n",
    "\n",
    "To prevent tracking history (and using memory), you can also wrap the code block in with torch.no_grad():. This can be particularly helpful when evaluating a model because the model may have trainable parameters with requires_grad=True, but for which we don’t need the gradients.\n",
    "\n",
    "There’s one more class which is very important for autograd implementation - a Function.\n",
    "\n",
    "Tensor and Function are interconnected and build up an acyclic graph, that encodes a complete history of computation. Each tensor has a .grad_fn attribute that references a Function that has created the Tensor (except for Tensors created by the user - their grad_fn is None).\n",
    "\n",
    "If you want to compute the derivatives, you can call .backward() on a Tensor. If Tensor is a scalar (i.e. it holds a one element data), you don’t need to specify any arguments to backward(), however if it has more elements, you need to specify a gradient argument that is a tensor of matching shape."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[3., 3.],\n",
      "        [3., 3.]], grad_fn=<AddBackward>)\n",
      "<AddBackward object at 0x000001E36BDF6208>\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "x = torch.ones(2,2 , requires_grad = True) ## Ya because gradient tracking is not obvious \n",
    "y = x + 2\n",
    "print(y)\n",
    "print(y.grad_fn)\n",
    "print(y.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So we can see there is a function object attached to the tensor y and the gradient respect to y for y at this point is None."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(3., grad_fn=<MeanBackward1>)\n"
     ]
    }
   ],
   "source": [
    "z = y.mean()\n",
    "print(z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "State of x :  tensor([[1., 1.],\n",
      "        [1., 1.]], requires_grad=True) None None\n",
      "State of y :  tensor([[3., 3.],\n",
      "        [3., 3.]], grad_fn=<AddBackward>) None <AddBackward object at 0x000001E36BDF8A58>\n",
      "State of z :  tensor(3., grad_fn=<MeanBackward1>) None <MeanBackward1 object at 0x000001E36BDF8A20>\n",
      "State of x :  tensor([[1., 1.],\n",
      "        [1., 1.]], requires_grad=True) tensor([[0.2500, 0.2500],\n",
      "        [0.2500, 0.2500]]) None\n",
      "State of y :  tensor([[3., 3.],\n",
      "        [3., 3.]], grad_fn=<AddBackward>) None <AddBackward object at 0x000001E36BDF8A58>\n",
      "State of z :  tensor(3., grad_fn=<MeanBackward1>) None <MeanBackward1 object at 0x000001E36BDF8A90>\n"
     ]
    }
   ],
   "source": [
    "print(\"State of x : \",x , x.grad , x.grad_fn)\n",
    "print(\"State of y : \",y , y.grad , y.grad_fn)\n",
    "print(\"State of z : \",z , z.grad , z.grad_fn)\n",
    "\n",
    "z.backward()\n",
    "\n",
    "print(\"State of x : \",x , x.grad , x.grad_fn)\n",
    "print(\"State of y : \",y , y.grad , y.grad_fn)\n",
    "print(\"State of z : \",z , z.grad , z.grad_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we repeat the above block a few times the operation gets repeated and the state of the variable gradient keeps changing. It's upto us what we do with them"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Onto NEURAL NETS then"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net(\n",
      "  (conv1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (fc1): Linear(in_features=400, out_features=120, bias=True)\n",
      "  (fc2): Linear(in_features=120, out_features=84, bias=True)\n",
      "  (fc3): Linear(in_features=84, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class Net(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(Net , self ).__init__()\n",
    "\n",
    "        ## Kernesl for convolution\n",
    "        self.conv1 = nn.Conv2d(1,6 , 5) ## where 1 is input number of channels 6 is output and 5 is filter size\n",
    "        self.conv2 = nn.Conv2d(6 , 16 , 5)\n",
    "        self.fc1 = nn.Linear(16 * 5 * 5 , 120) ## How is the in feature determined. Is it precalcylated ?\n",
    "        self.fc2 = nn.Linear(120 , 84)\n",
    "        self.fc3 = nn.Linear(84 , 10)\n",
    "    \n",
    "    def calc_feature_count(self , x):\n",
    "        size = x.size()[1:] ##dont take the batch size geez\n",
    "        num = 1\n",
    "        for s in size:\n",
    "            num *= s\n",
    "        return num\n",
    "    \n",
    "    def forward(self , x):\n",
    "        x = self.conv1(x)\n",
    "        x = F.relu(x)\n",
    "        x = F.max_pool2d(x, (2,2))\n",
    "        x = self.conv2(x)\n",
    "        x = F.relu(x)\n",
    "        x = F.max_pool2d(x, (2,2))\n",
    "        x = x.view(-1 , self.calc_feature_count(x))\n",
    "        x = self.fc1(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.fc2(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "    \n",
    "\n",
    "net = Net()\n",
    "print(net)\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have to define the **forward** functino and the **backward** function for tha autograd to work."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "torch.Size([6, 1, 5, 5])\n"
     ]
    }
   ],
   "source": [
    "params = list(net.parameters())\n",
    "print(len(params))\n",
    "print(params[0].size())  # conv1's .weight"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Well the parameter weights are actively available that is one good thing. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0108, -0.1214, -0.0612, -0.0247, -0.2039,  0.0635,  0.0200, -0.0822,\n",
       "          0.0041,  0.0708]], grad_fn=<ThAddmmBackward>)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inp = torch.randn(1,1,32,32)\n",
    "out = net(inp)\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net.zero_grad()\n",
    "out.backward(torch.randn(1,10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also note this that , Pytorch since have dynamic graph it always empties the graph buffers after one backward call unless explicitly specified. So one must specify retain_graph = True for sustaining a graph\n",
    "\n",
    "We now add **LOSS** function to the final output of the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.7244, grad_fn=<MseLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "inp = torch.randn(1 , 1 , 32 , 32)\n",
    "out = net(inp)\n",
    "target = torch.randn(1,10)\n",
    "criterion = nn.MSELoss()\n",
    "loss = criterion(out , target)\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((<AccumulateGrad at 0x1e309a45f60>, 0),)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss.grad_fn.next_functions[0][0].next_functions[0][0].next_functions\n",
    "## this is how you traverse through the entirety of graph in case you were wondering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "net.zero_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv1.bias.grad before backward\n",
      "tensor([0., 0., 0., 0., 0., 0.])\n",
      "conv1.bias.grad after backward\n",
      "tensor([-0.0050,  0.0027, -0.0013, -0.0004, -0.0089,  0.0115])\n"
     ]
    }
   ],
   "source": [
    "print('conv1.bias.grad before backward')\n",
    "print(net.conv1.bias.grad)\n",
    "\n",
    "loss.backward()\n",
    "\n",
    "print('conv1.bias.grad after backward')\n",
    "print(net.conv1.bias.grad)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding **Torch Optimizers** though we can loop throgh all the parameter data and optimize i.e change the values of the weights on our own we don't need to."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "# create your optimizer\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.01)\n",
    "\n",
    "# in your training loop:\n",
    "optimizer.zero_grad()   # zero the gradient buffers\n",
    "output = net(inp)\n",
    "loss = criterion(output, target)\n",
    "loss.backward() # Does a backward step  needs retain_graph = True for\n",
    "## Sustaining the training for the entire time\n",
    "optimizer.step()    # Does the update"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will load the CIFAR Dataset now and make some prediction  YAY !!!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "import torchvision.transforms as transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data\\cifar-10-python.tar.gz\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
    "                                        download=True, transform=transform)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=4,\n",
    "                                          shuffle=True, num_workers=2)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
    "                                       download=True, transform=transform)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=4,\n",
    "                                         shuffle=False, num_workers=2)\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat',\n",
    "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# functions to show an image\n",
    "\n",
    "\n",
    "def imshow(img):\n",
    "    img = img / 2 + 0.5     # unnormalize\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
    "\n",
    "\n",
    "# get some random training images\n",
    "dataiter = iter(trainloader)\n",
    "images, labels = dataiter.next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  dog truck  ship  deer\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAB6CAYAAACvHqiXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztfWmMJdd13nfr1dt7n+mefSWH+yJSiyV5iSIpkeQ4lgN4kWM4DCKAfxzEDgzEcvzDEZAfNhLYcQDHAWE5kgPDsiLbEaPIi8LIkUzbMilR3IbkcEjODGemp6f37a1VdfPjnFPnvO7XPcMeanq6fT9gMK9v1bt1761b9c76Hee9R0BAQEDAzke03QMICAgICHh7EF7oAQEBAbsE4YUeEBAQsEsQXugBAQEBuwThhR4QEBCwSxBe6AEBAQG7BOGFHhAQELBLcEMvdOfcR51zrzjnzjrnPvV2DSogICAg4K3DbTWxyDlXAHAGwD8AcBHAUwB+0nt/+u0bXkBAQEDA9SK+ge++B8BZ7/3rAOCc+zyAjwPY8IVeq9X8yMjIDVwyICAg4O8eJicnZ7z349c670Ze6IcAvGn+vgjgezb7wsjICB599NEbuGRAQEDA3z18+tOfPn89592IDd31aVtnv3HOPeqce9o593Sj0biBywUEBAQEbIYbeaFfBHDE/H0YwOW1J3nvH/Pev8t7/65arXYDlwsICAgI2Aw38kJ/CsAp59wJ51wJwCcAPP72DCsgICAg4K1iyzZ0733inPuXAP4MQAHA73jvX3yr/Rw+cZg+FAo6qEoJANBqreRtpRIdj2MacrfbzY8Vi0Uak7H4+CwDAFQqlbztxRdfAABMz12hc8zPWTtLAAAJfw8A4pT6y0xbJt+JyOKU9okSipx2LHapNE21D/4s/dpIoyyVz9pHFOnaCO6buLfn74/8yCP6h6P+nflexiNxZo2cozaPQs/3AKAAOaZIPY03SZK8LZF58UQLkVri4iLdq7ig26ws14quV5agEVxvMFa/qC1p61lnHvBXvvCZdedPj/89Osfed/4sa6YjA3wf42O/a8qn1Awxyvvj8+0+ydLeLwLIuD87DvuZrmk+Z3SsYJ4vfx19XAv9zj+69GTP39945Q/zzwuLCwCA5oruhaHhYQDAQK2YtzVWlwAA42MUPLF3opQfGx6hZ7nT0WtcmZqkax89kLctL5Fpd3GR3h9HjqkvsdOl/leWtY+py9Th9NV23nbxwiUAQLlOf584cTw/NjtHc+m09B20d88gzaWu75uBgQEAQJLQeUnWzI89cN/3AwDOnHktb7t05QwA4CPv/ufYKm7EKQrv/VcAfOVG+ggICAgIeHtwQy/0twON1ioAoJ2q1DdR2w8AmJmdydtGR4YAqHTRbuuv6ego/Zp3E/3pTliCHxis520iOIu0lXiVwBKsb3MpfcGnKk36Aku6jiSeotPzRTi1MqJIVJGVhlhayiVkb6VmltozlagilmatlLUWxaJKMklCa3N+Uh3jzW6Hx6FzEW2gldKxdqIakZw3MzOrfbRbdH5TndvdlKSO0X0kBY2P78+PVStVAEAp1m12dJTcLgdH9bxcEnaiRSj6Sbr9kB/dRELPzLGor0+/ty8roafy2fQh2plfHwuwucRr9pgXTcWvv6bO2fVpw7q2fueLGmo1xH5jXDvefuPv3afXlugH6rrXTpw8BABYnDPPV1QGALz2+ht5m2jehQZ99+oruteGWfr1RiU6ytL36opKv29eIA183759dH6qGkAxGgUAzExdydtmZmjfN1b1/REVaJxpQnO+enU6P1ap0rNWq+m7pc6fBwcGdbwjJKG32vSOq1bGdByO+hjgZwQADk7sxY0ipP4HBAQE7BKEF3pAQEDALsG2m1zKNVK7ludX87Zml9QnYx1AvV7v+V6zqSpWwuaaLFOzjajBPZohf85Y5bXqreO2otFoS9KddpureyU2cRSL1kmW8unaiZiSkj5qeT8TSpayqtejIYuZZzMzgfa/uLoIAHjyqa/ptYoUMloua+hoFNMCtxJSa5vthfxYkx1Kzz/zHXMVOj8x6yyO0oERcnDd+/AD+bF9Rw/xSbrOYBV2//CEjl2cdGIasc7t3KyhcLmD15gM+H9xdvaaXvrYwjYxucicrLPTiRM8W+94z/qYQfreK+k3NV493uTiwPZmXJlc6xrWDTFT+Ny8pw9OoVDivrJ134v6OKb7nZcP1cxJvrvZnpy+ovtpfnYOAHD0yKG8bd84mVEHytrGPly4Iq1Hc0X3a3OZ5tlqtfK2pSXap9W6mlXe+a77eS7Ux2tnz+THxPwxPaumnChi02BF16NSpf6aTbpmc1UdoPU6mVUy85AuLlG/pWI5b1u9TI7VPXtpnpcuah5mnJH5xZr+hgfVXLNVBAk9ICAgYJdg2yX06gCH+Sxp2+wCOUPrAyqVS2iiSAa9ziGRxk1b7qw0Eoccj+mXu5Tpr2PFkyRTNm0xS1RWUyhIiGRK5yVWyC7TXFpGghVp0xXW/3bqHIyjzfWb39rz1yMz4YLNLjlFl1fn8zZXIKkm7qom5Hgd0lxiVClE7svd992t1+dxJmYqGa/p6jzdwNNPP2P6IOlqdHQ0bxNp0hlZopDfU5F4jBTsev8HgITD+ZZW1YmbsiaU8tpbJ6A4NG24ZRRtvPVTvn6aaR+5Y7WP09L3CWmU69u2jMPXCl4l9PyWcmhn5nRcCUvosQ057HOtteGvdq28T9ad328ua6X1HofwmjBbQMOHN3PUz03pfpLw1saq5h6+euYCAKBY0LGNsjQ7vp8CHcQ5CQCpo707cXA4b6vVaJ+mma7pQI36W+LwRVfS/TRYJCl4dFTDHNutlMeh2sDkRXoHvcRjbLZ177SWyTpQG1JpPPU016UVfebSjPd6ga4/Na0ay7HDHHQwN5m3FUsc6KG+07eMIKEHBAQE7BKEF3pAQEDALsG2m1wiies2mtvMHKk7+07dlbctL1NqV4fTxKz5oc1OkiRTZ0lBsiSNGcaz+aXADpdKVy9aWKTvehPPOjFIqt2YMRm4Ei3ZpUlSlV59cyo/Vj9Ijr7ioKpupZhMNIXCeueRmB+sKaBfFp+gXyyxIDOxzRcuU1xvs6Fz8QVS56JUb7mPJB6e/o4i7aM4SnMf33c0b4siz3PRdZNY7DShsT31p3+ZH1uZJvVzz549eVuH71Ej0bEV+OaLyaWfuu/N/GZXSHX99svP5W3tNvWXpbKmJuOyT4ZrvUaqt0bvK8TJnpj1ljXqcYCKU7SPeUVNGD1R9TTfTE0Rg0O0zkvsfOtxSbL5LUvteqw3oci1dK3snl+/nzYzH/X7W74bm3yCwppcin4Y3TOQf15Zof2XGJOmOOoTE3VQG6D1aLboOZ+f1We62aR5Li5dzdvKJTJ7zK+ozfb8JXomT5ygnIesqA7TYY4FXzCO1YVJyrXwyVze1uB1q9ZpzrXhofzYfn4vLDXUfNnlvdUwph8xActcutZkVaZ+90yo+Qgw6atbRJDQAwICAnYJtl1Cl2zFobqG7CQd+rUrGQ4QXyDpo1TicMFYf3UXlkgSLBjHY2WQQpESFYbyaxXa9IvZmDS//pfoV3+spn3s4QzUiUHjqGJHYoslg2cvL+bHpufICXP47hN5WzxG4+jJFF0jUfVyhrCUZSQ7+a5zG4eUWW6bi1fYkWPoiveMkGSSGY+ZZNeJUzSuGjVpjO6H5X7JOux865hFZZmgUKX1qA/pfczHZIS42RWShr7x7De039ypJ+GkKhl3+V55s0atLt23ZcP1I05W8UdbiV4GkHrtt8BZr/0kdDnfOgr7ObBFOu0Xwief7b0VnpvYSL+X3qRs3rhGEmCprgVgXM6ns/6a/STj/Fo2NqAg2k8fziEzP/lstZi16JH8N9EkBadOHc4/n375dQBAoaiZkRlrG7WatklYbaNJ0vLBgwfzY9NXSTNb0duOYkxO0YkJlaBnFikLNGHNZjhWfpXxjN8bkWaat/aQZn3lkmaDHt9LkvORMXpuFk24aqFLa3Xl7IW8LeGQ5bFR1c4n9tOzsGcvPRuVqkrjZX7WKlXV/pvNG6cXDxJ6QEBAwC5BeKEHBAQE7BJsu8lFstpGBlUdGebPVpmTTNFcTTQHhzlLsV/mW4/mzQ6Z1jKpNstT6tTwK6QyDRxQB94VduQsTqtZpcK0nn/7CqlbpbKev8Bx382GOkYqQ6TUp0YPFjpZGa8dt6i8VpMtxOLg3Tjmd9U4aMZGaEwHTKztgw+8CwAw11SyrcnXKZNthh285VgNEJ02Oxe7Ovf2Eq1Ha0rjaYcH6Vr1A7Quki0LAG02a1i1vMs2sKsNJUdSilxeox4K2fXZkl12eMaxrofjrSyn98RYM0FVwTjIN4uf7hf7LkProcrdJIMz6xP7HjM9cdJRU59ns0O5RKYAG/ueZuspj/OB2Dj0Nc5QZwnY/HozjA5//QRkXfpRB/eYj/i8fs+cYHpG7/HoGJk9Flf02Zibpb01P6v97ttHJo55zmvwxmHqHe2/Rlv3ZJmtNUM1NbmM1on4rcj2t6ihYzw+SM/EIPQ5KB6mudRNbsJDdTL1XFiha5298HJ+LGlxv8YEWoqFYE6d/SOj/G4oUFvFJLvXOLP10gWdywq/P/aqlektI0joAQEBAbsE15TQnXO/A+CHAFz13t/HbWMA/gDAcQDnAPy4935+oz42w4VLxG/QGw0mTjKVVpRCVv6P1x1zfSQqi7Z4SMscTlUx4WAxSQ5TSyo9vcKhUN2ihknd/9CDAIBnr9B0hwx3Q7HG0o3J2pQQuLRHyuodt0W/DLzNHGGCVlslgzLzzOwZUe2hWiLxoBKpxDNap/kloDCvclVFiBWh1jWUxCOD1G9lSedS4uxSyWaUohYAkPZzsIkj0YRxFliibOfX0v6liECh2JOeCgDodo00y59F6iyY4h55dmpsJNdNOEskcbZH0pW+DJVyKlTHLEWWjWM6ZlnJmzC9DkuAdu61mPfHMu2nblXvWcp73PeEF/LYsn4i9/rnoJBLuNZhyw5km30rjmN+RnwfCb039NH1/N8P9957p57P2skb5zVTtMEZl8WSBjhIUYrVVZLGx8ZU8vYscXfVn4nJyyRpLy1p2OLJY+TkHB4jOtqFVX02Li7Tl62z9YExWufWkDpxryzS/rn8Gmni89Paf4ed8rHR+GoVcoA2V3VwLz5P77baAPU/ZsI4we+25SUdGzJ13m4V1yOhfxbAR9e0fQrAE977UwCe4L8DAgICArYR15TQvfdfd84dX9P8cQAf4M+fA/AXAH5hKwN47Q1KgvFG4ujmsYbrQ74KLMHaJId+RRCE+8XyYEhvBQ6NSiK1X3XZ9vs6l54CgCLb8mtVDUU6fJDscwcO0q//5UtqJ5wYp7aopGNzua1xfRJHPylRxm0ldLWrbywNdROVDDoswlh77BKHdlYMK12deT5WOcnHpxo+5kCaR9FIqRMd8j0UF7XwSLFCfcy1SfqIbNKR2MJtGJ2E89kQOG7rpDTuOFJb/uoKhxdWzJpyHcCsq+vX7fTaistl1Zxk2Xrs5pvYvyNmhCwWjZSfh5OafSrcN47GWI6MRslj85HunRbLT6stDU+bPUtlEQtF8kEM3/Ge/FgqA++pKbee6yffF/npfTiNeszwbN83z1euQfI8ezwMkYRPKoRPB9HGC9lcVV/LGJdoO3Vcy8GJL2FsTKXlV8+eo2F7ut/tVR1jY4nWu24YQ2t14SNS7ej866RRt5ZI1crMbF68TP1/tP7+vO3BY/cBAF66ogVhvvrsaQDAdy7Q+Y2m9l+uMK9UrGOLyxIaq89LkbX3OKbn6sJZlfLvPUnP3OEDWtRieIgLctxA9OJWbej7vPeTAMD/T1zj/ICAgICA7zK+605R59yjzrmnnXNPNxo3HjgfEBAQENAfWw1bnHLOHfDeTzrnDgC4utGJ3vvHADwGAAcPHlznyclyilqTtdaHJyL3cUmTcUgIZWq/UDWrJwo3R4epMEuGVvP4nZTdef6c1jecXiRuhVMnVCX8/oeJPL+9QJlsn/3CH+fHRveTolIZVOdHFtM1rWYa+V6zg1WR+/G1bObAE7Q66lyR4h9JR/tabpB5KZtVNbg0TQ6qQXZYtUp6D7pddtYZU1GZTSJjy2qqqrFauZRIiKLJruxjpshhzUe505f/tk5DKWxiumhyqmBs6qiKg1mygfvZVDITUtlvSIK4wFwuTV2rIjuTI2PaimN22Eb0f2S4c/LQtqpyodY5tC4x4W7799JxVyFnaNeaecTMaDll+nn71zRZZ2cbEhrbzzRiOV/YcZyHZ67nfrmeOqIW599QnqMJ5iwpGpPfoQNkvpyeVQ6VsWHaT9NzdI+LpuJMrUbfbbe1j6RDx+eWde3n5+m5LbLZ5r67b8+PNRfovDcuqHN26R3vBgC8cEVDGZ8/R87QRr6vjXmlQNcf26sO2+FRGvfUtKk9WqZ7OcYZ52nbBBOU6Fi1anhmRqltYRtMLo8DeIQ/PwLgS1sfQkBAQEDA24HrCVv8fZADdK9z7iKAXwbwKwC+4Jz7JIALAH5sqwMQ2cMKECIdOCOpeZYUhTTeOkFS5mywYYCpIwnJhj52WOqUxJUH7tFf7odvI2bHxx9Xx8WFWUrW6WQqCX79L/4aALDAP6PVYU2IGhijzyXzq9tiD4ctC5aHIWZ9xt1HQhdnnu8XqsZYNmxzMzMkGbWMxLg6R5Jlc0mduGMrNL+8wIAJOey2+LtlvQdldlYWLc8MlzhzrHVYjp1ui6XZHmGcHWymLcudedJo5snFAVxqr8mhqyUT2imFEPK4Pg2Z7HA4ZHNFRR8J4xzp8wj4jNYlaam0VS2yFL6iDuEIdN7x46TBTc/r2s5x+F050nswxpwes/MaGrvcpPPqQ+QsTAqGO4f3bk/YYh/CzbU8LZnZawmrtpG3shuvqSnEUvDymZ2uJuyzn4Tej6tmLebnde7y7F29qmu0tEjfLcU6/vIQ9XviBIU8HjikyXFPP0XsmlenVaIvFSnUzzf0ffDBuyi0+ANcnGWoonvyw3eSNL6wouN++fw5AMDZq1psosAhyLWE9kfS0bm32vwsNXSNKnXqb8RwuQwPkgRf5mfo1B06l/F9dF5qNL5S+dqa+LVwPVEuP7nBoQ/d8NUDAgICAt42hEzRgICAgF2Cbedykejw3oLsvTUSASBGLw2o5WgpMcdJwXB7pLGcb0wGrB4W2Zxwxz7Nynvg7pMAgCe/oQ7NhLMPW4ab5cxp4nTYO07fnTAE9QlIfaoaTgj2r/Vm2bHpJJKYaZP1mua/sXp+kc0aPtpYJVtaVnL8yasXAfTGeqctVpvLukbNUTI7lPiae+qaKZoskwnHxnM3a6wm7tuXt0V7KY423UPrUF5VTpkWf+4Ji+6T9bqWmtYW6yhzIYDUFKwoDRb5PDUZJFzTUWKbRS0GlJemXtP5nTxwnM47fxFrUebq7wOxqs9+gWKUJwZ0L0Qcf+7mzwIA9tX0Po7XaB/Fpu7k+CGKR14sqer9BnPalFlVn2/rfZRiEB7aRydlE4rlcuH1kgrymQ0m4Ocr6ZNFXciss5UzRNk0k9r6vH08yH1r+655mxw5rMEEy5zJ+fpZNWvcf987AQAje9Qk4iJaUymKYs0rjeYyH9P7UmAH5bhJsvyJ73sfAODuvWTGWllUc+TwYSp60TFZw3/8ta8BAIpl3U97Rjl7lHMzfKaTEwpqm1tS59qmUUHXdGSUvju6h0wvw4YivFShd5DkWQBAXDCZpFtEkNADAgICdgm2XUIXx4/NTMu5LlL99Y/4jPw8I3GU8kw2k8XHxyuR/vpXhNNjgRxRC2+8nh+rf+97AQAPP/xA3vbVJ5+laxoH7EP3UNhiXKRf52cnX9IxFiWj01SXF4eSjSWT31GRgvpIVJlxAhZk1puwLVrmvA5LswMmo67CUn4jU8nVVbmoATsZuyZscewgSeG26MVSnSSI4phm+3mWZuMKSSPxJY1glTJw1nEm69BPQhfnb0/5uC5JxFLUguZH69sxZcQa7AAWR2yrqfN86G7Kvvy+d/9A3rZ/lKTHr5z/bayF5/00NKbFB06epM+DHZUwsyZ9PnOGtLaZGZUE906QJDhUVK1gOCaNZWyfhrsVVkjDGTxyCAAw/4ZK6I0WrdtcS7WCbqb7WSBZ011ZR+PsLLL03o5Uygc/Ez0FLnJNljXbnrDP9eGKcs96nPhr3iZ7JrTh8mVaq4rR+CrseFxZ0ooVKYcJzk1SZvPMgq5pPt5In3PH4aT1qpFNmxRW25jh59aUhVudp+sXjPOyWKI1qpviNofKpHFeWaBxDAzpPasPUB9ppuM+duQ4AMCb56s+RPPbO057xxahQUz7tVQ1WjRs4ZitIUjoAQEBAbsE4YUeEBAQsEuw7SYXMZP0OEBZhbT+lrUVzW1Gojh8bKhtzOaaclfVrSrXwiyUSeW8wtS9AHD62WcAAHeeOpm3HTxIpoUFE+MdD1LWVyPhTDaTFVqusCpmBp47Jk2bdTQCQKejY0xz80RhXVuPA2qN9SU2GYYdpuRMjHO2xarpnn1KBhSPkRlmYZEcT7bwwiifF5tcAAiFsSErEy24zKRizqjDwv3Uo7wLfa4xQTVaZIpodfj/ho4jYVpg6xRdXqa1X57T+zLL9SBn35zmcehmODFA+QYHPnJE575JgYupeeo/6er6jbCT/dTJY3mbX+19fPxt+nejSXMYGlXCs/EBmvPKsqrqdxwn599SxHVdW5qdeqBCe61mHHjTvDZts2ccf6xUyDHnjOoeyd4x5kvJHs367FMnQQp9TGK2WIfcRyGT64fakJofTg6SSSlyev70zDnqPzP3gp/zlOdXr6upI67SszZjsjEHOdZ7yOwyie2fXiXz1d5hk43JjuvXZzUrtMmFWGplPW/vKJlJDt1OpsfUZIrW2Sk/PKLO7YEKjcMWUQE7eCUXxjBto8uEeLZtZlZi9O/BVhEk9ICAgIBdgm2X0DPO8rROvZzfxZ63JqtSCl0AWrjAm4yzSsJ9zCvvSJ2LE9SHyFHlu3r+q6efBwC87/DRvO173/cQAOCZb2v5qdfZuZPW6LvDEyrxCsVqZCSfbrY+tS9ip1VO52qkYKlenyTaRyJFB6yEvqZcvXUyVlhqahpSiCnO0LvHSNd7j5Pjxy/LWlpKVtGc9BoFviMHRtQpOlglySTjEK6XS8/nx6TARWakwwJL8I2GStdT05N8dTomXDQAsDpHUtb0G+qMnD5Pc+m2VRItOlqQJme/irYEAC8+RxS1ly4pNfLRo3qf16LLYYIzq9r/K00uD1bStqOjnKUYkRS+sKTrXWOH9DFTSCH1JLEevl2vXWbN6cwVkuYWLimX0KVp6u/k3bflbWPDFPo2N6fhfCKZpymt6ZDJXm5yhfrLLZUwO7kW2IfvRsrNrTuy9rw+TtE1OHJCw4KXmfo2NTVP7jtI2vDKskryKyxVl7gcYjtRba3AmczHTup61AdIaq8b2bQ9T6MXLsCoojGNU7Mk3T/3hu6FyxeJ1yUeVE3hjrvpviU1ml/bFi9hQT41PLdxiebgzfPe7UoIL/VbMrTawr1UMqGxAwN0bxf11r5lBAk9ICAgYJcgvNADAgICdgm23+TSh9wnSrg2oiGjkiy4fhlqRXaiWpUGy6y3LKnJZXCEqEprnH0YmxqaKatAs5PqKP3Rf/z3AQCjA5rh9eLrlFnY5vH4inVeMj2vKXrYr5pSm1X5UonUSuugkxjobqYqXk6zuwk5l40pln5bzoyD17ndWT+2vDBOn359jxmG6UAHVJWulEmdXW0zpa3Neu3nzOWLddvq1FueIlNBc5lU2MnLqg7PXyEyrIXLqoc6NpXVTOZdZYRU7y5TBtfNsYTzCKanNUb+6FF1bq5Fm7XmrvGyr7KJ76VzSruKLplamFUYp186kx86zFnI+/eour+fK9qnHXWKljim+RQTfJ0wmcd/9iQRwRU7es2JQ2TuOn5Ixx/FNIAuBwAMVdQROymBAC2T18Cmn8TpverI3uIYdftU5vvP2EuETK9QWG+2EQwYh3CV74fxbSNic+uevVofp9EhE544fctlNWmW+Dl3hsmvyc7nijU5xrSmtS7dg6m2mm0usyM9Mc5WsDO0OqzPYWGAzYW8tiVDmibmXpfqKi2v0v60NMURmxcrfD8SG2buOR4+Vtvp4BCNYxFbR5DQAwICAnYJtl1Cz6lhbQiVVBnPrHQY9RyzIVRFlkhdW9uWr5DzY8jUb3QZSQkdzrwrVc2vepV+RS+dfy1vG99P0tJHPvA9eduh204BAP78xScBAAtdzewTZcOGzEn9xk7HSNwiucrlbRYfS/62DylqkJoMwLWIjLQldMKZ4VqNi8J/YrI2JYTQ9ckclHH49Q7bWa5PCgDdlDM0mVfFhru1F+nY9DmVuMXhOXtZaWinz5PkPDdJtL+L89p/h0Mp7ThEi4mrKgEOM/eHOKVKJZWME1l7S6nbUil5LVKRRM1araa0Z2YjlfYuMCXxAycoJO/ggZH8WNaiOSwtaIjdXua7sULt1WmSx+rM+TJR0f5/4p9QZmvR1K584zwVXli+auq6smN6lDMS6zUTpsfhcWMmxDNOaJ07FQ25Xcpo7C1Pz5Llg5H9VDJcSQXhfsk2zm6MTJZ2ygELtkZtHAtNtt6XGnOiCOWxrR1cLg/weIzm7kTLNM8XO08vc5GMw+OqUc5MUVjoorkHtT20bnuOqLTcYIenZ3pbB52LT6VAjT5fxTJnW3cNv5AEXXABmYLJ8q3EZB2w77gkVe15qwgSekBAQMAuwfUUuDgC4HcB7AeZ1h7z3v+Gc24MwB8AOA7gHIAf997Pb9TPxv0zu5uNZ4Kwxxn4tYlF+hPbZd6OxoxanxIutTZiik10+NdWuF+6HZU4agO0FAVjPbx4lmyit92m9thxlrKEhyXt84tcqap0GEkYopEO11bH6y3tJcZJKy3TmfEmJcBiIw0lPKbEhHyJgNsxvCAdtmPL2kduvaYAI1G1mDvlwvQ5PY0LS4gkFRkpbuYiSd5P/q//o3PheSWmHNcS37c29182kneV/RztlmFPZFtx3fg2Em4rSdKWWSspYzc0qP3lFA19AAAf30lEQVQ2G5tYKkUyNxK6aAUd4/NZbJFUuNgkae4ffuzD+bESS8aZSWxrs6ZgbdFTUyQtH+LwtVPHTLLKIEmO//tPvpy3vXbmFQBAVNDyboOjVMrtQJvs8HtS1RQ6/NwcO6Cl8GYW6bsSkgcAnpOpOh1O6iuYvcB7PTE+Bc/a3GZF6TotU+KOw1oHSnoPJOFmyRYeqXCRGI4NzDLD3sl+rizVPSbdZU3to+1Ikk9YK1he0ZDXmPl0pqdUwxkcp2vWxkxYYZcL0zB7Yskk7mkGo02GFEeUKRKT0XlLCxxKaxLEBkq0vq2m7o+KWZut4nok9ATAz3vv7wbwXgA/45y7B8CnADzhvT8F4An+OyAgICBgm3DNF7r3ftJ7/23+vAzgJQCHAHwcwOf4tM8B+JHv1iADAgICAq6Nt+QUdc4dB/AQgG8C2Oe9nwTope+cm9jkqxui3RJnhqWRZBrQov7eFFmFzkOXTKhfwplmzSnlwRhKOZTR1EbscNxQuUyqjYtVrewmohIaEw2Tz6/OqCVpgZ1WwtVRNqqpl7gkE/MlWXyRob5tsxqedKU+qnVUspPY6LJFrm7fQ7+5BiVjYti3l9T216Y0jE4Od0wWZsbhkx3mS8kSow7LWE1d0ouvne85BgAHjh4GAAwNk8nAquBdDj1rL6npp8rmlHZb+43YbDO0h8wCY+P782MxX+3iuXPaMV9kYMAUF2F63dyhbkZSMyYwQavVXNem3XOYHoxJQjIjDQfNlTnaF1Kndaj6YH7s/fdTjdqqcerJHlheMeYediZfukjze+CBu/JDr56hrNE9o2pCue12ypJ85lm9t5fYbHPuArXd/87782O330u8ILWirkGXn435ptb3HOTs2IExMmOtJGoKmOPnoGOKPKSRFH7YxAzo1cnY5XDZRkdNI17MkV3towW6L20ONigWjeOxRHssimw2Zofb9F3R9GK6I1PbkjEzLjsyLR0/dShvW+2ucB96XswPoDzSBUMtLfu/XNKw55xO2BRWkXdVlQvJLMwpf0yDzy8XdY2suXKruG6nqHNuAMAfAvg57/3Stc4333vUOfe0c+7pRqNx7S8EBAQEBGwJ1yWhO+eKoJf573nv/4ibp5xzB1g6PwDgar/veu8fA/AYABw8eHBd7oqEH0Z9Qv2McI1ijSQCCUfLjDNydZFDv0zo437ms6jFxhnJTqmMKc7KZZVahJNi1ZRQi8v06zm7oL+sF1nya6eS+KASepGr0NvkAvk9N9PLJcamOHjNuDssaRZM2F3GY+sahr21d+7yhQv55yMHiVXw3MvKC+LYaZN1TYmsEjndlto0Z2eY8NrM6vfUn309b5s6R5JgyUh7F0YoEeudH3onz9OEtjHbY31InZdDg3RfpkzyULlGazh+4GDP92gcNLaBAQ2xa7ADrGsTuDKRqnm9rcaybz+frxqOZbhcCwmVLBZUcxJOkUXjfCuyw0wcrP/vb76ZH4s5BO17HnxH3jY8SBLmYEGTWh54iAqqXGG2SCkFBwDvfQ9xCR0wpdxm2cF2/4Nn87a/+uunAQDneQ8cP6yO1b0jJN37VDfg4CBpQgdNkk+xTG3RAK3VzJKuz9QC7YtzV/XZmOTnJCuolLoWZROOW2byoY5XzajNCV+jRgOR8F4X0R6r1lRrlNDcZlNDTqOIi66U1gc/ZBzmm5oSfkVmGK2bcosVdvzXjZM4Ye6ZotCaGi065X1UiVVDbHPIdKepfVT4/SHaiZ1LmyX5gtF3B/tokm8V15TQHYVgfAbAS977XzOHHgfwCH9+BMCXbng0AQEBAQFbxvVI6N8L4KcBPO+c+w63/VsAvwLgC865TwK4AODHvjtDDAgICAi4Hlzzhe69/0tsHG76oRsdgJgsCsbJKeaXyHgXyxy3KY6D2NDLxhxPPTGgKstohXlSzMilTKfEfVvqzxKbS7omblh8QNbkssAmhYgdtt4Q3wuXjCX9z+PmjcoWs4pXrdB51pRSZdXRGQeJjNOZzLS1+M63vp1/XpqnGFtv5leAxImbog3sVLy4SFwhLcN58eZ3yFzz6vOv5m1RIrH6aha4wrS2BXZU3fU+rclaq5GpxWXq+Mn43g6OqJq9whXhC3xfhgaVUjQeIpXa9WQSs3nCxIQX2OE4MELcH4WS3vhjx44DAMq2msAmymnEJj+bmRv1icXOpLIKmzOuLqlJ4smnSPZZXlYTw7vfSWapg/u1VukgO+gHpHCK6aPF2awHDqgJZXaBHJ/33nt73nb3XfR5mel7O4bCtSnedZOlKHw+q4sanz2xh+bQukL0x7eNq5nntjEa222HlFfl6y+QyezKysZ7Mm2bPR9JrVJFgTNgu6masQoRZ31zXH6aWNpfdt0Z52WVzXWtVTW/RWDzxwo5Vq9cUpff2IPyvtHnoMLO00pZ93WDzX6ZPN/GVCTmvMy8g6TubznWd5BwNBWc5KJo/57fbZYCemFRgzq2ipApGhAQELBLsO1cLuLMsEyCIrWXDOmFhDKm4jA1tG2DLHWOGbbFmKWrksm4lJJsOWOjkRe67JwtGQm20xSJR8fbYimvKHwisXXm0v8muVKS7FCv6S+3/GJ3OVwrMZLxYJ0kk8Ssh+cstay7MdfDydtUYnviT6ggxx5TtV64UGzpsm9/i5xpL71KBSDGxtXJMzV5la+p92CIJe5CxZSg4/W6yA7Y2og6L+usAbRWTIk91hrikkrtY8zDcvudd9J1hnTcErmaGG1jniUZm2E7PkFRs+94+GEAwMioOh4PHSFmwlJJJfTCJiXoMnZ4Z944TsVZaeJJI5Z60zYNcsGsbSROskydl7MN6uPOkypx33kbhSHuGaY5l42zrsUZlLa04u130vmthnIIrS7R5+V50iSHjQN5iJ3rKw3DUCjMi17XYP4qfbe5TBpCe0FjHI6cuBcAcHjvHXnbyQPE+nj1ZeWqWQtnQv3AzINFb/hdch4YlVIrzIkSZSIhG2fnGkZIQEN+vQlEkIAJrh2CNFMNoORJI1qZ05DNiLfbqnkfiOCcsWYxUDYh1Mzrknh9HmPeqGUjI4umnnFopzMBCWD2RhsOmSUbhyVfL4KEHhAQELBLEF7oAQEBAbsE225ykYy+1DhyJEbY1oXMz2dzyeqCOjqGhMwrsioLZwwWdYqSXZrHvhvbSCbqn1FDnZxnxpZxPLKEnxeMqufZkVk015TY50pF1cRSgT6vrNCxWsVki/G4B2pq1pDxZn3WQzA+vnddm43d9uKYMaacV18lh+eVsxS/PHNRx1HiLNqRmpo/ykzU70x88fAImUtmL1MfF06fy4+NjZAZpDagcxFTR2TMaffdfTcA4OQdZHIxnFiQ+1g05rQRpju1tUePcI3Qex8gp2ytpnMXh7d1Vtu8h7VwuTPeZCSyOpwZJ12Z90rS7jW9AOpYjY2VrHGBnJArTd27F5nm+cRhyh04uk/NMeOjFBuepDr3Ii9baUhNeDV2AFeZ0OqNN87nxxbYDDNx6N68LS4fp373qHlnZYbMNi4mR2xiggNSdlSeM3U4XzpDZq/llq4p1GJHY6waJ2ODY7Erek3naV+YtIa80EyerWtMNGLLtHU725wFWixrjHd9kMm5ymzKNPTDXSZGa6zoPagM0wBqpg9xeLb4fhfNIDtigzUBFxGbUGyGsmfnrGQvp+b1VIyECljNQVIQ40YQJPSAgICAXYJtl9AlrK9ctmXs2VlonAQRS7ievYzdljoSc1pZ+4spdK49IYT8y84/Y21TBg0cnlSp6W/cKNOXxkZCusr0BfNS0s0U2hDa3AHj2JKsRpiwO6G3FYdOzUjjKUvQDZOxKt/sqbC+xqe3vKB8M8vL65kZjh0nx+CyoV+4eImk6irTdsYmc1ZododGdC5S9CAxZewk+63MVLaGJRieCxHEZR3sKIcrFst6X47y2Kq8ByKzVqJVDQ+rk/P4SaoWb8sXDg7S9at1knwKpuBHzJqTdYQ6t5kssz67F7yfisYRW+Dt4z2tm3dm/Vjjm2vqHi7ztKrm9qwy9fMkO6HPDKqYexdXtz9yRKvcDw3xXkltSC9T2dbp+iePKU/Jm1OkAXTMPZtboD0wMqTXOnCYOHlqwlnj9UbOcqGSb76gkv/pSZag6xtTOEWxPl+OtWfraI7Z4dlt6LMsty1lut00NY5m1uq65r0gRSbseZJpXOCQxpWOanL7SqTJjo4pnXDbFC3J++Cwxoy/a98V7SbfSPP2dPxAdsx5orzLVut2rZOYwycjEywRBwk9ICAgIIARXugBAQEBuwTbbnIRp1enoyqexKGn3jqlOFuSK91kJnY75UDdRVMJJpV48oJxhIlZh1U3S/3Z4f47y9pvpUJqfK2sqtA4m0tmOEuwaUwMklTWMBVYxKTU7akdSF9a4WolBTN3iZVeMbS1eWarLZm+prhJ0tFxSybbc889l7fNzFJV8oqJh2+tkFmnwc7FzJh5ijyOODYmqzy7Tq/VYRKqQ+yULJrs3kUmkuoaU5FYLPbt25e3SUWhGlPrxobgK+a9UDRx62I6KZicAVkj+d9mAkZr8g/Wfl6LNgchG26unPq2be6jpB7L/sucjjERx52NPeZLXppU+twKm4NG2WTkTIz1t146DQB44aySrJ04Tut88qg6T0fE7FUXp7Wu34kq0+G2TG3Okpj11PbTXCanaMLV6JuZmg5eO0+Zxxcu633PEjKBpasb50a0W7qAUivUm2c64Xh4Z+RKCYhwRcnV0P4q7JxNjXcxLoo5zdS+5Wcu4hyRsX0aMFBhk5XNPO46moM1q5SjOo+XzrMOeJdRH62Gnu+5OlFq8jbymqlsPuqaCmIxx7UX7D70Ny5fBwk9ICAgYJdg2yV0cXR0zE/xABPHd41002TnYlEqaRsJtstSzbyRHFGlX1GbKTrE4UwlkRitdM0heQvG4bLKHpohkyVWbpOUMMrXXLLhlnmNTm2Tz21D9+sK63lVBCLT1GrqjMz78BtLQ9bhV+aiGotTmg330guUDWql33sfoEIIEs735ptv5scGmdK0WNUQxZjJ+AeNxDM3T5J/XaREs94ioWdmPRqrdB+rhip0hB2lUpdUioIAqrEUTSEAkZqsC2mthG4lcCkacr0SunQc+fXZpKnVtITXR7QHUyAk4vtRMpwy4geOU13TFaYHbknNVFvYhJ8NKaYCAG9OE4XxhcvKtXKIM20Pc7bs3lG9P8UyPQfDRR33QI0cgu2OhnZ283qntPYzRnqfnGJNbknrmNZqnAW5Sahd0jWFW/h/mxHeblK/Npw0Ya3H5YVK9D41VuiZt9TIsme8eVekcv8knHPYZJayFNwx0nKDv5uYlPDViDNmW/Tcdk1RjcGajNHW25XvmhvIz0KH723BGZ4oDseMnN6Dnso4W0SQ0AMCAgJ2CbZdQq+yBGhta1K4oGNsZWJDLbEk3zYsZYUqk9abKvADbHutmgIUVU7MGeREk3bHhseRZNI19u/zHP43MKihWXGdpMmHjhF3ysq51/JjFxcoRKxiwv8k9LJopHEJD/QshViekg4T3ztnQ+zoV99KMmsxOjpqPpMENjujlc0TljDKhmT/3vvvAwDcz2N75ZWX82NT/N3E2O0z5jM5eFDD4sTu3WRpq21KcLlovbQskuvRo8fyNik2IPwulaplrOOSg7Ymn5HQtGn9tfRY3HPOtSBJQTY00HHiSGT0giyV+8c+GRtLytJ9xzDyddnvU3OmMAeLk+JnmJxUKVj4gspGW4s5FPDiFdW+zr1Oe3CEzztg/BNHj1CI5969KrUPDNB59bqVDhnsG+ouWkZDnpcJZVxZJj+AS+2e7H2dVGJTgo59X52OYQDlbm1yXtYW2zL3aDQLOPqutyHAiSQi6dq3eEylOr8zLNMkJwp1jXqesaRteYsaXCQj4n1XqmofjQ75G0rm3ZJxyT5npGxhX/VcEs/WvYk4ZDMzhUfKlRt/HQcJPSAgIGCXILzQAwICAnYJrinjO+cqAL4OoMznf9F7/8vOuRMAPg9gDMC3Afy095Zv9PqQq7LGuSLV1gtFVaMq7BTLlrnKt+E/iTkjrGayD+vs5Kyb2pwFdmI1OJwpM2aQaQ5XvNLS0L2Ep1NaUNXqDs6oGxons0Z5Uh2JUoSjbELscgepyUwUR47UL7UOTeF+seyu0mZrYu7XxEn6e//+/PO73/MeAMDKis6lwSaR++5XTo8hrru6wo7K4RE12yyyA8qqyAmHV55+8bTOmcPu7mfzTWqcTWlK1LGJMZccOETmmjvv0ur29TrT8nKIZMGESsase/daS9ya//uZXAzNbRT1nAP0qu1rIXw6mQmDdezMyzl/qBM6X4oaWGpdcZgakanAIbSRMeVEEnbHppxSUb9Qkk1g9ukyZ1Va0xY4S7LB2dMXjTP8qeefpb7MnjzEJjO5ZwBwB/PoDPCzdHJCwyL/6nm6j3FRxz3CnCwtw3ezFnEfqlz7wpH70k2s+YO5UyTb0/CwSLiireMrWdeSQQ4Akti9yibCzDimS5xV6yvaVmeelswEDFT4uc2Y06ZYsgVWmKMl0XtQY1pt4UACNDw54mzQbk+mt7z3zF6Ibo7JpQ3gg977BwG8A8BHnXPvBfCrAH7de38KwDyAT97waAICAgICtozrKUHnAUiZ7SL/8wA+COCfcvvnAPw7AL/1VgcgrG4VI3EXWOpbMVKISLUJMxlW96uTJ2EpLjGcISk71lbMDFsdklKXmsJyqE6hBZYs2yXjoMyov0tNHUeBiwmce4mSdqYX1fGYj8f8EqskaB05wuWy3smjIY/6WyuS+WaV6i0fzAPMOFgxhSiW2Yl1+LA6NCUMcaVBnC5Nw48jrJaW+lAcvM1VU3qLE0b2sCN2nwlp3L+fpLyukVyPnzgBAJiYUMedSI8RO0ALJrlGSnU5I4EJD0s/x3H/sMWNHab9ELHUmdkSdPmldD2yTHhEZIzaf+7PN2F6dXbG23A+z1JckfewN1pBwsQ4rmTO57nYkF5xyC3zeIvW+8aOTN9YyZvmeC9cuKLsiRcm6fN9t5MT9dBx3ScDQ5zstqx8QaUCaXdDo1ZVXLM/I32mRRM39WAQFXgOqZV+mSOJuWeclVqZM8fk0KHJmX0mfxBRSQInqA9v+k/bnHhmiEuFUybLjFrc5D1TlIAEWwxH4iLNZMTx6tUiIFaFNmsnLrIBA6IFGo6Y9k3icnHOFbhA9FUAXwXwGoAF73O390UAhzb47qPOuaedc083DDFUQEBAQMDbi+t6oXvvU+/9OwAcBvAeAHf3O22D7z7mvX+X9/5dVooMCAgICHh78Zas8N77BefcXwB4L4AR51zMUvphAJe3MoAum1wi4+hoiepvfiISNju0WJeVGFMAWBQVecDwd/Bxq9KssslglePPh22WJ6tArmJUdalrWNUfonMLlBmZcSxxYhxcElO6ajhRxDlmeUeE9VXUy2bTOhJJ6RkY0PlJ/HmPI2wNrMNUHILHjh011+Taizasl01b4+xQnZrR+pCFRS54YJw24tSDMX8IHe4wO1htPdD9+ymbsWAKc0gWq+WIEXNaXJTzTC3Z3Jyx3snZM5ecw6Ww7vzNzDD9UGIHfYr1jrACrBOLP3iJnbZOV56T6UPMKd6o9uJ4zST71WTaZvzdTkPvrRSCqZgAgAoXZnBMJpQmhrZWmF5NgIGsb9uYbb7z3IsAgFdeJFPi2PhIfmxqkfbixITeW0guRWFjp6jlL2rys1cwwQ8Rm0JsoRmh0W52WJs3RSfErGFpk+v8bHYTzRxPmfq5zgJke0WPeckFGFNzq2NipMaSyaYtDvN4aWxFky2eMc9Np6P3xXFWb2QDHDigoMSm3WZLs2/TlMZh40ii68yT2AzXlNCdc+POuRH+XAXwYQAvAfgagB/l0x4B8KUbHk1AQEBAwJZxPRL6AQCfc+SBigB8wXv/ZefcaQCfd879ewDPAPjM1oYgrGqW/0TKjpVNGx2POLM0MmFYIvlk5vxFoWsxgmtXJCMultEykoEvUH9Vy+rHDsHEVCWXhLQOc8oYIR9Fljq7hrVNHFalvgUVOBPQaCdNJs/vGu4XkSyr1TUUiwbeSLUipdp+c6nXOCiF3G3/AXJeWslHDl6e1BC4BvPnlIx0eA/zwRySAgkm+1Ck8chIhyKNWzbEnD2R/8+MZBfxmvY4NN36uRRYsu0XorjuexsdF7BW1yONQxjzjOOPJUxh2Yx6tDV2aBptBrm2sT6jVPZ8wY5LwiLNve20acMtmaIQi+zKEsdtzWhE8mzY2cYcWNDDDMi3NOPJZG45P7bSon1XrWpRiJSdnGnBECKhd00tK2KJL1A13ECSaZuasEXJtBTHsUkgzx2qVVPSUDSnUqL9Ro5CKov8hahiOVL4s7mNPuNnvq0OzYzL40nJuk7Dao38nJsQ3RJnedrn1jkaU4sdtykM82uJ2gZHtN+qJgRvGdcT5fIcgIf6tL8OsqcHBAQEBNwCCJmiAQEBAbsE207OJWqadXCJOu4z65BjtZzVSRvrLUpLbGJWm0xZ2UsuxZXYy1xUw1oYuD9nSHsKHNduScK63d4Y18hZZyc7eI1jxEMyDG0cda9ZxWbxickgMWqomJs2M7lYB3L/zMj86trm5Hyaw9EjSpg1xLUtL19WX/f0LMXc26zU+znmvcoEW4WSUSvzWp7GWe36jY0Gl8/Zr3eA2jaH9eYSacn6OFH7WVeyTUwusayfcXLmZgFr+RGTiNjd7H7lvRZl652iXZtduSY2zBZdkezior0of0xtUQgeWsb2xcaKLcYg8c46NjG5tI25RPZgh01iDVOHs8HmxaSr5HeFmJ+XeD3FsF5bnfhDQ7Q/osjua6aXtQ7vRIpCUL+WRlrmZ+cSSSZx0TotuZ7rIjlWYxN0XmDTaqvVMG00pra5L+USxdd7IV6zmexcTLbHUeo4E9bSJTN1cVzl94fpo9lmgi9jHtu8zu31IUjoAQEBAbsE2y6hn6jdvrUv9hMMjOCTc9pvzDh77f76JWbKT+AmwvJ1o3SdbdeBV04/c0ND2Qh1s0Pq+ziUzasz6NVn//a7ct3txPv39XssrncjBQhefeaOG+5j4ezbMJBbGPPXPuUtIUjoAQEBAbsE4YUeEBAQsEsQXugBAQEBuwThhR4QEBCwS+A2I/p/2y/m3DSAVQDrOWd3FvZiZ89hp48f2Plz2OnjB3b+HHbS+I9578evddJNfaEDgHPuae/9u27qRd9m7PQ57PTxAzt/Djt9/MDOn8NOH38/BJNLQEBAwC5BeKEHBAQE7BJsxwv9sW245tuNnT6HnT5+YOfPYaePH9j5c9jp41+Hm25DDwgICAj47iCYXAICAgJ2CW7qC90591Hn3CvOubPOuU/dzGtvBc65I865rznnXnLOveic+1luH3POfdU59yr/P3qtvrYTXOT7Gefcl/nvE865b/L4/8A5t0UGmZsD59yIc+6LzrmX+V68bwfeg3/Ne+gF59zvO+cqt/J9cM79jnPuqnPuBdPWd80d4T/zc/2cc+7h7Ru5YoM5/AfeR8855/5YqrHxsV/kObzinPvI9oz6xnDTXuhc8eg3AXwMwD0AftI5d8/Nuv4WkQD4ee/93aA6qj/DY/4UgCe896cAPMF/38r4WVDZQMGvAvh1Hv88gE9uy6iuH78B4E+993cBeBA0lx1zD5xzhwD8KwDv8t7fB6KC+wRu7fvwWQAfXdO20Zp/DMAp/vcogN+6SWO8Fj6L9XP4KoD7vPcPADgD4BcBgJ/rTwC4l7/zX1xPaamdgZspob8HwFnv/eueKqN+HsDHb+L13zK895Pe+2/z52XQi+QQaNyf49M+B+BHtmeE14Zz7jCAfwTgt/lvB+CDAL7Ip9zq4x8C8APgEofe+473fgE76B4wYgBV51wMoAZgErfwffDefx3A3Jrmjdb84wB+1xP+BlRA/sDNGenG6DcH7/2fc2F7APgbUIF7gObwee9923v/BoCz2IEV2W7mC/0QgDfN3xe5bUfAOXccVIrvmwD2ee8nAXrpA5jYvpFdE/8JwL9BXkwRewAsmE19q9+HkwCmAfw3Nhv9tnOujh10D7z3lwD8RwAXQC/yRQDfws66D8DGa75Tn+1/AeBP+PNOnUMPbuYLvV+JmB0RYuOcGwDwhwB+znu/tN3juV44534IwFXv/bdsc59Tb+X7EAN4GMBvee8fAlFH3LLmlX5gW/PHAZwAcBBAHWSmWItb+T5shp22p+Cc+yWQSfX3pKnPabf0HPrhZr7QLwI4Yv4+DODyBufeMnDOFUEv89/z3v8RN0+JSsn/X92u8V0D3wvgh51z50Amrg+CJPYRVv2BW/8+XARw0Xv/Tf77i6AX/E65BwDwYQBveO+nvfddAH8E4P3YWfcB2HjNd9Sz7Zx7BMAPAfgpr3HbO2oOG+FmvtCfAnCKPfslkAPi8Zt4/bcMtjd/BsBL3vtfM4ceB/AIf34EwJdu9tiuB977X/TeH/beHwet9//13v8UgK8B+FE+7ZYdPwB4768AeNM5dyc3fQjAaeyQe8C4AOC9zrka7ymZw465D4yN1vxxAP+Mo13eC2BRTDO3GpxzHwXwCwB+2HvfMIceB/AJ51zZOXcC5ODdeeW4vPc37R+AHwR5ll8D8Es389pbHO/3gdSu5wB8h//9IMgO/QSAV/n/se0e63XM5QMAvsyfT4I261kA/wNAebvHd42xvwPA03wf/ieA0Z12DwB8GsDLAF4A8N8BlG/l+wDg90H2/i5Iev3kRmsOMlf8Jj/Xz4OieW7VOZwF2crlef6v5vxf4jm8AuBj2z3+rfwLmaIBAQEBuwQhUzQgICBglyC80AMCAgJ2CcILPSAgIGCXILzQAwICAnYJwgs9ICAgYJcgvNADAgICdgnCCz0gICBglyC80AMCAgJ2Cf4/Ir6TMNR24eMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# show images\n",
    "imshow(torchvision.utils.make_grid(images))\n",
    "# print labels\n",
    "print(' '.join('%5s' % classes[labels[j]] for j in range(4)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![This is why we put grad to zero before passing a mini batch](why grad zero.PNG)   \n",
    "\n",
    "**From the fast.ai forum ** [link](https://discuss.pytorch.org/t/why-do-we-need-to-set-the-gradients-manually-to-zero-in-pytorch/4903/9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net(\n",
      "  (conv1): Conv2d(3, 6, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (fc1): Linear(in_features=400, out_features=120, bias=True)\n",
      "  (fc2): Linear(in_features=120, out_features=84, bias=True)\n",
      "  (fc3): Linear(in_features=84, out_features=10, bias=True)\n",
      ")\n",
      "[1 ,  2000] loss 2.237\n",
      "[1 ,  4000] loss 1.890\n",
      "[1 ,  6000] loss 1.702\n",
      "[1 ,  8000] loss 1.555\n",
      "[1 , 10000] loss 1.502\n",
      "[1 , 12000] loss 1.473\n",
      "[2 ,  2000] loss 1.390\n",
      "[2 ,  4000] loss 1.348\n",
      "[2 ,  6000] loss 1.350\n",
      "[2 ,  8000] loss 1.346\n",
      "[2 , 10000] loss 1.332\n",
      "[2 , 12000] loss 1.286\n",
      "Time taken is  74.38872599601746  seconds\n"
     ]
    }
   ],
   "source": [
    "class Net(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(Net , self ).__init__()\n",
    "\n",
    "        ## Kernesl for convolution\n",
    "        self.conv1 = nn.Conv2d(3 , 6 , 5) ## where 1 is input number of channels 6 is output and 5 is filter size\n",
    "        self.conv2 = nn.Conv2d(6 , 16 , 5)\n",
    "        self.fc1 = nn.Linear(16 * 5 * 5 , 120) ## How is the in feature determined. Is it precalcylated ?\n",
    "        self.fc2 = nn.Linear(120 , 84)\n",
    "        self.fc3 = nn.Linear(84 , 10)\n",
    "    \n",
    "    def calc_feature_count(self , x):\n",
    "        size = x.size()[1:] ##dont take the batch size geez\n",
    "        num = 1\n",
    "        for s in size:\n",
    "            num *= s\n",
    "        return num\n",
    "    \n",
    "    def forward(self , x):\n",
    "        x = self.conv1(x)\n",
    "        x = F.relu(x)\n",
    "        x = F.max_pool2d(x, (2,2))\n",
    "        x = self.conv2(x)\n",
    "        x = F.relu(x)\n",
    "        x = F.max_pool2d(x, (2,2))\n",
    "        x = x.view(-1 , self.calc_feature_count(x))\n",
    "        x = self.fc1(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.fc2(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "    \n",
    "\n",
    "net = Net()\n",
    "print(net)\n",
    "\n",
    "\n",
    "import time\n",
    "\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)\n",
    "\n",
    "beg = time.time()\n",
    "for epoch in range(2):\n",
    "    running_loss = 0.0\n",
    "    for i , data in enumerate(trainloader , 0):\n",
    "        inputs , labels = data\n",
    "        optimizer.zero_grad()\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs , labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        running_loss += loss.item()\n",
    "        if i % 2000 == 1999: # print every 2000 mini-batches\n",
    "            print(\"[%d , %5d] loss %.3f\" % (epoch +1 , i+1 , running_loss/2000))\n",
    "            running_loss = 0.0\n",
    "beg = time.time() - beg\n",
    "print(\"Time taken is \" , beg , \" seconds\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This training was done on CPU mark that it took a lot of time. **LOOOOTTTT OFFF TIME**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12500"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(list(enumerate(trainloader)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GroundTruth:    cat  ship  ship plane\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAB6CAYAAACvHqiXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztfWmMHdl13ner6u2vX+/d7ObOITm7NDMajSRblmXJTkayLRmJ7Mgx7EGiYIDAQuzAQCzHPxwB+WEjgR0HcBQMLFmyY1hWJNlSZMWRPFq9jDScVZrhcBmuTTa72Xv321/VzY9zbp3TG9lkU2x2+34A0cVb9aruvXWr6pzzncVYa+Hh4eHhsf0RbHUHPDw8PDxuDfwL3cPDw2OHwL/QPTw8PHYI/Avdw8PDY4fAv9A9PDw8dgj8C93Dw8Njh8C/0D08PDx2CDb1QjfGPG6MOWGMOW2M+cit6pSHh4eHx43D3GxgkTEmBHASwE8AGAPwLICft9a+euu65+Hh4eGxUUSb+O1jAE5ba88AgDHm0wDeD2DdF3qxWLQ9PT2buKSHh4fHPz6Mj49PWWsHr3fcZl7ouwFcVP8fA/CWa/2gp6cHTz755CYu6eHh4fGPDx/96EfPb+S4zdjQzRptq+w3xpgnjTHHjDHHarXaJi7n4eHh4XEtbOaFPgZgr/r/HgCXVx5krX3KWvuotfbRYrG4ict5eHh4eFwLm3mhPwvgiDHmoDEmC+CDAL54a7rl4eHh4XGjuGkburW2Y4z5MID/ByAE8Alr7Ss3ep79818AABibpG3ZDHXLBPK9abWaAIBO3KZjstl0X5zQb20iFh8TxACAIFR9bpdoH2hfJttI94Vw15RzxEkHANDuSN+ShC1NJuL+iOWpyfu0LSrhcRkjra0WjSGOo1VjD7hvrUTaqtQN1Fpx2la67wlofPjDH063O53OqmveCtzw+eyKv7op0G3UGrhGbbgzbv4SdbybZznJtby11uq3O/5jH/vYqn37f5TnNu6kbdNXrwAAmg1ZM4fuOgwA6OmuAAAyofQnm6GFl9VtvJ4jo9ZYpw4AKJcyfA7pa8TboVrEs7MzAICurq60LZPJ8HnpOBPIOTpJCwAQrCG6BUYaa1Uyh0YRrcl8Pp/ua7XoHB1+BgGgkC/wtaRvv/+7v7Ps/Hv2DqXb5YGj9LtQnttKVxkAsNiUdV1dmOb+0v1O1GKIeBCFKJe25UN+hannNn0AuSlO5PyuLVFt7hpu7HR9nss11o7h+2cC/V6I1ziOfpvLUX+zgfQblrZNVuavNn0cAPD1Z76/6lwbxWZIUVhrvwzgy5s5h4eHh4fHrcGmXui3Ai2WsqytSyNLpzmU0qYA9CWLIpa8tcTBX12TkcamkyoS+QJGLAGG3BSpc5iEpGZ0RApx0nKiztEyJLnEIX1hW3pfHPC55GttWMrPq75FLBkFEXU8brdVRzo8JDmHk0jDcH0LWRiG6+67VbhZiV/PRypHKSkycSKV5TFY2ec0JgORhuQsm5fQ10K5SPc2sPJ4NKvUlrSE2M9n6bylAh0Xqcu4tZNTi6yQ5fuuxtKM3XG0rrJqnbgpiiK5t07yD5SU7+Ymx1qrXibVWpuvKXDarYWcN+CLZVhKdVI/ALSbTR6fGgtLnbjGmkisSPmdsJfOlZFnOg5JQg8ySkKvL1Hf4ir3Q87XtHRcW0nGDZ5fJbSj1SYtKuBnol6Td4t7TvT4nMYcBPIcWqfZ8GRqi0CnE/Mxck1j3PtJ1kxvL405V+ji88s9S9y6zkk/4qUyNgsf+u/h4eGxQ+Bf6B4eHh47BFtucrFskoAVU4dlMsrEohImbVKBwgKbNZTa6qwNmpjIskrVsaLSJO1w2XFOdQIAY1cQcwAMEzg2FNWxHpNud2Wa1LNqS9SopSVqC62ctyvP5Jgi9SpFIpQKORpnErTSfUFqXpGxuxG0k/XNBNqE8IOqE7uR8y4zb7jjl+mmbpc2EdGcN9s0H5HWs2P6bWjWunayRtvGcK2xRGz2CpTZKxvStTKBtOUCNqe5fYrQbNbJNBOGisCL6L63m0KsBmATW4farJFHMmbTUjZTkOPdPKg15sjhmM2GOt5j+upVAMDwQK8cz+aVMCvXCvlabp6V5QcRH99UJLEjbNttaVuJwMq+mPsbq+cgNjTmfJf0o3//MP12fhYAUK4tpftaDXpHxGV5HpNuijzvysrcu+sGbJdtNeX5cg4U+bzcl3RK1Zpw69j9DZSNt8NjTvTy48tnI1m7hQITx3BmQzHpJM6cq2XqW+DE4CV0Dw8Pjx2CLZfQo5gl81C+jgFLGrlQff0d48RfykAzP/zTjpZgHcmTFelm14G7AQALc1MAgKlpkWQyEUnjAeTL3erQ9NStBEQdP08Sj831AwDaoZA8LZYcluZn0rZLEyxp5JXkNT4HANi3i67Z36WlOOfKKGN3wkdsV7tGOWjJ+Fa4K94SKT/tt9Ie2LWzo8SbNmtKp86cAQAM7xJ3t4TJ7cE+kTDzTCQlm+jjteYoy1J40hHJLmTpKqMIuQy3BTGto2xGSX0hu8Yq7SsT0L1NjNLIEnbHbTA5qtZTg8deLMoaDh1TqsVDnocqu1Q+99zz6a42awq9lTenbbkcOweoKUhdZ1l7DZS7oLHOOUDWpE0cMbi+hN6BuFYGoLWehIoQZi0tVNpaidnNSpHv8fPPpvtaUyStjzxwt/TtKj1zTSPzVuaBLdaJWM2rseRYYw/6hYAMmBTVr5Rmkc4btVlzactkLZbovuTm59O2aO99AIBaT3falrDWFfM9yydCrKYWgVjawnjz8rWX0D08PDx2CPwL3cPDw2OHYMtNLk4vN5Gk1XXqcEdHUDIB1WI1OKvIpjh26p8ySfA5tF/vW378JwAAz/39PwAALrPpBQCqHRf5KarY+bFJAMDZsUtpW653BACwZ/ggXTMnamWL1cVMWbJcdhqkJk5PSpqbYi+Za8aWKPqwodTn4S5SCYsZUUPjNqnNOhhuJR24Fil6OyJFr22aYfIto6J62ce8viQk+Nw8qcYTU2SqKnSJ+tzPEZE6qtGRgDp6dI3OrujFxpFl855V58i4yY+l3yEceU9tGeXX3XbqdiLnCCs0D8aquAP2d05cNHIs63ppgUxz5aKQgAHPt47ajDiyeo7J0JkFMSUW2E+7pSwjrTZdK8rqNUNtMUdid5S5yUVpZ5WPteU1m8TrmwH1zDsTYqDGHnd4rMrWYdgk0jB03zOJrAUzQKa42qL0rX32JPXXiFkq4emqOv929Xxl2xw/clGR8jwf2tGiwebTsMFzJZdEcxf1sX5FTKtdhp550z0g4+PrtgNHNKvYC57vUJHsUbB5M6eX0D08PDx2CLZcQm8G9CWer6kIMpZuessiVlSYZIpYQtGEVep2pAgaR5rWarNp29e+RHljJuZI4phYku/Z+Ut03PnLkuI9zJO0HoeVtK1UoS9xpkj7orxIBjmWIvOBjGWqRVFqI3v2pW0NJmvOnCEJfWZO5ZTZTec9MCiaQoZd94xyGxP5jMervv42uTGZNA3MXENA0FJ5sIaEHrMUlrA0oqNZXQTe1emFtG2hSmOt6/wdNRpNkCPyuVqXe1suskSq+ubk/Y0qIDeqqeSMc7GT+XZk6JouhwlHJiqXw4g1ykgxj6Gh+bCxvns8PnYEiJVr29IizdsFfc3IRVaLNLm3QvPmXBRfevnldN8b7r8fAJBol8qY5jevXXpZU6jXWAOO5Pwd1hDDSJwD2pwvqNlcPyV2rKT3hNew1TIkOzG0tHsjX7d7kedqcDjdVxjaT/2xQkaCXS/twK60qZ7h3CxXKC8MlAtwlZ9XO9yftmUS6lNDafgl1hJbizS+ps6xU+CI3Krcl6iftAeTUW6ZnK+li38aKg2gY2juTaBcdLH5aG8voXt4eHjsEPgXuoeHh8cOwZabXK7WSc2YaQsp+s2/+wYA4L6jYrr4sfuJbOhlf3VNxrgkPIFSX2ImXxSXhrPnyc95pk6qkC32pfvCMpNvfWIeKHD905ZKmdpiIq7SS32rlKWPk1fIhLIwq8gSVgnzBTHNXJglMjZTIXVyclyqS5WvLAIAdlXk+IJL1ZsoMm0FqjWd3IxVTqVqutTCoUr05LZdOlCVEwtBsvpb76JYta1jic0BjhwtKOKswRF148rkMjlL24kizNpsT6ktEoE8OSXzN3ZpHABw35FDadtdB/ZQ/5VffkrOukhfbWVx3dZhCtegSkM2+SVtMScEbOKrz8tYwOYGy0mdwoKMPcv3Kqvm27TJ1BZrMwVHQ5uUiBVzU7VKpoWJCTm+VCnzNVViMp7z1hIdl1f+8FfniFh9/vtihinl6JqHD8mcRmz6adZo/RUilUiqSWsrVmmkY/eoNdR8rISaYpfCNlkWK8L71LOcYXNX7vQpOv1z3073dd7MpiqVhtZyjEh2UZ6NBmgeyhzvEebk+KRE5zdWEfWcHK+rX95BmUtsrlmiNZkZFucHXKR9UUXMoo2rNL9hUdqSo+Sb3uDEXoEi8bMdmpxI2RLtNTj+jcJL6B4eHh47BNeV0I0xnwDwUwAmrbUPcFsfgD8HcADAOQA/Z62dXe8c1+xAN0kJtWn5trSzRDzO1FTy9xa5EVWy7OaliBQnkYahkDaNFkm4VxX/NLVIX+diDxEivYNCVFYTkjQGoKLymEBpZURqalRJgmks0fH7FblSY2l8siXSsmFpaX5GSWUsrdT56x9mpd8TCzSN4/OiFewfYA3kGl/wuboMtFwkrSFQeSVcsY5lgrcja1wQ7rK0tWt869dwh7wyTi6dfX2k7RTyIvk0GzTmYk7adg2SpmWV+Fat0VhLLMm0GirdKQ96qSnj66R5NpQbXeo+6fatGuYyifFa3pZ5V8BAHeQk9JzSCspMPnczmRWw+yUA5Pge57VAylpU0JC1kBY94EIprQVZa10l2tfbJ5rk2THSAs9cvJK2nTz9NABgdook0qWGnKPWppozEZQbIkv+D959NG17308+DgDYzeu5mZdxNqpV/p1cs8IF6E19EeshE8r6c+mvHTkKSArZSMmV5Vm6VmeM3HwrSttYvEzXb+UlGtOC3gvmymTaVhplQrPCmifkWSqwu2x2TvrdYCK6MzWetmV5DjsLNFe5GXGMaNdZmyqIhjN3lpwpsgWR0LtGiMR1qaCsclFsOjJcreFWsnkRfSMS+icBPL6i7SMAnrbWHgHwNP/fw8PDw2MLcV0J3Vr7LWPMgRXN7wfwTt7+FIBvAPj1m+nA3W94DAAw9syJtK3cTV//x972lrStGJKducUSspY+DWeji63k++gaovrVL758Ss7bQ9Lh7v3kymWVLS7DUnjSnE7bWq1k1bVC/qK+8tJLAICKSlBfLJFkUFJ2tMtXJgAszzMTstTRx+5mc7Ni/5udoe2z4+KaNTpMLllRVkU3rEBUEU0hZum6revvsW0y/Quxa7pgFS2R2jV8GJ0Arzwk0wAXl+8DynW0h12/2m11LpbaimWxSToJ3XCwmFEuYrmCc+9SZdWYGFlmc1zVN7lmZvkhvHt9Ef3iuXPcb5nvxQVad3FbNIVLl0g7meU1UF0Se/JQP0nV5ZIEBYVcnKWlMhRGnGso4FxCVSW9N9xgVKGNC5eJfzk7JjxDtUW/zXez61xJJsatxFJWZLfx8xSMc/nyRNr27W//HQDgXuYqBntEIq0vkeTvysMBQPteyqeyNL++Yp7Lytitk9YTpTKzhhMoN9slDgRcevSNAIBK9KZ0X22R7kFb5X0yOZ4bVZ4xU6DrVtk9U7vbtjlfSkY9G3WeG+00WGe7fm2JrlkqyFgafHyuLM95Xxe9e2L1rljitQt2oyy0VcZG7pP2MG7fgvxJN2tDH7bWjgMA/x26zvEeHh4eHj9g/MBJUWPMk8aYY8aYYzpPs4eHh4fHrcXNui1OGGNGrLXjxpgRAJPrHWitfQrAUwAwOjq6SqcodpOpYP8hIWjqbIHYd/Bw2jbAavvc2XMAgLaOLuuQ6eKxd/xM2rbv0KMAgIMPnkvbnnuBzCS9ZTJhXJ6UXC4RuzHldHEF7u1SVciuuRlSO/vKGX0I9YPNKgODksvFFW2YmhUTiuFoyi52eYxCRYywyv36xbG0bbCX1PIje5Tr1Ap84o//l5yf+5FR6l+5i1TGwweFCH7zG8itypW9tMos5EhGq+0rLseOMqs4wi6bo/NrsjObJRNKf69yn3S1YVWNxjRHSIbO0ejI+eeYJJ5TqUoX58kE0Naumkxk9rPr2ZHDQlhlXDShLgwfLDPALMO3//4ZHq4qsOKI7LqshXNXiLhLa38q8ai3m0wWJUUS5/i4jHJljNilLuCaojVFaEZ8DqvyFl2ZISK9rdjtYpdzt+N8R0vK3ZLvR6Mh/a500Xnf+qYH07Yqp3xusIvuhQtiSnn99ddp7MrF7vw0zX29JueNckLuA0CpJA4GHZ6HdqzvGReaUWSgYRNUYZiIz4WqjOXqPI3dKHfcFtdMzWpycY5+43JB5bLyHCzwGs9n1KvPpTVWkaJNjl4G1wyer8uadGl0iiqatmsPmXhDbQZM6+HyvdK1LNybQy3K5Bb4Ld6shP5FAE/w9hMAvrDpnnh4eHh4bAobcVv8MxABOmCMGQPwWwB+G8BnjDEfAnABwM/ebAfCHBELlyeOp20PvYmS8Ze65YsfLhIBFbOUEKnyWWcuEnHx9t6DcuIiBZ90lVSV9oiuVWA3wXxWlQrnr/Pu0ZG06VWWTLKK3FlgYubgXtIojt5zX7pvZoaLWVQkQOEyu1MZRcL09JJUO8/Sp85/UijSb+uL0u9TFzjYQxFbw5K6go6vqeCnOm1nVJDPIgu4RdUW33sPAKBhmTxSEnqOJSUt1bpCFToLYXcfaSMp8aTcHZ0bVqikcRfppWWRhKWVcxz4dWlSFL6ZadKI6nWR7OImS6Iq54vLKbJnLwVr7du7J91XSteKJn3Xl9BfPEX9KBZEI7KsETY7cl+6OWumI/9aSgq+ukT3IFRz1ZUnjawTCwlumAQM2bfNRBKolquSZNlqC9k6M+PIUF0ujf62OEfMYlXmqsXurHsHxfWxv5cWjwtcAoCZWcoD099D/Xj0jfen+8bYNXW+Lmv4tTG6L4Fa1wcl7QoAIFKZTgtd9MwtqZJyEas0scoyGHHwTcBrMlHuloYL3kTqmm6r3VIZJlnLjljy1hqRI0NjpQW60nYdtSozBSYt49VZW13ul0xHaQrsMaAzNuZjl6GTr6WWnAusW+5FvPnsqBvxcvn5dXa9e9NX9/Dw8PC4ZfCRoh4eHh47BFueyyWTJ4Km0dDqM9dvVBGUxZIjmcgUoOuNliNSmT751MfTtp/+Fx+mc6jotizXUnTFMg4e2p3um5whgquxJGrzriHyW9cFA5pc5/HQYSJs7zosZO78C1TLsbooaqUjdToqQq7OJpEerj8YW4la6+4ldbGjKhKEAY1v7LKYIobfgGX4uX/2z6WPTBaWVP4YR8IUlKnKpZZYWOD8Kh0xBWSYpIuU/61l1bWu/LNtQudzVdE1ERvx8ZmMjkBdbbZx/rcNzn9SUjkyejmfTtySvuVDGtfctJgMxi6dAwAcZiI9DJRpybqK9irF8DVcfhfYrGc18cixBYVQ5mPP3ruo/y5N8BVZa1NsKhoeFo/e3ACZgapz4s+dcCRsdy/ZK3I5iaVo8JBrHTG55Pk5iNuyxkImF13Rl0xWFdrI0/Zjj4gJ5ej+UTp/S9b62ddpXK+feBUA8LY3C2G6dy8df+FlyTnUjl1OpfVrimZVP7JcUzexYuYsMAneUWmKFzlSNmbiM98tpqLhEpvAFHno1rU2V4RwNVPpry7MsRYsP5va5BKzr7tLUxyoa2adoUclimryO0XnjorY5BiD88fooiv83Oi6rtr0erPwErqHh4fHDsGWS+iGI8hqSjJusISZ0XkcptmliPO1ZDCX7hvpoS/mqeMSFXp57DRt1KT02/mxcwCAh3dRdOru/cIsjk6ShFQ9LVJIX46kw64eKSv1+utn6ZqjJN3PLYj01OYv/cRVJYE5skS5JtZYQjec20FTISWXvTGRyM+sofloTV3BekjaIkGkEoraX87SeQt5mdM6Z8qrtakf586ck2syKbrv4P607exFmssv/fXTaVubM1zmOV9LUZ3fRdd1VyTqsKebpKyHHxYVY3CApNK79tCcBspd0ElZjrgChOyqD4n0NjpC92p0N5HaOoNfjV3blmks1xBlMkzUDw6Npm15JqSnpsSdtMpRyy7cr6EiQLsHaW3tVq63Xd00zsqASO3TTKTHLLG1VUU35yJZU0Riq+0IT9FYsi6jZ47uccaKBjXEcz/YK/cgzwTfYK+wmBV27Zu+cAEAcP71c+m+XX20/ucnnknbMkyGt8L1XyGRyl0SchbJvMrvMjdJBO/MkuRQuTpO89vbRev/gftEU8iwdt5UhHCbNQRN6Lv174q+BIqod1KyLp0Yp0SsZi2X5wbSmVyRnkOeuYiP12vX/SbjNCf9oPPpA+WCGV/DlXaj8BK6h4eHxw6Bf6F7eHh47BBsucklTX2r1JeRAVK3tPr+tZfJJ7yXk+wf6RMVKJ9jUigSX+yrk+fo9E2JeNt3F/mph3zeYkUIqIFhIqymZ0S9nWcyVBc2HxoidTlic1BDkZcu6VJdmQc6/OOOOkmjyak5O/Q97VcquOFag1kjY8kxaRTb5ZF4Gn/5f76SbiecsD9QPrxlJpi7lPnjwBEa82A/mRj6RySKtI/7lFfJpeaOkznqe8el7mrdumIa9P9IqcMV/u3hfWK2edtjj9C1SuLjXWK13Wm8LTWnHfatrs2Lia3NftyFovStp4fMDROcDG1KFckocMTi8C6Z52JRxSCsQC+b2EJlTmhyIQ+jZKCZaerTwgKnQVYmwpAjDM9fkgRYlQUyl3R3S5yC8z9vslOAUQRhzkUzluS+F6yLLNW5gOmZKBXYHGnFHLOnn+alqAjK6gL1u6NMOa74x0E2ER1/7Uy67+hRSsQFRYBevky+6fleMXsBens5CeiKrSTK/LHIMR1Xr4opcW6Wznvy5e8CAF576R/SfYcPU8zHgcP3pm29A2w2UuYKlyraFTvRhoww9WFXfUsLvUibq5ErhXQU6crHa149jaxeg21PSddlye/4rOp+63fJzcJL6B4eHh47BFsuobsoru6yEFY9XbRtVM6QBUuSxtQsfSkHuqTrJSZ04kAkk3OXzwEAhnslGf5+/sI7d7DvPifRqZfGSZLvKovUnmG3qldOX1A9dpGO9LepvqpLHKHXowoSdFjsHJ9QCfi7qE8Ru0YViyKBufwnaAuxGlepb8ND6+dyefaF76fbhQwRlM2mELZZJvXe8tY3p23nL5GkPc2c1AP3i2tblgnNWlOk/AxrNo88IoRmgyMRsyxNHjkk0br3c4rV0QGRSCtFureJclO9eIWiFCdnubjH1NV0X5XJ8rk5kdBbnMI2o1wwXS4ZF0ncVgRlsYfm7QHI+Lq7159LJ2nXVCRqaFwJP9EKYk7FGnEEcmJFPsrm6PwDAxJ5XOY1nleuoN3c74jvmXbntOwa2FHupN3s0hmo6MqE08RGLrqyKZJ3NyeQsR3RGmPWeloq0rHO96PIa/P8FVl/r75O2l+zKRGo7QbNrw019b4+nFSbz8vY77mbIpUP3yvuw7VFktZfeZ5cgF84JkTst79FGuLxV2WtH733IQDAkbtFau/ppfXmyOJwWR/d/K6Re1mTra5kXmd12UcXPRorEjVJ3SfXx7L01MaVzZQ1rFNs3yy8hO7h4eGxQ+Bf6B4eHh47BFtucnHRe7uGxCfc1RhMFLk4sodU+WNsSpkzkqLWhqSWdw8I8dhdYR/QvKjWB9jkUuaUvX/0iT9J99X4Wgt1IdNq7AesM23u4kjOxgypf9WcviaZhV47If7wExNkPlhQ0aM9PXTCSonU51CRWBmO3gtrl9K2wRLt786LQqeSkAIArl5U/vN9ZDbas0dIwPvecITOn5NzvPIiEU/DrAaXVTWjSa6vWKqIyaq/Qse97/F3pG0BO3R3d9NxA/3iPz/DqYbPnpf5mJ8jM9DCvETHLjL5PMdpimcWJAK0wwRvRqU1znKFoEBF1nVXaFw9HFnaq8xTOTZpZQti2lqqC+m8Ev3sQ659+8tcfSZR6V8zAc3HEPurGxUlm2WfaWcKAoA8R0uGKs+uM7GkVZqUycX54NeqsnZcxGJOLUrL5pfaPM33pXMy3zPs/NxTkOOHOcVwPq9r8LIJJSJzU1QU8vwq1/fcOyLPXBdX81pork/kJSotrkviZQPdRn0LlW96Tz+loX37O2ntHj4sJry//eY3AABnz8qzUX2Bn9sFMck9+AaqdrR3L51Lp6eOO7TGY9W3hE27y6p0pfVz3V/Z5ertaoLcWUu0z7sjSNNrLSNF+R2nzDbahHOz8BK6h4eHxw7BlkvojgSs9IqE3ompW7lI3MCOcmGGY8+R5LWQkQi8xJC0N7xbvvSvHid3px/60X+Vtv0DFy6oVklKbLekwMXkFeeKJ9+4Ja4BGKmovN6AJPjdBTrH/FWRhjohScbDQ0KsxuzqVVcSYaNOEmmVybdOIhJYu0GRckMZkQRHyyRJNTvStlJCv3TylXR7gYmzn/4n/zZte/xxSo75N18T98YhJguHihxFqlzh8hw9N9wtkloXb+eVu2CHpRonieqcNVdOkCR1YVJc91pcqCTKS5rYri4ikYdYYmy3VhNRGVWkwOW80LkvurpoLJVKF+9TdSo5n87EhNzvRmP96llFlk7birgtsAtmT0W0niRN5UyEZkHVSU1JLyUdJpbbtBzliou4v4qs6/D97sTS14VpGoN+cDMsoS/NkzY4flmio4f7aCw9JYl2rrF0nShNocNndETsbi7YAAB3c53Rh+6ToiEnz9Dz8sL3xLFgJXTK6IALUASRaN0ZdgqIVXSlSz8bMEl85KgQ8Am7+Y6Pfy5tm52isZ5qilY3cYnqE991hEjXe++XcwwNE0kdqXdLp83FN1RK3Zhr5Lr7uGZBlGU5ZVbvT1M08zzoU6TFZJTovywa9SbhJXQPDw+PHYKNFLjYC+CPAewC+fo8Za39fWNMH4A/B3AAwDkAP2egHt4UAAAgAElEQVStXb8E+DpwuUt6B0SC6PDXvBFIYYR8mSUNzlB44aIEI7z9zeSO1liSL2axi9wExy9J7o3TJ6naecdVA1feTFW223b1i5vZ/DxJRt1lkUjvPkq5JZ596TUAwPPHz0o/fuy9AJZniTxzmiT4OZWx0bk8Nuokme8fFsmuwEEkfX0iGduIJIdOa323poYqBfbgG6mP73r3u9K2/h6ybf/wW5T9myW7LtYUKmWRmkMu2uCq0gNiq9VFB+ZnyW5bYYknURlkDt39AABgaI9kpJyZJc2mq0dcGV3mPmNXV2R3dlhXGg0AltimbFXJMFc44eI42f6dFgQAbS7+ofO7FEvrBxZVWZvqUgUuXJDRpMrTs8DBTglnZTzsAnAA9HD+kzCjpU/a1lpMi+uZ1Zg7aTSl350WzZVRBTFsk44vKY2lp4c0nEKWbNyRkXXSw9pdd5esyRafo6aySbY4w2nAgS69SjMrcpbSMcXTsHCN++8+krZdVe6mdC7NB7C9XPUty7sT/SCy5OpszC2lre3ZewAAcODAgbTt2Qm63x1VHu/q5Bz3h6T348dfTve5wKm77pJ+Dw+T22RXl/BF4AC/Rott7urZy7BGpoOInNuijiuyRrtG0qjS06cFMQThLShwsREJvQPg16y19wJ4K4BfNsbcB+AjAJ621h4B8DT/38PDw8Nji3DdF7q1dtxa+zxvLwI4DmA3gPcD+BQf9ikAP7P2GTw8PDw8bgduiBQ1xhwA8DCA7wAYttaOA/TSN8YMXeOn6yLhGo3dfVLUoFonNacWi4riCDBXK/LkK8oVrkaqTbkkuUi49gDOnxQ18RKTRW97G6XP1WlJuzgdbt+ouEldmCGzSr2pktuXSL2tDBJp9HCX1K68yur4ufMvylhqZJ6Ym5drDQ2SatxtqT/7y+LqN1ThohBGTCguZWpJqbDi9Ec4dM9D6fYHf+nf0PhiUctPnCZiMjEqBw6Tp21W/2bmVNKaxOWxEfrVFVZPIMTW4gL1JJwg1fiyqgfqCpUkDSGbSkzAnjklprCznLLVuf31Dch8OPPA/LyQXtNTRAxaZUIJ2B3OBC6viYo8ZgI2r1MHL62klQU5dpGcnpKxvD5L13RRlgDQ00vk98gI5RNpqajCdovMNomVPi6wWayuzEExR3CGbM7StSudWSVfkrEU2F2xodZuwkRiqcxusGqdZDlKUhPIjmBuKBLQ8HGOlGyrIiZj02RJrakapI5U3DUi638lQmVySLfVNWF4vpa587nfmFX7XJRpV5eYg1KyclnxEmfCo2stzsp9fIFTUL/y0rNpW18/3cddu4QI3jVygK9JZph+ZYod5IK+RhHv7j53lBmww6Rp6raoXR/Z3GWV+c0mK000N44Nk6LGmDKAzwH4VWvtwvWOV7970hhzzBhzrFZb37PAw8PDw2Nz2JCEbigF4OcA/Km19vPcPGGMGWHpfATA5Fq/tdY+BeApABgdHV3F6i1yIpGCylSXZp5LVLk0JlMG+kh6OxlINrjJGZJ8pkP5wnWX6St6zwNCdJw5R5KgKyKgicojR4gkOXLwrrTt/DhJJK+88r20bXqKg1S4CEKvclUbe4Uk+vEp+d4ZJnZDFeA0spfcv/bzF3tfl0hgeS5l1WzowAeSqLRb1Up84Bf+Zbrdu4ukppe+L1KwI5daSgqImaRzpdY0KeNKe8VaguC2YJkYwLlTOAvm1LS4KDq3OxVLgp5KD/dHJN2ZadZGWEqcmhICtMnaSUe5fcZcBjBUuVyKeZrnnHNp1BXZXfIeiPRUUFkkV2KOid7Ll8T9r8Rk9T2q4ILLSFnk/DSNumhVs7Pk3tpuyzhrnGulqNw+uyu07ks5+ltQZGfEUmesSNFOp8XnVdk7XfmztBiDKprAWm5bPXlRyKReolxpOZvk9FXSRKamxcXTZUWcVfl0nKaV6xJtaiWM1RI6/dVEoWGpVuc4SSVt/usISACoL1E/rlyRghiXL9P2fFGOy/A6ciR/SeWPKUZ0nCbIL3FRjVPn5J1Sr1MRl05M5xoYlGInDz5IAYpHDotEPzhIa6HSLc4duQJpEhZ8ffXsddIkjoqYvh2kqKGckh8HcNxa+7tq1xcBPMHbTwD4wqZ74+Hh4eFx09iIhP7DAH4RwPeMMc44/B8B/DaAzxhjPgTgAoCf/cF00cPDw8NjI7juC91a+7dYPyvkuzfbgTOnSc3Zd0TSX+YDTgPaEuIqYrVJiBEhUctctOGee8QP+G++8mUAQG1e/NWL/URenR4j69DePUKiHrybCi/klBp/aB/tn5sR9/pXuW5pwoTL2KyQRwtM5jZiMR8tzJFZZ0gRLuenqa1vL5kfpnPKJzphElWZV2zEtRQTUd9XelG/8OKxdPvl79F310BMOS5fRqSLMKSpYDN8jKjqEafb1elOXT6VrOpvwH7qoaV9laxEyQZslmqHyjzAkbPKbRhZzrXSrrF/dFVMVi0mDU1bRY+yzaelSPOYo0Gri3R8Ud3HwW7qR6RMHc6ysRY12jdI66RXFR5xBRoiNR+LS0RMLi1Rf3M5MZc4UlGnXx0dJjI8lxfzgCNDLecTqTakRw0mnOdmJb/Q9Az5eteVeedeTlOcYd/+5QUduN6pWk9NroU6lkZHiw95i81Ztaqcf36OTI9ZFfXqxv70176Wtr3jLQ9jGVTxhsT5l3dUhCabZJQ7PExqDqJ9oYqcfen55wAAS7Pi797P/vUXx6Wtwj70WX5uEhVhXSmzP7yKD8hGXBgkp+IwAjbjzpKZ6dxZicSem6V5e/6Yyt3DcRt790o07SgXjBkZpWd/dFjeNyVO020Kqt5psH5sxEbhI0U9PDw8dgi2PJfLi6dJWt73wGNpWwL6OhpNAvIXfoEJmrk5IW36+8hl772P/1ja9tAbKY/DZz7/F2mb4bwM3Vx9ffeouFyVmawLOyKZ9O2i6Rk5KFLWPBcneP5FkoLHl5S7VIYI2O4RIYoGDlPbssII7CZ4got2nL4iEmyW2aO6ioys8jR0EpEq3rPCSfTb3/xqul3jzHPZjCpdVnSkrNzy0HL+DlclPaMldOpHPqcIW3b7y6osfVGJxprP0jhzKh+FSxViVJZIR263VeGMBhOeqVSrI+z4eF3aLg3xVRJxT4m2u0s0pnJBpOBchs6XMXIfjXI/XIk2k3TazTFil8p4GdHnyu/x/CnROM9SeL0q46xzhsm68jl1mlCQcW5ssuZPHH8VAHD+3Lm0zUU5W+UOOTpCDgB9nPGyrrzJ3PbcrBCa00z61pUG7HIOOU+0uQXRkgKe+2Ika8fli7lyRTTglRJ6WxXVcKS86cg5XFSqdtazoDZHoi4tyWS5Yip3HxVt/pGHHgUAPPeyFL145lnKIjrHxVHijtyDoREiN9/+9renbRHf53PnxcX5mWcoF9QD91EUeqVbnCsmeMwTE+IA4NburmFxbzx48ABdnx0Lqovi9ukcDDKRaAWNNXIY3Si8hO7h4eGxQ+Bf6B4eHh47BFtucjk5Tyr9VKxSj2ZIBQ9aSkVJXA0++js6IjaHH/khIjTzGVFDD+6nyM+f/MAH07bP/sVf0bWu0HnH50XZazROAwCyEJV3pk7bp8+LWglWi+wgmXR6h8X8kNYVVNGYCZsnEiMmAJeMap4jOfMZlYSMU9hWjUouxWSkTbRKtlw9Gx6U6LnxOhFEcSxqdoXrnEaqbwtTRPYuLlS5X6KaJk5dXit6TZlVMgW6DzZD13eJ1QAgYJtLUSUrc5Xp4/Zqcxo4CZTJiu0iz+RmQZk/+rpITd2rYgD2jJD/r+M9mw1R1QNL6ylSkX09FVp3Ncm1leLkSUoJe//996VtBTah6OkImH5MODpwQkXJumRvzboya7AJMVZmlUOHDwAABoeo/7rwQobNPD0qUZYjVHWZTOdD/toJShu7pApiuH06hiFhk1J1Ueaoxv2scTRrS5nEXDGNCxNCPLoar/E16mDaZRGg1m2kcFGeKogViSNS+VYVVL3dH3nnu3mX/MAVrzj6kJhsH3gT1c11ZVcDRRO7AiyHDkm8ScRzeuCIpNkd3UdEc4EjjruVycWNyxVwAcSsMjQoacBdsq+QTVWBYn9jdnBoKztdYtafy43CS+geHh4eOwRbLqGfmKNvyhf+VqIxH9pP0squrBAGRZYSRnbRF3BkQKSWuw4xuWlFqhjnvCqf+PRfpW3PvUgkk4tEXRZ4aR0pJeeIc3SNWBN97ArYYYK1EyjS0M2mKiXVaPF51Zc4YoI0ZGnMqlwnHaaIMupr7kqRtdrrR5LZtkj03SWSOBYVsdqOSWq7594H5DejJK1McnTgpIoOXOK8Ljpdg5MsbSznLUUkhdzzRkpLelmVlru6QBpAvSUSY50LS+io1By7UpZYE+lRuUsGuYL7yKhIPod3k1vhUE7E1CV2dZxht74wK/NXLBEJXlYRuf2cv+PyWSHCHNos3TeWRMMJHBmpRExXvCJm18RTp06m+xbnHTEtj5grAhIp8TrhkMGAI22hXDH7WavSZGuNUy7X6zKnFy+OLTtOBR/CsotnrSX3zEnX1SnRgDPcT1fyr6MiKavstthRrpISabm+VFlX2knILpiRVRG8/Lx2VARvh+fBnV+XsXMCf0dpOK4cXEvlUBndx/mYEk5Rm6giEvycn70grqD1lssDpAqmdB9cdv3ZeblmxBJ3qXJABuvyIc3LmC9PzPA5qOM5lQ7cBcCasqyPxuz6ZRE3Ci+he3h4eOwQ+Be6h4eHxw7BlptcllgN+ZvnRV09+TpFj77nTUJK3TVKqv3ZMxSp+Y43i+kgz6r6YkvUuc/8NaXHfP5VSbBUc1FqbPIIVKpSpxYFKrrNmUlipc412RTSZpXQKN/mJkdcajIoilbXvyxyIqEsXAXydBdiJhV1UqwOE4jZLqnyszIX2vRlScQVt0l1qyt1uHaREpP1qQrrg5xWNsNVcgoqi1Y9dBVYtF1qtZpdq5OZ5h1cNer+eyV51YULZM6YnpNI26Yj2xSZFjHRXWAWa0ARoD2lEl9Z7sGVKRrLiSlJ0mSY2KoMkRmpUBHCtMgkqk7LW1Yk10oU+J61lFnDkdXL6mQ6/3M2V1QqEr2cZ5/+cklIvZDHVVTRps7Eceo1Suw2PyOmgHmO6IyVz3kmyxGraj3lWH83PH81FW06ycRdrSnqfMhj6O2W9dRi81yNneQ7KvlXkppXdP5Xng+zvkz4rW99XcbSoapBpUjmI+Z111ZmFUfMu4Rk+llqs2lLP4+OcGw0pS1OK2BxKmpVP7Svh8y55bKumEVj0PyuScfnEp6piE4ec6BMKBEn/QrM6uPcEJaFVxh+fxTl+KDB5kJFeN8ovITu4eHhsUOw5RJ6/wDlt5iZlc/jOEe1/T3X7QSAuL2ft+hLOLhLojxNSF/g7x6TaLG/+hpFejUTkQjAX+ogWP0di1lytOoz7dzRtJTgojwzLBkY/TnlPBSa9HK1KHXumZCvH1qWOKzSFFjK12L7yC6SJrsqSqqsLZfQd430pdtjF8Z4TLqYAG2fPXkibZpnd0J39apyi6yyNJTEy5hjOl4VE2g1SaJ7/m+/AgB4Z0nG+QCPs94t0rIjAXUUcIMJu3mO3tTk7PnXKBpvqi6Ri40MXb8wJGPu3UUSV65CYwpVpGiR3f5yRSHZTbj+0neusXFH7oGLMk46SlvjsTtStKAiKQPWGusqJ0pzhrTFC7o4Bc+DSyHr8uUAQp5n8kor4Eu0WjJ/i7MkkTcaS/xXiGx3p/JqzbfrnIJX1X91BKb7q8lI517YUdqJZak2m1mfqM+rSOV2yPdFpcTOsdNBolxdndtmwNfUJHTC+W60VuAiZhOrooB51NbV7TSKhObbF6i6uFHIKaubEtmaEqQ8PF2ztM0as9a63Zox6tlY+Z5pqahXy+doqNdHLiRtanR0P24WXkL38PDw2CHYcgndSbMZlQWw0yDp6uyESGXNKgV7vOMRqiBf6JGcCfNcDOKb35GMg3W2/bZVtrscu4056WOtCkqhkhbSj62yreVYsjNOVArU8TmSQgqq/JlzcWqrQJpFltpcUEZTSYLdveyyOSKJ8svsD1lXgSArP8X7jkomtwV24auOTakjOOueckeb4etmecwtZS8Xu+1qt7RlBQkYp16m/BkXF0XyGQxoPpZpOCy1LCl7/RVLUuFptqmOqRwgtSJrOPukwMDwQZJg8j3iupreB5aaymXRFIpsTw/UGrPXsP0ucJ6g2qK4LU5epjXZaEjfXPk4l8dD32On6QUqmCnDgW+OVwEkw2XENnftothmO7LOB9Ns0tpZVO5x7raVKuwOqyRD26Z5bi7JWndFMuaVROokc2efNspentjVwWUut41J1i+6kqj7uFQlHqUY6ntAf2O1mF0AVIvdcDsd5crHhTysksYlq6U8hx22ocdOG1T32gVVaeHZWupns6Fz28TLjteau035nFi1uaBCXSRm+TXDlu43587p1YVvaHsUXkL38PDw+EcP/0L38PDw2CG4rsnFGJMH8C1QTYUIwGettb9ljDkI4NMA+gA8D+AXrVWhmhtESjJpYjAk1bGlSJuJJVKLnj9BxNJ7a6ICLVoyRVyaFZNEnlXuTk3O0WAV09WAjFQUn9u3zC3NOLcnOc4Gy1POZnLigrbErl4tlYLXmV+02cGZWKocsVruEfNKL+eCaKmUn6+xS1tGuWu9aYVWVukVgnBwmPKrjCuTS6r+qd802azi6k1q18D4GhGAy/bwidusslenJN9HkOOUxMpl7jJf40WIOn464vkokxpf2itFMgZHKSdPPxedAIAcuwK2VE8smwVyEVe5jzQx7doUaXkN37Ar58iFVldhdyq40RG/nL7XVX/X6naWzTs6j43brwnHDpsYlpa45mtT51xhlzmjXQhpXWRVMYbh3aN8DoroXJgVN9EOF6ywioR25pRaS5thnDnD+dhh1fEZNXZXeKJWU2bAFbh4UZwUTo1TP0qqRmjEtqJ4WUkOmlMXDZoooj7LuX50mzPRxDq1Ec+zIy2NypHiyFZt23L5YPR9ce61SeyiSBXZySbKZTmbXAEPuzqy1f2yrfJExX20LnY/KK7Z3e6WbiKly0Yk9CaAd1lr3wjgIQCPG2PeCuB3APyetfYIgFkAH7r5bnh4eHh4bBYbKUFnATg/qwz/swDeBcCVmv8UgP8E4GM33ANHNujCARz8kqi8Dy6fytlJkgg+8Zkvp/ve9U5Kcn/2skiHVRcsoL5ZGZepjqWEonI7ynLhivqiSNeOuLCKtMwwQekkQE2EOUkwUQRKnV3UdJs7roel6n6VFP/qNAWWzE1Jhse58xRMdfjQQayHQl4kthwHsGRUPpOYyTH98e+kkguPT++8hpSwjCJjaWiJx/eakvq6uTzdaw0pBPAKay/TFZFc+/fSuEYOkjTeo1wwc+wGGah8HG1eK2GkSrmxRBylQTZyfCpda5eya5CiYcKue8p1NHUv1OdlbS2wTmKTczTZBbPTlvXkJG5dcd7BkeeZrC4RyGUDNanMazGfU+5/BfrNzDRdU2dRzLDGGerq8qyNdrQ0uYLUWxZI4wp+KK1niYuo1KqSD2YlAqvKFzppNRap1mkDy4KTQnZbtM41UGlaLBmrOKt07q1yTXQ3woqPYgonhWvX4g5fv62cAhJ+B1lXIlA9D2leJtURg9VjsUx+dziAsaLyEe15kJw7IiP3e+4k57PaI9rojWJDNnRjTMgFoicBfBXA6wDmrIQRjgHYvc5vnzTGHDPGHFvLq8TDw8PD49ZgQy90a21srX0IwB4AjwG4d63D1vntU9baR621jxZVbmMPDw8Pj1uLG/JDt9bOGWO+AeCtAHqMMRFL6XsAXL7mj9dBP1cqb6iCBFWOZMuG4s/t0mo6X+JvfvfldN9Zrm84VxVmZGaJ1GbFLaLE6nuH1a6cql7vVPV8QeWJCJyPsKj2zme2wyYGo/1TWQWLVYX6FvvJFlT+Dpdkv2+ATC0tRQg3uaBDPSfXTDh6UFeEX4m2iuiscj6Orh65ZqNKarYuoBCzephmbFWpW81qq0AKq9IDWyaUquwj/G1VlOR8jdqmVb6KaJgqoI/sGUzbDg7Sdn83zUugok2rLCc0FLEVseqva37mOQo04urr+YIIDzmeex2FeS0ka+QRccqoVaYfy2xyatJR53CRhrE2GfA60uvOrTFH0i6zeiVuPQmpHDP53MrIva1zWltnakk0Acq5XxpKO3bjstoX2x3vzBWqHxGPxbaEyJ6dJjNau7X+muwoP/SYj2sFmhB2eX10URRu4mcpUPfApchNtGmEzWKJSjftCGln/dDHO5OZtvIkzj9cmdicmSk1zWj/cjYLQRO2zmyj3gdtTmPddzcV09h9YG+6r8H1SF9/TWJnCm22bEsQ/A3juhK6MWbQGNPD2wUAPw7gOICvA/gAH/YEgC/cfDc8PDw8PDaLjUjoIwA+ZSghQgDgM9baLxljXgXwaWPMfwbwAoCP30wHGix15tSnpckSUiYUKbXDH0qXsD8oiBR3jsnQQJE2HZaeOorQbHBGuSpHamrix0lNpaxIcQUmSgMlVTjCsVCk6+ucGlc5U16i3JMiJkR6K0Ja7uojrWTXLiL/5qoiySxwZsKleYlS7OFCB1NXdeTnADTaqop9mKWx9w7KNdtlmstOW2W2S9xfJkyVhO6GrCMGU+lNs3+OuONshG2VQ6XZTf2+q0dInt4+iu4sV2TplYt033JMODdUvpQWuzlaJV2Hzt1U94O3M6xpabdFV7xBE2z2Gqxvg139Iu2u6lzhtOsjj90VutDraaXkzR2grupITp575zYYq8jLNs9DqDSzNucDiZV7balJmo2TzHWunWadpfs1SsUla0T8un5Eer653zMTkj+ozRGr+hasgh4653wJsnLNjMt2Gi+ryME/5blSp7MuQ6HSEPOsgfRWhEh3JedcQRY9pyG7mOaUBuzytCyLjuX74iJnFxdUHhZenkkkczTPqRSjAenH/qNEfPZy9Pel106n+6ZOU0bZSPUtf428OBvFRrxcXgbw8BrtZ0D2dA8PDw+POwA+UtTDw8Njh2DLk3M5lTCnkhgVHTHSFlXTuZkm7AWtEwYlrJ51WorEil0KTU1s0XaSpuiU79nsDJk6ZtQ1K1wYoVtFYVbYdz0PMse46t0AELFKGKpal01O5uQKJOjjOjWu1VhTSYzmpnnswubmOSKxcY3oxlCpaz39ZA4ql5QfepNNUMrk0omdb7rzPVaJxvhbHyxLB8pmBJVcKmIVusgmjq4uFcHIRQTKOSG3S+ybns2JutrizSX2m68rgtcRt3ml3mZD57MtanOwwpyh73uLSa9sVpFYmfXn0kX/BsqskXGmPm0u4b65GVpWtD2NHFTJq+LVxLSLlHaFLlotue91NrXEdRXRyaRoSZmlCt2k0nd4nO2GnCNYwyaS+uNrgtyFg7ApqqRiNKpcG3ZhQcyAzmKl18xKhB01x1y3M1ERwhbU3xAqZTBvS1StIjSNXfYXABJOvleLJJGfRHu79Ndqvjmau9GWvrm1bpb5sqed5DOpUFS+via8K5zKefCoxIoE/K468ex36JqTYjIN+f7pQiVrmcBuFF5C9/Dw8NghMPYWfBU2itHRUfvkk0/etut5eHh47AR89KMffc5a++j1jvMSuoeHh8cOgX+he3h4eOwQ+Be6h4eHxw6Bf6F7eHh47BDcVlLUGHMVQBXA1PWOvcMxgO09hu3ef2D7j2G79x/Y/mPYTv3fb60dvN5Bt/WFDgDGmGMbYWvvZGz3MWz3/gPbfwzbvf/A9h/Ddu//WvAmFw8PD48dAv9C9/Dw8Ngh2IoX+lNbcM1bje0+hu3ef2D7j2G79x/Y/mPY7v1fhdtuQ/fw8PDw+MHAm1w8PDw8dghu6wvdGPO4MeaEMea0MeYjt/PaNwNjzF5jzNeNMceNMa8YY36F2/uMMV81xpziv71b3ddrgYt8v2CM+RL//6Ax5jvc/z83xmSvd46thDGmxxjzWWPMa3wv3rYN78G/5zX0fWPMnxlj8nfyfTDGfMIYM2mM+b5qW3PODeG/83P9sjHmka3ruWCdMfwXXkcvG2P+wlVj432/wWM4YYz5p1vT683htr3QueLRHwB4D4D7APy8Mea+23X9m0QHwK9Za+8F1VH9Ze7zRwA8ba09AuBp/v+djF8BlQ10+B0Av8f9nwXwoS3p1cbx+wD+2lp7D4A3gsaybe6BMWY3gH8H4FFr7QOgWj4fxJ19Hz4J4PEVbevN+XsAHOF/TwL42G3q4/XwSawew1cBPGCtfQOAkwB+AwD4uf4ggPv5N//DLMunuz1wOyX0xwCcttaesda2AHwawPtv4/VvGNbacWvt87y9CHqR7Ab1+1N82KcA/MzW9PD6MMbsAfCTAP6Q/28AvAvAZ/mQO73/FQDvAJc4tNa2rLVz2Eb3gBEBKBhjIgBFAOO4g++DtfZbAGZWNK835+8H8MeW8AyogPzI7enp+lhrDNbar1hJUv8MpCTz+wF82lrbtNaeBXAa27Ai2+18oe8GcFH9f4zbtgWMMQdApfi+A2DYWjsO0EsfwNDW9ey6+G8A/gMAl+W/H8CcWtR3+n04BOAqgD9is9EfGmNK2Eb3wFp7CcB/BXAB9CKfB/Acttd9ANaf8+36bP9rAP+Xt7frGJbhdr7Q16qAui1cbIwxZQCfA/Cr1tqF6x1/p8AY81MAJq21z+nmNQ69k+9DBOARAB+z1j4MSh1xx5pX1gLbmt8P4CCAUQAlkJliJe7k+3AtbLc1BWPMb4JMqn/qmtY47I4ew1q4nS/0MQB71f/3ALh8G69/UzDGZEAv8z+11n6emyecSsl/J9f7/RbjhwG8zxhzDmTiehdIYu9h1R+48+/DGIAxa+13+P+fBb3gt8s9AIAfB3DWWnvVWtsG8HkAP4TtdR+A9ed8Wz3bxpgnAPwUgF+w4re9raMrqJEAAAF9SURBVMawHm7nC/1ZAEeY2c+CCIgv3sbr3zDY3vxxAMettb+rdn0RwBO8/QSAL9zuvm0E1trfsNbusdYeAM3316y1vwDg6wA+wIfdsf0HAGvtFQAXjTF3c9O7AbyKbXIPGBcAvNUYU+Q15cawbe4DY705/yKAX2Jvl7cCmHemmTsNxpjHAfw6gPdZa2tq1xcBfNAYkzPGHAQRvN/dij5uCtba2/YPwHtBzPLrAH7zdl77Jvv7dpDa9TKAF/nfe0F26KcBnOK/fVvd1w2M5Z0AvsTbh0CL9TSA/w0gt9X9u07fHwJwjO/DXwLo3W73AMBHAbwG4PsA/gRA7k6+DwD+DGTvb4Ok1w+tN+cgc8Uf8HP9PZA3z506htMgW7l7nv+nOv43eQwnALxnq/t/M/98pKiHh4fHDoGPFPXw8PDYIfAvdA8PD48dAv9C9/Dw8Ngh8C90Dw8Pjx0C/0L38PDw2CHwL3QPDw+PHQL/Qvfw8PDYIfAvdA8PD48dgv8P8QITwTAXGKoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataiter = iter(testloader)\n",
    "images, labels = dataiter.next()\n",
    "\n",
    "# print images\n",
    "imshow(torchvision.utils.make_grid(images))\n",
    "print('GroundTruth: ', ' '.join('%5s' % classes[labels[j]] for j in range(4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted:   ship  ship  ship  ship\n"
     ]
    }
   ],
   "source": [
    "outputs = net(images)\n",
    "_, predicted = torch.max(outputs, 1)\n",
    "print('Predicted: ', ' '.join('%5s' % classes[predicted[j]]\n",
    "                              for j in range(4)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device ## we still have this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = net.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 ,  2000] loss 1.067\n",
      "[1 ,  4000] loss 1.068\n",
      "[1 ,  6000] loss 1.069\n",
      "[1 ,  8000] loss 1.048\n",
      "[1 , 10000] loss 1.066\n",
      "[1 , 12000] loss 1.054\n",
      "[2 ,  2000] loss 0.976\n",
      "[2 ,  4000] loss 0.978\n",
      "[2 ,  6000] loss 1.015\n",
      "[2 ,  8000] loss 1.021\n",
      "[2 , 10000] loss 1.043\n",
      "[2 , 12000] loss 1.020\n",
      "GPU Time take is: 171.316162109375 seconds\n"
     ]
    }
   ],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)\n",
    "\n",
    "beg = time.time()\n",
    "for epoch in range(2):\n",
    "    running_loss = 0.0\n",
    "    for i , data in enumerate(trainloader , 0):\n",
    "        inputs , labels = data\n",
    "        inputs , labels = inputs.to(device) , labels.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs , labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        running_loss += loss.item()\n",
    "        if i % 2000 == 1999: # print every 2000 mini-batches\n",
    "            print(\"[%d , %5d] loss %.3f\" % (epoch +1 , i+1 , running_loss/2000))\n",
    "            running_loss = 0.0\n",
    "beg = time.time() - beg\n",
    "print(\"GPU Time take is:\" , beg , \"seconds\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In case you are wondering , it is because at each point the data of batch size 4 needs to be move to the GPU , a pain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}