pytorch_practice.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "imdb_movie_review.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "TgTMaizV4rz9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torchvision\n",
        "\n",
        "import torchvision.transforms as transforms"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TXATAK140Yv-",
        "colab_type": "text"
      },
      "source": [
        "## Pytorch basics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y96Fw0PVwocx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4MpaBGIGwvqk",
        "colab_type": "code",
        "outputId": "43ec030a-bb6f-425a-e4f1-fbd0fbf831f5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "x = torch.empty(5, 5)\n",
        "x"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.0000e+00, 0.0000e+00, 6.8664e-44, 7.9874e-44, 6.3058e-44],\n",
              "        [6.7262e-44, 7.7071e-44, 6.3058e-44, 6.8664e-44, 7.5670e-44],\n",
              "        [1.1771e-43, 7.0065e-44, 6.8664e-44, 8.1275e-44, 7.2868e-44],\n",
              "        [7.9874e-44, 8.1275e-44, 7.4269e-44, 7.1466e-44, 6.4460e-44],\n",
              "        [7.7071e-44, 7.7071e-44, 6.7262e-44, 7.2868e-44, 7.5670e-44]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sBj3td2Pw1u_",
        "colab_type": "code",
        "outputId": "71b42e99-e9a5-4e64-b5a0-fef51066aa6a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "torch.rand(2,2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.0462, 0.0250],\n",
              "        [0.7451, 0.8604]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZQwrR_omw91O",
        "colab_type": "code",
        "outputId": "e8d6191c-765f-4212-c2e7-40daea907d83",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "torch.zeros(2,2, dtype=torch.long)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0, 0],\n",
              "        [0, 0]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QrdBGrH7xCUU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "??torch.zeros"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WWWkqUk8w_z9",
        "colab_type": "code",
        "outputId": "130dffb4-1c49-4010-b281-16295c670a42",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "torch.tensor([[1,2],[4,5]])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1, 2],\n",
              "        [4, 5]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jusd4Ay2xTXA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = torch.tensor(range(4), dtype = torch.double)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xbj8lZOlx-uP",
        "colab_type": "code",
        "outputId": "d9039d6e-2b1a-4113-8646-b9348dd44164",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "print(x.new_ones(5,3))  # what is the point of this ? \n",
        "print(x)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[1., 1., 1.],\n",
            "        [1., 1., 1.],\n",
            "        [1., 1., 1.],\n",
            "        [1., 1., 1.],\n",
            "        [1., 1., 1.]], dtype=torch.float64)\n",
            "tensor([0., 1., 2., 3.], dtype=torch.float64)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZhXP-6sEx_ov",
        "colab_type": "code",
        "outputId": "7242b01b-da31-428f-adcc-a51342912c0d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "torch.randn_like(x)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([-0.6031, -0.6388, -1.2493,  1.4617], dtype=torch.float64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hyd8hHFDyItQ",
        "colab_type": "code",
        "outputId": "b63a7c51-ad9e-4c08-8065-ab29705a87a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "print(x.shape)  # same \n",
        "print(x.size())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([4])\n",
            "torch.Size([4])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8DkQJFfqydNO",
        "colab_type": "code",
        "outputId": "81cf0eb1-ae26-47c8-eff3-ba92f3c7aba5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x + torch.tensor(range(4), dtype = torch.double)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0., 2., 4., 6.], dtype=torch.float64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RElnyRg0ypRO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = torch.tensor(range(4), dtype = torch.double)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2nOjYQ0cyzkv",
        "colab_type": "code",
        "outputId": "869eec61-8346-4226-b3de-4178ff204d4f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "y.add_(x) # y = y + x"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0., 2., 4., 6.], dtype=torch.float64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HT_muasky2F_",
        "colab_type": "code",
        "outputId": "e6688d22-e72b-4982-ea59-e7f6d0792b21",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "y"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0., 2., 4., 6.], dtype=torch.float64)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UzRdT2Rvy3Jv",
        "colab_type": "code",
        "outputId": "02eaa492-1e04-47d5-9be8-0fd983a1a4f8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x = torch.randn(4, 4)\n",
        "y = x.view(16)\n",
        "z = x.view(-1, 8)  # the size -1 is inferred from other dimensions\n",
        "print(x.size(), y.size(), z.size())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([4, 4]) torch.Size([16]) torch.Size([2, 8])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VQaIUiwFzBA9",
        "colab_type": "code",
        "outputId": "52e4bc6d-8934-4c76-da6f-c355cf9958e1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "x"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-1.3778, -1.2523,  0.2914,  2.0074],\n",
              "        [ 0.0446,  0.7125, -0.9100, -1.5203],\n",
              "        [ 0.7891, -2.1536, -2.2918,  1.0709],\n",
              "        [ 1.2489, -0.3730,  1.3713,  0.4874]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kww6B8qSzHBv",
        "colab_type": "code",
        "outputId": "ce200cf8-e571-4602-b073-403683eed441",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "x.numpy()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[ 0.62224495,  0.7477238 ,  2.2914345 ,  4.007374  ],\n",
              "       [ 2.044599  ,  2.7124665 ,  1.0900159 ,  0.47973967],\n",
              "       [ 2.789056  , -0.15356159, -0.29184008,  3.0709243 ],\n",
              "       [ 3.2488632 ,  1.6269643 ,  3.3712604 ,  2.4874258 ]],\n",
              "      dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TlgeU_5tzIfO",
        "colab_type": "code",
        "outputId": "0ab857ba-db91-4fc4-f565-589be552d447",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "y = x.numpy()\n",
        "print(y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 0.62224495  0.7477238   2.2914345   4.007374  ]\n",
            " [ 2.044599    2.7124665   1.0900159   0.47973967]\n",
            " [ 2.789056   -0.15356159 -0.29184008  3.0709243 ]\n",
            " [ 3.2488632   1.6269643   3.3712604   2.4874258 ]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gqMTLxTMzUSO",
        "colab_type": "code",
        "outputId": "ff4cf1e4-2c27-4219-8e13-1e4d5d329b68",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "x.add_(1)  # x and y both change values\n",
        "x, y"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[1.6222, 1.7477, 3.2914, 5.0074],\n",
              "         [3.0446, 3.7125, 2.0900, 1.4797],\n",
              "         [3.7891, 0.8464, 0.7082, 4.0709],\n",
              "         [4.2489, 2.6270, 4.3713, 3.4874]]),\n",
              " array([[1.622245 , 1.7477238, 3.2914345, 5.007374 ],\n",
              "        [3.044599 , 3.7124665, 2.090016 , 1.4797397],\n",
              "        [3.789056 , 0.8464384, 0.7081599, 4.0709243],\n",
              "        [4.248863 , 2.6269643, 4.3712606, 3.4874258]], dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Y-oc096zJJ7",
        "colab_type": "code",
        "outputId": "3f98acf6-dba1-4c55-dc4a-f4d1f1b30126",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "x = x + 1 # only change x\n",
        "x, y"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[2.6222, 2.7477, 4.2914, 6.0074],\n",
              "         [4.0446, 4.7125, 3.0900, 2.4797],\n",
              "         [4.7891, 1.8464, 1.7082, 5.0709],\n",
              "         [5.2489, 3.6270, 5.3713, 4.4874]]),\n",
              " array([[1.622245 , 1.7477238, 3.2914345, 5.007374 ],\n",
              "        [3.044599 , 3.7124665, 2.090016 , 1.4797397],\n",
              "        [3.789056 , 0.8464384, 0.7081599, 4.0709243],\n",
              "        [4.248863 , 2.6269643, 4.3712606, 3.4874258]], dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZFmdW840zZ29",
        "colab_type": "code",
        "outputId": "4e367d9b-134f-41fd-d9ee-0e42717bc1b4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import numpy as np\n",
        "a = np.random.randn(5)\n",
        "a"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 1.27008064,  0.38176109, -1.86975933,  0.49286453,  0.08534811])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K61DSknmzokh",
        "colab_type": "code",
        "outputId": "cad8a600-e14d-45c0-d074-0b501e087c34",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "b = torch.from_numpy(a)\n",
        "np.add(a, 1, out = a)\n",
        "print(a)\n",
        "print(b)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[ 2.27008064  1.38176109 -0.86975933  1.49286453  1.08534811]\n",
            "tensor([ 2.2701,  1.3818, -0.8698,  1.4929,  1.0853], dtype=torch.float64)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EmX3FC4Wz30x",
        "colab_type": "code",
        "outputId": "4f6dc310-3857-4364-a473-ceb36031d430",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "a = a + 1\n",
        "print(a)\n",
        "print(b)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[3.27008064 2.38176109 0.13024067 2.49286453 2.08534811]\n",
            "tensor([ 2.2701,  1.3818, -0.8698,  1.4929,  1.0853], dtype=torch.float64)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mvs04Ux-z7Iy",
        "colab_type": "code",
        "outputId": "21794afd-4b38-4ce7-da45-c3bac6bc2fe2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "torch.cuda.is_available()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-C5jGlRmz_yQ",
        "colab_type": "code",
        "outputId": "59a7a7e8-f1f7-4f92-e443-1b316b87c1d3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "device = torch.device(\"cuda\")\n",
        "device"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uF0x1x7g0FA-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = torch.ones_like(x, device=device)  # directly create a tensor on GPU\n",
        "x = x.to(device)                       # or just use strings ``.to(\"cuda\")``\n",
        "z = x + y"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qU7dyoQY0MnB",
        "colab_type": "code",
        "outputId": "57d2e55e-22c3-4f3e-ee54-d8da6714afa0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "z"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[3.6222, 3.7477, 5.2914, 7.0074],\n",
              "        [5.0446, 5.7125, 4.0900, 3.4797],\n",
              "        [5.7891, 2.8464, 2.7082, 6.0709],\n",
              "        [6.2489, 4.6270, 6.3713, 5.4874]], device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fy5X-QDn0Nex",
        "colab_type": "code",
        "outputId": "ae0d4684-892a-474b-956a-539c251912ad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "x"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[2.6222, 2.7477, 4.2914, 6.0074],\n",
              "        [4.0446, 4.7125, 3.0900, 2.4797],\n",
              "        [4.7891, 1.8464, 1.7082, 5.0709],\n",
              "        [5.2489, 3.6270, 5.3713, 4.4874]], device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gWR6L_wl0cnj",
        "colab_type": "text"
      },
      "source": [
        "## Autograd"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1U0MbwJB0eJR",
        "colab_type": "code",
        "outputId": "a850fd54-2ca4-4fe2-d34a-6c63e06af6d9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "x = torch.ones(2,2, requires_grad=True)\n",
        "x"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1., 1.],\n",
              "        [1., 1.]], requires_grad=True)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yCLBDF-P0z9B",
        "colab_type": "code",
        "outputId": "86b7a910-ab9e-4cf4-95d4-c709e4a48c17",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "y = x + 2\n",
        "y"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[3., 3.],\n",
              "        [3., 3.]], grad_fn=<AddBackward0>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tm2-sRZv1G6x",
        "colab_type": "code",
        "outputId": "b94dc8e9-8ae2-477f-a560-440a014b2ae2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "p = x * 2\n",
        "p"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[2., 2.],\n",
              "        [2., 2.]], grad_fn=<MulBackward0>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_PWG1wM01JWx",
        "colab_type": "code",
        "outputId": "c62268ac-2197-4f21-f02b-ccc3ad479108",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(y.grad_fn)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<AddBackward0 object at 0x7f65739eefd0>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "80C6m-sZ1MOS",
        "colab_type": "code",
        "outputId": "e9d71e5d-4262-4c83-d7d2-ff04eae1b5da",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "a = torch.randn(2, 2)\n",
        "a = ((a * 3) / (a - 1))\n",
        "print(a.requires_grad)\n",
        "a.requires_grad_(True)\n",
        "print(a.requires_grad)\n",
        "b = (a * a).sum()\n",
        "print(b.grad_fn)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "False\n",
            "True\n",
            "<SumBackward0 object at 0x7f5f32429f98>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j2q6wAaM63-6",
        "colab_type": "code",
        "outputId": "4ad6f66f-366e-4b2b-e14f-afd70cf7b813",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "x"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1., 1.],\n",
              "        [1., 1.]], requires_grad=True)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kZe8X-yN1WiP",
        "colab_type": "code",
        "outputId": "4be1d2d8-87a6-48db-ae03-40f982521b57",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "z = y * y * 3\n",
        "out = z.mean()\n",
        "z, out"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[27., 27.],\n",
              "         [27., 27.]], grad_fn=<MulBackward0>),\n",
              " tensor(27., grad_fn=<MeanBackward0>))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bZt_eFHt6zqI",
        "colab_type": "code",
        "outputId": "8fd4e4b2-58d5-41b4-8981-9e729f289744",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x.grad == None"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "foJDc_-G1sCh",
        "colab_type": "code",
        "outputId": "de2ac530-f993-4a33-8bc6-3251e235ea0a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "out.backward()\n",
        "x.grad"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[4.5000, 4.5000],\n",
              "        [4.5000, 4.5000]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C7cUaxc510Jj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = torch.ones(2, 1, requires_grad=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OpvYvHIG9Rpi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = x + 2\n",
        "z = y * y * 3"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5Vg2v2P19SJj",
        "colab_type": "code",
        "outputId": "f1459cb3-3dc4-489f-aeb2-462af2c8690c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "out = z.mean()\n",
        "out.backward()\n",
        "x.grad"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[9.],\n",
              "        [9.]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 102
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sUY3DFPA-Kxo",
        "colab_type": "text"
      },
      "source": [
        "### y = g(x) R^n -> R^m vector valued \n",
        "### z = f(y)  R^m -> R^1 scalar valued\n",
        "### dz/dy = (dy/dx).t() @ dz/dy\n",
        "\n",
        "### dy/dx m * n\n",
        "### dz/dy m * 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0sC9C_vf-j6T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YPZviaRn-JJj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VLlTXKXC9luy",
        "colab_type": "code",
        "outputId": "88749475-f8c1-4f89-a8b7-79292cd29f20",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x = torch.randn(3, requires_grad=True)\n",
        "\n",
        "y = x * 2\n",
        "while y.data.norm() < 1000:\n",
        "    y = y * 2\n",
        "\n",
        "print(y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([-444.7076,   27.8274, 1279.7865], grad_fn=<MulBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sYu8qeIa9zkA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## Dont run \n",
        "y.backward()\n",
        "## RuntimeError: grad can be implicitly created only for scalar outputs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uGyoUtKP999D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aV39cG4C-yba",
        "colab_type": "text"
      },
      "source": [
        "## Pytorch NN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zV9yezB35glZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "?? nn.MaxPool2d"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4_EZnZAP5cKf",
        "colab_type": "text"
      },
      "source": [
        "nn.Conv2d(in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True, padding_mode='zeros')\n",
        "\n",
        "nn.MaxPool2d(kernel_size, stride=None, padding=0, dilation=1, return_indices=False, ceil_mode=False)\n",
        "\n",
        "nn.Linear(in_features, out_features, bias=True)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "abOUUv7Z5QP4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "?? nn.Linear"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HsUDjnYA-0HD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "996dd454-32c7-449b-ca32-b13508acefbb"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "\n",
        "class Net(nn.Module):\n",
        "\n",
        "    def __init__(self):\n",
        "        super(Net, self).__init__()\n",
        "        # 1 input image channel, 6 output channels, 3x3 square convolution\n",
        "        # kernel\n",
        "        self.conv1 = nn.Conv2d(1, 6, 3)\n",
        "        self.conv2 = nn.Conv2d(6, 16, 3)\n",
        "        # an affine operation: y = Wx + b\n",
        "        self.fc1 = nn.Linear(16 * 6 * 6, 120)  # 6*6 from image dimension\n",
        "        self.fc2 = nn.Linear(120, 84)\n",
        "        self.fc3 = nn.Linear(84, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        # Max pooling over a (2, 2) window\n",
        "        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))\n",
        "        # If the size is a square you can only specify a single number\n",
        "        x = F.max_pool2d(F.relu(self.conv2(x)), 2)\n",
        "        x = x.view(-1, self.num_flat_features(x))\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.relu(self.fc2(x))\n",
        "        x = self.fc3(x)\n",
        "        return x\n",
        "\n",
        "    def num_flat_features(self, x):\n",
        "        size = x.size()[1:]  # all dimensions except the batch dimension\n",
        "        num_features = 1\n",
        "        for s in size:\n",
        "            num_features *= s\n",
        "        return num_features\n",
        "\n",
        "\n",
        "net = Net()\n",
        "print(net)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Net(\n",
            "  (conv1): Conv2d(1, 6, kernel_size=(3, 3), stride=(1, 1))\n",
            "  (conv2): Conv2d(6, 16, kernel_size=(3, 3), stride=(1, 1))\n",
            "  (fc1): Linear(in_features=576, out_features=120, bias=True)\n",
            "  (fc2): Linear(in_features=120, out_features=84, bias=True)\n",
            "  (fc3): Linear(in_features=84, out_features=10, bias=True)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "98OUHzD-46si",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "14685c29-ac05-4593-8678-00f6dd5e49d2"
      },
      "source": [
        "params = list(net.parameters())\n",
        "print(len(params))\n",
        "print(params[0].size()) # conv1"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "10\n",
            "torch.Size([6, 1, 3, 3])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5fSVFTGy7iiE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "32cdbdab-4662-4782-c4f1-6ca06ecc718a"
      },
      "source": [
        "print(params[2].size())  # conv2"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([16, 6, 3, 3])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_pAJ9G2L9Cyd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fx6FiPbR9D2z",
        "colab_type": "text"
      },
      "source": [
        "#### Conv2d takes in nSamples x nChannels x Height x Width."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cg2Dk_ud7__4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "63b988c2-f02e-495b-f9d7-0b2730a51c52"
      },
      "source": [
        "input = torch.randn(3, 1, 32, 32)\n",
        "out = net(input)\n",
        "print(out)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[ 0.0132,  0.1461, -0.0693,  0.0979, -0.0507, -0.0881, -0.0144, -0.1017,\n",
            "          0.0246,  0.0951],\n",
            "        [ 0.0261,  0.1233, -0.0621,  0.1154, -0.0379, -0.0913, -0.0386, -0.0807,\n",
            "          0.0452,  0.0676],\n",
            "        [ 0.0348,  0.1506, -0.0708,  0.1415, -0.0518, -0.1054, -0.0157, -0.0928,\n",
            "          0.0348,  0.0882]], grad_fn=<AddmmBackward>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p_CCT-XU8PNo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "8087f40f-0e5c-4776-894a-00bf48045b29"
      },
      "source": [
        "out.size()"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([3, 10])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vegAtX7V8k9Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "input = torch.randn(1, 1, 32, 32)\n",
        "output = net(input)\n",
        "target = torch.randn(10).view(1,-1) # 1 by 10 "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "owlqIp4f9SI1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "criterion = nn.MSELoss()\n",
        "loss = criterion(output, target)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mtC3Av8K9WG5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c2e7f548-049f-4c5d-a7b8-aff8024e10d0"
      },
      "source": [
        "loss"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(0.6606, grad_fn=<MseLossBackward>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fyux7-bG91rF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "41dd837d-fc5c-4d7a-d750-c590913e2fad"
      },
      "source": [
        "print(loss.grad_fn)  # MSELoss\n",
        "print(loss.grad_fn.next_functions[0][0])  # Linear\n",
        "print(loss.grad_fn.next_functions[0][0].next_functions[0][0])  # ReLU\n"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<MseLossBackward object at 0x7f6570e7bc18>\n",
            "<AddmmBackward object at 0x7f6570e7bc18>\n",
            "<AccumulateGrad object at 0x7f6570e77e80>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MsSo7C06957r",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        },
        "outputId": "cb3ade5b-4f46-4996-8f94-971c261968db"
      },
      "source": [
        "net.zero_grad()     # zeroes the gradient buffers of all parameters\n",
        "\n",
        "print('conv1.bias.grad before backward')\n",
        "print(net.conv1.bias.grad)\n",
        "\n",
        "print('conv1.grad before backward')\n",
        "print(net.conv1.weight.grad.size())\n",
        "\n",
        "\n",
        "loss.backward()\n",
        "\n",
        "print('conv1.bias.grad after backward')\n",
        "print(net.conv1.bias.grad)\n",
        "\n",
        "print('conv1.grad after backward')\n",
        "print(net.conv1.weight.grad.size())\n"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "conv1.bias.grad before backward\n",
            "tensor([0., 0., 0., 0., 0., 0.])\n",
            "conv1.grad before backward\n",
            "torch.Size([6, 1, 3, 3])\n",
            "conv1.bias.grad after backward\n",
            "tensor([-0.0044,  0.0030, -0.0070,  0.0054, -0.0065,  0.0049])\n",
            "conv1.grad after backward\n",
            "torch.Size([6, 1, 3, 3])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O8Bv9Wro_XXN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "f64e83e6-16d5-4c28-b6c2-b0bf69d40a05"
      },
      "source": [
        "[f.size() for f in net.parameters()]"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[torch.Size([6, 1, 3, 3]),\n",
              " torch.Size([6]),\n",
              " torch.Size([16, 6, 3, 3]),\n",
              " torch.Size([16]),\n",
              " torch.Size([120, 576]),\n",
              " torch.Size([120]),\n",
              " torch.Size([84, 120]),\n",
              " torch.Size([84]),\n",
              " torch.Size([10, 84]),\n",
              " torch.Size([10])]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rIEFf4G8AJI3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "[f.grad.data for f in net.parameters()]\n",
        "# tensor([-0.0044,  0.0030, -0.0070,  0.0054, -0.0065,  0.0049])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WJ6cTYI_Ag_X",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "eb3e3e68-80ba-4ffe-da76-de2d40ba16a5"
      },
      "source": [
        "[f.data.sub_(f.data) for f in net.parameters()]"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[tensor([[[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]]]),\n",
              " tensor([0., 0., 0., 0., 0., 0.]),\n",
              " tensor([[[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]],\n",
              " \n",
              " \n",
              "         [[[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]],\n",
              " \n",
              "          [[0., 0., 0.],\n",
              "           [0., 0., 0.],\n",
              "           [0., 0., 0.]]]]),\n",
              " tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]),\n",
              " tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
              "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
              "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
              "         ...,\n",
              "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
              "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
              "         [0., 0., 0.,  ..., 0., 0., 0.]]),\n",
              " tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]),\n",
              " tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
              "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
              "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
              "         ...,\n",
              "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
              "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
              "         [0., 0., 0.,  ..., 0., 0., 0.]]),\n",
              " tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]),\n",
              " tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]]),\n",
              " tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6FgUYsOd-bqn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "# create your optimizer\n",
        "optimizer = optim.SGD(net.parameters(), lr=0.01)\n",
        "\n",
        "# in your training loop:\n",
        "optimizer.zero_grad()   # zero the gradient buffers\n",
        "output = net(input)\n",
        "loss = criterion(output, target)\n",
        "loss.backward()\n",
        "optimizer.step()    # Does the update"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zU12G59ABjkP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = torch.eye(2,2, requires_grad=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6cU9_MvnBFPu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "fc103f65-5fb8-4a19-d965-92ba74e80057"
      },
      "source": [
        "x.data.sub_(x.data)"
      ],
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0., 0.],\n",
              "        [0., 0.]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 94
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GcooZcBGBd04",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "5f0e6e9f-2575-4156-939c-f69dffed9899"
      },
      "source": [
        "x.data"
      ],
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0., 0.],\n",
              "        [0., 0.]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T3cev-xsBFeX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xthnssHdaRE8",
        "colab_type": "text"
      },
      "source": [
        "## CIFAR10"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-RhldTUyaV7r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "??transforms.Compose"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qhy_muOMa5_s",
        "colab_type": "text"
      },
      "source": [
        " * ```transforms.ToTensor```\n",
        " Converts a PIL Image or numpy.ndarray (H x W x C) in the range\n",
        "    [0, 255] to a torch.FloatTensor of shape (C x H x W) in the range [0.0, 1.0]\n",
        "    \n",
        " * ```transform.Normalize```\n",
        " transforms.Normalize(mean, std, inplace=False)\n",
        "Source: class Normalize(object):\n",
        "    Normalize a tensor image with mean and standard deviation.\n",
        "    Given mean: ``(M1,...,Mn)`` and std: ``(S1,..,Sn)`` for ``n`` channels, this transform\n",
        "    will normalize each channel of the input ``torch.*Tensor``\n",
        "    \n",
        "* ```transform.Compose```\n",
        "transforms (list of ``Transform`` objects): list of transforms to compose."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zVLpWcKCaz08",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "a44b9490-9d6b-437b-f1f4-1dec67e107fa"
      },
      "source": [
        "transform = transforms.Compose(\n",
        "    [transforms.ToTensor(),\n",
        "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
        "\n",
        "trainset = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
        "                                        download=True, transform=transform)\n",
        "trainloader = torch.utils.data.DataLoader(trainset, batch_size=4,\n",
        "                                          shuffle=True, num_workers=2)\n",
        "\n",
        "testset = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
        "                                       download=True, transform=transform)\n",
        "testloader = torch.utils.data.DataLoader(testset, batch_size=4,\n",
        "                                         shuffle=False, num_workers=2)\n",
        "\n",
        "classes = ('plane', 'car', 'bird', 'cat',\n",
        "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
      ],
      "execution_count": 106,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r0it [00:00, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "170500096it [00:06, 27768524.84it/s]                               \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "miWP6tpL2hGX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "def imshow(img):\n",
        "    img = img / 2 + 0.5     # unnormalize\n",
        "    npimg = img.numpy()\n",
        "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
        "    plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mz5SgZXq2yC5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dataiter = iter(trainloader)\n",
        "images, labels = dataiter.next()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3A09iElU3THZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.nn as nn\n",
        "import torch.nn.functional as F"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6EpI_2QV24D7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "fd34e25f-e05f-4820-f173-6ac467a124ab"
      },
      "source": [
        "imshow(torchvision.utils.make_grid(images))"
      ],
      "execution_count": 122,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAB6CAYAAACvHqiXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztvWmsXdd1JvjtO8/3vnkk+UiKlETN\nFq3Isa04suPYZSeqwW3YHaRdXQYEdFejq4ICup0OgoqB/lFBF2oCUuk2Kik7rlTsxHZix+XEceSp\nIluKSEskRXGmHofHNw93nu/uH2vtsxaHR1KPMh/fy/4AQZf7nHfOns45a/yWsdbCw8PDw2PrI7TZ\nHfDw8PDweHvgX+geHh4e2wT+he7h4eGxTeBf6B4eHh7bBP6F7uHh4bFN4F/oHh4eHtsE/oXu4eHh\nsU1wRy90Y8yHjDGnjDFnjTGfebs65eHh4eHx1mE2mlhkjAkDOA3gFwBcBvAKgE9aa994+7rn4eHh\n4XG7iNzB3z4F4Ky19jwAGGO+BOA5AOu+0FOplC0UCndwSw8PD4+/e5idnV2y1g7d6rw7eaFPALik\n/n0ZwM/c7A8KhQKef/75O7ilh4eHx989fPazn71wO+f91J2ixpjnjTGHjDGHarXaT/t2Hh4eHn9n\ncScv9BkAO9S/J7ntKlhrP2etPWitPZhKpe7gdh4eHh4eN8OdvNBfAbDPGLPbGBMD8AkA33h7uuXh\n4eHh8VaxYRu6tbZjjPnfAHwbQBjA71trj7/V6/zmb/6mu56+9vX34/+3ul0AQLlUCo698tqP+P8/\nCNouX74IAGg2ukFbMpUAAEQjLQDAWH8mOJbODAMAis1VucaVIgAgYmSaBkfoGxgN9wMAcrl8cGxl\ndQ4AsLQgfTOhDp2fkDG12/T/0moPABAOR4NjIf7EZjLSt1g0Tv2Iyfc30x2Fxm/91m8FvzstHrOR\n411LY15euBK0RWMxAEC+f5DPl3FaS38cVhdxv/TqGHv1QavuefWZ68Ncd566yI0uYcwNGm8Nva8i\nERqrnjeH2NJRAEC32wvanLkwElH35p+NFq3x1N77gkOD/QMAgEuXZuX+UdJQBwcHg7aQpb9tNGl9\nyuV6cKzT5bbKUtDW69L56bRou8P9FGiQiNGYqo12cKzFQ2jU5bpt3oDNVkuu26O56XS4P41GcKzR\nbNI5iAdt0RDdq16pBG273vEsNL52ZE7Gzs9BqlcM2ro92s82JoESveYaXT/K40zIXJkejbltV4K2\ndKgKAAgb6VuzRwsT71Df6k0ZZzczDgAIqY1VbNNYopmBoC1SWwQAlFZp7uM5Odbt0vzlEuq55Xmr\nt2TPNHt0vJfI0b87csy0mzwmtcEjaQDAx9+Rw0ZxJ05RWGu/BeBbd3INDw8PD4+3B3f0Qn87YPmj\nZZVo12NJqsPSiG47d+EkAODVIz8Ojl2eeRMAcGn6zaCtUScptd2Tr3OnS1JHlMXgXEKk4HyBJZKy\nSNfpaBgA0GxJP8rL9PWMRukL22GJAgBKNbpGB6IVlIsk2aVaaelHh8ayvEx/29cvX+R4lLSIipJ8\n4nGWkKoylkzuagldw0ZZi1CS7OqleQDAt//kD4O2WJLu9YvP/SMAwOC4uEQsi59GS8vB7+slZCf9\ndq+W39fto76uSPnUFtJ/dqM8ibdBQr856PpOkgVEcrVKojJOYeJN3GyIFNxlCSyTCAdtmf4+AID2\nJTVqtM5d3mu5vOzJcJj60VB7rMbXjUZEOkylaG+lk6RxNdoiBccjIR6Rkg55wvU0uuevyxpwLCqv\nhniMpN9OT8bSYqk3HJW26xCSY9bQ/UOq34ZfP51oUvWD5tlEWPIOy543vB5RVIO2KGutja70N8K3\njbGG1ezJ8+teMwaxoC0cp/vbmEj5nUaIr0UXi+nJitEYIlaec/AzH4/ImOtVeh/YEN0rYmTsludG\nv+NMt4k7hU/99/Dw8Ngm8C90Dw8Pj22CTTe5VFukppZK4oyslOn32pqomtUamUJOnTsCAHjxR/9d\nLmJJlYlGRGUyrFv1lHrWqLAzdJgcoLWmxMWfPLsMAIhFxckZDrNKpcw2uSx9A2MJajO9xHX3RFfU\n8lQsy30Ula1SKQMAQmGa/l5HlqHFDp1oXFSxcJTaOk2t4q0Pa0k1XZqfD9p+8OcUgHTs+9+XE9m2\n0WTTz2M/867gUCLLZqCwfPPDkZj7EbT1D1Dy2tDYGAAgEhK18uadVL/dtLGK3NV+xxtYVzZmcLl9\n9HrUD21yMdwRpVEjwuaMbILWOGTFrDE3SxG8lYrsv04oydeVa9SrtBdq7eudoqzto1ZTDs0W/XGj\nKU7LNv9tN0Z/YELK+Rt211ImBnbstpuqI9z1wFGqTCkJdp7XVYBBp8VOvZCM+Vr0jDK5cJ/asq0R\nT9DzamLyDIUsjavbqfHYxAyRZqdhWJvO2IwRioupKhVmByyc2URMOgu8tggpE02S1q8VkzkyvLbx\nKF0/EVLPQcxc1w9nRo2p5yXOZqsun+cCOuj2dK9uWJse5T2zUXgJ3cPDw2ObYNMl9G/+1ZcBAFdm\nz0tjj77Oxsj3ptGgL/XqGoURaYl+YY6koHhCvrAjo/TFjsaUs5WdhU4yL1bli9ms01RM7ZGveadD\nWkFESRAxdojUWWNIK8dqX541hZTcc22J+70iX99alSSuJktbyZQKn0yzQ1MJuh0+r7gmkhpEkbgO\n515/FQDwZ3/whaDt4rHXAQDhumglI6PkWD1z+CcAgBe//Z3gmGUpLppRGkiUnW4tmbedU3sBAL/w\nkV8CADz0xEHpN4ui1apIqcUiOezCSpYYGKDQtNHJSb6PbMs2azvhsEh7d0tC73RknFHuUyopfYuy\ndNVix1ZpTcLpuj3624Vl2aer7Gjbu+dA0DYzQ+Fx5So5R0tFmasea4ahiIi1hT7aKz0VUum0Hdfv\nkPYqh64+BwBiHN7Y7Sgt0Emglo61OyK9h1lrTMXl+Wq1aIN2uuu/QkJaW+Nnz1g5vxdI0Er6Ndwn\nlrIjRsbphOqQURJ9ih6EjlVJi23SONNx+gOrQgmd+mfVq89wIEJPhQ+HeQ5jQYCBGhf/jmnpOswS\nekKsBJWG0zLYgRxWfeTBWKPWwKyv7dwuvITu4eHhsU3gX+geHh4e2wSbbnJ56ZUfAgAiUVFvnWOh\nURe1r8tqbY1NHb2eUoc5C7NnRX1xmX2NZTkvnSJHX5udNZ2eXD/GTshuS1Rel8W3uiax6UvLlPk3\nPEzX2rnjieBYvUkmkbV5UZ3WitP0Q6uO7EhqcNxyqyMOrmaLliSkYtmdjSGRFFXzWnRUv8//5BUA\nQOnCuaDNZbVdKIqTaeUyOe4e3E9mk0HI/E2fO0N/15Z7tjkG24ZFrbxwnObm/ztBScITu/cHx1xM\nc1U5BpdXyPncl80GbRPj5FDd/+jjAICDPycZh1P3P0g/ftp2Fg0eZyYjKrIztSWiynbB8c0z82QG\nrCmnYa6PMgu7IZmrizOUOVmqyDXmLlPmrjPp6Dj3Lscl5wqyBhE2jSSU+SPKJhTnuE0k5Hxn1rsq\nhdc5b9Uz1HUmFj5UV2ayLvcpne4P2hIcS98Lr78wyYiMxZlLrMo2jUQ4czuknltDY07H6PmK9OT6\n1tJza1W2bpsdnrat4vJdTH2L3gFrVXnOIznKFM1kxbRaZTNQSAUndDn2vy9FE2KU7Gu7dN3+hPR7\nic0rZfW8dHjQPWdCjMl+CvE7KNyWea5VHBXWJDYKL6F7eHh4bBNsuoRer9MXKmnl21KqsOSqOCkQ\npi9lnaXaal2+/jF2auivk8s+7LZFMu5yaFGtRk4KC5V5yVJnqyL3XFkjB15ZSb99A+SUck6QUESk\nnEyEs0hVRliCnWgNFSLZcqFYLGi0myrDMMmhcHnhtzCG+mQqKubrGixeEWr6VIj6NDYsntMaZ7H2\nzHDQdvSNU3RszxSPSfqd5PnYPTIStLGAhCWlsbiMt5kFkrxPlV4NjkU4Zq6uaJNdCF5VOT6Ls8S7\n8+pPDgEALly8GBz7x//L/woAGJqULNbODbJY3dpLRuSNMlxvDxmXyRlWTiq33lY0HMt3rdXpvNWi\n7MmFNXKGNpUW2OD9fOmiOE97LIGGOUxuoE80l2yO+pFVmoLTCnod6Uenc/W+sCoz0jk0Q0Yk+mqd\n9nWnLvvaKbdlzkbuKOk9naU9v6b4k0YnSdKt18pBm3LZ0zXVc+OyXps9lbHKvyMqi9WyVBthadbU\nNeU2PwcqA7XTobvGVBiiy/AO8/OeUhxIUQ5FHpBpRrPOYZw9yeZu8WbvdmgMO0YkMzvK3EoTQ8Lv\ncuQ8rfdyRea5y9muzQY5vI3KCnUOW/T0Gkh2+EbhJXQPDw+PbQL/Qvfw8PDYJth0k4uLte225dtS\nZ/OAVeQ3bc6Ma3IMdEuZKWybVKukorN09Kgho9TgRYr57XF8bDYv58eSdF5dpSlGY6TWFlT2aD7D\ntKEcNzx7WShCw5xBWVqTNuPMLypeN2xYra2TCtZJienH8TvVaqJmdzi9bpHNGgCw435chZA2P3DM\n9u69e4K2eXa+xZQZqzRMTq6lhUXuj86MJNVxdVHU7JEJihe/b2qnXIMzG2s1+tvFqjh4s2lSTQsZ\nmb9uk6lElSrdz+RkPValr5x+PTj2wp/8VwDAz//D/yFoG9xFTtxOT2dEOrpfx/Z2lQEObwUuLrqt\nyZccvay6brVCYw2420LKrMHz0r0qJJyOp1W+RMc6hz7tgV5XxX+zQ7Wr4uFDfA+r+tFq03EXmh42\nKsCAzQM6Dt1lX+o8j7USmU7mF8kZGE/K/lur0lotLS1KP3j9+gtiu6iLv5PO0f1gauaWMrl0+Nls\nKIdtGOzkbLn5UJmzHI8fUsRaUX6FRdR8OIvM6BiZC+OKcK/GGai2rIj/2LFaiYoJJZwmk1JjhRze\ne8bF2dk/Rua/k8dlPqotMsmEkrLX6zUKFGi16F2RiUi+Sa/J+6klprNez8ehe3h4eHgwbimhG2N+\nH8BHASxYax/mtn4AXwYwBWAawMettavrXeNmKJXo69VLyee90aCvc60mDgPDYX9dlkYSimuiwc6P\nSlkk3TCHQWruiFbzah6MfiU5DvbR/YdGhMo2nyaJdHFBruskmRhTlXaheTaYZrQrfatyVmitJF/i\nbJKz0EbICdOxcn6lxBS/YeGxSbrss5t8wAcnp4Lf8QKFPfUpL9XwfUTd+ldzLwRtDeZ8CXPY5Hvf\n/0xwrMjhha+/fDhoO/MyOTzf9/PvCdoeeeRhAICNnAUA1Kcvy005Qy6k1splhSq/KiKchhdlrSqf\nFelw/uxpAMCff/HzQduHP/mrAICJPfuCNlc7wDnM7kRScRwttiePR4dD/WIqM/jyHIWZLa/RPu10\nReNzUnM8LuFxTnNqKZrUKEupYeZhiSmyGMPSe1VRKbtMTqO4RSo1XasdKOTFuZfkfZpMSj+yhTxf\nVyTdco322yrv09KMaIOxNNPF6vDCRZJcY01F+SpRjXS+kecm6jiKVL+7McflIv1ION6kNjsSe7KJ\nQzwfYSvnWw7vtapvq5xVHuVnc6xfnvNGia5xfla06PgQPfO5iMxblDWysTStd7wna1BbodDlsCIa\nctwvTaVNOQ0oyg+uUSGKoQivh6YIV1wvG8Xt7PvPA/jQNW2fAfCCtXYfgBf43x4eHh4em4hbSujW\n2h8aY6auaX4OwPv49xcAfB/A/7mRDszNkWSQiEtXnE28Utb2JS7lFqdvUHFFScZsj4qocKZmgyV6\nxXnRZk6HaoWuewXy1XVsdOU1kSrc/RcWpGDA0BB97Q3b8yz6gmM7x8hm3e1XfBXWcViIDS7BoY7R\nJEkolYp8mV3pr9UVkdArHEIYia6/XGHFNLn/IEnQZ1/5UdC2/OYxAMAkJxEBQLlO96oskbg8d0kK\nhBjuW2pYwhbPsh3+x6+8FrTVWJsKccijUSFzcV7TtpLi2jGyYSbiqnyXYxUs0zxnoyKBOVx8Xe75\n/a+TBP9zH/kHQdvk/gcACNHlxoMWgRDb4zttZQNmSbvWlKudfZOkvGb7+oQey+F8bcWKGGbbfCwi\nclSObdDJFI1Z07AkmNBHS11tDrVdK2nOFw7p5TJzi0uyF5LJKPdf5ntwgEJX2y0xrM/MUSjl3CKH\nISpJuskxjcm4tM3zvco1maOJB3EVbFeVuONnOq6KWTQ4WaulQjujLJEno9RmVUimKxBhetpYz5wy\nSrPpsv/n1MwFAMBwQXw++3ZNAABWV1XRix6NOd2WcNkIS9PjcbpXY03eATmWrgfyip1xjaT2kJX3\nQYbnvFllrVGxtsZCtFc0j00iJXOzUWxUMx2x1rpiiXMARm52soeHh4fHTx937BS1VNdr3dpexpjn\njTGHjDGHarXaeqd5eHh4eNwhNhq2OG+MGbPWzhpjxgAsrHeitfZzAD4HAOPj49e9+EcnuDK2ykwz\n7EBJK5UmxFW9XbGHwTFVm5BrHXbacvk2F4PQVK+ubmK1zCqQUtOqHMplJfENYTZ1pArKqcIOWNvk\nLMGq8lQ6jhjl28jmaAxt5fColpjjpM71JFU1cBfBpfvdCpH6mTDrfjevQnaIvFP73vWOoO3yEjkX\nW7Oi9u1jR+p8jwphVJdFrWyzqn7u8kzQVnPdVMUEDr1GoVn9XC8zpjhGsjlyMlll9nLhfGFVW9LV\nm4yy0zChTEsJNk+EWmK6OMUZpctrYjL7OabvffDRR+n6cWX+MNfLLeYmdUndXmyrWrJxrg16Vjl9\nF5nOOJumcdYa0kfLjrCI2mNJzhpOpmTs2Sw7BtnMo8NPXT1Vze/iTCFG0Qm7QgouRFabijrMT1JX\nWaEL8+TArlVlfKUSh1myI9iqfjQ5vLannP0V97wqh+0ErkZHhS122NwQVVwuCecQ7CjemAqZscJJ\nNrGqTOwwj0/Xhm3z89hV9UCTOcqyTtbJsVtck3iNVh/t08Eh2cMLZyljui8r18hzyOhIjp79viEZ\n3fk1eh4vLIgz2pmeIqq/NTZptfk1m1Imqw5nHIcUnXBI8dFsFBuV0L8B4FP8+1MAvn7HPfHw8PDw\nuCPcTtjiH4EcoIPGmMsA/iWAfwXgj40xnwZwAcDHN9qBhx4nB02zqapfM+tfSDlmoiyZ91ia66lE\nAscJ0VGl35w0Ya9KuHEk/vQ/7TB1vjyjpiTCEnpXSUjhwIlBbYm4hDrFOTkpARU2ZpN8b+W0YUdP\n4LhVjh8XmuVKWgHiCMtkbk+haqySeyNqRWJ87LFHAADfPiLOxfk5TozgsDGjOGhcKbJVxZTY5u9/\n34jwqhSXSLovcxX4rHIMZgvMONgSraDMoaXFFYlbHBgkKSiRoHmrqpJrLQ5B6ygHW2WNJK+LL70Y\ntK2skhPZfJy24sNPPR0cC7H0piW7m8FxtMTTkjSzyNrAufPTQZsruuGk8KbSIvr7aSwFFYKZZYZJ\nnTDnOFxckpSWxgNpXXlKrdP0lBbY4rlxDH7plGblpBNjadEyI8xnUqksBW1OQ4jEWOK11+95V/6O\nO07nR9dnAG2razSZYTSsmDrBmnVUlcwLR7hIBz/nVpX16/C8RWKqDiCX9etFZHw2Rm3hOLUtla4E\nxw6dIQbSXluVx2vSHs8UpG+xON2r1CCVvTgnUv7ZCo29op7zUp2cyiYqz0siSdpAgxkYWyHZH06P\njah3XCYt19sobifK5ZPrHHr/Hd/dw8PDw+Ntg88U9fDw8Ngm2HQulzZ7EI2qrecyrDodVQ+UVUxn\nctHl4E2Pzg9H1HD4U6XbuuyYMazCJpOiYjmHakjFsvdcJqXKAAwzjapzWMXioi4mElx7saMoNDlV\nNZMUOtxWnvpUq5DJo6Wqr5dWyTGZzEjfMpztV69dS1Aq6DbEhDF9+G/o/BXhmkhxcYzhAYmTfeVv\nqZboIptBUjnpY5znJpsVNTDHmbX5AaHgrTCnx8o8mXlMVMby6rGTAIC0KswxWKBM0WRSzBmFPFdd\nZ8ddV6nZlh1nOofOOZhbNYlvnz1PqvQr3/1r6itnpALA1AHKZr3tePQw9XduTmhuj7x+gu6p6JgL\n7DDLZHl8qtalswrklPmjn+uBastPjefP0b+2deYgmxLDek82rzdF9NhMY5zTXNepZJNOwOkCwHLc\nd1+/rLflzOQUP1fDQ7LGjkvm7Emp+5vg4hTJ1Poml7CiY24yl4sJy34KcRELbUGJ99a4v7QnG6o4\nRYtr1BZyioPJPegdeTZiLvuWM3LDRm5QqtNY6oq3yNUlieaEy6XMWepNzl0ZHFEmK7eA6nnssgk4\nYrQJhQMiOG6+UpWIi0EOtGj2FPfRTYqF3C68hO7h4eGxTbDpEnqDWfoiqqy2kziukkLY4emkhWhM\nxJygNFVT8yhw9l5SO8Loei5DLqayK11lbk2Y0uV71uvSlk2TRJJm51+nLhJKkSO4hkcfDtr27p4C\nAJSWRJp89dhRAMCliySNVxsSLhiJOclYwqrSGf7660rv18A0xcE1f5ZCCc++cVxOcJqEciY/eZDK\n5/3gx8TRUq2IBFEs02Aqip0xwdpJrSLaQLZAUt65SxTCtTwrUm2NM1FTKgt4R42kx9F+0RTCrgI6\nO8ciSvtqc1ZvRa3tMvPjTO7cFbQ1mbVulrMDjx8/Kvfc/yDfR4mCN8Hrr5O0f/GKROM2WTLOpWVd\nBlnb6bAk2NNZspwFGVXiZ7N1vYblBLooM4UmVMk6V7iip1kluQp9Szma3V53TtqukvLdvu4ozcKy\nRhtXHCrjo8QWGOFw0qTSqlwpwZ07JXTvCnO9rK7K3r0WLR3Ry9m/RhWiiLDGHO7KvotyIZgYaxFd\nVRimwZnbyxVVzpEnMKwyLrt8j0KXHJlhK47KIrO6rjVUUAW/U06cOR205fso9LdvmMZ87vJscKyb\nIkl+MC9aTCJCz0G9LuvS5VDoAhfYGMpK/uXkMLE5RidkTju1DdFhXQUvoXt4eHhsE/gXuoeHh8c2\nwaabXFy8s45PrbOqHlFOFadG9gInqo5FJfUsGlXx3wlyOuQywunpYlyXV5hUSZFGZTjmuKYKNLTb\npO5F1Hcv1CJn26nj5Lw5quK6a1yPcd8DjwVtn/6fqRLFM+8Ratr5BVK9//zid2i8Sq3sG6S45Y6K\naY6FSMXrqOIH12JlVoi1lmbJeVUpiTpcZqfR0rzQoqaYHjjKDqu+nJgTwCYDrCjaUK4peeWSkBhZ\nl8nJDp12XRFx8Vr14rK21bojPBMTUY0plF2y365JUUPz7EyePydZeatFUqF7lyRrc2SU1N8YUw03\nG2LecHVGjb09k8sbp6bp3gXZOxMT9LtakTlt8bo5U0tGOZAzbJrTpHOOwlabS1JMwOWKe7RUrHeU\naYdjKguyweNqKtIvd113nja5OLaNclmRebH1SlPwugzfKJshNWlUi++Vzug4abd+N3Hk6WeUTSNx\n5d5OdblzFYkTNyv0u2NpnCFVlzTH2eJ1lUW9wpmqZbXeQxnaxyNDNC9ltWYXmHzPxCQ/YKSfnv1s\nWkxQLmDizQvTdE8ra/DMB4n8bs+eh6TfHDgxe0XGslakoISVMme/KrNyCLQ/SooOpWvV87dBeAnd\nw8PDY5tg0yX0Cn8xq0XlTGAfkI5CFIcWSVsNJaHE2aGknZyWOSyyybGgbe9uypassfOhWhUH3mA/\nOSxOnTsStJ04Tb9TUQmBO3WE/ub1I2cAXC3ldzlj78jhHwdtvz1H0uz0Jz8RtJ0+Tc6X+YvEk5JM\nqaIGhSS3CZXmwhJJ1XVFs/uwVJcDAFy5JCFlAwP0t8P9jwdtqxWa1GOdY0HbGyeI06PDYXohyFjC\nnJW3c8d40FZcZgrZqkjtjrI4xdLH2A6pjr5aZApZJXW6TFgdqmlYO1papXXpdIU/5r4povtdrYok\ns8TVMbR0k2Tp8RJT/O44IH20HSeV3R5XRpR5YHTl+1UuRaYlbsNOeyfN6vDMBJeZC6ksyG7POTlV\naUUWpuPM79JQTuhIsJ9VZqnjfFHXdYJ2p+OcsyrLmG8QUuF0jgdGO3HBIZJOqyuuiibnSivWGqqY\nBT+Hvd76Eno0pkIUwcEPbbluHLSOnZXpoG2A57TJDtOO0mZyrInkM7KOiRhp4pfkUcYcO/fjWTqv\n3pA1a7OW1p8QCb3AGZqP7JO6jjHW8BNJ6uOu+58Ijn34g88BAFZXpW+rRdIQjCrysXuE3j3l0gj3\nQ/akG1ZkXOYom6d7njn+fWwUXkL38PDw2CbwL3QPDw+PbYJNN7mUi6SKtRRVaSpNqlUnomLC2cEW\nZpKflFK7hnJUreaJh8XxODJMdTXHR4VIqpAj00mQqafi3COsVo70iS2jzX6WE0cuBG3nTl3m/pLJ\nJ6SIkzquxLu67twcxa9+8YtfDNpaTGSVY+eNjk9emuMMNUWBmmIK3kZ9faeopqR/4DGKuw51Rf97\n6SXK2hwfl+ottQbH2fOtyg1xQJ08RyalCaVSJzi7s1dXWXns8KyUyVzSlxdT0QjXciwuS3xtk52i\nqaSomo5sKZak+ZjVNL5dcvYWlZknzDaGoQHJ7HOLevoUUaH2FBla/yA5TN//keek36n1iZC6uD4b\n062zuixcYp9zoKUS4lTLswmoo+qHurjypKqf6+AIvhKJ6+PQKxWZb5c9GlH2SFepyJHP6YxiR/YV\nV3TCLhchm5FcAGe2cQ7balX2gstw1AR60YDieH2ZMKQI1ZL8LEfaUomrXqJnKdcV8+lAhvbYBXbi\nNvT+YxpfExFTW4zNaNm4nLfGJr4Tc5RHkFYZvEMJun5MZWU68rjpk2eCtr37iIb5Zw++m65ZVw5N\npspNqph6m6Os14E+yb7l8HPUlmkvmI4yH3GmaETRTVvOJ/AmFw8PDw+PzZfQi8v0dW6pUKQIhwAl\nFV2s43RwjsdsUiTvJx/6RQDAM+/6YNCWTJCkqOsYONpS5zTSVbadNLRr4oGg7b1PfhQAcOzF3wna\nypwlKVl8IsW5yt8TKvvr4QMU2uQchABwnilY73+InLTHjv5Ers8hfLoGVL1KUl63rRlNrsauffuC\n35EOScRllcQ3MMnSuqqOfmWOwqkWZliSUdmbOyZpfo+fPhu0ZfKk4bgMUAB4eGA3nT9Fkn9lVSSw\n8UGSoIcmhoK25XnyXu3ae19NwSloAAAgAElEQVTQduYCaUCjY+RESmaFN2NpgTScnVMyp+Oc9eg0\nOQCo8GDjLHnVVyUs8jvfJLr+A+94MmjbtVccYNdCEnJlvtNpcqIV8tK3cIQr2fM2TSZEfA9xcQdd\nk8QV8NBtTvJv1XjdVXZlmH/HFUVzD6zNxLWUTyc6rTGutMYm859AZck67SGt6Vo5bHh1hdanpqT8\n/jztizWVIVwss4Z6k0Ihaciej3ARF9MR6bq4SGuUVL7qRpz2Z51r1WoeFlegpq2euSJnDRdVf8Mc\nhptkjSWlwp/dWhloBy/LtRFxlO59gJ6n8T2k6f/gxUPBsZkr9ExkByTgIsphkO22PBsd1hpzGX4X\nNRWvFI9vbUW8uRm1tzYKL6F7eHh4bBPcToGLHQD+AFQI2gL4nLX23xtj+gF8GcAUgGkAH7fWvmUy\nAhc5FVLflhbbio2SKjpO0mDRdef4geDY3imydzWVjdmwnTAaletaOEbFkBtccCwcEPbLlIS5enip\nqIo8tJ2mQBKYrvQ+MEIS7OSOyaDNJYD0VKmu3YN03sAAfZFHx4TjocrnOb4NAGhzIYxobH1pKDWo\nrsG8MSYp8/HIu0lTePGbc0Hb/CKFB87Nk6QULYsElhskib4wINL1/Cr1ra2Kb1xZpCW/byeFK5aV\n7dX0kb1y97hoU/cxZ0imT+zfl2cpaSjJ9uOlFdlG+QLZJp9+6p1B26ULFAraUslXPU5A6eeq7hMT\ncs9LKyS9v3lObKQ3k9BdiOzgkISrRsOOXVNsnukMrb0NWP10gRVXtu1G/DuKc4ht4U3WenTCnHWl\nGJO5oC3GRUD03nXhih1DUp+2r4O5WboqqSrCduei2pNVLuSwskx7wajQxwcfIK01tiJaT5W5bapF\nuca1ePZJKYEYYa3hxKt/K9fgZynQIgCU+XkyLGVnUiI1x5iHZbUme8ztWG1rj3PRjQxrVVFV+CbC\naxRX74VYjJlIJ3YHbUPseyuzZm1VH4sllqpjoiF2I6Qh9PWJlpvkxLBQjdalp/rhiq0k1PjWiuvz\n4twubkdC7wD4F9baAwCeBvBPjTEHAHwGwAvW2n0AXuB/e3h4eHhsEm75QrfWzlprf8K/ywBOgOrB\nPgfgC3zaFwD8/Z9WJz08PDw8bo235BQ1xkwBeALAywBGrLWOU3IOZJJ5yxid5JAefR/WUiMhUU0t\nmz8cZ8j4uKjsjqtDh4hZcAX0q/yIrCYyX4tTyQChJdUdmT5LNKqrRXH0ORV5ahdRtz7z7PuCYxM7\nSN23KgNv7jKZE5bS4pjJcHje4Pg431tUscUlMoM0GqrQARffiEbW//52QzKWMNePTCYlTApcrGPH\n/aJW3neGQgL7OPwvmROnzMUrZJopKJVwYZnUz4YydUxPM8dKjdTFqSExD+ycYKeRmo9FDmGsqAzA\nNmcgGjY7LK6Ko+jxR0nd37tbTCgXz5NTKqnCBJGncDFHF2uUV7nENUjnLgv3y80QZX6VnHaAutqc\ninLW1Yl15gGjzCDOb6ciHwOellhMO+noekkO40zERY2fmaE16HVl/6UyzvwmJpQ4Z6/WObQzW5B+\nFwbJxFVRtK71KhkqKsp0cfrcNI2Tx/AoO+wBoM5ZvfpZ6i+QaSGkHOTX4tknnw1+LyzQ3JemJaO5\n1Uf9jEP6lsjS/um6UExtKgrRvLV70u8yZ93WlYkow87THvc7qrJ74zH6nVDPUjREcz8xKWa4vn5y\n8rus2tlhMQMO9JEZMpGRea5zyHJTcQa3ajQ3UXs9HXiD6Z5rKuN9uajSXTeI23aKGmMyAL4K4J9b\na0v6mCWD0A0r8BpjnjfGHDLGHKqpVG0PDw8Pj7cXtyWhGyoH/1UAf2it/Ro3zxtjxqy1s8aYMQAL\nN/pba+3nAHwOAMbHx6976Y+OpPg8cToMpeir2xdWX682fQ3LPepyNCzOSMeU2K9KanVYnNAl5WZn\nSeI5eYIk72d+7t3BsaD6ekekSctilnaE7ZyaAgB87GMfo3/vFYnXVW43iqtjmJ2Kld17gzYnRToW\nu8ERIcrPsVOlfFmm0znHujcpcJFI6SQRkmCqRXGARtnpu/eB/UFbt0L3P/YyFbiY2CVJR5Nj5BD8\nb98SXpp+ZhCsqvC18WHqe5a5RZId6WOCEy9iijejwW6s2TnpW5slqssXSIqbnpsPjo2M0pqePvVG\n0La2Rs65MeVsbYVp3dbYiXVFXaPIHDG9lg5VWx8ucamrRFLnXI8q6TqXo31XKZIGUFMJV86x2lSa\nlpPuo1HNntjiY9QWUskqY2Ok8TWUdO3yYfKK+bDBrI8J7tvIsCjLjlWwYxWfSYPOTyVFGzjA+yLB\n4ZBZpZlVK7SfNB9MlcMFQ5H1XyFtFaYXT5AW+OhjT8k15ug5TEZkXWoczBDmuWrrEEzW6rpKi46y\nHJmNyb1S/HuUHfu9jnK6lphfqCtjf+q9xJ74nvf/Q7luhOa32eQghZC8A9q8x8cL8sxVuY5dSzlP\nA2cvB1o0VWJRj52zSRVUcf8eCuU9deZlbBS3lNAN6ZG/B+CEtfbfqEPfAPAp/v0pAF/fcC88PDw8\nPO4YtyOhvxvArwI4Zoxx5N//F4B/BeCPjTGfBnABwMd/Ol308PDw8Lgd3PKFbq39G6zPYv/+O+1A\nskMqXiuq6iCC1L2lVVF5mxFSV8o1UncuXVS1Bp9gSl0dh87OnUhSun7uPDlkVtjpFo3efPhtzsqr\nNaQf+x4knpQRdmjWlWOpw7HpKUWjGuZairG4qLBBlfYwqZr9KTEtHdhB89CuinUqzupeza6fKapj\nm50nrtMUFa/F5oDZk1I3sbJAZp06U6UuK/U5xY7VyR2ivlfaLmNViPhHOBb7Ps7y3D8p2XOuUMla\nXeJrV1o0p8ttRRmcI9W1WqKCAGuKQvbw6xQ7nlDFPQbypA73F8QB62hnXU3WFZWB18+cMrWyOLZq\naze0EAIQZ2dXZRLHs6Si6xhv5zwtcF3VtqJHrVWd+UXGmeY57SgTgItNTqVccIDOvaD5jse0edEV\noJB9HWETUV+e5iOtTIQNNhu2lTkoyeaMzICYKJ3zz5mn6iWZvyY791bWZK/PXCaTVja9PidOOKmK\n1rC5q39cTI/DE/S7vCDFWYa4Tmt+lHI5ehGZD+dUHq2JiWaWi63ooh75Apl3ImzauqAKoZSZj+aB\ngxIj/8yH/wEAIN2nqKJX2DTjLhuR+S5V6Xlpqn0d6lI/tekHzE+V4TlKK/6gPNemjSoTW1dRSm8U\nPlPUw8PDY5tg07lcmuwMbSgHVKtMX7aYFcdFq0FfwONHKazv3IkfBcfe+x7iculTEofjVGgrKWti\nkpx++++nUDhdKdxxsmj2xGiKnVhK4nGRaRUuuBDVPBTsQOm0RVLrsCOzqQo0OMmrxbwW2c50cOzg\nfZztFxWmxFaVrjGztr5TtKulvg51MhXPBm3z8+wYVFmeTnuIMIvimTelfFa15RgmZY6GB0jCGMyL\nxFhdppA6Jy2P7Ra2yukZWqtXfiJl+l47T9KSVc7Ffmaqq9do3WtqL1jO+supzNIHH6B77NojDt5y\n4ygAYJGLX9RUKbz7mWdmZUEqt68tiVP2WkQjrgCEzLdj6MxlRStosqRY4zDAdFL2idt2LmMZAFJJ\nx/ch0qTjBgoYQFWwWIbZOHVZuipL2rWG9G3XLnIOu6IXIZVtGmVpPDakJF3HqmmlHy5L0Qn+ZbVf\ni0Xq48KsaDgulFFFcV6PjqxBnh3qmvslkqOAgZYKU90zThwqaQ5wuLgg0rVz/k6MC69PlwMMGoqR\n8vF3HAQALC3TdctKor9vP2WYf/B9YlyI8TovXRZNIcKhqAUuPlNIyqvyxEkKIogqLanBloO62nf5\nAq1frclZoWnZwzsnSQPJZeQZrVdEA9oovITu4eHhsU3gX+geHh4e2wSbbnJxjsd2RRUTYCIkk1bk\n9q5uYo3aclMSA5rg+pulmsRH15ukvqRUXcNRJoaKc5adrpzu4n97yvG4g0m23vWzB4O2Ny+QGWFl\nhRx4k6pCfThCKlZFV4bn2NlOW1RC52wrFime+uibEjO9vEgD1f4Rl0l5s/oWJqQcUA2mdQ2LeaAw\nTJmtjrgIAKJ7aB7yoxT/+sPvixmryBmghaw4c6NcNzSs1OY6U9h2eX0OnTwZHDt+kopNnFMx513O\n2iuWlQORf7cbjvhMBlph5+LZaakzWmDzTgOi70+z42tmhsxGUzslRt0w0dn0aSHnchm8N0KKHY9x\nVWzCOdjaTR1XTnPZCrk1lnkpsDkhJLYUFDgTN6rsFMeO0x5w+yQaEVNUPE79tipbEpx/YZVpps7U\n066+bUNRybq8ilRGzJfOz9hRjkRmJMZqma5x9NR0cKxYYvOikb4VOPcjbNc3AxqVuV3guPmGmj9H\nuJfKybM8xM7QMhd9efWI5B/EOTP42fcK9fLEBD3DJ45JrdwGmz3CTAlcyIupIxaleTh3UplXOtS3\nkHodGjbDjAxS3zoNMUEtztNeu6xonuNRMhFFVH7McD/tAZeKsFKS+Zifp2d+dVWbsWTdNgovoXt4\neHhsE2y6hF5l50tEhQWF447EX743Iyn6cv/ar/0TAEB6QCQwMOfL+QtSjME5UIYHJOwuEiIJ3fRI\n6jRGpAsnGHWUEzXHX/ZnP/CBoO3wq+QQSbDKEFJ8My57tRlTGYmOV0JlibX598oqfaWrypHS5XJZ\n7ZY4SNIsBd83uv5yhUISSmgiJFVEEiJRpfKU0VlviBYTZgdwZpCkrXROJNIEz/2508K9EeOaWmMT\nktkaYanpGEvj5ZpoPXXnxFLO6gHm5JlTkbAhVkfi7CCqKYE0wuF0FxfFcbb84mE6P3IkaGs16L5D\n/eRMTqdEerrCdLtzyhH6+msi0V2LKNOedrSjnvdFU/HSBA5PzqAsrcncpnju0ylZgwZrjYUB0Zy6\nrBEu8fjGxiV0zkUL91TmcYYdryWl4Ti64Uyext5UxRteP0Ea0+KCUN+GueCDDgpYZqdolbNSe4qO\nNpmidYnHZH8Y3jvmxowfAIChfhlnNERrvLS6GLRlexQaONAv2sMqZzf/zWEq+nLhsjiyn3onZXY3\nW3J+NEr30Gt1/jw59Cd3TNE1V0VjHk/SvtizR7hq8gV6L1RLwpnT6dEcVjmTt6aydV3Bj/17xSk/\nPkK/U0nZ6467pcwl/AblEGKcITozJ+NrN9enIr5deAndw8PDY5vAv9A9PDw8tgk23+RSI1UsoRyg\nztHSLIqas5MzzN737C8DAFYV1eShV4lA6tXXhEiqzhVYHMERADzy4BMAgP17HwMA5BVdrOUakEVV\ntafK7JDpjOhKTx78GQDA8hqpju22qLcuxltXqXFEXJ2WqOpNNnu02qQqTwwoQrA0XW8lI87IA1Ok\nYvZlRb1dUsy4BFnKXIGyNbuqQGWXKUfjKosVHAfvuLMeekAqLe1gs8Brao6Gx5g2NCVOvVNnpgEA\niyukLo4MSJWffJYpYRUB0QKbFkZjMj5XBSrG3rpmStW/5Pjz5SVRm4+8QY6yvqyYmaK8Z+LscFxa\nEmfTpYvkKA3HZD5qQcyvsu+4axnO1lUq8NISmXSaBYkb3ncf7UnLmb9NFe/sEhb27BMHnpt7vQZ7\nuRbsyTeIqOrSjKjgExNkfkkk1GPa42peKob8wiUy3S0fpyzgXF76uMjmhnBc5qrVpL1eKYujLxym\neUu7eHGVwehqdEYU5WwYdP/wTbKX31COynaN+tgoSq4D6pRxOT4h9XBneA6vsHPbtmXPjwzQfIwO\nT0k/2KS6pEwXrgrQwXfSs3rqjDhA77+f4tD33/9Y0GbY2ds3KLkfnRbNzTyTvLV6Mh8HDlD1rIlx\nWdtUosB/J8MzLC/HmYiruComvxq3DefVHr5JfdbbhZfQPTw8PLYJNl1CL65ysQklCbY4C66mMiP3\nfOhhAECzRY6Lb//l14Jj/+1bfwoAKNfE8eOy5Y7gaNB2/Bg5Wp547BkAwFPvfF9wbMcOkmrXyiL5\nF9mZ0emowgUskHRb9C100g71jSTvRkMktVKJpJCq0iiKRfrqT45PAQAe2S31UXfmiWPkr1+SUMYL\nM9SPmQWZj1HlE14PYfW9NlwAozAs9ypxP3ptuufQpOZLoXsm+0Ta28+V0HXmYo9DDFvspCutiCRd\nahr+v4gtWS7gMDElnC/zXKsyleeQxppUi69zjcsnpiQD9b5hkoYWF8XB5mQTlyF6eUUyDNMFkj5H\nR0UaevAAhXHOHb2+6MWTD5HkvVwSCXaZaXldHVEAyOfoeudOk3QdUaGjmRQd27lLuEsWlqm/i8sy\nR4ucabtacpS2co2FFdo7/WoNCny8qwo6uDTT1SW6fqUo2kmOQ3pHR0X7Kq7RWOYXZP46LlOanxu9\ndxp16ltY8aq4xMmBrNL4rkFxTRVsYKdoLCtjqVwkKX9xSdZ7ZBfVvv2lDxKV7elTp4JjZ47Rs/ww\nZ3oDwEMH6PyleZF+G5ypPcTUzuPjMvaRIZLyeypbt9Old1C3J+sd4/DaSQ5/3blT9l+WQzZrFTm/\nxM+3LrrieH9qXRrfG2dfDY7V+N2iknpx5Ci9nwb3PoqNwkvoHh4eHtsEmy6hx3scvnNJ7JVxDo+b\nGpOwoGyG7Fvf+Q5J5t/7gdCvz7KdK9+niFU45Ktela//ygp9Uf/g85RgMntFJIj3f4CkdqvszobL\ng3UU/8mpk29wf8kul1T25DonD6VSIgmuLJNt78K5c0GbK1jx1JMUhmWiYk8uhokFrtz9i6DtIkuw\ncWWLHsX6uLEljgsHRFVCFudcLS/RPFQVy2GEWQBHJ0S6MWxX1VJIPkvSsmOYLMZEOix1yE49MiLS\neH+CJMa4qlIwmSC7+wJrMwllX++fIGnosUcflutW6bo/flkqyE/uJL6W02dpbTsdSdJ415P0t1NK\nKxjqcxrh9RL6zmFav4wKOcxxslGek0UAIMVEPsu8PmenxY77s09RIYez5y4Gba8cPkTjXBbJ2LFE\nFldpTI7/AwAqNbpnOCz9Tidp5bvK9j/ADIWugEZC2b9TcZckpZKwKnT/alT2erHGTKGcTBVXrJKx\nHmlYecXhM9JH/Yx01ucfGRoVaXyZNXGdeJPN07pXynKNHIe6jo3QPD/8iIQXfuOb3wIA/MW3/lz6\nxqGX6ayEMsbd88pcMnt2iB8tzwlWy3MSjpvmUpRdlQjVq9Pa5nMk5adVCHBtiZLSplU5vSQXCxkc\nEh/SWeYyurhA2sObV2SvFXj/1Sry3rMJWfuNwkvoHh4eHtsE/oXu4eHhsU1wS5OLMSYB4IcA4nz+\nV6y1/9IYsxvAlwAMADgM4FettdcF090K+x8gdefKjDg0Lavl+RExdfzwR18GABz+W1KprRGVet8B\nzoJUYWPxGF2jmhN1zjjHDJt03nxTeEcO/8SpbqIm5gqsPinPxdwS172cpn5Ua6oiO1eeHx4QFfn0\niRMAgPlZUbPzOTrvq1/9MwDAnj1yfogda9MXhWskzPdvVNevsH5LWJfZJ0ter7NjyIVtDYshp8YO\ntp5y9LU61I+kor6dmCTnojMzLRXFmXv4BJmndJGMPSM0p8mezGmjRnP/yutcy9NIH8vMzzOzqBze\nXP093yfhpCscRmpBc/ShDzwdHNs5VuDxibkknVVpe9fAqdcdFX6aStC+SOXk75zVaJDDBKfDMvY3\n2dSydEE4aBod50QTR3qSOWJ67GAtV8RE2O7QWBKq9mcqTffPplX/q1z3MsOZv8o0V2an8uq8mH7i\nnAWZjsrz1WQTS4gNdjoYMZZy/DGqyAMTCxVi679CTk4Ld06UC7zkFD9Ogp+vdlWcxKvLNIdr/CYZ\n3CUhjb/08V8BAHz9T78atP2XP/kjAMD9eyVKYM847bfKMjn7M3H1/F6ZBgBUi/JeGBogc26fqkmc\ncZnGzCmzuCLP77EjP6C2JVnvDPPR7NoroYyOYjjFccFPPiz1VPcwzbRRoYpVXse/fOm72ChuR0Jv\nAnjWWvsYgMcBfMgY8zSA3wbwb6219wFYBfDpDffCw8PDw+OOcTsl6CwAZ7mP8n8WwLMA/kdu/wKA\n3wLwu2+1A9k0fYo/8rR8W0pMFv/KZXE6hAx9Mfu4yEInLA68lVUOAQqJxDE8zuxuLXEGVYo0jL4h\n+v/S2nRw7MjrJCXsmpIv/WKRvqwxI47E4T6Sli4zr8Ul5cwdYdL6UlGSkxZm6bqdlv4Sk6R28iQ5\nEGcuS7KF5UQNowpthCI0H9HozaoJ3ALB5WSeY3GaozA7ZfuHJLFiaITCuxpVGV+7TvPcVUktfSO0\nLjl2mDVPi7SVYh6WnirTt3OCwsyiiivk8GGS5AxzjPzNj8XZWW/TXJVU2Ocgc2kMDgpXyAJzlezf\nTdpatl85sSI0p/epJJ+B3VP860Vci/wgSfJtyD1tj9Y7pESgHjvBdwyRZHdFhRe62hgVVXghmqb5\niKuqKG3mvgkzH0w2o0q6cTjiwqxoJz1mjhwbEj6dOu+nOme1NPIiXw/1U2LWoEpUW1sgrcGEFM8M\nawMtdijqpDSE6Xp1lVzjlMW6Yk+cwtU4dFppJ1zGMRFRZf1Az8lEv4y5t0b7baFO9zrzTZFWDWsz\nzY70Y3qRk6pWpoO2zn4OpuAyeiHFMJobpjUaG5YwRJccWG2Itl2q0N/Wp4kfqqKKTyxzKUOrNKeL\nJXo2mosSFDA8zoVHWBttd2XzLFdprUKKwbLblettFLdlQzfGhLlA9AKA7wA4B2DN2uDJvgxgYp2/\nfd4Yc8gYc6hWq93oFA8PDw+PtwG39UK31nattY8DmATwFIAHbvEn+m8/Z609aK09mEqlbv0HHh4e\nHh4bwluKQ7fWrhljvgfgXQAKxpgIS+mTAGZu/tc3xswVUodOvSGq+ugIqSgtxaPqnKCGHTplVcXc\nsNMwn5fhVOpc37MpqmMsRmp+PEXfsUhMmXnKdP/jR8UpNTRMKtBHfkb6u2eCsx/n6G/PnBbTz8Is\nOUx1AYNEklSqluJy6bp0U1azY8rJ2K4b7r84QGNJdoo237LP+TpYI2YpV2l+dIxiuONxbU9gro6k\nzEenSedX1xaCtsUrZC6anSanW3lFzn9wP8V/5/NiGolEyURz+g1xSB85QdmAq6yW73lgKjiWYVPB\nOx+TeORakVTjZUWHe//9ZE5ZW6NY9osqC/LnP0r8P2MPPB60JXPrR/K3Q7TuKeUgN2yKqKl95+Kc\nx4dofE8+en9waHaBVPSLl2SuVjgbNKJoa3Ostie5eEjnqn3CtXVVTHiRCyJUVsU80ObsWC48D7Mg\njsclrku6W5mDnLFLMVYjzLHmbF1BNCr3jERo7MmMos/lwjGu3zfCS69JlmeXqYYzSVXwI13jY7I/\nkmxaPXGBnseFpgQHtFn+zKfleRlnuuS8kX3XZCrluVkyx/QPS9BBH5v1jp8Wqu0e8+Nks3LdmRl6\nlrtMy7t7r8iwO0dpL16eE6dokq1GM2timjl0mgrGuGIuUbWOA8x5FFM5Fw0OUhgXC+9bxi0ldGPM\nkDGmwL+TAH4BwAkA3wPwMT7tUwC+fuMreHh4eHjcDdyOhD4G4AuGxOAQgD+21n7TGPMGgC8ZY/5v\nAK8C+L2NdGClRl/iiirf1SmRhBvqSvdq/NVtsyNK0aUgHObzFL1Fr97l80WCTsTpC9zkEm1dFZbm\nGO00T0lpmX7r0lQzLBSurtK1Mnn5qvfYgeccTAAQTtK9+pRjps2StuOB0c5c16e2IuyPcVageYts\nbPp8G5RCk3uFWFJMuhAtJb13mOsipBw/iQRJUmFV1q+/zpw2bepjqk8klGyWRI2cKjHmGBVTAzJH\nDz3J5d3Y0T08KtJzOkF9G1WsmW2WjCtFJaW26HqXLlIGb7Zfyo6NsCPWhhS7JnRW8dVwlcIyCTk/\nypphrSl+IMuZhVnOFn5o/67g2NgYrWOxKucX52jdi6qWYJSzNZPsKI3ERQq2PWrrXVVEgs+PyX6K\nh2hOqy3qT1s5nIvsZDyvSp1l45bPE20gzkyKIXbKZ1Ii6/X3kRQZVud3QM9tnwr/vLY8wyN7JVy1\nxmXyQkbGnsvQfopHlRM3Rc/as0+Q03KtqcJbmb9G+RExzBmiY6oc3GA/3dc5kNNZ2QtFLlhR7Mqc\numcjAcnU3P8AhRj2sQO+kJeAgStzrHU1JPNzkgMKrJKRH54kzTcS5tDsK2LEMKz15AvS73KV5ne1\nsiFjB93rVidYa48CeOIG7edB9nQPDw8Pj3sAPlPUw8PDY5tg08m5KlVSwbSD0tXiq5TECdiL0vEC\n1ylcWRQnSI/dPLWKqtvZ4iIZilirlXQmFmqLhJWZh6ujQ1VpL1bp/K98V5yczrm5ViLTQkiZSxJM\nUKXrDyYSpDImlJNpkVXvSOT62pXRGKtiSvUOsUodjtyeyaXL9S+vMrngepOLOypDVtcPYpRFv3XV\n56OKtGpsD6mM/WNEE+tqMdIfcG3YmJhtwhzI3b9DnExPsJrf41qYRokZEXZQRpTzqMsdDikTkaN4\nvf8gmV6M0stTeUdeJSp1+CYml+MnyME7OSKZpf1MwZvqE/IlTr5FgwmWum1Z9wyTee3dLQ65Kwtk\n9tDrvVoi00XUsiM2JXMVYWd5PCx7x7gKCqr2bcg4cwl1KKrmJeLWVM1HmTMYowlFaBXjGHk2axQU\n2VWWHftoipmszQUg2qpoA8TCAgAYU0EKlnMddJGMKKgfawtiupitOOpbfuZUgYuBApvw+sVM0c/1\nTp96VIwI45NT9Kcc9z2jsrS/+/3/DgB4403JYh0bpzWKx5V5lgvuLC6RWU8XrgizCTQWVu8FzieY\nZDMLAMSjZJp0JpeCMl/G2MQ7NCqTNsf748XXNm5y8RK6h4eHxzaBsdbe+qy3CePj4/b555+/a/fz\n8PDw2A747Gc/e9hae/BW53kJ3cPDw2ObwL/QPTw8PLYJ/Avdw8PDY5vAv9A9PDw8tgnuqlPUGLMI\noApg6Vbn3uMYxNYew1QQe9IAAAUKSURBVFbvP7D1x7DV+w9s/TFspf7vstYO3eqku/pCBwBjzKHb\n8dbey9jqY9jq/Qe2/hi2ev+BrT+Grd7/G8GbXDw8PDy2CfwL3cPDw2ObYDNe6J/bhHu+3djqY9jq\n/Qe2/hi2ev+BrT+Grd7/63DXbegeHh4eHj8deJOLh4eHxzbBXX2hG2M+ZIw5ZYw5a4z5zN2890Zg\njNlhjPmeMeYNY8xxY8w/4/Z+Y8x3jDFn+P93UDTqpw8u8v2qMeab/O/dxpiXeR2+bIyJ3eoamwlj\nTMEY8xVjzEljzAljzLu24Br8Gu+h140xf2SMSdzL62CM+X1jzIIx5nXVdsM5N4T/wOM4aox5x+b1\nXLDOGP4f3kdHjTF/6qqx8bFf5zGcMsb84ub0+s5w117oXPHodwB8GMABAJ80xhy4W/ffIDoA/oW1\n9gCApwH8U+7zZwC8YK3dB+AF/ve9jH8GKhvo8NsA/q219j4AqwA+vSm9un38ewB/aa19AMBjoLFs\nmTUwxkwA+N8BHLTWPgwgDOATuLfX4fMAPnRN23pz/mEA+/i/5wH87l3q463weVw/hu8AeNha+yiA\n0wB+HQD4uf4EgIf4b/6jccWKtxDupoT+FICz1trz1toWgC8BeO4u3v8tw1o7a639Cf8ug14kE6B+\nf4FP+wKAv785Pbw1jDGTAD4C4D/xvw2AZwF8hU+51/ufB/AMuMShtbZlrV3DFloDRgRA0hgTAZAC\nMIt7eB2stT8EsHJN83pz/hyAP7CEl0AF5MewybjRGKy1f8WF7QHgJVCBe4DG8CVrbdNa+yaAs9iC\nFdnu5gt9AsAl9e/L3LYlYIyZApXiexnAiLV2lg/N4Tpq/3sK/w7A/wFXjBIYALCmNvW9vg67ASwC\n+M9sNvpPxpg0ttAaWGtnAPxrABdBL/IigMPYWusArD/nW/XZ/icA/oJ/b9UxXAXvFL0NGGMyAL4K\n4J9ba0v6mKUwoXsyVMgY81EAC9baw5vdlztABMA7APyutfYJEHXEVeaVe3kNAIBtzc+BPk7jANK4\n3hSwpXCvz/mtYIz5DZBJ9Q83uy9vJ+7mC30GwA7170luu6dhqJbZVwH8obX2a9w871RK/v/CZvXv\nFng3gF82xkyDTFzPguzRBVb9gXt/HS4DuGytfZn//RXQC36rrAEAfADAm9baRWttG8DXQGuzldYB\nWH/Ot9SzbYz5xwA+CuBXrMRtb6kxrIe7+UJ/BcA+9uzHQA6Ib9zF+79lsL359wCcsNb+G3XoGwA+\nxb8/BeDrd7tvtwNr7a9bayettVOg+f6utfZXAHwPwMf4tHu2/wBgrZ0DcMkYcz83vR/AG9gia8C4\nCOBpY0yK95Qbw5ZZB8Z6c/4NAP8TR7s8DaCoTDP3FIwxHwKZIH/ZWltTh74B4BPGmLgxZjfIwfu3\nm9HHO4K19q79B+DvgTzL5wD8xt289wb7+x6QWnkUwGv8398D2aFfAHAGwF8D6N/svt7GWN4H4Jv8\new9os54F8CcA4pvdv1v0/XEAh3gd/gxA31ZbAwCfBXASwOsAvgggfi+vA4A/Atn72yAt6dPrzTmo\nuvjv8HN9DBTNc6+O4SzIVu6e5/9Xnf8bPIZTAD682f3fyH8+U9TDw8Njm8A7RT08PDy2CfwL3cPD\nw2ObwL/QPTw8PLYJ/Avdw8PDY5vAv9A9PDw8tgn8C93Dw8Njm8C/0D08PDy2CfwL3cPDw2Ob4P8H\np0pBYqLAxQ8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vTrs3VIiBlb8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7d530780-e57a-4b8e-f865-3e1f257714a3"
      },
      "source": [
        "14 - 5 + 1"
      ],
      "execution_count": 141,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 141
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "luXIr2b93Kj8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Net(nn.Module):\n",
        "  def __init__(self):\n",
        "    super(Net, self).__init__()\n",
        "    self.conv1 = nn.Conv2d(3, 6, 5)\n",
        "    self.pool = nn.MaxPool2d(2,2)\n",
        "    self.conv2 = nn.Conv2d(6, 16, 5)\n",
        "    self.fc1 = nn.Linear(16 * 5 * 5, 120)\n",
        "    self.fc2 = nn.Linear(120, 84)\n",
        "    self.fc3 = nn.Linear(84, 10)\n",
        "  \n",
        "  def forward(self, x):\n",
        "    x = self.pool(F.relu(self.conv1(x)))\n",
        "    x = self.pool(F.relu(self.conv2(x)))\n",
        "    x = x.view(-1, 16*5*5)\n",
        "    x = F.relu(self.fc1(x))\n",
        "    x = F.relu(self.fc2(x))\n",
        "    x = F.relu(self.fc3(x))\n",
        "    return x\n",
        "  \n",
        "net = Net()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rZWI--Oy8lwu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iQPxLsr-9sXJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "36d9cddd-4bc6-48c8-9174-56dc02d69398"
      },
      "source": [
        "a,b = iter(trainloader).next()\n",
        "a.size(), b.size()"
      ],
      "execution_count": 154,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([4, 3, 32, 32]), torch.Size([4]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 154
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DL5jMio6Bz81",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "e93f2bd3-3464-47aa-cdcf-dc6022be4ef2"
      },
      "source": [
        "print(a.size())\n",
        "print(nn.Conv2d(3,6,5)(a).size())\n",
        "a = nn.Conv2d(3,6,5)(a)\n",
        "print(nn.MaxPool2d(2,2)(a).size())\n",
        "a = nn.MaxPool2d(2,2)(a)\n",
        "print(nn.Conv2d(6,16,5)(a).size())\n",
        "a = nn.Conv2d(6,16,5)(a)\n",
        "a = nn.MaxPool2d(2,2)(a)\n",
        "print(a.size())"
      ],
      "execution_count": 155,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([4, 3, 32, 32])\n",
            "torch.Size([4, 6, 28, 28])\n",
            "torch.Size([4, 6, 14, 14])\n",
            "torch.Size([4, 16, 10, 10])\n",
            "torch.Size([4, 16, 5, 5])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bJSfHzq6Ajnb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# CPU is slow ..."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IGlhfaVg9K2w",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "8c558594-8e3d-4973-a72f-6cb838f25b02"
      },
      "source": [
        "for epoch in range(10):\n",
        "  running_loss = 0.0\n",
        "  for i, data in enumerate(trainloader, 0):\n",
        "    inputs, labels = data\n",
        "    optimizer.zero_grad()\n",
        "    \n",
        "    outputs = net(inputs)\n",
        "    loss = criterion(outputs, labels)\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    \n",
        "    running_loss += loss.item()\n",
        "    if i%2000 == 1999:\n",
        "      print('[%d, %5d] loss: %.3f' %\n",
        "            (epoch + 1, i + 1, running_loss / 2000))\n",
        "      running_loss = 0.0"
      ],
      "execution_count": 158,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1,  2000] loss: 2.219\n",
            "[1,  4000] loss: 2.150\n",
            "[1,  6000] loss: 2.091\n",
            "[1,  8000] loss: 2.048\n",
            "[1, 10000] loss: 2.012\n",
            "[1, 12000] loss: 1.970\n",
            "[2,  2000] loss: 1.946\n",
            "[2,  4000] loss: 1.945\n",
            "[2,  6000] loss: 1.910\n",
            "[2,  8000] loss: 1.829\n",
            "[2, 10000] loss: 1.752\n",
            "[2, 12000] loss: 1.717\n",
            "[3,  2000] loss: 1.642\n",
            "[3,  4000] loss: 1.643\n",
            "[3,  6000] loss: 1.607\n",
            "[3,  8000] loss: 1.623\n",
            "[3, 10000] loss: 1.592\n",
            "[3, 12000] loss: 1.580\n",
            "[4,  2000] loss: 1.514\n",
            "[4,  4000] loss: 1.530\n",
            "[4,  6000] loss: 1.524\n",
            "[4,  8000] loss: 1.494\n",
            "[4, 10000] loss: 1.499\n",
            "[4, 12000] loss: 1.494\n",
            "[5,  2000] loss: 1.418\n",
            "[5,  4000] loss: 1.441\n",
            "[5,  6000] loss: 1.434\n",
            "[5,  8000] loss: 1.440\n",
            "[5, 10000] loss: 1.448\n",
            "[5, 12000] loss: 1.443\n",
            "[6,  2000] loss: 1.361\n",
            "[6,  4000] loss: 1.370\n",
            "[6,  6000] loss: 1.295\n",
            "[6,  8000] loss: 1.247\n",
            "[6, 10000] loss: 1.276\n",
            "[6, 12000] loss: 1.260\n",
            "[7,  2000] loss: 1.178\n",
            "[7,  4000] loss: 1.184\n",
            "[7,  6000] loss: 1.171\n",
            "[7,  8000] loss: 1.208\n",
            "[7, 10000] loss: 1.207\n",
            "[7, 12000] loss: 1.193\n",
            "[8,  2000] loss: 1.118\n",
            "[8,  4000] loss: 1.138\n",
            "[8,  6000] loss: 1.138\n",
            "[8,  8000] loss: 1.147\n",
            "[8, 10000] loss: 1.094\n",
            "[8, 12000] loss: 0.988\n",
            "[9,  2000] loss: 0.905\n",
            "[9,  4000] loss: 0.925\n",
            "[9,  6000] loss: 0.926\n",
            "[9,  8000] loss: 0.935\n",
            "[9, 10000] loss: 0.931\n",
            "[9, 12000] loss: 0.929\n",
            "[10,  2000] loss: 0.843\n",
            "[10,  4000] loss: 0.859\n",
            "[10,  6000] loss: 0.905\n",
            "[10,  8000] loss: 0.881\n",
            "[10, 10000] loss: 0.915\n",
            "[10, 12000] loss: 0.926\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mQBlJVpv-ZHV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dataiter = iter(testloader)\n",
        "images, labels = dataiter.next()\n",
        "\n",
        "outputs = net(images)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I3luyIavDNXx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        },
        "outputId": "4d6edf02-8220-459f-8734-01c90e462a3c"
      },
      "source": [
        "outputs"
      ],
      "execution_count": 160,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.0000, 1.0408, 2.5326, 6.0399, 3.8863, 4.2835, 1.7533, 2.4266, 2.6470,\n",
              "         0.3653],\n",
              "        [8.5865, 6.4794, 0.0000, 0.0000, 0.4180, 0.0000, 0.0000, 0.0000, 7.6094,\n",
              "         4.5536],\n",
              "        [2.9252, 3.2191, 0.0000, 0.2945, 0.0219, 0.0000, 0.1825, 0.0000, 3.4059,\n",
              "         3.2888],\n",
              "        [2.4846, 3.0032, 0.0000, 0.6693, 0.6655, 0.0000, 0.3466, 0.0000, 3.3824,\n",
              "         1.7961]], grad_fn=<ReluBackward0>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 160
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L9aLkNviDPrJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DocwflLFKu1a",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "efa7b306-0bee-4935-d2ca-88bc45674c5b"
      },
      "source": [
        "device"
      ],
      "execution_count": 169,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda', index=0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 169
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SY17oV5YDgGY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        },
        "outputId": "688c0d57-b600-4be1-e066-69095502c134"
      },
      "source": [
        "net.to(device)"
      ],
      "execution_count": 170,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Net(\n",
              "  (conv1): Conv2d(3, 6, kernel_size=(5, 5), stride=(1, 1))\n",
              "  (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
              "  (fc1): Linear(in_features=400, out_features=120, bias=True)\n",
              "  (fc2): Linear(in_features=120, out_features=84, bias=True)\n",
              "  (fc3): Linear(in_features=84, out_features=10, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 170
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ycEQca7iKm7v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KrRSYeOQEd1m",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "e360deb9-88b8-47b7-8e74-bdf29a0f9802"
      },
      "source": [
        "for epoch in range(10):\n",
        "  running_loss = 0.0\n",
        "  for i, data in enumerate(trainloader, 0):\n",
        "    inputs, labels = data[0].to(device), data[1].to(device)\n",
        "    optimizer.zero_grad()\n",
        "    \n",
        "    outputs = net(inputs)\n",
        "    loss = criterion(outputs, labels)\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    \n",
        "    running_loss += loss.item()\n",
        "    if i%2000 == 1999:\n",
        "      print('[%d, %5d] loss: %.3f' %\n",
        "            (epoch + 1, i + 1, running_loss / 2000))\n",
        "      running_loss = 0.0"
      ],
      "execution_count": 172,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1,  2000] loss: 0.779\n",
            "[1,  4000] loss: 0.825\n",
            "[1,  6000] loss: 0.860\n",
            "[1,  8000] loss: 0.878\n",
            "[1, 10000] loss: 0.845\n",
            "[1, 12000] loss: 0.870\n",
            "[2,  2000] loss: 0.746\n",
            "[2,  4000] loss: 0.788\n",
            "[2,  6000] loss: 0.809\n",
            "[2,  8000] loss: 0.829\n",
            "[2, 10000] loss: 0.820\n",
            "[2, 12000] loss: 0.847\n",
            "[3,  2000] loss: 0.718\n",
            "[3,  4000] loss: 0.779\n",
            "[3,  6000] loss: 0.779\n",
            "[3,  8000] loss: 0.780\n",
            "[3, 10000] loss: 0.812\n",
            "[3, 12000] loss: 0.837\n",
            "[4,  2000] loss: 0.704\n",
            "[4,  4000] loss: 0.737\n",
            "[4,  6000] loss: 0.751\n",
            "[4,  8000] loss: 0.780\n",
            "[4, 10000] loss: 0.814\n",
            "[4, 12000] loss: 0.808\n",
            "[5,  2000] loss: 0.691\n",
            "[5,  4000] loss: 0.711\n",
            "[5,  6000] loss: 0.756\n",
            "[5,  8000] loss: 0.738\n",
            "[5, 10000] loss: 0.774\n",
            "[5, 12000] loss: 0.763\n",
            "[6,  2000] loss: 0.661\n",
            "[6,  4000] loss: 0.710\n",
            "[6,  6000] loss: 0.727\n",
            "[6,  8000] loss: 0.739\n",
            "[6, 10000] loss: 0.751\n",
            "[6, 12000] loss: 0.762\n",
            "[7,  2000] loss: 0.639\n",
            "[7,  4000] loss: 0.664\n",
            "[7,  6000] loss: 0.729\n",
            "[7,  8000] loss: 0.720\n",
            "[7, 10000] loss: 0.737\n",
            "[7, 12000] loss: 0.746\n",
            "[8,  2000] loss: 0.636\n",
            "[8,  4000] loss: 0.677\n",
            "[8,  6000] loss: 0.693\n",
            "[8,  8000] loss: 0.705\n",
            "[8, 10000] loss: 0.761\n",
            "[8, 12000] loss: 0.715\n",
            "[9,  2000] loss: 0.619\n",
            "[9,  4000] loss: 0.635\n",
            "[9,  6000] loss: 0.678\n",
            "[9,  8000] loss: 0.698\n",
            "[9, 10000] loss: 0.700\n",
            "[9, 12000] loss: 0.748\n",
            "[10,  2000] loss: 0.599\n",
            "[10,  4000] loss: 0.629\n",
            "[10,  6000] loss: 0.700\n",
            "[10,  8000] loss: 0.682\n",
            "[10, 10000] loss: 0.715\n",
            "[10, 12000] loss: 0.711\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ehqNU2rDQXuf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dataiter = iter(testloader)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q0csbfq1ElUZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "outputId": "05dfba0a-cf6c-47c6-e85e-7a37ac396597"
      },
      "source": [
        "\n",
        "images, labels = dataiter.next()\n",
        "imshow(torchvision.utils.make_grid(images))\n",
        "print('GroundTruth: ', ' '.join('%5s' % classes[labels[j]] for j in range(4)))\n",
        "\n",
        "images = images.to(device)\n",
        "labels = labels.to(device)\n",
        "\n",
        "outputs = net(images)"
      ],
      "execution_count": 207,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAB6CAYAAACvHqiXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztfXmMZel11+97+1avXu1VXb1U93TP\nPvZ4Mo4n8cJ4nBDHCRihxLITBSMsRkggCCCBQ5CIJZBAoLAJjCwSYlAUJ3FCMgSSOAyTjPdZehbP\nTO89vdS+19uX++7HH+d895xau6Z63NVV+X7STL3+7n33ftu975zzO4ux1sLDw8PD4+Ajtt8d8PDw\n8PB4d+Bf6B4eHh6HBP6F7uHh4XFI4F/oHh4eHocE/oXu4eHhcUjgX+geHh4ehwT+he7h4eFxSHBb\nL3RjzMeNMReMMZeNMZ9/tzrl4eHh4fHOYfYaWGSMiQO4COBHAUwCeBHAZ6y1b7173fPw8PDw2C0S\nt/HdHwRw2Vp7FQCMMV8B8EkA277Qc7mcLZVKt3FLDw8Pjz9/mJmZWbTWDt3qvNt5oY8DuKn+PQng\nAzt9oVQq4emnn76NW3p4eHj8+cMXvvCF67s57/tOihpjnjbGvGSMealer3+/b+fh4eHx5xa380Kf\nAnBM/fsot62DtfZL1trHrbWP53K527idh4eHh8dOuJ0X+osAzhhjThpjUgA+DeCZd6dbHh4eHh7v\nFHu2oVtrA2PM3wHwxwDiAH7VWvvmO73OF//jFwEAQSwZtcU6DQDAIyeEA/jcT38CADBaot+gDtrR\nsU4nDQBotrpyjRiZd4rpuNwstPoPtIOP+2ji8hvX7VLrarkVtV26MQcAuDK3BADoHR6PjsVD+u5D\nZ45GbQ+cGqHrNipRW8oYGjN3wMSMXMPQNWw3lL51u9xvafvabBoaP/Ppvxp9brWovx3UorZ2m+Z0\noEf6G7Td+XTe8spcdCy0AQCgsirz3FxtUh8zMqdLXfpuIkfrV67IOOM8l6Ojso6jw6MAgKPDR6K2\nBOi7QbfOf2W+87leAIC1srbtFvXp5qyYFdcqqwCAQqEAAOgpiDZ4Y3YaANBoNqM2w1Nz5eI8NuJS\nd3hTmzFmU1uM20xss1zkzk+qr8V5/fSlYvxdw+sei2uvM/q83hHNqv8TQv5uhhuTkBu0+WZd04na\nEqC1taFcxYbuO3Strn42eH8GXVmDrtufaizj4ZLuKH7pl/6pnB92eSxy4XDzlCLGh030jKo+bnU+\nz58+z829cZ3b4nvr4O6FzR5/W637DpeQlwuAc699DwCwtLoMAHjiQx+MjqVSqW37/YVf+he7uudW\nuB1SFNba/wPg/9zONTw8PDw83h3c1gv9XYH7BdRiCLctrpajplqDpLbicZLwau1qdKxj6deuHSjJ\nhH/+ux2RTIq5DAAgHicJMwxF4u0GJLWESZF8YxmS9rI5OS/foOPtOZL23r4hUu3EyCAAYPyISJ+F\nPEmKJiF9S7FE1Ymx5B1Tv9L8MQxEGrKub3az9OaQzIvUnMjSPa3JRG2dZoevq8bC0myXNaJ6WebU\nZKmP/UN9UVuYi3E/5BrHB+ga6SzNSzwuWypXon7kCtKPVIKOGy3UOimyQ42thozFWpLGE+q6KZba\nh/tEE+rv7adrhE0eWz46trJMWkMpLS6zxaEeAFtL6DG+l8Fm6WkrxLaQ0N2YYkr7ipntz3Nru/4x\n4O+qrxk4iV5OTHM/E+5x1vcx7rqpqCkMSSNKxoOoLZWk/VHI0TVKPJ8AEIDW49rkbNRWYyXKxtQr\nRLYFfS9UEj3v4YvnLkRtg8OkuQ2OiEbU6dB6v/qdFwAAzUYjOvbEX/gIjTMp9+zwdRNqzKEJ13Un\nptYuer6w+VkyW4jyNpLy1TGnWesT3Wnq+bp+7iIA4MrVqwCAx54QJ8A4PwfYQkK/HfjQfw8PD49D\nAv9C9/Dw8Dgk2H+TC0OrtzE2iazVhcSaXiRi4b33jwEA2kq1qbfpu514r1wwT+r1WmUmamrV6bxS\nb5ZvKtcwMf6cFpOLSdN5fWyaAICH86QmVpp0rRdeejk6lsmSaaFvaCBqi6dYHVYml4j3smwGMaL6\nRmSQUeqqYRNRV84DOtAoZEVFdkRLqHRgk2M1UTFLqSSNr9mieW535FiDVV90ReVtNdiEkhU5oFGl\n82oV6u/oiJg1kpauX1uVvqZKpPo3GtIWJokMTbClpWtlnI0am1DSYkLJZmksgwMyz2G3CAAoN8iE\n0uoIOTtQorlJxbNRW3wH9TY6tBUZqU0i7khkgjKbjoXrVOotbubWKtxMmDpok44j0GPKXAier8Dd\nVZl5whjNs1UMZBjSY68J2HyKjk+M0TM0NDQSHbt2k81SgRDksExGbjGk6JR1/2Iz6oyYbdJs2hoe\nFZPLzRtvAwAunKXnSpvw1h5+GABQ6JPnPM5m1PL8ctSWytEeS/CxEFs8e2qenamj3ZS97sjNZC7P\n11DryJ+1+dIZeBJJMRf291M/52/SGKprK9GxXJbMsnYrz4zbgJfQPTw8PA4J7koJ3blwNZVAOslu\ngo4crdck6rTJkndWEXjoIWkyzIg75Mo8ua9lLP1yl3pE6kvG6Vc6kda/cfwrbRXhwu6SpSxN3dGx\nwejYxOmTdOsBkZYTliRM25Zf7qDNA+uytGUVoelEtIT8+odgl6+YSO3AGjScex+w4VffXYPd/jRB\n5Ny08j1Om5H5qLLrY6smfZueIUnt6DEhfctrRFw32vT3yIK4HHZD0o5yBbnGqZNEZDZrIqGXjtCc\nhjGSYNo16WMStI6FtLQ5t8Z4XNY2ZPK0G9K+qNdXo2ODTJ4mlQZXVQTwRjjyMrR6vtkVbgvJbido\nV7twC5dDdbHtj2nJP7qg3pNOWuY21e+gRfMQV496Mk4aZz4tbUfHSPPsL9Gx5WWRJienaR2126Ij\nXneaAU2eu6mKx2QvrCwuAABunpdRP//M/wIArE3Rszo4Lnvt7Le/CQDI9fZEbe//yIcAAN967v9F\nbafvuw8AcN+jjwAAOnodWXtp1sSl17kQXrl0OWprsXXg/T9MroYt5VyRYq1hem4yaltcovfTqfvu\nidrmFmjeFq5fAQC8/Cd/HB176qc+DQAwaj7i8duXr72E7uHh4XFI4F/oHh4eHocE+25yiW3hh+58\nP0MVPTo1SyrN0iqZGjotMbk0aqTa9R6R8zPFfr6+EJo1VvNnF0idrJSFBClkSe0aGBB1Lpuh6Qna\nQgbVG6yqse/48LgQOkdOHQcAmJz4/CbZ3zRmJXIxYJ9wuOsGYlvqdtjnXLdxZJ/2qd8Jdgu/2hgT\nq9pM4O61ME+mlHPnJNB3uUzkcDYv41taJp/72QUZX4ujLxuNRQDA5YvSx2aTzDC5gtzze0McTWiF\ntBw7Tqaek/fT+qnYXvSlad7SJVnHZp32QBCTdTFMRjWbtC86gaxtu0v3cr7yAJDr2T6vkCMezRah\niXr+NppctjZ1qc+7Yr3WOejT9xT35vzP9Z1t5H/O/vOKyF6ZuwYAyMTlGxPHHwIAnJ44GbUN9ROp\n7IjBq5OLco0GmSwCo18XHNm6g0i4ro88EUFTTHJff/Y5AECPPLbIdMgUFtTIVHT1osSirNwkE0aq\nKKbB+x4h84qtiQktcHugRXuxpdbF8cXnXns1aisWaOytqrxT6hV6zrtsegxbsq9bzN5Xy7KH6xzr\nMHPlRtT25gvfAQAMsoPD1NmX5NgpmvvjZx6QfhRvP7W4l9A9PDw8Dgn2XUJ3Uk5Ck3XusxFZbXaZ\nfrGn5klyGOoVKTHO7oftuhAdA+xymFJkYb5A0uHMJJMVdfn1j7Pr14giOYtFlgq70rcVJtNi/Kt+\nZGQ0OpYr0b3mVoVQ6mHyNJ+RaMlkhvoWMKmXUpFvbsSdlkifcc6bEXSU2xgWsR22jHiL8sbIb3jQ\nIckhy1JwNi/k0dQ5IqX6h4VojiVIWu8Esi4mRtJSgaWmREKkYBuj+atURPKprJEEGFPRsdevEzk2\nN0Pz8eSPnIiOFUs0VzEdKWrY3TIQqazVpP3RqJBEtbgs0pObtkRK1mCgIPfYiCgXyBak504S+tZQ\nmueuqoNtdmOzZvPhdU1Ok2VyPancVfvynCenJm59Q7103uiQJtId0Uda2PSyclcFranRbr78N7aD\n1vHia6LxTd+gRKydaSESV65eAgAU1R4b4MjjOEcXz5XFdbleofXWms5zX6V8gI2pBbnuLGmc33vp\ndQCSM4n6HXB/rkRtOX7+eviZBoAae2RMXaRoT6Mit5NFel5qVXnf1BbpmU+oN2qRNaXhftL6O8rN\n8Y2v/RGNb3I6avvIJ/4SbhdeQvfw8PA4JPAvdA8PD49Dgn03uST4JyXoqGhJ9s2MKV2z2iS9eTmk\nY2dOCKGTrnDEZVzUIpeGVmn26Cm4ZFukMq0sSmKmbJpU04UlUeOvTdLxYkGI0lqDVMD+IfLbffCe\n+6Nj/QNkrqnWVcKnkExDcStqpeGovZjLrZpSrBCnOY0nxbziiJxsXNOF29fi3kq1d+aBoC3kzjKT\nw5TOHpg4+QPRsdffeAUAMHpETEr9/UTaDPQJKVWrk4mj2aG+5UuitrpES01FhDWZiNapjlNJTlO8\nRur4lUtiLjnB+bf02iYKZHIxa2os8zd5fHSt1pqot8sBmdZ6e8XE1p+7NcG8PvnS3hIn2XX2kltf\nQ5s1sBU5a52/ukrzzMsdA/vnqyjiYU5TXJ6X/RR2ynwNmdOZJTIfXJoi00zDqiR13KdMTJtc6Kbt\nDQm5NP7wuW9FnxfnyCRyb0r6dv8g7aMeleI6aNF7IM9J8vrS0sebHIPSDGTsa5yiVrm8I1Ol5zzH\nGcS6oTJVMlleyEnUcCqk82xDTKXVCvWjzGaVVkXMK33HKFp9cETSQl+7Qj7sGXWvE6PkmLHG/Uir\nuIn4Kr0jbtg3orb2Uz+K24WX0D08PDwOCW4poRtjfhXATwKYt9Y+zG39AH4TwASAawA+Za1d2e4a\nO+GjH30cAPDaq9+L2lZWiOBKJoXEevJjFBH22IcphWaPcoWrdkg667REEuywi5+Jy093iaXHe06f\nAQCkEiIZdLv0y6ojUCsLJK3E4iq/CxO1NiAJJa1cK1MxknRD5V5YKFK+kfFR0Sg6XGwixlpBqyu/\n6rPsZpZMyfiy2Sy3iVSxEXaH1Lo0BlrqlWWRUhcXiCQucc6JTltcA52WNHFSJO4HTpO43KvS4SaT\nJBO8cp6uNTUrkgxYo7DKEXFgmLSdekO5HPIc3vcAzdHUlEgtzz9POT2e+IHHorZijjSFRkO0qSLn\ndelyZG6yR2SV6RkiwacvSR6RvswOBdTtuj/Ux61O2w3Jue6UXchPOho4clFUuVxcT5TU7vZukt04\n+/LyWCc4vW0mK2tbLpPb58KyEKWXrtP6lVlzSibE6SDFROK9xyVq0/F7F64LqbcR1y9fiz4vl+mZ\nPj4k/Rjm/VFLSX8Na7lxlozTKi3uSD9JvPk+cVxwUc5GEd7JPN1j/Cjt10xGnl/nWBBX2q5hElk7\nItzP8+vywNi2LGQnwfmZrLxvJs8zeVoWZ4UWd30tRhaBotLE8wl6zxjlEt2ubh+9vFvsRkL/NQAf\n39D2eQDPWmvPAHiW/+3h4eHhsY+4pYRurX3eGDOxofmTAJ7kz18G8KcA/vFeOvCpz9BvxZMfFfvt\nG2+QfTiTkeCPD3BOhR52Jey2RdJssm1taupm1JYr0K/z6LhkjUtyzobBQfqFz6tAk+UlsmktzIn7\nU3dkswSWcIFCLiNkWfqR7eP+qkyJaS4YYOMquyBLUh12u1talV/1pRUaQ07Z+Ao9x7n/yta+CZtt\ntety8rvoFJXXIipKkaR5mZsTic1dY3VZ3Ma++Z3rdESZ8j/wGM1zpUwS1dqKSOjZPElN2o4c8Lzl\nsyqnB9vV212S3nv7pXDFGtv5V+YlN4vt5Rw7WZHK+nvJpbLCczkzL/XKW6scHBKK/LI4K+t8NyGE\nlhw353dxtmttJ0+x7fz4CPE0J0bFHfHGxdcACFcFiOvtxYuSu6TaYq7H0B4rxJW0yvl3RkaFTzl/\nlTmLHWzoczfERfHqNcqieOzMWNR2gouMZHpEC0z20/OaOUprm80KX9PNcDlCs5k/iCt33DYHHcaS\n7BaclfdID2feTCQ0H7W5VF2X89a4poRVRTU4ZePaguyxgR56LwU1KXgT4+yoyfT6wjoAkGDOMF6R\nZ+7KK9/B7WKvNvQRa63LSzsLYGSnkz08PDw8vv+4bVLUkiFxW2OiMeZpY8xLxpiX6vX6dqd5eHh4\neNwm9uq2OGeMGbPWzhhjxgBsLszIsNZ+CcCXAODIkSObXvx9faSqFPJiYhgcJrUok5a2NJM6MU4x\nGTNC2rgK5J2OkBT1Brm+VWuisqWS7rukSOWUKtbiRParaXGZW6uQStVbkhwLjiRpceRYS7kBtjgP\nxsCgql3JLn4dRaAks47YIrWr2hSTS2DpR6/akPNX1riwRF7GshE21KleXTihUk2dCqlU01V2zbrI\n6vPiivzgxpioTKTELLVWpfMbZRnzuYtLfKzNd1bkEc9NUqm3jgPKpmT9XGCjc6nMpGT+1uKkoocq\nJ8/A8DgAIJ0Tk0s05l6uiVmS9MKlHrpGr8qVkYz2mxQoeTexZY2MXUGZAiJ3ROWyyea8jHpyjw7Q\nuB66l6JfUxAz2VSXPpuurFmNU8e2uvLYxrNEEvewS+/DE6J0T3C+opsL4vcwNUOmhXAHV8wb169H\nn92z8Yoywxw/+QQA4NFHpdZmaojWqsmOBeW2PAcuD0zYFTtPENB5CWXOsCEX/GAz49qamEWnp88D\nEKcJAGjzPXp7ZX/k+VlbWiDTXKclK9nlYjUDWRn7aoXeG0aRpz3sopkJyVwYa6sobZfWWxWt+fbX\nv04fSmJyfKfYq4T+DIDP8ufPAvj9PffAw8PDw+NdwW7cFn8DRIAOGmMmAfwzAP8SwG8ZYz4H4DqA\nT+21A0kug2ZD+cV0XFdS5QVJM7GQ5EIHTZUpsc2S3ZAq/Vbo4euqiIPQrq/E3myqPBFsDtIkpyML\ntZTlZKWQJd6uSoVX4VwT+ZIKWmDXxHhcSaSGCzowSZdUJd1MbTPLtFYjyaintn2GwECV64tx4YK4\nktAdadRU7oLf/ga5ina5VNzaqlyjw+XGlpeFjHS//6GSmq5epXlzWQVjapytBklN6awilDgAqVaV\ncXa4gEilTNcqFlUARoYI7Jtz4tJV6CFpaFB5HvaxlBrngJSHH3siOhbJUUp82dHjMCr0vnMgUGyn\nVINb3MhuVS3eHXPXVP2Ks9tiRrne9udpfsdV/pN7uDhFqYfGvjQnGl8YcLCR6qphCbajchkVC/Ts\nHBmmeRwqyh5eWyHi7toNcfuscwAQVLAMNmzdlnLJsyxJHzn5SNQWO0pBebW0kLgBlyZ0eVICpVn0\nsXttW+2/TtuV35OJiwprsOvt1/5ICkt843mSgnuUttZkd8UPsuMFADz44IMAgG99k4jKmsq26N4j\nH3m/jGXsOEnViZzK6Mn7foVLTSY6MkGpOu3npirrt+yE9dtIurgbL5fPbHPoY3u/rYeHh4fHuw0f\nKerh4eFxSLDvuVycKcBFQwKA4UT6nbbKzcIJ+l0W1a6qb5hlf/VMVmp5lvrIxzUWF+IsdMQha2dr\nq0KczXORh5UVIX7SGSJGioosaTRc+lf3WyhkTIcTW5RXxTzQdilqU7oG5PrCBdmcEH4uf0dXmaCa\nbbrezLwkz9+IlSnxf80Pk/qc7ZXljbG6H6ho2tMTRHYFIc3RufPiE7u8SOTVKy+Kmj06RhG22Yzk\ntolSgrIaGlckasi+u9WaMjuwiqlJqTSnFq5XaO5LSh3OsD/+yoz4jVdXyOTy4Y9IcYDBITYHuX2x\nrnbqxg87Yyv/7+jYO8zpstt0u+5IXN3VRWgO5sWscaSf5mNUpY/u48hd4+rGKjFtgCMvdVStK0pS\nrau6rmm6b4HNgc2GxBPMrdB3Fyti5gRHkiZ2mg+VcjbDz/d7H5d4k94i+Z83lfdbsof2lisy0lXv\ngCmuM6qffZefaZ15kUn4OU4F/Pzzz0fHPvD+HwQAnDwltT+XOGJ2dFSI4OERmrcPPfUkjTMpa+DM\nrAllx+oGZPZKHL83anMFehKcP6Y7JSRxY86NRdZgYZneRz1j92Gv8BK6h4eHxyHBvkvoJsaZEhWh\nEueK984lCQDaHZYwOTSsq7IzDg7Qr2NTuQVVq0T4JFLya+7ytBiWVmPqF9ZV/i6oqLUc54kYPSLR\nbbOzJLEW2K1JF14IOk66kl/zgHO+dFUtMsMSRJwJ4Z6CaBbFIkkL5cpS1OY0i0C5OG1EeUW0DcO5\nK+IqpLPBUtnMdYluK2ZpnkMmtvJ50WbaHZKCOxXlrjVE0puWyUrsYlXlAgqVtZnomJOWdUGCGDNn\n2YxoZCXOnhh0udDFtGSSLLJba19JJP+BEZKCcjmdi4dz9+wxK+J62A1/AURSu93Qgoj43PrWO/fH\nbPiQU8VOhrloyJnjsj9qiyTlnf22uFsWnvyLAIC+PiINU0pL6h8iqbO2Jnu9mKfjtbrsp1qV1ntt\njq4fhuPRscUqrWNHuQpH7sPQ7rLrod2IB0eYNExK35pMmjopG1D5cXgyF+dE81ycn3M3j9pOnzlN\n182Lw0A8Rvc4+yKVmavXpB8jY9SPr39TojJfe4WiaT/xiU9EbXV+lifn2LVzc92RyF0aADrc77Sa\n+wHOKVPgaPG+vJC/k0zEpppCiiZU/py9wkvoHh4eHocE/oXu4eHhcUiw7yaXBEdvNpVvqfMjDZXJ\nxSXTWbhJJoNuTY6NHaO0q9dnhdSbmaHzqnVtpiA17sgRTgPaFT3K+UKPjUiU1gAXsWirREgZ9s/N\ncTKemiKPOFMpxlKSxAislplQfLwdgZhk1bCYF//5I6N0fqMh/U5wRfvBfkkbCsxAoxnIWIIZMtcE\nStWc5VTAC5PyvVUmYWJpJpBVwO9j7/shAEDHyj2zHCDQbgrB5talv48I005L1NBGnfyhM0oNLfXS\nWEtqLP0cP1AtU7KoTluI2KC5xtcVlXp2lgjb116XuXepkft66boudSoApNIcT2B3yCSlEOdoTGN0\nKlsmXZUJxZkdClE6VVGf02zCC7XMxNdIK7OK4T65aNqhohDOg7005pFBiRC+xpz9kirOcvMmJb7q\n7X2IrpUUk0SOk1v15ESdL2TJxKaDi29ep33xylmqwzn5+rno2NGH3k/jiykHAzan7cQzh8qOOjxM\nz1JH+XMnCtS3lIoadt+x/NxYRUbmBsj3XpulkvzZGpXimp/lVTZD6jq3FU5MdvXK21HbGpOiRvU3\nzXvGJeFbXBRnCRdp22zJM51i0nd4QOIDBofJxDJYondFqyH9vrBGz/eQ8kPPDtx+SiwvoXt4eHgc\nEuy7hO6KG9iY/HI7csl0FWkZ0nn1Ga7urtK03nvmvQCA/kE5v7dEF6nUVLQaS0j9fSQRupSvAFDn\n6tuLc5Kwf3SYfjFtUhFbcY5MY1ekvn5VWo4j+gIVIZdid8sEROLu8rgsR4pmUyJ9dnMshWTkunkm\nx3J55d64EYp0TXGy/6oqmzV9k8iujk6oXyMizLIr42nhwdAI6V6XrotUlua0oQ2VZ6ZSuUDji5Nk\n2WhK9GGLta5GU9bWfXdGzTPOsxtnwERsW5UjZGI3nZZrzE3TuKZuiCTaV+J0pKy5PPqY9PH9T4h7\no2B7sjJkv7/1BCgTnyr9cJaLURQ51exwQaTJ8aNEpMfU2iaZENQSuhOTnaSu6pqgxdrf2oIQg92A\nU7Iqrefadapgf/wEaZ5FtU8s5ysyunwc5yJJJuRmg6yNjrEDQBkSIWw451BM5SMy/Oro7CATZjKy\nd06eIi06FldunPzVUKkKkaMCh8wez8oaFA3vVyVJNyxJwTWIZuNcP0fHaCyZCxdUr+hYrSL7NGTt\ntt2RZ6PCLs2rXBDm3HlJNexcblsNHZ1Na18bEuLzdIGjog3NX72rnCUsaQDNtkj+y8u0zqOnpKzl\nO4WX0D08PDwOCfwL3cPDw+OQYN9NLi4yshMoNdtV3FF+4jE2T+TjpE6uVkQljFlSZfr6h6O2Bieh\nyhbkGnVWkRYXiazTCbOOHafUo+W0kE0rS0TODR0XAq/E5FKZ655OOIIVQI0rpSwvSHKka5dJnTt1\nRmqKJp0azGRaEIoqu1omgqZQVFVWelmF3oGByqVFlS3l6PONS+JzfvbllwAALRXZeu8o9f30g+TX\nnVIE1Ff/mM7XCbvWynS9OU5BCgD1GqmkMSb1egpK9eX5DVRkX4L99gOVqKjTcdGjTCArs0OeIwdd\npSgAqFVoXaanFNlaIntRIU1z1axL9apT9xBBOjQifQt34EfbcSK44ioNbZzrdvarRGONWaojOct1\nJCcef190bIATZblqNYD4KOsUxq52a4xNSwlF3LZczdm2SmG8RvfSc1Rv0L5b5FSv6YSsY+hiF5RJ\nrsMmhjBUBG+a0+a+jwjQsCTk+eQSEYlhfIvXxQ570iXBA4BBTik9MirPUtUlBzMSGRywSStkgrln\nWaIrs9dpT3aMMrk8yo4LKTEzxdnp4PwFMpO0VVmleMqR2yoFL78GpxXRnC7RXllgMtTVOQaAkGNg\njI4tYTNMJSVtS/ydRZfwryhjj3OsSKItc7TAJmDlUvGO4SV0Dw8Pj0OCfZfQ4yx5hSrNaJkT0pum\n/N4kWKrpzRGhOR0KUbS0RARG6Z4Tco0KSb3LyyItZzl5v8v1UK2I+11PgdzeCuMSFXr2BUq1aRJy\n3TGuUbo0SdLqzI1rcg2O1FuaFbe7b//p1wAA9z0gOR5++MmPAgBGj5GE3GiKdrKyTFJWRkVtZtI0\n9nZ7+0jR2pqMZYWr3FdU7o25Fc4noXLEdEZoLI64vT4p/Z5fpPOCQKSWgSL1Y+jMo1FbKkfS4MwM\nR6oqybHZJPfJK5dfiNoanLdDk2MuynOMc1iEyv3vwjmSyrT2EAQkOR87JhJxqXCMbp+jOWqp1MhL\nC0SmDY9KFLC120c4OhIwrVxNH+CCDycGRHNazZA2lc24ghsS/brIeyCVVulUmSTMFYQ4izOBnXRV\n65X07rSSlJI+XYRtsSjX6HAgzCN/AAAgAElEQVQ6XJe7JKEiKW1n855ZZde9qkql7LZWk7WkaVXs\nJJHn1MTquk7ANTuI6DF1fp6J2t5eWYNqg57zsKvdQ9fXwzVqLwS8n+IpmWdwrpeY0jYspx12mktb\nuUrGWXsZGBFtvl6mayTV5q3Wa9xHmqtir7iOZngvZhUxXWcHgEKPnDfNBWTcXjNtif6uck6oMdVv\nVXZ4z/ASuoeHh8chwW4KXBwD8N9BhaAtgC9Za/+9MaYfwG8CmABwDcCnrLUr211n2+uznTyRFgms\nzOWcuqrYg3MdHC6QtHDPfQ9Gxyo1+rXLqnJw/QOU/6JHSTIuo+MC27gNpLsue+I6lzl2nZq6IXa8\n8RHSEJx7WdASKXiw9zgAoLEqUlwfSybnXntT+svaw/3vIXe6rAr6aLOtdmRcLGlJ/t01WkLagEK/\nBCdV2VWu2Cu//n2DlwAAaeU+WeD8F29zWbCvPPN70bGumQAADA8fj9pGh7jggrKTjx2jXBp9/aQV\npNMy9ukpsjGnY49FbYNciKLYpyQ1ltoffZTst1YFcn33u9/k66oiGbxGxZ5TUdv4OHEU6SzNUbEg\nczo3S1rG6fskcCMe316WybH75IPHJIfKBx+heVidkoCUOkuC6awrMyh7uMmFGvrSutgJB6XlZI4M\n26WdllJryNgtS65pVW/OaZc9PSK1r5RJO5phrSCr7PYtDoKZnhaN9txFGkNLuQWfevA9AIBkntYl\nXZT9FHL2U2WKjuTynVLnPPjQRPQ5xbblQHEELkOiKxkHAEnmElycXDujnt8R2mtdVVSjy/Z0q2zi\nljuV5neKfmxccNyPfOzJqO3GBO1/XQbzxk1yq73G5RnTShMJWJtaU1aFNnNJayoT6VSMnglXHU/n\nHjrDhUoSKTm/mN9eA98tdiOhBwD+obX2QQBPAPjbxpgHAXwewLPW2jMAnuV/e3h4eHjsE275QrfW\nzlhrz/LnCoBzAMYBfBLAl/m0LwP4K9+vTnp4eHh43BrviBQ1xkwAeB+A7wIYsdY636ZZkEnmHcO5\ncKVVVJkralBvijkjWWBXvxSpt4Mq5eYqu/2tVsSE4vKe9BQUmcHugo6gyWVF3alwVGVNuUQNjxPZ\ndeOy5LVw6ruL+ksoF7E2R/bZlhA5ZybILDCgqtDPLdLn6+evAgBK/WJ+aHWIiOuqtJqFJM1NsU/U\nz43IKhNGg/NgrKp6oDU2a3RVdGCGUwC/zQTvW5cuRsfGj5BKePz4w3LdDpmq3j73StT21iUy5Qz0\nkXniqMqhEnbpXj/xkz8RtQ2NkslldU1IN2d26+mh8QVdceP8iz9G3zVGp8qlOeotiklkiQuTXLpM\n5rHJSSHaFuZobh55VIoaDAzJfG1Elt1Jh0qyP5bnaau/evZs1DY5RSTXmfdQVO/AEXkEcqxnxzLK\nhTBD+84mZa+DTWxgstook4RzBV1XJCMi7qStwe64rtbmvEo5e+Et2rvTN4XwnpyluVpW+YKGT5PJ\npcR1OzvavMKWBavc9FxeHLuDzWV8RExLxRTnrAlUtCnbVVqq2Im7XsBuzGt5cQteOknPdNKofsQ5\nH0xM1aHliOYi58UZGZHis/0c2d1bkPV3RTRC5TDw5JkPAADOHKN7xutCaGayZI4KcrL/2qB11O+D\noR4af7+h91hKmX4mAzp29hv/N2pr7KZG7S2w6ysYYwoAfgfAz1try/qYpSTGW9LdxpinjTEvGWNe\nqqvKJB4eHh4e7y52JaEbY5Kgl/mvW2t/l5vnjDFj1toZY8wYoFL1KVhrvwTgSwBw5MiRzS99/lWK\nq8CR/gEmHq2q9F4iyc6CJGOrsguWcvQrPVuWbIvzcyR1ZlRF8TS7fCU5+1pOJcV3AS+NltxzfIKI\ntjAQ97X5eZJSj52YoOur8nGLXMaurMrY9bJmkU2JVNZXNNxvGlNelcnrMik2+7YEdlSX6Pdz4j4h\nATfCWpGUkjyXOuueKwwytSxzNLvEGRiXqb+FgrhsuiIg01OXojaX9XFtRbSHRovIo6k4EUuXzl+J\njh07Ru6e6ZxIQ2dfJq3k7WvXorYcawq9veRK1lAZLIMOa2lKQneFT554QsqZ3bxBUulL3yY3R102\nMHEPuUPWqtLvweHtJcs2Z+47d02kWtugOVpYkL3QjpO0d71M15oLRc7JcZGWdEr2dW8v3X+sX/ZM\nb5ILfnAeIBvKOB0J3qiLphplI1SEnBOUXIbAtTXRBqemaF0qVZGMW1wqsaQC8RI5ek6acASo0hRY\nVtP3tC7/yg6kaK4k+7qXM0f29AjxuLRK92qprKftxvoShfTqIQRMzqq8pYjxvo/VRRtNMGnq3iPF\nouy/0VFyNshn5bqDnH9lcUlcnE9O0N59/8O0d+qXvhcdi/fS+TNJ0eAuvU2aoS68E3LBirLL+dKW\ndWxwHqWxUVmDTnZ7F9Dd4pYSuiF971cAnLPW/rI69AyAz/LnzwL4/dvujYeHh4fHnrEbCf2DAH4O\nwPeMMa9y2z8B8C8B/JYx5nMArgP41Penix4eHh4eu8EtX+jW2m9ge8XqY7fbgRhXhUjGVUQdmzHq\nSRXJxjkpUhlSQ5uBqC+G/ZJLBTGhtOqk5gQq5WenQm3lNVJ3BofF1zvLJGe/Us8KJSJkxgaF/Lj0\nJhUAKDCx6qLRAGCJc8Q0m3LPHJuDEklRNUfYlz3P/sitllyj7lRjRfxUgjrfW1J4bkSo/OfjPKf5\nospvwWaYSeWP/GKMU7G6sZdkPpypY+rGa1Fb39AEACCrCGlXwKHdZrUylHFO3rwBAPj6178dtfX2\nEmla7BWyyxU0WS3TOHUtWVfh3RXSAIBkgu7xxhsyH7UqfTfFxKMqAh9Flq5Py4tt0Q5prubKKj1q\nyH7DA5KTJ87mgHJAfSsrojdhnclA1jG5TOa8qRnp3EPHaC9MDNO+sypfSocdv+tVMQOCc/HMLIh5\nZ7lK43p0goqMHFdq/MnjEwCAWksG/NYVMiUFCTGJFHrJfNRgS4tW/h0XGmpSNPq8/UT2nJYYhhZH\nVa7MiFljheNN1havRW15Nk06M1wyK++FhCs6EVN++Wyy1b2wcH7odP78/EJ07PnnvwUAmDghBHaC\nHS1yOZmPtXMvAgB+75tvAABuKHNddZXMO0urlaitzJGloar7a/nZcOa/fmVuuofNK+/9+PulH/3U\n36kLe/dH95GiHh4eHocE+57LxaXUiEFIijRHOibSIplUuSp5NkESWEZF2zX51zGrqmb3cBGLjnLT\nm71OBNHKIhGOiYz8noWsIYRWpqTJfkaJpFx3lLMrdjjHw/y0ZPWrcD90dCo470RS5Z9wLpoNdst0\nkbEA0A5oQjIZkUwc4Tl7U/HOMYnkA4CmcvEMODNlXLlylTiHxtEj4la4vELSkquHkE3LOJc4E16t\nLtJNd7HBfRMyqL+fcqg46T6XFw1nZobcIF999RtR2+Ag3T+v1q/OpF+TC2HEFZnrXPEmjks+nUab\nJOE3b0rU5ugYHb/nHsqZk1Bjb7eI0GwqTcjsIFnGmPyziogNOQOjzjlko2tw8Qvl8hryd3XJOufN\n2lbRxSeYh7ZM5GkSMHQujUpcDpg0rQeyP5CmvTA6Tm6Zp0+K9uPutdaW69Yy9Bwsl2XfOS2Jg6O1\nghiRoZoUDR0Jv4Oqo8vHdS31u62cGWaniFD/0//9R1Fbkq+b4CIgsYwqncd7uKQI7/5+0p5HR5XL\nKEv3r75IBHlF5TkqcG6dcq88j90qPcMD46JRpNmxYekSRXgvx8X1MejQnsyoiNWeAToeVyS4iz53\nrtkJFc2a4KIXbeXGmVCRxnuFl9A9PDw8Dgn8C93Dw8PjkGDfTS6O9FLaXFTJO6Wi7FYWOWFXgc7P\nKbUrywmhQlUkI8asWCzUNSDpulk2oXQ7ou449bobyPmVNVK7YirVqkvav7ZGxMjcnPiL59jUUuhR\nRR5Y9deqekThsLqqj6VZJazVxDxQYZNMQxGwsaH1JpdAqepdJqw6KitphWso5lSBhqMjlBzs/FUi\nF5MqgrGHk6A1muJT7wpRNJpCrI6NEUk4zIULHnpEkqb92XOkUr99VRKT3TNB5oAPf/g9UdtN9pVe\nWiLzQG+v1FOtVqnfP/A+OX9sjPr2b37530VtjQb51J88SYTg/LxE9s3PLvI5QrYGne1lGTGkqJSs\n0cGtTAx23ff0d/U13HetlqOMM485ck8VXuCkVV11/kqd2jK9QnwOF2mhs2zuMorsDFp0/uySql3J\nppZOV1f5oM+h3cGEsuXn7c9PGtmA8Th97u+TZ6PJKWR7B8TpoFOlNpfa2qio6ysXKcbBJdkDgCSb\ndeKqmEaczawVvkZXpaid5wRmP/PJp6K2Uz30Pqisyp5p8Tyk2eSTUoVhnElYk8TOZAUV6dvtuBrD\nNHaXgA0AGhP0rlhQ7yBwMZkEZP+/U3gJ3cPDw+OQYN8ldCecalc1V/k7lxVCxKVUDdgtKFS/Ra7K\nvUnqXBP0q9iuqui9GEm/o5znBSrFqvsFjitiC+wqpyX/JkewrTBpElNuZj3szqfLpQUdp4FI31z6\nVMu5I3R1dBcJqKP9XPX5gkpbuzGJQqstroQBE4kNRQKusKvmhQuvR20feuKHAQBHRomojCWF0HS5\nNKqKFHXSckxJqZevkFvj9PQ1Oqcm0vvMDElDsbhINy5vR0JpA9kc59xIkdRSVNJnbIGuEU/KnC4x\nydVsqWhJ1hqeffYPAYhGAgB9PbTeLsoYANrt7SXLvWOLa27RZNdpa4RIQlfPQRiQdNpSRP18hduM\n7N0Eu592WBLtKkK4HjChviiRlAFL5voZ6nK0psVWZOfeIhhjVvbf3DJFCJuWKhtYoD3Q1y8SetnQ\nPrWscRbzKmfTqlvTzf2uViW62EW2Ru6Nav+tVqhPv/2H34ra+vhytiRa/4lRkpJX2vTd1ZpEWDvN\npq1U4C6vaaej4lj5WU6y9vDgA2fk+o8Qgb3YVuvC/R72ErqHh4eHh3+he3h4eBwS7LvJxZkYuoq8\n7IZMhMSkzVWiCQ0n71F1CLsukQ90QiE6XpkXVWnqApEqI8fI+bcwpOoysh93XKfm5Gg/XX+yxqp8\nl9Uu5/MKSDRmoEw0dU7MEzPKv53rXTpSJa5UQkeAajOMI0o7qiLTRpPL6pqQR46gqdWl3/U6fbda\nF5/cs69TJofHHv0QAOD0vVIr9OIlSruazckcNZt011ZbJXpqkcpY5dTFiwuT0TE3H+msjOXiJSJg\n5+fFpHTiFPn/9nPqVhsqcxpXqrpwSaJCz774GvdH+fCy7/rMNCVJMipUdKBvjM+XeVYWrU2wezQx\nbHMxwQ5WHuccEDbFdBBaGkO5LRdZrNI6dmNicnFEXJN9vIOYmDVmuDboYll2TOjmRkVcusfPmVy2\n6qrZihDewnzkcF9B/OHBppFcSe2naepT5iFJ0fzWeYpd6PYzKalMbbhIe1I/XyaqQSrr7SohuYRu\nOs2yqxrl6hADwAK/U2ILYv5IM4H5Uz/9VwEAX/wv/y06NjlFjhDaZJXhRH/33i9mlRzv+/NvklPA\nkqpvvLxMZkUVQI6Erynq4eHh4eGw7xK6+4EPVbrOLhesaCppJZVhwsdwAQhVDdxaVw9U5XdhKf/i\n629FbRdfJsnuAx/9IACgOC6/3M6lKFBSn5M9Gg25bq3GEi5LBh1FjLgxtFpyDSdNaCJxYYGIRldL\nMQhkLA4xley+yrk8dD+g3DYBoFwVMqYbeVDJNR59lAhQnVY2GVsfUdrtSj9c9GZvr0TIdThfS6st\n9wqjwgxcH1JpSU5TCFQBg7UaSUEjI5I35lOfISnoBLs0hsqlcHaG5iqmaoDeuHGN/k5K4ZFkl3MC\ncWrkgX6JiM0XiGSqVmRdyuUd8mXsUUC/lTC+1WVdmthWi+bUqvSrLUsS3qwqVFLnfDS6vmzYpc/L\nXKTlrcvXomOXr5PraFs96o6kDuxml0oX2RpT7nfYIlI0alvnlrl+bPcVJXrTaVONuuydRJu0uiMj\nEqH55nn6e/KeCQBAXtUa/vbzzwMQ7ViPJaZu7oqhJFOuCI2qnXqPiziWa7z+Jr0XMmo+WjXq5xtv\nUfro2rpaDjx2VRAjxtp8Ia0KsbTpeXKpvs++IHmRKqwN/IOf/1tRWz5Pa3T26jT2Ci+he3h4eBwS\n7LuE7hCqyt9OWm4ribvV5sAiTjbRtSLhpZLsBqh+pdt1kggSKqggy65ecbbntZU07mzjnbpOn88B\nClXJedHgX+oaV1PvKOnTBSXpgANXnCCdltwl5TLdq85J7nWWtxK7TmmNxbk5Ols6ACxuEPfabQmK\nMLysBVVm6wNPkFYyPi5FMq5eIemjXne2PZlvZ6/PZnVxgAkAQKslmtNclWyRke1VaSIuQWIiJfbe\nIpflSmdFlnDzsMDBQC21Ls0mcywq4Kvdpvs3GxV1Hk3IwADllikURbNIcDbJjsoj0tE11u44ZI6c\n7bzF8x1TASxrXCJubrm86btGcU6uSMwsz9/snLiaNjj3i4krm7vzTFTPhonz59BJn+qWvBf1npRg\nOHXiBgm9oWzdFdYuV1YlwMkN9dwb4kp79QKJ6EnOy+RyEAGyB/RtIs1X2dCdJuEO9fQIz1VnDunk\nyYmobXCKNPXZSZGMz6/RnH+PuRvteusCw/Reb9ZIGn/hWy9Lf/k91luiMTz1sQ9Fx97zXuINRk4c\ni9oiD2gvoXt4eHh4+Be6h4eHxyHBLU0uxpgMgOcBpPn8r1pr/5kx5iSArwAYAPAygJ+z1m5m924F\nVo9C5YboSLR2WxW44AisNuuCjZaYRnJpJvesqjvJKunYifGobbCHVJ+BoxQ5uKzciJzbnVVFEBpN\nR87KvZqsGi8sklrbo/K2dFjF1KRNpFKr/rqamS46Vl0+upeug+jS7bYV2brRb7HZkrH3c6L8hIqc\nLa+R+2a7LV+07ErpIt+SKTk/6NK9dDGBBBOOKZ0KmFPuOhU8rlR7V9uxt1fI59ERciHMZiUC8M+e\npWICKfbhKqr0w0PDRKx1VM4Ll48DOn8Hk4Qut4dWwQNeg0CtS6utTBZ7xEY3vlvFnpoo54vKDeSo\nUuNIf5nvRc4X1FDuqo4MtbrYBI+1xeOM6bS//Nkowi+aNkW8O29d18f1BCjnoFHXkMPbj7qiimqU\n2STWCIXkdOa8oYI8AI+dJjJ7eppSI194Q8xN3e7mNXP5hXR3Y2yucavdoyKs336bIlZ1YZrePiLN\nlxbkfZDrpWfO1SBdK4ub4+wsRSU39IPrzDDKZNZl0vTYUdrzn/ubn42OZbhwR6Dyu6hMunvGbiT0\nFoCnrLXvBfAogI8bY54A8K8A/Ftr7WkAKwA+d/vd8fDw8PDYK3ZTgs4CcNEoSf7PAngKwM9w+5cB\n/BKAL77TDrh8Fe2G/Np1WHK1ytHeSacul4tKwobA/RQrqSzkghnJYfl1zg0SOVJlCdllUwSABhOf\nWpJxJct0AMsK/1K74hQDg4PSj8jlTBGDHFxj17NMAETiMCoQxJEvmvPs8pjXSwQFaBR6Rap1xQQW\nloW8rNc4ICoUyaTUxyXUytSft69ORceaPN+xhCIjuS2VVOUCs3m+Lp2nJXQn3bc7slizXA6skJO2\nRoNG6zI8xo0QyLkM57tRpLKTrtdl6FTFTaj/MnanRSV0wYXu9m6Luw0s2jLQZuM56nOcr5tWYlTK\n9Ym1n2pT5nuO8wXpgDnLc9pVUrhxWi6c66G+/+ZAoTCSuDe7JsYcAaqeAxfgtG7CIzfV7edAF/cw\nzPjp3Ee5LK3pkfGxqK2/n6TlE8ukUeqsmasn6byGciF0WnxVZSdtdDiviiN9VWDgAw8+BABYXhNC\nfXKaNL622hMf+zA5EZw4Thr+yrL0o84OFy+88FLUdv0aBdRZ6GBI2pMPP0JZTbvKyaPM76CEcj9N\nmNu3gO/qCsaYOBeIngfwJwCuAFi11roZmAQwvs13nzbGvGSMeale3xjf6OHh4eHxbmFXL3Rrbdda\n+yiAowB+EMD9u72BtfZL1trHrbWP51TZMQ8PDw+PdxfvyA/dWrtqjHkOwA8BKBljEiylHwUwtfO3\nt4bzyXbpaAFRW5KqPp8zT6xx6tR6r6gv7QIpCnGVUD/GqkxXRYk12GTQYIKtpew2zvc5qUwGjhSt\n1eReyxy1l0yuV/HpvDofE9Xe5WTR6rkjnNJcN1T/0LljFVVn1LW1FTm2ceUCxaiscb6PQGnU7W6S\nz1O1KDklcYLV4CtXpEZnm7+cUmlaO1GEqMphweRmk0nfhKqz6NbAKPOAkMQqqteNk2MRLBQx3aD5\n1j71fSX2G567Ltfl7zpXae2jns5SdGAypfyRG9vz9xEhqEwMuzGvbIWYSsec5sILSUXeL6zQ/m/z\n/DWVr3yVoxWhzStR4QyoNlcDdXd9dOPTfuVSNzTkv5v36xaBojuap1IqL1LAfu7raoRyfWATqoIc\nAa3R6CCtd/eUKP6uPuq6mARXdEWZbOv8vK6u0rvi9cuS0rlayfK1ZOzl1TXuh/T37IvkT/7mK+Qj\nn8/LM5pnv/aMyjOTSjDhrcxY97KP+XCJzp+5LvmIcjlqy6mcTTq3zl5xSwndGDNkjCnx5yyAHwVw\nDsBzAH6KT/ssgN+/7d54eHh4eOwZu/lJGAPwZUNiVgzAb1lr/8AY8xaArxhj/jmAVwD8yl46ML9E\nJNnaquSryGRJyutJCvHnJN36NJ0/x8UTAGCQc0akU5rkoT/rJGl2K1xg96SpayLhWf6lT6fFJc8w\nQVlvasKWpIMBdnVqKanZuoT2SkKPcw4STay6ElruPO2OVeY0gJrAy/LY83npW3mDi1O5orQN7pPZ\nInouldHZITnCNjYPAFhakQjDdIaOqVoJCAKah07QUOfRGsViGf4rX0jEt99e6bRIJi6S1BGrjbq4\nqoXsPtlsidtYhjWbjF6rmBsTrVmhIOM8eYoshEl1fruzvWTpXAL3KJSvQ1wRci5Pi46gnFqkdZtl\nF0Kd86fD/Qi1dhdJxO+sc9oN0W6Rm2Vj2/pjm68RSfc7TFJWRR4nkzzmpJIhWTLXBWSc62/Q4Xwz\nXdlDnYxoKg5OcwpUDpxOi/Z/nYte5JUG/OqFmwCAtir+MpJnrVWvVY32m3uquhUZZ32JM0Eqdvu9\n99A7SBfkGBkhh4lEh+YhGcj+c1HrGSWVp5O3T4ruxsvldQDv26L9Ksie7uHh4eFxF8BHinp4eHgc\nEux7cq6FRfLvXFkRP89cns0rDfEljrPK1uTkWdNV4WDHh4h8KBZVzcgoylNUq+UFIkeuvU2mlpmb\nN6Njzg89r8i3YonNKsqPOsdRZ87HtqGiWRNbJKhqdxxZJ+Ygd3yVzUw6Yb+LCu1RaqLzVdaJwDYi\nUERbhv1ftYkmxsVCAmUiqvKYBweHuF+qFiWbPeoNMX84E06PimKNRy7K9N1EQu7piLVApRl1ZqaS\n8pvPsnkn6NL5cWWqiRlH4Im6n8zSXJ48LYUROkx4p5mkfeQ9Uqzj5AkiReNQBTES2yfn2ooUdVGV\nW5HbDlsRp9rXO2STklUkZ8CFKgInW4XaP579yu1mk4vZ3mK0a2xlQnHzrKY7atuqFupWbQ5xNR0J\nlwZXn87X1SNO8Dxb3ieB8g3vdGhfrDcl8vym1VgyNM+9PbQXBocHomMPPzABAKiUpdBLk50jtOnT\nRXG7msDrks6xo0VcpXROsdkwvs4hgt5HLqmeI0IBIMOfUyrhXipKYidFed4pvITu4eHhcUhgdvqF\nfbdx5MgR+/TTT9+x+3l4eHgcBnzhC1942Vr7+K3O8xK6h4eHxyGBf6F7eHh4HBL4F7qHh4fHIYF/\noXt4eHgcEtxRUtQYswCgBmDxVufe5RjEwR7DQe8/cPDHcND7Dxz8MRyk/p+w1g7d6qQ7+kIHAGPM\nS7tha+9mHPQxHPT+Awd/DAe9/8DBH8NB7/9W8CYXDw8Pj0MC/0L38PDwOCTYjxf6l/bhnu82DvoY\nDnr/gYM/hoPef+Dgj+Gg938T7rgN3cPDw8Pj+wNvcvHw8PA4JLijL3RjzMeNMReMMZeNMZ+/k/fe\nC4wxx4wxzxlj3jLGvGmM+Xvc3m+M+RNjzCX+27fffd0JXOT7FWPMH/C/Txpjvsvr8JvGmM319O4i\nGGNKxpivGmPOG2POGWN+6ACuwd/nPfSGMeY3jDGZu3kdjDG/aoyZN8a8odq2nHND+A88jteNMY/t\nX88F24zhX/M+et0Y8z9dNTY+9gs8hgvGmB/bn17fHu7YC50rHv0nAD8O4EEAnzHGPHin7r9HBAD+\nobX2QQBPAPjb3OfPA3jWWnsGwLP877sZfw9UNtDhXwH4t9ba0wBWAHxuX3q1e/x7AH9krb0fwHtB\nYzkwa2CMGQfwdwE8bq19GEAcwKdxd6/DrwH4+Ia27eb8xwGc4f+eBvDFO9THW+HXsHkMfwLgYWvt\newBcBPALAMDP9acBPMTf+c9GF8M9ILiTEvoPArhsrb1qrW0D+AqAT97B+79jWGtnrLVn+XMF9CIZ\nB/X7y3zalwH8lf3p4a1hjDkK4CcA/Ff+twHwFICv8il3e/97AXwEXOLQWtu21q7iAK0BIwEga4xJ\nAMgBmMFdvA7W2uexOTH3dnP+SQD/3RK+AyogP3Znero9thqDtfZrXNgeAL4DKnAP0Bi+Yq1tWWvf\nBnAZB7Ai2518oY8DuKn+PcltBwLGmAlQKb7vAhix1s7woVkAI/vUrd3g3wH4RwBcpYUBAKtqU9/t\n63ASwAKA/8Zmo/9qjMnjAK2BtXYKwL8BcAP0Il8D8DIO1joA28/5QX22/waAP+TPB3UM6+BJ0V3A\nGFMA8DsAft5aW9bHLLkJ3ZWuQsaYnwQwb619eb/7chtIAHgMwBette8DpY5YZ165m9cAANjW/EnQ\nj9MRAHlsNgUcKNztc34rGGN+EWRS/fX97su7iTv5Qp8CcEz9+yi33dUwVJftdwD8urX2d7l5zqmU\n/Hd+v/p3C3wQwF82xtDSh0UAAAHESURBVFwDmbieAtmjS6z6A3f/OkwCmLTWfpf//VXQC/6grAEA\n/AiAt621C9baDoDfBa3NQVoHYPs5P1DPtjHmrwP4SQA/a8Vv+0CNYTvcyRf6iwDOMLOfAhEQz9zB\n+79jsL35VwCcs9b+sjr0DIDP8ufPAvj9O9233cBa+wvW2qPW2gnQfP8/a+3PAngOwE/xaXdt/wHA\nWjsL4KYx5j5u+hiAt3BA1oBxA8ATxpgc7yk3hgOzDozt5vwZAH+NvV2eALCmTDN3FYwxHweZIP+y\ntbauDj0D4NPGmLQx5iSI4H1hP/p4W7DW3rH/AHwCxCxfAfCLd/Lee+zvh0Bq5esAXuX/PgGyQz8L\n4BKA/wugf7/7uouxPAngD/jzKdBmvQzgtwGk97t/t+j7owBe4nX4PQB9B20NAHwBwHkAbwD4HwDS\nd/M6APgNkL2/A9KSPrfdnIOqWf8nfq6/B/LmuVvHcBlkK3fP839R5/8ij+ECgB/f7/7v5T8fKerh\n4eFxSOBJUQ8PD49DAv9C9/Dw8Dgk8C90Dw8Pj0MC/0L38PDwOCTwL3QPDw+PQwL/Qvfw8PA4JPAv\ndA8PD49DAv9C9/Dw8Dgk+P96EoNLrh/dNAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "GroundTruth:    cat   car plane truck\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tc47HUnOPY7V",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e4897903-187b-4ec2-f822-459ddaa8055d"
      },
      "source": [
        "outputs.size()"
      ],
      "execution_count": 208,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([4, 10])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 208
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NN8t5QDlNqME",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "val, pred = torch.max(outputs, 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3obgivCXPksp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d388022f-c02b-4b24-9f44-91e8f39f3d1a"
      },
      "source": [
        "pred"
      ],
      "execution_count": 210,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([3, 9, 4, 9], device='cuda:0')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 210
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lmnwMwVEP52g",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "24a8272a-b1ef-4a9f-dab1-7eaf570f26fb"
      },
      "source": [
        "[classes[p] for p in pred]"
      ],
      "execution_count": 211,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['cat', 'truck', 'deer', 'truck']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 211
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F_PglcCUQEy-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zoYm9f1dPwrZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "401b2024-c4d6-49ea-826b-7305e1659e85"
      },
      "source": [
        "val"
      ],
      "execution_count": 212,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 7.7879,  6.9551,  5.8587, 15.5622], device='cuda:0',\n",
              "       grad_fn=<MaxBackward0>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 212
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WXfKhXz_PLPy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "??torch.max"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S44EczmAQ4Ga",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "f443922a-16f0-4509-dbe6-2c609971b442"
      },
      "source": [
        "(labels == pred).size()"
      ],
      "execution_count": 217,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([4])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 217
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Xoien5gPUok",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6f60e547-3ebb-4f71-8353-f110ca23a39a"
      },
      "source": [
        "(labels == pred).squeeze().size()"
      ],
      "execution_count": 218,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([4])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 218
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zirZYQBJRRs9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "c = (labels == pred)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sgo6BHVwRZUC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "b473735d-9dd8-40f7-ef5d-f5908b4b0c8f"
      },
      "source": [
        "a, b = iter(testloader).next()\n",
        "print(a.size())\n",
        "print(b.size())"
      ],
      "execution_count": 223,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([4, 3, 32, 32])\n",
            "torch.Size([4])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kOEBIf8IR4Ub",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class_correct = np.zeros(10)\n",
        "class_total = np.zeros(10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IgNScYBwQvBR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "correct = 0 \n",
        "total = 0\n",
        "for data in testloader:\n",
        "  images, labels = data[0].to(device), data[1].to(device)\n",
        "  outputs = net(images)\n",
        "  _, predicted = torch.max(outputs, 1)\n",
        "  c = (predicted == labels).squeeze()\n",
        "  correct += c.sum().item()\n",
        "  total += labels.size(0)\n",
        "  for i in range(4):\n",
        "    label = labels[i]\n",
        "    class_correct[label] += c[i].item()\n",
        "    class_total[label] += 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EGjatLWISS0H",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "71234e71-5138-4c95-82ff-4807e028e1df"
      },
      "source": [
        "class_correct/class_total"
      ],
      "execution_count": 251,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.693, 0.751, 0.545, 0.463, 0.616, 0.439, 0.684, 0.62 , 0.74 ,\n",
              "       0.616])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 251
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bO_52fgRS7NJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6e4d4e98-0afb-43be-a30c-a983925ddd8e"
      },
      "source": [
        "correct/total"
      ],
      "execution_count": 252,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.6167"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 252
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "weSshKhRSGdz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d79949e9-d234-4d91-e159-2e136ad8ed14"
      },
      "source": [
        "correct"
      ],
      "execution_count": 255,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6167"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 255
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DnuL8WwKSrCO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "192c3601-5044-4e00-a202-659d05edb1e5"
      },
      "source": [
        "total"
      ],
      "execution_count": 254,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 254
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hw2pKO7-TFES",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ukN_nh97TdkZ",
        "colab_type": "text"
      },
      "source": [
        "## NN from scratch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HS_QtVRdBXk3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = torch.eye(3,2)\n",
        "a[1,1] = 2\n",
        "a[0,1] = 10\n",
        "a.unsqueeze(2).shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "COZoT8kgTgIA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from pathlib import Path\n",
        "import requests\n",
        "\n",
        "DATA_PATH = Path(\"data\")\n",
        "\n",
        "PATH = DATA_PATH / \"mnist\"\n",
        "\n",
        "\n",
        "PATH.mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "URL = \"http://deeplearning.net/data/mnist/\"\n",
        "FILENAME = \"mnist.pkl.gz\"\n",
        "\n",
        "if not (PATH / FILENAME).exists():\n",
        "        content = requests.get(URL + FILENAME).content\n",
        "        (PATH / FILENAME).open(\"wb\").write(content)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jNydNSNEUDdo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pickle\n",
        "import gzip\n",
        "\n",
        "with gzip.open((PATH / FILENAME).as_posix(), \"rb\") as f:\n",
        "        ((x_train, y_train), (x_valid, y_valid), _) = pickle.load(f, encoding=\"latin-1\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RoROj_hVUBNB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 303
        },
        "outputId": "58d01e8f-6dbf-40c2-f17d-1b32f1c724a4"
      },
      "source": [
        "from matplotlib import pyplot\n",
        "import numpy as np\n",
        "\n",
        "pyplot.imshow(x_train[0].reshape((28, 28)), cmap=\"gray\")\n",
        "print(x_train.shape)\n",
        "print(y_train[0])"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(50000, 784)\n",
            "5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADgdJREFUeJzt3X9sXfV5x/HPs9D8QRoIXjUTpWFp\nIhQUIuZOJkwoGkXM5YeCggGhWkLKRBT3j1ii0hQNZX8MNAVFg2RqBKrsqqHJ1KWZBCghqpp0CZBO\nTBEmhF9mKQylqi2TFAWTH/zIHD/74x53Lvh+r3Pvufdc+3m/JMv3nuecex4d5ZPz8/pr7i4A8fxJ\n0Q0AKAbhB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8Q1GWNXJmZ8TghUGfublOZr6Y9v5ndYWbH\nzex9M3ukls8C0FhW7bP9ZjZL0m8kdUgalPSqpC53H0gsw54fqLNG7PlXSHrf3T9w9wuSfi5pdQ2f\nB6CBagn/Akm/m/B+MJv2R8ys28z6zay/hnUByFndL/i5e5+kPonDfqCZ1LLnH5K0cML7b2bTAEwD\ntYT/VUnXmtm3zGy2pO9J2ptPWwDqrerDfncfNbMeSfslzZK03d3fya0zAHVV9a2+qlbGOT9Qdw15\nyAfA9EX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+\nICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUFUP0S1JZnZC\n0llJFyWNunt7Hk0hP7NmzUrWr7zyyrquv6enp2zt8ssvTy67dOnSZH39+vXJ+pNPPlm21tXVlVz2\n888/T9Y3b96crD/22GPJejOoKfyZW939oxw+B0ADcdgPBFVr+F3SATN7zcy682gIQGPUeti/0t2H\nzOzPJP3KzP7b3Q9PnCH7T4H/GIAmU9Oe392Hst+nJD0vacUk8/S5ezsXA4HmUnX4zWyOmc0dfy3p\nu5LezqsxAPVVy2F/q6TnzWz8c/7N3X+ZS1cA6q7q8Lv7B5L+IsdeZqxrrrkmWZ89e3ayfvPNNyfr\nK1euLFubN29ectn77rsvWS/S4OBgsr5t27ZkvbOzs2zt7NmzyWXfeOONZP3ll19O1qcDbvUBQRF+\nICjCDwRF+IGgCD8QFOEHgjJ3b9zKzBq3sgZqa2tL1g8dOpSs1/trtc1qbGwsWX/ooYeS9XPnzlW9\n7uHh4WT9448/TtaPHz9e9brrzd1tKvOx5weCIvxAUIQfCIrwA0ERfiAowg8ERfiBoLjPn4OWlpZk\n/ciRI8n64sWL82wnV5V6HxkZSdZvvfXWsrULFy4kl436/EOtuM8PIInwA0ERfiAowg8ERfiBoAg/\nEBThB4LKY5Te8E6fPp2sb9iwIVlftWpVsv76668n65X+hHXKsWPHkvWOjo5k/fz588n69ddfX7b2\n8MMPJ5dFfbHnB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgKn6f38y2S1ol6ZS7L8+mtUjaLWmRpBOS\nHnD39B8618z9Pn+trrjiimS90nDSvb29ZWtr165NLvvggw8m67t27UrW0Xzy/D7/TyXd8aVpj0g6\n6O7XSjqYvQcwjVQMv7sflvTlR9hWS9qRvd4h6Z6c+wJQZ9We87e6+/h4Rx9Kas2pHwANUvOz/e7u\nqXN5M+uW1F3regDkq9o9/0kzmy9J2e9T5WZ09z53b3f39irXBaAOqg3/XklrstdrJO3Jpx0AjVIx\n/Ga2S9J/SVpqZoNmtlbSZkkdZvaepL/J3gOYRiqe87t7V5nSbTn3EtaZM2dqWv6TTz6petl169Yl\n67t3707Wx8bGql43isUTfkBQhB8IivADQRF+ICjCDwRF+IGgGKJ7BpgzZ07Z2gsvvJBc9pZbbknW\n77zzzmT9wIEDyToajyG6ASQRfiAowg8ERfiBoAg/EBThB4Ii/EBQ3Oef4ZYsWZKsHz16NFkfGRlJ\n1l988cVkvb+/v2zt6aefTi7byH+bMwn3+QEkEX4gKMIPBEX4gaAIPxAU4QeCIvxAUNznD66zszNZ\nf+aZZ5L1uXPnVr3ujRs3Jus7d+5M1oeHh5P1qLjPDyCJ8ANBEX4gKMIPBEX4gaAIPxAU4QeCqnif\n38y2S1ol6ZS7L8+mPSppnaTfZ7NtdPdfVFwZ9/mnneXLlyfrW7duTdZvu636kdx7e3uT9U2bNiXr\nQ0NDVa97OsvzPv9PJd0xyfR/cfe27Kdi8AE0l4rhd/fDkk43oBcADVTLOX+Pmb1pZtvN7KrcOgLQ\nENWG/0eSlkhqkzQsaUu5Gc2s28z6zaz8H3MD0HBVhd/dT7r7RXcfk/RjSSsS8/a5e7u7t1fbJID8\nVRV+M5s/4W2npLfzaQdAo1xWaQYz2yXpO5K+YWaDkv5R0nfMrE2SSzoh6ft17BFAHfB9ftRk3rx5\nyfrdd99dtlbpbwWYpW9XHzp0KFnv6OhI1mcqvs8PIInwA0ERfiAowg8ERfiBoAg/EBS3+lCYL774\nIlm/7LL0Yyijo6PJ+u2331629tJLLyWXnc641QcgifADQRF+ICjCDwRF+IGgCD8QFOEHgqr4fX7E\ndsMNNyTr999/f7J+4403lq1Vuo9fycDAQLJ++PDhmj5/pmPPDwRF+IGgCD8QFOEHgiL8QFCEHwiK\n8ANBcZ9/hlu6dGmy3tPTk6zfe++9yfrVV199yT1N1cWLF5P14eHhZH1sbCzPdmYc9vxAUIQfCIrw\nA0ERfiAowg8ERfiBoAg/EFTF+/xmtlDSTkmtklxSn7v/0MxaJO2WtEjSCUkPuPvH9Ws1rkr30ru6\nusrWKt3HX7RoUTUt5aK/vz9Z37RpU7K+d+/ePNsJZyp7/lFJf+fuyyT9laT1ZrZM0iOSDrr7tZIO\nZu8BTBMVw+/uw+5+NHt9VtK7khZIWi1pRzbbDkn31KtJAPm7pHN+M1sk6duSjkhqdffx5ys/VOm0\nAMA0MeVn+83s65KelfQDdz9j9v/Dgbm7lxuHz8y6JXXX2iiAfE1pz29mX1Mp+D9z9+eyySfNbH5W\nny/p1GTLunufu7e7e3seDQPIR8XwW2kX/xNJ77r71gmlvZLWZK/XSNqTf3sA6qXiEN1mtlLSryW9\nJWn8O5IbVTrv/3dJ10j6rUq3+k5X+KyQQ3S3tqYvhyxbtixZf+qpp5L166677pJ7ysuRI0eS9See\neKJsbc+e9P6Cr+RWZ6pDdFc853f3/5RU7sNuu5SmADQPnvADgiL8QFCEHwiK8ANBEX4gKMIPBMWf\n7p6ilpaWsrXe3t7ksm1tbcn64sWLq+opD6+88kqyvmXLlmR9//79yfpnn312yT2hMdjzA0ERfiAo\nwg8ERfiBoAg/EBThB4Ii/EBQYe7z33TTTcn6hg0bkvUVK1aUrS1YsKCqnvLy6aeflq1t27Ytuezj\njz+erJ8/f76qntD82PMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFBh7vN3dnbWVK/FwMBAsr5v375k\nfXR0NFlPfed+ZGQkuSziYs8PBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0GZu6dnMFsoaaekVkkuqc/d\nf2hmj0paJ+n32awb3f0XFT4rvTIANXN3m8p8Uwn/fEnz3f2omc2V9JqkeyQ9IOmcuz851aYIP1B/\nUw1/xSf83H1Y0nD2+qyZvSup2D9dA6Bml3TOb2aLJH1b0pFsUo+ZvWlm283sqjLLdJtZv5n119Qp\ngFxVPOz/w4xmX5f0sqRN7v6cmbVK+kil6wD/pNKpwUMVPoPDfqDOcjvnlyQz+5qkfZL2u/vWSeqL\nJO1z9+UVPofwA3U21fBXPOw3M5P0E0nvTgx+diFwXKekty+1SQDFmcrV/pWSfi3pLUlj2eSNkrok\ntal02H9C0vezi4Opz2LPD9RZrof9eSH8QP3ldtgPYGYi/EBQhB8IivADQRF+ICjCDwRF+IGgCD8Q\nFOEHgiL8QFCEHwiK8ANBEX4gKMIPBNXoIbo/kvTbCe+/kU1rRs3aW7P2JdFbtfLs7c+nOmNDv8//\nlZWb9bt7e2ENJDRrb83al0Rv1SqqNw77gaAIPxBU0eHvK3j9Kc3aW7P2JdFbtQrprdBzfgDFKXrP\nD6AghYTfzO4ws+Nm9r6ZPVJED+WY2Qkze8vMjhU9xFg2DNopM3t7wrQWM/uVmb2X/Z50mLSCenvU\nzIaybXfMzO4qqLeFZvaimQ2Y2Ttm9nA2vdBtl+irkO3W8MN+M5sl6TeSOiQNSnpVUpe7DzS0kTLM\n7ISkdncv/J6wmf21pHOSdo6PhmRm/yzptLtvzv7jvMrd/75JentUlzhyc516Kzey9N+qwG2X54jX\neShiz79C0vvu/oG7X5D0c0mrC+ij6bn7YUmnvzR5taQd2esdKv3jabgyvTUFdx9296PZ67OSxkeW\nLnTbJfoqRBHhXyDpdxPeD6q5hvx2SQfM7DUz6y66mUm0ThgZ6UNJrUU2M4mKIzc30pdGlm6abVfN\niNd544LfV61097+UdKek9dnhbVPy0jlbM92u+ZGkJSoN4zYsaUuRzWQjSz8r6QfufmZirchtN0lf\nhWy3IsI/JGnhhPffzKY1BXcfyn6fkvS8SqcpzeTk+CCp2e9TBffzB+5+0t0vuvuYpB+rwG2XjSz9\nrKSfuftz2eTCt91kfRW13YoI/6uSrjWzb5nZbEnfk7S3gD6+wszmZBdiZGZzJH1XzTf68F5Ja7LX\nayTtKbCXP9IsIzeXG1laBW+7phvx2t0b/iPpLpWu+P+PpH8ooocyfS2W9Eb2807RvUnapdJh4P+q\ndG1kraQ/lXRQ0nuS/kNSSxP19q8qjeb8pkpBm19QbytVOqR/U9Kx7Oeuorddoq9CthtP+AFBccEP\nCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQ/weCC5r/92q6mAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4F411skFUGSy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "\n",
        "x_train, y_train, x_valid, y_valid = map(\n",
        "    torch.tensor, (x_train, y_train, x_valid, y_valid)\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CKa2EV-RUOzF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "3226504d-dc6e-4c87-aa02-8d0be1f7cd9c"
      },
      "source": [
        "x_train.shape"
      ],
      "execution_count": 124,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([50000, 784])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 124
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "StfWzkFrUQr0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "weights = torch.randn(x_train.shape[1], 10) / np.sqrt(x_train.shape[1])\n",
        "weights.requires_grad_()\n",
        "bias = torch.zeros(10, requires_grad=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zADiQvJw_rjs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "??weights.requires_grad_"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IjveviuX2mZn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def log_softmax(x):\n",
        "    return x - x.exp().sum(-1).log().unsqueeze(-1)\n",
        "\n",
        "def model(xb):\n",
        "    return log_softmax(xb @ weights + bias)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "awKwDNes-ajz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "751f4845-1074-4771-9bd8-7a5cbbb0c5e2"
      },
      "source": [
        "bs = 64  # batch size\n",
        "\n",
        "xb = x_train[0:bs]  # a mini-batch from x\n",
        "yb = y_train[0:bs]\n",
        "preds = model(xb)  # predictions\n",
        "preds[0], preds.shape\n",
        "print(preds[0], preds.shape)"
      ],
      "execution_count": 132,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([-2.3252, -2.7084, -2.5848, -2.5580, -2.1519, -2.3117, -1.9825, -2.7354,\n",
            "        -1.9035, -2.1567], grad_fn=<SelectBackward>) torch.Size([64, 10])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B8Bvr40w-pE-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "3f3c7b04-e5f9-4d79-e19d-461738641326"
      },
      "source": [
        "def accuracy(out, yb):\n",
        "    preds = torch.argmax(out, dim=1)\n",
        "    return (preds == yb).float().mean()\n",
        "\n",
        "\n",
        "print(accuracy(preds, yb))"
      ],
      "execution_count": 133,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(0.1094)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SsDymrjP8i-C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def nll(input, target):\n",
        "    return -input[range(target.shape[0]), target].mean()\n",
        "\n",
        "loss_func = nll"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EBYAFMT4Brvu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "weights = torch.randn(x_train.shape[1], 10) / np.sqrt(x_train.shape[1])\n",
        "weights.requires_grad_()\n",
        "\n",
        "bias = torch.zeros(10, requires_grad=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3LonxwsR7Nt5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lr = 0.5\n",
        "epochs = 10\n",
        "bs = 64\n",
        "n = x_train.shape[0]\n",
        "\n",
        "for epoch in range(epochs):\n",
        "  for i in range( (n-1)//bs + 1):\n",
        "    a = i * bs\n",
        "    b = a + bs\n",
        "    xb = x_train[a:b]\n",
        "    yb = y_train[a:b]\n",
        "    pred = model(xb)\n",
        "    loss = loss_func(pred, yb)\n",
        "    \n",
        "    loss.backward()\n",
        "    \n",
        "    with torch.no_grad():\n",
        "      weights -= weights.grad * lr\n",
        "      bias -= bias.grad * lr\n",
        "      weights.grad.zero_()\n",
        "      bias.grad.zero_()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "llPEzw_b_RBV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "91c74f4b-4593-4170-e8a7-88e22b6c9246"
      },
      "source": [
        "print(loss_func(model(x_train), y_train), accuracy(model(x_train), y_train))"
      ],
      "execution_count": 157,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(0.2733, grad_fn=<NegBackward>) tensor(0.9230)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0yH6aI9C5NqU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.nn.functional as F\n",
        "\n",
        "loss_func = F.cross_entropy  # using cross_entropy fun\n",
        "\n",
        "def model(xb):\n",
        "  return xb @ weights + bias\n",
        "\n",
        "lr = 0.5\n",
        "epochs = 10\n",
        "bs = 64\n",
        "n = x_train.shape[0]\n",
        "\n",
        "for epoch in range(epochs):\n",
        "  for i in range( (n-1)//bs + 1):\n",
        "    a = i * bs\n",
        "    b = a + bs\n",
        "    xb = x_train[a:b]\n",
        "    yb = y_train[a:b]\n",
        "    pred = model(xb)\n",
        "    loss = loss_func(pred, yb)\n",
        "    \n",
        "    loss.backward()\n",
        "    \n",
        "    with torch.no_grad():\n",
        "      weights -= weights.grad * lr\n",
        "      bias -= bias.grad * lr\n",
        "      weights.grad.zero_()\n",
        "      bias.grad.zero_()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RbHLP0JEBv5v",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "56749fbe-668d-40c0-f660-ecd21b1faa7f"
      },
      "source": [
        "print(loss_func(model(x_train), y_train), accuracy(model(x_train), y_train))"
      ],
      "execution_count": 159,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(0.2640, grad_fn=<NllLossBackward>) tensor(0.9253)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qVuIhzy5JD_0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tt = 10"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6fWOPwpwHjIs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Rectangle:\n",
        "    def __init__(self, length, width):\n",
        "        self.length = length\n",
        "        self.width = width\n",
        "\n",
        "    def area(self):\n",
        "        return self.length * self.width # + tt \n",
        "\n",
        "    def perimeter(self):\n",
        "        return 2 * self.length + 2 * self.width\n",
        "\n",
        "# Here we declare that the Square class inherits from the Rectangle class\n",
        "class Square(Rectangle):\n",
        "    def __init__(self, length):\n",
        "        super().__init__(length, length) # --> use of super() <--\n",
        "        \n",
        "class anotherSquare(Rectangle):\n",
        "    def __init__(self, length):\n",
        "      self.length = length\n",
        "      self.width = length"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CbJKHKxqHkIc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "66da62e9-f940-4dde-f9ab-1ab3042f9b94"
      },
      "source": [
        "f = Rectangle(5,4)\n",
        "print(f.area())\n",
        "print(f.perimeter())\n",
        "\n",
        "s = Square(5)\n",
        "print(s.area())\n",
        "print(s.perimeter())\n",
        "\n",
        "s = anotherSquare(6)\n",
        "print(s.area())\n",
        "print(s.perimeter())"
      ],
      "execution_count": 176,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "30\n",
            "18\n",
            "35\n",
            "20\n",
            "46\n",
            "24\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EcO4m1OIB9Ra",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch import nn"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y4PAEvogB1og",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Net(nn.Module):\n",
        "  def __init__(self):\n",
        "    super().__init__()\n",
        "    self.weights = nn.Parameter(torch.randn(784, 10) / np.sqrt(784))\n",
        "    self.bias = nn.Parameter(torch.zeros(10, requires_grad=True))\n",
        "  \n",
        "  def forward(self, xb):\n",
        "    return xb @ self.weights + self.bias\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RIkbahrTJX5l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = Net()\n",
        "\n",
        "lr = 0.5\n",
        "epochs = 10\n",
        "bs = 64\n",
        "n = x_train.shape[0]\n",
        "\n",
        "for epoch in range(epochs):\n",
        "  for i in range( (n-1)//bs + 1):\n",
        "    a = i * bs\n",
        "    b = a + bs\n",
        "    xb = x_train[a:b]\n",
        "    yb = y_train[a:b]\n",
        "    pred = model(xb)\n",
        "    loss = loss_func(pred, yb)\n",
        "    \n",
        "    loss.backward()\n",
        "    \n",
        "    with torch.no_grad():\n",
        "      for p in model.parameters():\n",
        "        p -= p.grad * lr\n",
        "      model.zero_grad()\n",
        "      "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uvl0J5tGJ_ye",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "02eed7b7-a842-458f-ad0d-0db410abf878"
      },
      "source": [
        "print(loss_func(model(x_train), y_train), accuracy(model(x_train), y_train))"
      ],
      "execution_count": 183,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(0.2730, grad_fn=<NllLossBackward>) tensor(0.9227)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a3RjE0pHKI6m",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = Net()\n",
        "\n",
        "lr = 0.5\n",
        "epochs = 10\n",
        "bs = 64\n",
        "n = x_train.shape[0]\n",
        "\n",
        "for epoch in range(epochs):\n",
        "  for i in range( (n-1)//bs + 1):\n",
        "    a = i * bs\n",
        "    b = a + bs\n",
        "    xb = x_train[a:b]\n",
        "    yb = y_train[a:b]\n",
        "    pred = model(xb)\n",
        "    loss = loss_func(pred, yb)\n",
        "    \n",
        "    loss.backward()\n",
        "    \n",
        "    torch.optim.SGD(model.parameters(), lr=lr).step()\n",
        "      "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FmLu8KfHKSuh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b3e3a256-7126-4b9e-adc4-e58e59d6d643"
      },
      "source": [
        "print(loss_func(model(x_train), y_train), accuracy(model(x_train), y_train))"
      ],
      "execution_count": 186,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(405.5279, grad_fn=<NllLossBackward>) tensor(0.8888)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1EOUxnzgqdQ4",
        "colab_type": "text"
      },
      "source": [
        "## with torch.no_grad():  It will make the code run faster whenever gradient tracking is not needed.\n",
        "\n",
        "### for variables with grad, use temp += 1 or temp.add_(1) may gives a error\n",
        "\n",
        "```a leaf Variable that requires grad has been used in an in-place operation``` \n",
        "\n",
        "temp = temp + 1 is fine but grad will be set ```None```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VjwAz-Cwx_52",
        "colab_type": "text"
      },
      "source": [
        "* case i"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W9eIjz8TsoNu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "b375181c-d6c4-4f5a-9ee2-6a6cb6dc9b4a"
      },
      "source": [
        "temp = torch.ones(10, requires_grad=True)\n",
        "print(temp)\n",
        "print(temp.grad == None)\n",
        "\n",
        "loss = temp.mean()\n",
        "loss.backward()\n",
        "\n",
        "print(temp.grad)\n",
        "with torch.no_grad():\n",
        "  temp += 1\n",
        "print(temp)\n",
        "print(temp.grad)\n",
        "loss.backward()\n",
        "print(temp.grad)"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], requires_grad=True)\n",
            "True\n",
            "tensor([0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,\n",
            "        0.1000])\n",
            "tensor([2., 2., 2., 2., 2., 2., 2., 2., 2., 2.], requires_grad=True)\n",
            "tensor([0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,\n",
            "        0.1000])\n",
            "tensor([0.2000, 0.2000, 0.2000, 0.2000, 0.2000, 0.2000, 0.2000, 0.2000, 0.2000,\n",
            "        0.2000])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LXbSg4XryBx2",
        "colab_type": "text"
      },
      "source": [
        "* case ii"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s0L7bFO98Lpz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "6aaaa6a4-168b-41e8-f776-70ef88a24520"
      },
      "source": [
        "temp = torch.ones(10, requires_grad=True)\n",
        "print(temp)\n",
        "print(temp.grad == None)\n",
        "\n",
        "loss = temp.mean()\n",
        "loss.backward()\n",
        "\n",
        "print(temp.grad)\n",
        "\n",
        "temp = temp + 1\n",
        "\n",
        "print(temp.grad == None)\n",
        "\n",
        "loss.backward()\n",
        "print(temp.grad == None)"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], requires_grad=True)\n",
            "True\n",
            "tensor([0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,\n",
            "        0.1000])\n",
            "True\n",
            "True\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tQfLGZZEyDhV",
        "colab_type": "text"
      },
      "source": [
        "* case iii"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vjdMTyVIxBCR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "dd54cc10-5d94-4104-b7b0-1eb60edd6549"
      },
      "source": [
        "temp = torch.ones(10, requires_grad=True)\n",
        "print(temp)\n",
        "print(temp.is_leaf)\n",
        "print(temp.grad == None)\n",
        "\n",
        "loss = temp.mean()\n",
        "loss.backward()\n",
        "\n",
        "print(temp.grad)\n",
        "\n",
        "with torch.no_grad():\n",
        "  temp.add_(1)\n",
        "\n",
        "print(temp.grad)\n",
        "\n",
        "loss.backward()\n",
        "print(temp.grad)"
      ],
      "execution_count": 84,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], requires_grad=True)\n",
            "True\n",
            "True\n",
            "tensor([0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,\n",
            "        0.1000])\n",
            "tensor([0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,\n",
            "        0.1000])\n",
            "tensor([0.2000, 0.2000, 0.2000, 0.2000, 0.2000, 0.2000, 0.2000, 0.2000, 0.2000,\n",
            "        0.2000])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oEieTA_DyfvA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "\n",
        "# Creating the graph\n",
        "x = torch.tensor(1.0, requires_grad = True)\n",
        "y = torch.tensor(2.0)\n",
        "z = x * y\n",
        "z.backward()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TDgwurMr-Obv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "d34498ba-986f-4257-b828-c7315d5b2402"
      },
      "source": [
        "print(x.grad)\n",
        "print(y.grad)"
      ],
      "execution_count": 122,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor(2.)\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4xRV6ZBDJ92Q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I0xpqI4_-Va_",
        "colab_type": "text"
      },
      "source": [
        "## Torch linear regression from scratch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DrFzKN5d0fw_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = torch.randn(100,5)\n",
        "beta = torch.ones(5)\n",
        "y = x @ beta + torch.randn(100)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wVNyDrFN2oYS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "w = torch.zeros(5, requires_grad=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NOoj_Ipv2pNR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "loss = torch.sum( (x @ w - y)**2 )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4K1PSY3L3uT2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "loss.backward()\n",
        "w.grad == -2*x.t() @ (y - x@w) # exact equal"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yHucMxWA3JkL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "527f969a-9b26-4051-83e5-b31b0d987f04"
      },
      "source": [
        "lr = 0.001\n",
        "for i in range(10):\n",
        "  loss = torch.sum( (x @ w - y)**2 )\n",
        "  loss.backward()  \n",
        "  with torch.no_grad():\n",
        "    w -= lr * w.grad\n",
        "    print(w)\n",
        "    w.grad.zero_()\n",
        "    "
      ],
      "execution_count": 152,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([0.9980, 1.0695, 0.8108, 0.9776, 0.9324], requires_grad=True)\n",
            "tensor([0.9980, 1.0695, 0.8108, 0.9776, 0.9324], requires_grad=True)\n",
            "tensor([0.9980, 1.0695, 0.8108, 0.9776, 0.9324], requires_grad=True)\n",
            "tensor([0.9980, 1.0695, 0.8108, 0.9776, 0.9324], requires_grad=True)\n",
            "tensor([0.9980, 1.0695, 0.8108, 0.9776, 0.9324], requires_grad=True)\n",
            "tensor([0.9980, 1.0695, 0.8108, 0.9776, 0.9324], requires_grad=True)\n",
            "tensor([0.9980, 1.0695, 0.8108, 0.9776, 0.9324], requires_grad=True)\n",
            "tensor([0.9980, 1.0695, 0.8108, 0.9776, 0.9324], requires_grad=True)\n",
            "tensor([0.9980, 1.0695, 0.8108, 0.9776, 0.9324], requires_grad=True)\n",
            "tensor([0.9980, 1.0695, 0.8108, 0.9776, 0.9324], requires_grad=True)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KwywfPp03LFT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "15014f00-02e2-4ff2-9998-a8dcd59fd967"
      },
      "source": [
        "torch.inverse(x.t() @ x) @ x.t() @ y"
      ],
      "execution_count": 151,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0.9980, 1.0695, 0.8108, 0.9776, 0.9324])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 151
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VpGwuWCTA5R6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}