04_CNN_Advanced/2_GoogleNet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# [GoogleNet Implementation](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43022.pdf)\n",
    "\n",
    "- 2014 ILSVRC 1nd place \n",
    "- Inception Module\n",
    "\n",
    "![alt text](./networks/googlenet_diagram.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 1. Settings\n",
    "### 1) Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.init as init\n",
    "import torch.utils.data as data\n",
    "import torchvision.datasets as dset\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2) Hyperparameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size= 1\n",
    "learning_rate = 0.0002\n",
    "epoch = 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "img_dir = \"./images\"\n",
    "img_data = dset.ImageFolder(img_dir, transforms.Compose([\n",
    "            transforms.RandomSizedCrop(224),\n",
    "            transforms.RandomHorizontalFlip(),\n",
    "            transforms.ToTensor(),\n",
    "            ]))\n",
    "\n",
    "img_batch = data.DataLoader(img_data, batch_size=batch_size,\n",
    "                            shuffle=True, num_workers=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Model \n",
    "### 1) Base module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def conv_1(in_dim,out_dim):\n",
    "    model = nn.Sequential(\n",
    "        nn.Conv2d(in_dim,out_dim,1,1),\n",
    "        nn.ReLU(),\n",
    "    )\n",
    "    return model\n",
    "\n",
    "def conv_1_3(in_dim,mid_dim,out_dim):\n",
    "    model = nn.Sequential(\n",
    "        nn.Conv2d(in_dim,mid_dim,1,1),\n",
    "        nn.ReLU(),\n",
    "        nn.Conv2d(mid_dim,out_dim,3,1,1),\n",
    "        nn.ReLU()\n",
    "    )\n",
    "    return model\n",
    "    \n",
    "def conv_1_5(in_dim,mid_dim,out_dim):\n",
    "    model = nn.Sequential(\n",
    "        nn.Conv2d(in_dim,mid_dim,1,1),\n",
    "        nn.ReLU(),\n",
    "        nn.Conv2d(mid_dim,out_dim,5,1,2),\n",
    "        nn.ReLU()\n",
    "    )\n",
    "    return model\n",
    "    \n",
    "\n",
    "def max_3_1(in_dim,out_dim):\n",
    "    model = nn.Sequential(\n",
    "        nn.MaxPool2d(3,1,1),\n",
    "        nn.Conv2d(in_dim,out_dim,1,1),\n",
    "        nn.ReLU(),\n",
    "    )\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2) Inception Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class inception_module(nn.Module):\n",
    "    \n",
    "    def __init__(self,in_dim,out_dim_1,mid_dim_3,out_dim_3,mid_dim_5,out_dim_5,pool):\n",
    "        super(inception_module,self).__init__()\n",
    "\n",
    "        self.conv_1 = conv_1(in_dim,out_dim_1)\n",
    "        self.conv_1_3 = conv_1_3(in_dim,mid_dim_3,out_dim_3)\n",
    "        self.conv_1_5 = conv_1_5(in_dim,mid_dim_5,out_dim_5)\n",
    "        self.max_3_1 = max_3_1(in_dim,pool)\n",
    "\n",
    "    def forward(self,x):\n",
    "        out_1 = self.conv_1(x)\n",
    "        out_2 = self.conv_1_3(x)\n",
    "        out_3 = self.conv_1_5(x)\n",
    "        out_4 = self.max_3_1(x)\n",
    "        output = torch.cat([out_1,out_2,out_3,out_4],1)\n",
    "        \n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3) GoogLeNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('layer_1', Sequential (\n",
      "  (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3))\n",
      "  (1): MaxPool2d (size=(3, 3), stride=(2, 2), padding=(1, 1), dilation=(1, 1))\n",
      "  (2): Conv2d(64, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (3): MaxPool2d (size=(3, 3), stride=(2, 2), padding=(1, 1), dilation=(1, 1))\n",
      "))\n",
      "('layer_2', Sequential (\n",
      "  (0): inception_module (\n",
      "    (conv_1): Sequential (\n",
      "      (0): Conv2d(192, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "    )\n",
      "    (conv_1_3): Sequential (\n",
      "      (0): Conv2d(192, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(96, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (conv_1_5): Sequential (\n",
      "      (0): Conv2d(192, 16, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(16, 32, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (max_3_1): Sequential (\n",
      "      (0): MaxPool2d (size=(3, 3), stride=(1, 1), padding=(1, 1), dilation=(1, 1))\n",
      "      (1): Conv2d(192, 32, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (2): ReLU ()\n",
      "    )\n",
      "  )\n",
      "  (1): inception_module (\n",
      "    (conv_1): Sequential (\n",
      "      (0): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "    )\n",
      "    (conv_1_3): Sequential (\n",
      "      (0): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(128, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (conv_1_5): Sequential (\n",
      "      (0): Conv2d(256, 32, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(32, 96, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (max_3_1): Sequential (\n",
      "      (0): MaxPool2d (size=(3, 3), stride=(1, 1), padding=(1, 1), dilation=(1, 1))\n",
      "      (1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (2): ReLU ()\n",
      "    )\n",
      "  )\n",
      "  (2): MaxPool2d (size=(3, 3), stride=(2, 2), padding=(1, 1), dilation=(1, 1))\n",
      "))\n",
      "('layer_3', Sequential (\n",
      "  (0): inception_module (\n",
      "    (conv_1): Sequential (\n",
      "      (0): Conv2d(480, 192, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "    )\n",
      "    (conv_1_3): Sequential (\n",
      "      (0): Conv2d(480, 96, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(96, 208, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (conv_1_5): Sequential (\n",
      "      (0): Conv2d(480, 16, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(16, 48, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (max_3_1): Sequential (\n",
      "      (0): MaxPool2d (size=(3, 3), stride=(1, 1), padding=(1, 1), dilation=(1, 1))\n",
      "      (1): Conv2d(480, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (2): ReLU ()\n",
      "    )\n",
      "  )\n",
      "  (1): inception_module (\n",
      "    (conv_1): Sequential (\n",
      "      (0): Conv2d(512, 160, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "    )\n",
      "    (conv_1_3): Sequential (\n",
      "      (0): Conv2d(512, 112, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(112, 224, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (conv_1_5): Sequential (\n",
      "      (0): Conv2d(512, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(24, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (max_3_1): Sequential (\n",
      "      (0): MaxPool2d (size=(3, 3), stride=(1, 1), padding=(1, 1), dilation=(1, 1))\n",
      "      (1): Conv2d(512, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (2): ReLU ()\n",
      "    )\n",
      "  )\n",
      "  (2): inception_module (\n",
      "    (conv_1): Sequential (\n",
      "      (0): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "    )\n",
      "    (conv_1_3): Sequential (\n",
      "      (0): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (conv_1_5): Sequential (\n",
      "      (0): Conv2d(512, 24, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(24, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (max_3_1): Sequential (\n",
      "      (0): MaxPool2d (size=(3, 3), stride=(1, 1), padding=(1, 1), dilation=(1, 1))\n",
      "      (1): Conv2d(512, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (2): ReLU ()\n",
      "    )\n",
      "  )\n",
      "  (3): inception_module (\n",
      "    (conv_1): Sequential (\n",
      "      (0): Conv2d(512, 112, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "    )\n",
      "    (conv_1_3): Sequential (\n",
      "      (0): Conv2d(512, 144, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(144, 288, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (conv_1_5): Sequential (\n",
      "      (0): Conv2d(512, 32, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(32, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (max_3_1): Sequential (\n",
      "      (0): MaxPool2d (size=(3, 3), stride=(1, 1), padding=(1, 1), dilation=(1, 1))\n",
      "      (1): Conv2d(512, 64, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (2): ReLU ()\n",
      "    )\n",
      "  )\n",
      "  (4): inception_module (\n",
      "    (conv_1): Sequential (\n",
      "      (0): Conv2d(528, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "    )\n",
      "    (conv_1_3): Sequential (\n",
      "      (0): Conv2d(528, 160, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(160, 320, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (conv_1_5): Sequential (\n",
      "      (0): Conv2d(528, 32, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(32, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (max_3_1): Sequential (\n",
      "      (0): MaxPool2d (size=(3, 3), stride=(1, 1), padding=(1, 1), dilation=(1, 1))\n",
      "      (1): Conv2d(528, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (2): ReLU ()\n",
      "    )\n",
      "  )\n",
      "  (5): MaxPool2d (size=(3, 3), stride=(2, 2), padding=(1, 1), dilation=(1, 1))\n",
      "))\n",
      "('layer_4', Sequential (\n",
      "  (0): inception_module (\n",
      "    (conv_1): Sequential (\n",
      "      (0): Conv2d(832, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "    )\n",
      "    (conv_1_3): Sequential (\n",
      "      (0): Conv2d(832, 160, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(160, 320, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (conv_1_5): Sequential (\n",
      "      (0): Conv2d(832, 32, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(32, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (max_3_1): Sequential (\n",
      "      (0): MaxPool2d (size=(3, 3), stride=(1, 1), padding=(1, 1), dilation=(1, 1))\n",
      "      (1): Conv2d(832, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (2): ReLU ()\n",
      "    )\n",
      "  )\n",
      "  (1): inception_module (\n",
      "    (conv_1): Sequential (\n",
      "      (0): Conv2d(832, 384, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "    )\n",
      "    (conv_1_3): Sequential (\n",
      "      (0): Conv2d(832, 192, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(192, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (conv_1_5): Sequential (\n",
      "      (0): Conv2d(832, 48, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (1): ReLU ()\n",
      "      (2): Conv2d(48, 128, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "      (3): ReLU ()\n",
      "    )\n",
      "    (max_3_1): Sequential (\n",
      "      (0): MaxPool2d (size=(3, 3), stride=(1, 1), padding=(1, 1), dilation=(1, 1))\n",
      "      (1): Conv2d(832, 128, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (2): ReLU ()\n",
      "    )\n",
      "  )\n",
      "  (2): AvgPool2d (\n",
      "  )\n",
      "))\n",
      "('layer_5', Dropout2d (p=0.4))\n",
      "('fc_layer', Linear (1024 -> 1000))\n"
     ]
    }
   ],
   "source": [
    "class GoogLeNet(nn.Module):\n",
    "\n",
    "    def __init__(self, base_dim, num_classes=2):\n",
    "        super(GoogLeNet, self).__init__()\n",
    "        self.layer_1 = nn.Sequential(\n",
    "            nn.Conv2d(3,base_dim,7,2,3),\n",
    "            nn.MaxPool2d(3,2,1),\n",
    "            nn.Conv2d(base_dim,base_dim*3,3,1,1),\n",
    "            nn.MaxPool2d(3,2,1),\n",
    "        )\n",
    "        self.layer_2 = nn.Sequential(\n",
    "            inception_module(base_dim*3,64,96,128,16,32,32),\n",
    "            inception_module(base_dim*4,128,128,192,32,96,64),\n",
    "            nn.MaxPool2d(3,2,1),\n",
    "        )\n",
    "        self.layer_3 = nn.Sequential(\n",
    "            inception_module(480,192,96,208,16,48,64),\n",
    "            inception_module(512,160,112,224,24,64,64),\n",
    "            inception_module(512,128,128,256,24,64,64),\n",
    "            inception_module(512,112,144,288,32,64,64),\n",
    "            inception_module(528,256,160,320,32,128,128),\n",
    "            nn.MaxPool2d(3,2,1),\n",
    "        )\n",
    "        self.layer_4 = nn.Sequential(\n",
    "            inception_module(832,256,160,320,32,128,128),\n",
    "            inception_module(832,384,192,384,48,128,128), \n",
    "            nn.AvgPool2d(7,1),\n",
    "        )\n",
    "        self.layer_5 = nn.Dropout2d(0.4)\n",
    "        self.fc_layer = nn.Linear(1024,1000)\n",
    "                \n",
    "        \n",
    "    def forward(self, x):\n",
    "        out = self.layer_1(x)\n",
    "        out = self.layer_2(out)\n",
    "        out = self.layer_3(out)\n",
    "        out = self.layer_4(out)\n",
    "        out = self.layer_5(out)\n",
    "        out = out.view(batch_size,-1)\n",
    "        out = self.fc_layer(out)\n",
    "        \n",
    "        return out\n",
    "        \n",
    "    \n",
    "model = GoogLeNet(base_dim=64).cuda()\n",
    "\n",
    "for i in model.named_children():\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Optimizer & Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss_func = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(),lr=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 6.8770\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 0.2369\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n",
      "Variable containing:\n",
      " 0.8947\n",
      "[torch.cuda.FloatTensor of size 1 (GPU 0)]\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process Process-51:\n",
      "Process Process-52:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 35, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 342, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 35, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 343, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-6a77fdb1d7ca>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m10\u001b[0m \u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m     66\u001b[0m                 \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m                 \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m                 \u001b[0mbias_correction1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m \u001b[0;34m**\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'step'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for i in range(epoch):\n",
    "    for img,label in img_batch:\n",
    "        img = Variable(img).cuda()\n",
    "        label = Variable(label).cuda()\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        output = model(img)\n",
    "        \n",
    "        #print(output.size())\n",
    "        loss = loss_func(output,label)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    if i % 10 ==0:\n",
    "        print(loss)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}