06_Prob&Solutions/3_Weight_Initialization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Weight Initialization\n",
    "\n",
    "- https://discuss.pytorch.org/t/weight-initilzation/157/9\n",
    "- Initialize with small numbers\n",
    "- Xavier Initialization\n",
    "- Kaiming He Initialization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Settings\n",
    "### 1) Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.init as init\n",
    "import torchvision.datasets as dset\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2) Set hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 16\n",
    "learning_rate = 0.0002\n",
    "num_epoch = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Data\n",
    "\n",
    "### 1) Download Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mnist_train = dset.MNIST(\"./\", train=True, transform=transforms.ToTensor(), target_transform=None, download=True)\n",
    "mnist_test = dset.MNIST(\"./\", train=False, transform=transforms.ToTensor(), target_transform=None, download=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2) Check Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 28, 28]) 60000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(torch.Size([1, 28, 28]), 10000)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(mnist_train.__getitem__(0)[0].size(), mnist_train.__len__())\n",
    "mnist_test.__getitem__(0)[0].size(), mnist_test.__len__()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3) Set DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(mnist_train,batch_size=batch_size, shuffle=True,num_workers=2,drop_last=True)\n",
    "test_loader = torch.utils.data.DataLoader(mnist_test,batch_size=batch_size, shuffle=False,num_workers=2,drop_last=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Model & Optimizer\n",
    "\n",
    "### 1) CNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class CNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(CNN,self).__init__()\n",
    "        self.layer = nn.Sequential(\n",
    "            nn.Conv2d(1,16,3,padding=1),  # 28\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(16,32,3,padding=1), # 28\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(2,2), # 14\n",
    "            nn.Conv2d(32,64,3,padding=1), #14\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(2,2) # 7\n",
    "        )\n",
    "        self.fc_layer = nn.Sequential(\n",
    "            nn.Linear(64*7*7,100),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(100,10)\n",
    "        )       \n",
    "        \n",
    "        # initialization\n",
    "        \n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                \n",
    "                '''# Init with small numbers\n",
    "                m.weight.data.normal_(0.0, 0.02)\n",
    "                m.bias.data.fill_(0)\n",
    "                \n",
    "                # Xavier Initialization\n",
    "                init.xavier_normal(m.weight.data)\n",
    "                m.bias.data.fill_(0)'''\n",
    "                \n",
    "                # Kaming Initialization\n",
    "                init.kaiming_normal(m.weight.data)\n",
    "                m.bias.data.fill_(0)\n",
    "                \n",
    "            elif isinstance(m, nn.Linear):\n",
    "                \n",
    "                '''# Init with small numbers\n",
    "                m.weight.data.normal_(0.0, 0.02)\n",
    "                m.bias.data.fill_(0)\n",
    "                \n",
    "                # Xavier Initialization\n",
    "                init.xavier_normal(m.weight.data)\n",
    "                m.bias.data.fill_(0)'''\n",
    "                \n",
    "                # Kaming Initialization\n",
    "                init.kaiming_normal(m.weight.data)\n",
    "                m.bias.data.fill_(0)\n",
    "\n",
    "    def forward(self,x):\n",
    "        out = self.layer(x)\n",
    "        out = out.view(batch_size,-1)\n",
    "        out = self.fc_layer(out)\n",
    "\n",
    "        return out\n",
    "\n",
    "model = CNN()#.cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2) Loss func & Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss_func = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Train "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable containing:\n",
      " 2.6892\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 1.3642\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 1.3983\n",
      "[torch.FloatTensor of size 1]\n",
      "\n",
      "Variable containing:\n",
      " 0.7934\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(num_epoch):\n",
    "    for j,[image,label] in enumerate(train_loader):\n",
    "        x = Variable(image)#.cuda()\n",
    "        y_= Variable(label)#.cuda()\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        output = model.forward(x)\n",
    "        loss = loss_func(output,y_)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if j % 1000 == 0:\n",
    "            print(loss)          "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#param_list = list(model.parameters())\n",
    "#print(param_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Test Data: Variable containing:\n",
      " 89.2700\n",
      "[torch.FloatTensor of size 1]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "correct = 0\n",
    "total = 0\n",
    "\n",
    "for image,label in test_loader:\n",
    "    x = Variable(image,volatile=True)#.cuda()\n",
    "    y_= Variable(label)#.cuda()\n",
    "\n",
    "    output = model.forward(x)\n",
    "    _,output_index = torch.max(output,1)\n",
    "        \n",
    "    total += label.size(0)\n",
    "    correct += (output_index == y_).sum().float()\n",
    "    \n",
    "print(\"Accuracy of Test Data: {}\".format(100*correct/total))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}