notebooks/cnn_customdata_vgg_finetune.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN FINETUNING WITH PRE-TRAINED VGG NET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Packages loaded.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.misc \n",
    "import scipy.io\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "%matplotlib inline  \n",
    "print (\"Packages loaded.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LOAD DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['trainlabel', 'trainimg', 'testimg', 'testlabel']\n"
     ]
    }
   ],
   "source": [
    "cwd = os.getcwd()\n",
    "loadpath = cwd + \"/data/data4vgg.npz\"\n",
    "l = np.load(loadpath)\n",
    "\n",
    "# Show Files\n",
    "print (l.files)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PARSE DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "69 train images loaded\n",
      "18 test images loaded\n",
      "37632 dimensional input\n",
      "2 classes\n",
      "shape of 'trainimg' is (69, 37632)\n",
      "shape of 'testimg' is (18, 37632)\n"
     ]
    }
   ],
   "source": [
    "trainimg   = l['trainimg']\n",
    "trainlabel = l['trainlabel']\n",
    "testimg    = l['testimg']\n",
    "testlabel  = l['testlabel']\n",
    "ntrain     = trainimg.shape[0]\n",
    "nclass     = trainlabel.shape[1]\n",
    "dim        = trainimg.shape[1]\n",
    "ntest      = testimg.shape[0]\n",
    "\n",
    "print (\"%d train images loaded\" % (ntrain))\n",
    "print (\"%d test images loaded\"  % (ntest))\n",
    "print (\"%d dimensional input\"   % (dim))\n",
    "print (\"%d classes\"             % (nclass))\n",
    "print (\"shape of 'trainimg' is %s\" % (trainimg.shape,))\n",
    "print (\"shape of 'testimg' is %s\" % (testimg.shape,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GENERATE TENSORS FOR TRAINING AND TESTING "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of trainimg_tensor is (69, 112, 112, 3)\n",
      "shape of testimg_tensor is (18, 112, 112, 3)\n"
     ]
    }
   ],
   "source": [
    "trainimg_tensor = np.ndarray((ntrain, 112, 112, 3))\n",
    "for i in range(ntrain):\n",
    "    currimg = trainimg[i, :]\n",
    "    currimg = np.reshape(currimg, [112, 112, 3])\n",
    "    trainimg_tensor[i, :, :, :] = currimg \n",
    "print (\"shape of trainimg_tensor is %s\" % (trainimg_tensor.shape,))\n",
    "    \n",
    "testimg_tensor = np.ndarray((ntest, 112, 112, 3))\n",
    "for i in range(ntest):\n",
    "    currimg = testimg[i, :]\n",
    "    currimg = np.reshape(currimg, [112, 112, 3])\n",
    "    testimg_tensor[i, :, :, :] = currimg \n",
    "print (\"shape of testimg_tensor is %s\" % (testimg_tensor.shape,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DEFINE A FUNCTION FOR USING PRETRAINED VGG NETWORK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def net(data_path, input_image):\n",
    "    layers = (\n",
    "        'conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1',\n",
    "        'conv2_1', 'relu2_1', 'conv2_2', 'relu2_2', 'pool2',\n",
    "        'conv3_1', 'relu3_1', 'conv3_2', 'relu3_2', 'conv3_3',\n",
    "        'relu3_3', 'conv3_4', 'relu3_4', 'pool3',\n",
    "        'conv4_1', 'relu4_1', 'conv4_2', 'relu4_2', 'conv4_3',\n",
    "        'relu4_3', 'conv4_4', 'relu4_4', 'pool4',\n",
    "        'conv5_1', 'relu5_1', 'conv5_2', 'relu5_2', 'conv5_3',\n",
    "        'relu5_3', 'conv5_4', 'relu5_4'\n",
    "    )\n",
    "    data = scipy.io.loadmat(data_path)\n",
    "    mean = data['normalization'][0][0][0]\n",
    "    mean_pixel = np.mean(mean, axis=(0, 1))\n",
    "    weights = data['layers'][0]\n",
    "    net = {}\n",
    "    current = input_image\n",
    "    for i, name in enumerate(layers):\n",
    "        kind = name[:4]\n",
    "        if kind == 'conv':\n",
    "            kernels, bias = weights[i][0][0][0][0]\n",
    "            # matconvnet: weights are [width, height, in_channels, out_channels]\n",
    "            # tensorflow: weights are [height, width, in_channels, out_channels]\n",
    "            kernels = np.transpose(kernels, (1, 0, 2, 3))\n",
    "            bias = bias.reshape(-1)\n",
    "            current = _conv_layer(current, kernels, bias)\n",
    "        elif kind == 'relu':\n",
    "            current = tf.nn.relu(current)\n",
    "        elif kind == 'pool':\n",
    "            current = _pool_layer(current)\n",
    "        net[name] = current\n",
    "\n",
    "    assert len(net) == len(layers)\n",
    "    return net, mean_pixel\n",
    "def _conv_layer(input, weights, bias):\n",
    "    conv = tf.nn.conv2d(input, tf.constant(weights), strides=(1, 1, 1, 1),\n",
    "            padding='SAME')\n",
    "    return tf.nn.bias_add(conv, bias)\n",
    "def _pool_layer(input):\n",
    "    return tf.nn.max_pool(input, ksize=(1, 2, 2, 1), strides=(1, 2, 2, 1),\n",
    "            padding='SAME')\n",
    "def preprocess(image, mean_pixel):\n",
    "    return image - mean_pixel\n",
    "def unprocess(image, mean_pixel):\n",
    "    return image + mean_pixel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EXTRACT FEATURES FROM THE VGG NETWORK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TYPE OF 'train_features' IS <type 'numpy.ndarray'>\n",
      "SHAPE OF 'train_features' IS (69, 7, 7, 512)\n",
      "TYPE OF 'test_features' IS <type 'numpy.ndarray'>\n",
      "SHAPE OF 'test_features' IS (18, 7, 7, 512)\n",
      "PREPROCESSING DONE\n"
     ]
    }
   ],
   "source": [
    "VGG_PATH = cwd + \"/data/imagenet-vgg-verydeep-19.mat\"\n",
    "with tf.Graph().as_default(), tf.Session() as sess:\n",
    "    with tf.device(\"/cpu:0\"):\n",
    "        img_placeholder = tf.placeholder(tf.float32, shape=(None, 112, 112, 3))\n",
    "        net_val, mean_pixel = net(VGG_PATH, img_placeholder)\n",
    "        train_features = net_val['relu5_4'].eval(feed_dict={img_placeholder: trainimg_tensor})\n",
    "        test_features = net_val['relu5_4'].eval(feed_dict={img_placeholder: testimg_tensor})\n",
    "print (\"TYPE OF 'train_features' IS %s\" % (type(train_features)))\n",
    "print (\"SHAPE OF 'train_features' IS %s\" % (train_features.shape,))\n",
    "print (\"TYPE OF 'test_features' IS %s\" % (type(test_features)))\n",
    "print (\"SHAPE OF 'test_features' IS %s\" % (test_features.shape,))\n",
    "print(\"PREPROCESSING DONE\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VECTORIZE CNN FEATURES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SHAPE OF 'train_vectorized' IS (69, 25088)\n",
      "SHAPE OF 'test_vectorized' IS (18, 25088)\n"
     ]
    }
   ],
   "source": [
    "train_vectorized = np.ndarray((ntrain, 7*7*512))\n",
    "test_vectorized  = np.ndarray((ntest, 7*7*512))\n",
    "for i in range(ntrain):\n",
    "    curr_feat = train_features[i, :, :, :]\n",
    "    curr_feat_vec = np.reshape(curr_feat, (1, -1))\n",
    "    train_vectorized[i, :] = curr_feat_vec\n",
    "    \n",
    "for i in range(ntest):\n",
    "    curr_feat = test_features[i, :, :, :]\n",
    "    curr_feat_vec = np.reshape(curr_feat, (1, -1))\n",
    "    test_vectorized[i, :] = curr_feat_vec\n",
    "    \n",
    "print (\"SHAPE OF 'train_vectorized' IS %s\" % (train_vectorized.shape,))\n",
    "print (\"SHAPE OF 'test_vectorized' IS %s\" % (test_vectorized.shape,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DEFINE NETWORKS AND FUNCTIONS (ADD 2LAYER MLP)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network Ready to Go!\n"
     ]
    }
   ],
   "source": [
    "# Parameters\n",
    "learning_rate   = 0.0001\n",
    "training_epochs = 100\n",
    "batch_size      = 100\n",
    "display_step    = 10\n",
    "\n",
    "# Network\n",
    "with tf.device(\"/cpu:0\"):\n",
    "    n_input  = dim\n",
    "    n_output = nclass\n",
    "    weights  = {\n",
    "        'wd1': tf.Variable(tf.random_normal([7*7*512, 1024], stddev=0.1)),\n",
    "        'wd2': tf.Variable(tf.random_normal([1024, n_output], stddev=0.1))\n",
    "    }\n",
    "    biases   = {\n",
    "        'bd1': tf.Variable(tf.random_normal([1024], stddev=0.1)),\n",
    "        'bd2': tf.Variable(tf.random_normal([n_output], stddev=0.1))\n",
    "    }\n",
    "    def conv_basic(_input, _w, _b, _keepratio):\n",
    "        # Input\n",
    "        _input_r = _input\n",
    "        # Vectorize\n",
    "        _dense1 = tf.reshape(_input_r, [-1, _w['wd1'].get_shape().as_list()[0]])\n",
    "        # Fc1\n",
    "        _fc1 = tf.nn.relu(tf.add(tf.matmul(_dense1, _w['wd1']), _b['bd1']))\n",
    "        _fc_dr1 = tf.nn.dropout(_fc1, _keepratio)\n",
    "        # Fc2\n",
    "        _out = tf.add(tf.matmul(_fc_dr1, _w['wd2']), _b['bd2'])\n",
    "        # Return everything\n",
    "        out = {'input_r': _input_r, 'dense1': _dense1,\n",
    "            'fc1': _fc1, 'fc_dr1': _fc_dr1, 'out': _out }\n",
    "        return out\n",
    "\n",
    "# tf Graph input\n",
    "x = tf.placeholder(tf.float32, [None, 7*7*512])\n",
    "y = tf.placeholder(tf.float32, [None, n_output])\n",
    "keepratio = tf.placeholder(tf.float32)\n",
    "\n",
    "# Functions! \n",
    "with tf.device(\"/cpu:0\"):\n",
    "    _pred = conv_basic(x, weights, biases, keepratio)['out']\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(_pred, y))\n",
    "    optm = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "    _corr = tf.equal(tf.argmax(_pred,1), tf.argmax(y,1)) # Count corrects\n",
    "    accr = tf.reduce_mean(tf.cast(_corr, tf.float32)) # Accuracy\n",
    "    init = tf.initialize_all_variables()\n",
    "\n",
    "print (\"Network Ready to Go!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN FINETUNING "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 000/100 cost: 3.248429298\n",
      " Training accuracy: 0.760\n",
      " Test accuracy: 0.444\n",
      "Epoch: 010/100 cost: 0.146418720\n",
      " Training accuracy: 0.980\n",
      " Test accuracy: 0.722\n",
      "Epoch: 020/100 cost: 0.000040202\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.778\n",
      "Epoch: 030/100 cost: 0.000000000\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.778\n",
      "Epoch: 040/100 cost: 0.000000000\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.778\n",
      "Epoch: 050/100 cost: 0.000000000\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.778\n",
      "Epoch: 060/100 cost: 0.000000000\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.722\n",
      "Epoch: 070/100 cost: 0.000000000\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.722\n",
      "Epoch: 080/100 cost: 0.000000000\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.778\n",
      "Epoch: 090/100 cost: 0.000000000\n",
      " Training accuracy: 1.000\n",
      " Test accuracy: 0.889\n",
      "Optimization Finished!\n"
     ]
    }
   ],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(init)\n",
    "\n",
    "# Training cycle\n",
    "for epoch in range(training_epochs):\n",
    "    avg_cost = 0.\n",
    "    num_batch = int(ntrain/batch_size)+1\n",
    "    # Loop over all batches\n",
    "    for i in range(num_batch): \n",
    "        randidx  = np.random.randint(ntrain, size=batch_size)\n",
    "        batch_xs = train_vectorized[randidx, :]\n",
    "        batch_ys = trainlabel[randidx, :]                \n",
    "        # Fit training using batch data\n",
    "        sess.run(optm, feed_dict={x: batch_xs, y: batch_ys, keepratio:0.7})\n",
    "        # Compute average loss\n",
    "        avg_cost += sess.run(cost, feed_dict={x: batch_xs, y: batch_ys, keepratio:1.})/num_batch\n",
    "\n",
    "    # Display logs per epoch step\n",
    "    if epoch % display_step == 0:\n",
    "        print (\"Epoch: %03d/%03d cost: %.9f\" % (epoch, training_epochs, avg_cost))\n",
    "        train_acc = sess.run(accr, feed_dict={x: batch_xs, y: batch_ys, keepratio:1.})\n",
    "        print (\" Training accuracy: %.3f\" % (train_acc))\n",
    "        test_acc = sess.run(accr, feed_dict={x: test_vectorized, y: testlabel, keepratio:1.})\n",
    "        print (\" Test accuracy: %.3f\" % (test_acc))\n",
    "\n",
    "print (\"Optimization Finished!\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}