nw_compression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from set_gpu import *\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting data/MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('data/MNIST_data/', one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set model math to save/resotre\n",
    "model_path = \"models/mnist-shallow/model2.ckpt\"\n",
    "\n",
    "hidden_units = 1024\n",
    "x = tf.placeholder(tf.float32, [None, 784], name='x')\n",
    "y_ = tf.placeholder(tf.float32, [None, 10], name='y_')\n",
    "weights = {\n",
    "    'h1': tf.Variable(tf.random_normal([784, hidden_units], stddev=tf.sqrt(2/(784+hidden_units)), mean=0)),\n",
    "    'h2': tf.Variable(tf.random_normal([hidden_units, hidden_units], stddev=tf.sqrt(1/hidden_units), mean=0)),\n",
    "    'h3': tf.Variable(tf.random_normal([hidden_units, hidden_units], stddev=tf.sqrt(1/hidden_units), mean=0)),\n",
    "    'h4': tf.Variable(tf.random_normal([hidden_units, hidden_units], stddev=tf.sqrt(1/hidden_units), mean=0)),\n",
    "    'h5': tf.Variable(tf.random_normal([hidden_units, hidden_units], stddev=tf.sqrt(1/hidden_units), mean=0)),\n",
    "    'y': tf.Variable(tf.random_normal([hidden_units, 10], stddev=tf.sqrt(2/(hidden_units+10)), mean=0))\n",
    "} # xavier init\n",
    "biases = {\n",
    "    'b1': tf.Variable(tf.zeros([hidden_units])),\n",
    "    'b2': tf.Variable(tf.zeros([hidden_units])),\n",
    "    'b3': tf.Variable(tf.zeros([hidden_units])),\n",
    "    'b4': tf.Variable(tf.zeros([hidden_units])),\n",
    "    'b5': tf.Variable(tf.zeros([hidden_units])),\n",
    "    'y': tf.Variable(tf.zeros([10]))\n",
    "}\n",
    "\n",
    "# for batch norm\n",
    "epsilon = 1e-3 # a small float to avoid div by 0\n",
    "scale = tf.ones([hidden_units])\n",
    "beta = tf.zeros([hidden_units]) # offset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# linear combination -> batch_norm -> activation -> dropout\n",
    "\n",
    "hidden_1 = tf.nn.relu(tf.matmul(x, weights['h1']) + biases['b1'])\n",
    "#batch_mean, batch_var = tf.nn.moments(hidden_1, [0])\n",
    "#batch_norm_1 = tf.nn.batch_normalization(hidden_1, batch_mean, batch_var, None, None, epsilon)\n",
    "#dropout_1 = tf.nn.dropout(tf.nn.relu(batch_norm_1), .4)    \n",
    "\n",
    "hidden_2 = tf.nn.relu(tf.matmul(hidden_1, weights['h2']) + biases['b2'])\n",
    "#batch_mean, batch_var = tf.nn.moments(hidden_2, [0])\n",
    "#batch_norm_2 = tf.nn.batch_normalization(hidden_2, batch_mean, batch_var, None, None, epsilon)\n",
    "#dropout_2 = tf.nn.dropout(tf.nn.relu(batch_norm_2), .4) \n",
    "\n",
    "hidden_3 = tf.nn.relu(tf.matmul(hidden_2, weights['h3']) + biases['b3'])\n",
    "#batch_mean, batch_var = tf.nn.moments(hidden_3, [0])\n",
    "#batch_norm_3 = tf.nn.batch_normalization(hidden_3, batch_mean, batch_var, None, None, epsilon)\n",
    "#dropout_3 = tf.nn.dropout(tf.nn.relu(batch_norm_3), .4) \n",
    "\n",
    "hidden_4 = tf.nn.relu(tf.matmul(hidden_3, weights['h4']) + biases['b4'])\n",
    "#batch_mean, batch_var = tf.nn.moments(hidden_4, [0])\n",
    "#batch_norm_4 = tf.nn.batch_normalization(hidden_4, batch_mean, batch_var, None, None, epsilon)\n",
    "#dropout_4 = tf.nn.dropout(tf.nn.relu(batch_norm_4), .4) \n",
    "\n",
    "hidden_5 = tf.nn.relu(tf.matmul(hidden_4, weights['h5']) + biases['b5'])\n",
    "#batch_mean, batch_var = tf.nn.moments(hidden_5, [0])\n",
    "#batch_norm_5 = tf.nn.batch_normalization(hidden_5, batch_mean, batch_var, None, None, epsilon)\n",
    "#dropout_5 = tf.nn.dropout(tf.nn.relu(batch_norm_5), .4) \n",
    "\n",
    "y = tf.matmul(hidden_5, weights['y']) + biases['y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=y, labels=y_))\n",
    "train_step = tf.train.AdamOptimizer(.0001).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# saver obj to save/retrieve model\n",
    "saver = tf.train.Saver() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session(config=config)\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# train func\n",
    "\n",
    "def train(mnist, sess, train_step, cross_entropy):\n",
    "    i = 550\n",
    "    epochs = 50\n",
    "    display_step = 1\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        avg_cost = 0\n",
    "        for _ in range(i):\n",
    "            batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "            _, cost = sess.run([train_step, cross_entropy], feed_dict={'x:0': batch_xs, 'y_:0': batch_ys})\n",
    "            avg_cost += cost/i\n",
    "        if epoch%display_step==0:\n",
    "            print(\"Epoch:\", '%02d'%(epoch+1), \"\\tcost={:.9f}\".format(avg_cost))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 \tcost=0.308158701\n",
      "Epoch: 02 \tcost=0.104051820\n",
      "Epoch: 03 \tcost=0.064420025\n",
      "Epoch: 04 \tcost=0.041513531\n",
      "Epoch: 05 \tcost=0.028730659\n",
      "Epoch: 06 \tcost=0.019593434\n",
      "Epoch: 07 \tcost=0.016509421\n",
      "Epoch: 08 \tcost=0.011214017\n",
      "Epoch: 09 \tcost=0.012797758\n",
      "Epoch: 10 \tcost=0.009444008\n",
      "Epoch: 11 \tcost=0.009825363\n",
      "Epoch: 12 \tcost=0.007510452\n",
      "Epoch: 13 \tcost=0.009674547\n",
      "Epoch: 14 \tcost=0.007070621\n",
      "Epoch: 15 \tcost=0.008752603\n",
      "Epoch: 16 \tcost=0.005820285\n",
      "Epoch: 17 \tcost=0.006214108\n",
      "Epoch: 18 \tcost=0.003818299\n",
      "Epoch: 19 \tcost=0.008404477\n",
      "Epoch: 20 \tcost=0.005452839\n",
      "Epoch: 21 \tcost=0.004848849\n",
      "Epoch: 22 \tcost=0.008018450\n",
      "Epoch: 23 \tcost=0.002518609\n",
      "Epoch: 24 \tcost=0.002082388\n",
      "Epoch: 25 \tcost=0.008568506\n",
      "Epoch: 26 \tcost=0.003341781\n",
      "Epoch: 27 \tcost=0.006861400\n",
      "Epoch: 28 \tcost=0.003680519\n",
      "Epoch: 29 \tcost=0.003584975\n",
      "Epoch: 30 \tcost=0.003585211\n",
      "Epoch: 31 \tcost=0.003785598\n",
      "Epoch: 32 \tcost=0.005266891\n",
      "Epoch: 33 \tcost=0.002438171\n",
      "Epoch: 34 \tcost=0.004274180\n",
      "Epoch: 35 \tcost=0.003406585\n",
      "Epoch: 36 \tcost=0.004417368\n",
      "Epoch: 37 \tcost=0.001154811\n",
      "Epoch: 38 \tcost=0.000765570\n",
      "Epoch: 39 \tcost=0.000143267\n",
      "Epoch: 40 \tcost=0.000012292\n",
      "Epoch: 41 \tcost=0.000006502\n",
      "Epoch: 42 \tcost=0.000004339\n",
      "Epoch: 43 \tcost=0.000003020\n",
      "Epoch: 44 \tcost=0.000002129\n",
      "Epoch: 45 \tcost=0.000001509\n",
      "Epoch: 46 \tcost=0.000001072\n",
      "Epoch: 47 \tcost=0.000000760\n",
      "Epoch: 48 \tcost=0.000000537\n",
      "Epoch: 49 \tcost=0.000000379\n",
      "Epoch: 50 \tcost=0.000000268\n"
     ]
    }
   ],
   "source": [
    "train(mnist, sess, train_step, cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# test func\n",
    "\n",
    "def test(y, y_, mnist):\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = 100*tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    return sess.run(accuracy, feed_dict={'x:0': mnist.test.images, 'y_:0': mnist.test.labels})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 98.55%\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {:.2f}%'.format(test(y, y_, mnist)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save model\n",
    "save_path = saver.save(sess, model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from models/mnist-shallow/model2.ckpt\n"
     ]
    }
   ],
   "source": [
    "# restore model\n",
    "saver.restore(sess, model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# dict of lists to store s, u, v values of svd\n",
    "svd = {'s':[], 'u':[], 'v':[]}\n",
    "\n",
    "for layer in weights:\n",
    "    if layer!='y':\n",
    "        s, u, v = tf.svd(weights[layer])\n",
    "        svd['s'].append(s)\n",
    "        svd['u'].append(u)\n",
    "        svd['v'].append(v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Having a look at the shape of the S matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(784,)\n",
      "(1024,)\n",
      "(1024,)\n",
      "(1024,)\n",
      "(1024,)\n"
     ]
    }
   ],
   "source": [
    "for val in svd['s']:\n",
    "    print(val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 done\n",
      "20 done\n",
      "50 done\n",
      "100 done\n",
      "200 done\n",
      "full done\n"
     ]
    }
   ],
   "source": [
    "# feedforward with different values of D \n",
    "\n",
    "d = [10, 20, 50, 100, 200, 1024]\n",
    "accuracies = []\n",
    "\n",
    "for dval in d:\n",
    "    if dval==1024: # for d=full\n",
    "        hidden1_svd = tf.nn.relu(tf.add(tf.matmul(x, tf.matmul(tf.matmul(svd['u'][0][:,:784], tf.diag(svd['s'][0])[:784,:784]), \n",
    "                                              tf.transpose(svd['v'][0][:,:784]))), biases['b1']))  \n",
    "    else:\n",
    "        hidden1_svd = tf.nn.relu(tf.add(tf.matmul(x, tf.matmul(tf.matmul(svd['u'][0][:,:dval], tf.diag(svd['s'][0])[:dval,:dval]), \n",
    "                                              tf.transpose(svd['v'][0][:,:dval]))), biases['b1']))  \n",
    "\n",
    "    hidden2_svd = tf.nn.relu(tf.add(tf.matmul(hidden1_svd, tf.matmul(tf.matmul(svd['u'][1][:,:dval], tf.diag(svd['s'][1])[:dval,:dval]), \n",
    "                                              tf.transpose(svd['v'][1][:,:dval]))), biases['b2']))\n",
    "\n",
    "    hidden3_svd = tf.nn.relu(tf.add(tf.matmul(hidden2_svd, tf.matmul(tf.matmul(svd['u'][2][:,:dval], tf.diag(svd['s'][2])[:dval,:dval]), \n",
    "                                              tf.transpose(svd['v'][2][:,:dval]))), biases['b3']))\n",
    "\n",
    "    hidden4_svd = tf.nn.relu(tf.add(tf.matmul(hidden3_svd, tf.matmul(tf.matmul(svd['u'][3][:,:dval], tf.diag(svd['s'][3])[:dval,:dval]), \n",
    "                                              tf.transpose(svd['v'][3][:,:dval]))), biases['b4']))\n",
    "\n",
    "    hidden5_svd = tf.nn.relu(tf.add(tf.matmul(hidden4_svd, tf.matmul(tf.matmul(svd['u'][4][:,:dval], tf.diag(svd['s'][4])[:dval,:dval]), \n",
    "                                              tf.transpose(svd['v'][4][:,:dval]))), biases['b5']))\n",
    "\n",
    "    y_svd = tf.nn.softmax(tf.add(tf.matmul(hidden5_svd, weights['y']), biases['y']))\n",
    "    \n",
    "    \n",
    "    accuracies.append(test(y_svd, y_, mnist))\n",
    "    \n",
    "    if dval==1024:\n",
    "        print('full done')\n",
    "    else:\n",
    "        print(dval,\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA34AAAJQCAYAAADR+LbmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3X+U3nV95/3nywTL5B41LC3TGvFobzGlRfmRa1l+HOgMUKmtq4DbiqUtbZXZ7k0hYMtWz902d9ube1G27Wq979pUtLa6GYUEqO22wIEdbLuFdkJQAohoEZakKhaCjow2wPv+Y65sxzQ/Jsl8r+/km+fjnDkz1+f6Xtf35Tlvw3md769UFZIkSZKk7npB2wEkSZIkSc2y+EmSJElSx1n8JEmSJKnjLH6SJEmS1HEWP0mSJEnqOIufJEmSJHVcY8UvyYeSfCXJ5jlr/yrJbUke7v8+or+eJO9L8vkkn0lyUlO5JEmSJOlQ0+QRvz8EfnintXcCt1fVMcDt/dcArweO6f+MA7/XYC5JkiRJOqQ0Vvyq6lPAkzstvwn4SP/vjwDnzVn/o5p1F7A8yfc0lU2SJEmSDiVLB7y/kar6B4Cq+ockR/XXVwD/c852j/fX/mHnL0gyzuxRQYaGhlYdffTRzSbeT88//zwveIGXUKoZzpea5HypSc6XmuR8qSmLebY+97nPfbWqvmtv2w26+O1OdrFWu9qwqtYCawF6vV5NTU01mWu/TU5OMjo62nYMdZTzpSY5X2qS86UmOV9qymKerSSPzme7QdfWL+84hbP/+yv99ceBuYfuXgZsHXA2SZIkSeqkQRe/PwEu7v99MXDznPWf7t/d8xTg6R2nhEqSJEmSDkxjp3omWQeMAt+Z5HFgDXAN8IkkbwMeA36sv/l/A34E+DzwDPCzTeWSJEmSpENNY8Wvqt66m7fO3sW2BVzaVBZJkiRJOpQtzlvTSJIkSZIWjMVPkiRJkjrO4idJkiRJHWfxkyRJkqSOs/hJkiRJUsdZ/CRJkiSp4yx+kiRJktRxFj9JkiRJ6jiLnyRJkiR1XCvFL8nqJJuT3J/kiv7a8Un+Jsl9ST6Z5MVtZJMkSZKkrhl48UtyHHAJcDJwPPCGJMcAHwTeWVWvAW4Erhp0NkmSJEnqojaO+B0L3FVVz1TVs8CdwPnASuBT/W1uA97cQjZJkiRJ6pxU1WB3mBwL3AycCswAtwNTwCrg3VV1c5J3AL9eVS/axefHgXGAkZGRVRMTEwPLvi+mp6cZHh5uO4Y6yvlSk5wvNcn5UpOcLzVlMc/W2NjYxqrq7W27gRc/gCRvAy4FpoEHmC2Avw+8DzgS+BPg8qo6ck/f0+v1ampqquG0+2dycpLR0dG2Y6ijnC81yflSk5wvNcn5UlMW82wlmVfxa+XmLlV1XVWdVFVnAk8CD1fVZ6vqdVW1ClgHfKGNbJIkSZLUNW3d1fOo/u+XAxcA6+asvQD4FeADbWSTJEmSpK5p6zl+65M8AHwSuLSqngLemuRzwGeBrcCHW8omSZIkSZ3S1qmeZ1TV91fV8VV1e3/tvVX16v7PO6uNiw8lSZIkqe+mTVs4/Zo7uG/L05x+zR3ctGlL25H229K2A0iSJEnSYnPTpi28a8N9zGx/Do6GLdtmeNeG+wA478QVLafbd22d6ilJkiRJi9a1tzw0W/rmmNn+HNfe8lBLiQ6MxU+SJEmSdrJ128w+rS92Fj9JkiRJ2slLlw/t0/piZ/GTJEmSpJ1cde5Khg5b8m1rQ4ct4apzV7aU6MB4cxdJkiRJ2smOG7jMXtP3dVYsH+Kqc1celDd2AYufJEmSJO3SeSeu4LwTVzA5OcllF422HeeAeKqnJEmSDmpdetaa1BSP+EmSJOmg1bVnrUlNaeWIX5LVSTYnuT/JFf21E5LcleTeJFNJTm4jmyRJkg4eXXvWmtSUgRe/JMcBlwAnA8cDb0hyDPAe4Ner6gTg1/qvJUmSpN3q2rPWpKa0ccTvWOCuqnqmqp4F7gTOBwp4cX+blwBbW8gmSZKkg0jXnrUmNSVVNdgdJscCNwOnAjPA7cAU8P8BtwBhtpCeVlWP7uLz48A4wMjIyKqJiYkBJd8309PTDA8Ptx1DHeV8qUnOl5rkfGmhbZvZzpanZni+ipEh+PIMvCBhxRFDLB86rO146ojF/G/X2NjYxqrq7W27gRc/gCRvAy4FpoEHmC2AS4A7q2p9kh8HxqvqnD19T6/Xq6mpqcbz7o/JyUlGR0fbjqGOcr7UJOdLTXK+1ISbNm3h2lse4sKjv87E/3zRQf2sNS1Oi/nfriTzKn6t3Nylqq6rqpOq6kzgSeBh4GJgQ3+T65m9BlCSJEnao/NOXMFfv/MsXrPiJfz1O8+y9Em70NZdPY/q/345cAGwjtlr+n6wv8lZzJZBSZIkSdIBaus5fuuTHAlsBy6tqqeSXAK8N8lS4Jv0r+OTJEmSJB2YVopfVZ2xi7W/Ala1EEeSJEmSOq2VUz0lSZIkSYNj8ZMkSZKkjrP4SZIkSVLHWfwkSZIkqeMsfpIkqXE3bdrC6dfcwX1bnub0a+7gpk1b2o4kSYeUth7nIEmSDhE3bdrCuzbcx8z25+Bo2LJthndtuA/AB21L0oB4xE+SJDXq2lsemi19c8xsf45rb3mopUSSdOix+EmSpEZt3TazT+uSpIVn8ZMkSY166fKhfVqXJC08i58kSWrUVeeuZOiwJd+2NnTYEq46d2VLiSTp0NPKzV2SrAYuAQL8QVX9lyQfB3b8F2A5sK2qTmgjnyRJWjg7buAye03f11mxfIirzl3pjV0kaYAGXvySHMds6TsZ+CfgL5L8WVW9Zc42vwU8PehskiSpGeeduILzTlzB5OQkl1002nYcSTrktHGq57HAXVX1TFU9C9wJnL/jzSQBfhxY10I2SZIkSeqcVNVgd5gcC9wMnArMALcDU1V1Wf/9M4Hfrqrebj4/DowDjIyMrJqYmBhI7n01PT3N8PBw2zHUUc6XmuR8qUnOl5rkfKkpi3m2xsbGNu6uO8018FM9q+rBJO8GbgOmgU8Dz87Z5K3s4WhfVa0F1gL0er0aHR1tLuwBmJycZLFm08HP+VKTnC81yflSk5wvNaULs9XKXT2r6rqqOqmqzgSeBB4GSLIUuAD4eBu5JEmSJKmL2rqr51FV9ZUkL2e26J3af+sc4LNV9XgbuSRJkiSpi1opfsD6JEcC24FLq+qp/vqFeFMXSZIkSVpQrRS/qjpjN+s/M+AokiRJktR5rVzjJ0mSJEkaHIufJEmSJHWcxU+SJEmSOs7iJ0kC4KZNWzj9mju4b8vTnH7NHdy0aUvbkSRJ0gJp666ekqRF5KZNW3jXhvuY2f4cHA1bts3wrg33AXDeiStaTidJkg6UR/wkSVx7y0OzpW+Ome3Pce0tD7WUSJIkLSSLnySJrdtm9mldkiQdXCx+kiReunxon9YlSdLBxeInSeKqc1cydNiSb1sbOmwJV527sqVEkiRpIbVS/JKsTrI5yf1JrpizflmSh/rr72kjmyQdis47cQX/6YLXsKJ/hG/F8iH+0wWv8cYukiR1xMDv6pnkOOAS4GTgn4C/SPJnwMuANwGvrapvJTlq0Nkk6VB23okrOO/EFUxOTnLZRaNtx5EkSQuojcc5HAvcVVXPACS5Ezgf6AHXVNW3AKrqKy1kkyRJkqTOSVUNdofJscDNwKnADHA7MAWc0V//YeCbwC9V1d/t4vPjwDjAyMjIqomJiQEl3zfT09MMDw+3HUMd5XypSc6XmuR8qUnOl5qymGdrbGxsY1X19rbdwI/4VdWDSd4N3AZMA58Gnu1nOQI4BfjXwCeSfG/t1Eyrai2wFqDX69Xo6OgA08/f5OQkizWbDn7Ol5rkfKlJzpea5HypKV2YrVZu7lJV11XVSVV1JvAk8DDwOLChZv0t8DzwnW3kkyRJkqQuaeMaP5IcVVVfSfJy4AJmT/t8HjgLmEzyauCFwFfbyCdJkiRJXdJK8QPWJzkS2A5cWlVPJfkQ8KEkm5m92+fFO5/mKUmSJEnad60Uv6o6Yxdr/wT8ZAtxJEmSJKnTWrnGT5IkSZI0OBY/SZIkSeo4i58kSZIkdZzFT5IkSZI6zuInSZIkSR1n8ZMkSZKkjrP4SZIkSVLHWfwkSZIkqeMsfpIkSZLUca0UvySrk2xOcn+SK/pr/1eSLUnu7f/8SBvZJEmSJKlrlg56h0mOAy4BTgb+CfiLJH/Wf/t3quo/DzqTJEmSJHXZwIsfcCxwV1U9A5DkTuD8FnJIkiRJ0iEhVTXYHSbHAjcDpwIzwO3AFPCPwM8AX+u//sWqemoXnx8HxgFGRkZWTUxMDCb4PpqenmZ4eLjtGOoo50tNcr7UJOdLTXK+1JTFPFtjY2Mbq6q3t+0GXvwAkrwNuBSYBh5gtgBeA3wVKOA3ge+pqp/b0/f0er2amppqOO3+mZycZHR0tO0Y6ijnS01yvtQk50tNcr7UlMU8W0nmVfxaublLVV1XVSdV1ZnAk8DDVfXlqnquqp4H/oDZawAlSZIkSQeorbt6HtX//XLgAmBdku+Zs8n5wOY2skmSJElS17RxcxeA9UmOBLYDl1bVU0n+OMkJzJ7q+UXg37eUTZIkSZI6pZXiV1Vn7GLtp9rIIkmSJEld18qpnpIkSZKkwbH4SZIkSVLHWfwkSZIkqeMsfpIkSZLUcRY/SZIkSeo4i58kSZIkdZzFT5IkSZI6zuInSZIkSR1n8ZMkSZKkjmul+CVZnWRzkvuTXLHTe7+UpJJ8ZxvZJEmSJKlrBl78khwHXAKcDBwPvCHJMf33jgZ+CHhs0LkkSZIkqavaOOJ3LHBXVT1TVc8CdwLn99/7HeA/AtVCLkmSJEnqpFQNtmMlORa4GTgVmAFuB6aA24Czq2p1ki8Cvar66i4+Pw6MA4yMjKyamJgYVPR9Mj09zfDwcNsx1FHOl5rkfKlJzpea5HypKYt5tsbGxjZWVW9v2w28+AEkeRtwKTANPMBsATwNeF1VPb2n4jdXr9erqamppuPul8nJSUZHR9uOoY5yvtQk50tNcr7UJOdLTVnMs5VkXsWvlZu7VNV1VXVSVZ0JPAl8EXgl8Ol+6XsZcE+S724jnyRJkiR1SVt39Tyq//vlwAXAH1XVUVX1iqp6BfA4cFJVfamNfJIkSZLUJUtb2u/6JEcC24FLq+qplnJIkiRJUue1Uvyq6oy9vP+KAUWRJEmSpM5r5VRPSZIkSdLgWPwkSZIkqeMsfpIkSZLUcRY/SZIkSeo4i58kSZIkdZzFT5IkSZI6zuInSZIkSR1n8ZMkSZKkjrP4SZIkSVLHtVL8kqxOsjnJ/Umu6K/9ZpLPJLk3ya1JXtpGNkmSJEnqmoEXvyTHAZcAJwPHA29IcgxwbVW9tqpOAP4U+LVBZ5MkSZKkLmrjiN+xwF1V9UxVPQvcCZxfVV+bs83/BlQL2SRJkiSpc1I12H6V5FjgZuBUYAa4HZiqqsuSXA38NPA0MFZVT+zi8+PAOMDIyMiqiYmJgWXfF9PT0wwPD7cdQx3lfKlJzpea5HypSc6XmrKYZ2tsbGxjVfX2tt3Aix9AkrcBlwLTwAPATFVdOef9dwGHV9WaPX1Pr9erqampRrPur8nJSUZHR9uOoY5yvtQk50tNcr7UJOdLTVnMs5VkXsWvlZu7VNV1VXVSVZ0JPAk8vNMm/xV48+CTSZIkSVL3tHVXz6P6v18OXACs69/gZYc3Ap9tI5skSZIkdc3Slva7PsmRwHbg0qp6KskHk6wEngceBX6+pWySJEmS1CmtFL+qOmMXa57aKUmSJEkNaOVUT0mSJEnS4Fj8JEmSJKnjLH6SJEmS1HEWP0mSJEnqOIufJEmSJHWcxU+SJEmSOs7iJ0mSJEkdZ/GTJEmSpI6z+EmSJElSx7VS/JKsTrI5yf1JruivXZvks0k+k+TGJMvbyCZJkiRJXTPw4pfkOOAS4GTgeOANSY4BbgOOq6rXAp8D3jXobJIkSZLURW0c8TsWuKuqnqmqZ4E7gfOr6tb+a4C7gJe1kE1a1G7atIXTr7mD+7Y8zenX3MFNm7a0HUmSJEkHgVTVYHeYHAvcDJwKzAC3A1NVddmcbT4JfLyqPrqLz48D4wAjIyOrJiYmBpJ7X01PTzM8PNx2DHXItpntbHlqhuerGBmCL8/ACxJWHDHE8qHD2o6nDvHfLzXJ+VKTnC81ZTHP1tjY2Maq6u1tu6WDCDNXVT2Y5N3Mnto5DXwa2HGkjyT/Z//1x3bz+bXAWoBer1ejo6NNR94vk5OTLNZsOjidfs0dbNm2BIBffM2z/NZ9s//3XbF8CX/9ztEWk6lr/PdLTXK+1CTnS03pwmy1cnOXqrquqk6qqjOBJ4GHAZJcDLwBuKgGfShSWuS2bpvZp3VJkiRph7bu6nlU//fLgQuAdUl+GPhl4I1V9UwbuaTF7KXLh/ZpXZIkSdqhref4rU/yAPBJ4NKqegp4P/Ai4LYk9yb5QEvZpEXpqnNXMnTYkm9bGzpsCVedu7KlRJIkSTpYDPwaP4CqOmMXa69qI4t0sDjvxBUAXHvLQ8DXWbF8iKvOXfm/1iVJkqTdaaX4Sdo/5524gvNOXMHk5CSXXTTadhxJkiQdJNo61VOSJEmSNCAWP0mSJEnqOIufJEmSJHWcxU+SJEmSOs7iJ0mSJEkdZ/GTJEmSpI6z+EmSJElSx1n8JEmSJKnjWil+SVYn2Zzk/iRX9Nd+rP/6+SS9NnJJkiRJUhcNvPglOQ64BDgZOB54Q5JjgM3ABcCnBp1JkiRJkrqsjSN+xwJ3VdUzVfUscCdwflU9WFUPtZBHkiRJkjotVTXYHSbHAjcDpwIzwO3AVFVd1n9/EvilqprazefHgXGAkZGRVRMTE4OIvc+mp6cZHh5uO4Y6yvlSk5wvNcn5UpOcLzVlMc/W2NjYxqra66VySwcRZq6qejDJu4HbgGng08Cz+/D5tcBagF6vV6Ojo03EPGCTk5Ms1mw6+DlfapLzpSY5X2qS86WmdGG2Wrm5S1VdV1UnVdWZwJPAw23kkCRJkqRDwcCP+AEkOaqqvpLk5cze0OXUNnJIkiRJ0qGgleIHrE9yJLAduLSqnkpyPvC7wHcBf5bk3qo6t6V8kiRJktQZrRS/qjpjF2s3Aje2EEeSJEmSOq2Va/wkSZIkSYNj8ZMkSZKkjrP4SZIkSVLHWfwkSZIkqeMsfpIkSZLUcRY/SZIkSeo4i58kSZIkdZzFT5IkSZI6zuInSZIkSR3XSvFLsjrJ5iT3J7miv/avktyW5OH+7yPayCZJkiRJXTPw4pfkOOAS4GTgeOANSY4B3gncXlXHALf3X0uSJEmSDlAbR/yOBe6qqmeq6lngTuB84E3AR/rbfAQ4r4VskiRJktQ5qarB7jA5FrgZOBWYYfbo3hTwU1W1fM52T1XVvzjdM8k4MA4wMjKyamJiYiC599X09DTDw8Ntx1BHOV9qkvOlJjlfapLzpaYs5tkaGxvbWFW9vW038OIHkORtwKXANPAAswXwZ+dT/Obq9Xo1NTXVaNb9NTk5yejoaNsx1FHOl5rkfKlJzpea5HypKYt5tpLMq/i1cnOXqrquqk6qqjOBJ4GHgS8n+R6A/u+vtJFNkiRJkrqmrbt6HtX//XLgAmAd8CfAxf1NLmb2dFBJkiRJ0gFa2tJ+1yc5EtgOXFpVTyW5BvhE/zTQx4AfaymbJEmSJHVKK8Wvqs7Yxdo/Ame3EEeSJEmSOq2VUz0lSZIkSYNj8ZMkSZKkjrP4SZIkSVLHWfwkSZIkqeMsfpIkSZLUcRY/SZIkSeo4i58kSZIkdZzFT5IkSZI6zuInSZIkSR1n8ZMkSZKkjmul+CW5Msn9STYnWZfk8CRnJbmnv/aRJEvbyCZJkiRJXTPw4pdkBXA50Kuq44AlwE8AHwEu7K89Clw86GySJEmS1EVtneq5FBjqH9VbBnwD+FZVfa7//m3Am1vKJkmSJEmdkqoa/E6T1cDVwAxwK/CTwBeBN1fVVJL3AmdV1Wt28dlxYBxgZGRk1cTExMBy74vp6WmGh4fbjqGOcr7UJOdLTXK+1CTnS01ZzLM1Nja2sap6e9tu4MUvyRHAeuAtwDbgeuAG4AvAe4DvYLYM/mhVnbin7+r1ejU1NdVs4P00OTnJ6Oho2zHUUc6XmuR8qUnOl5rkfKkpi3m2ksyr+LVxA5VzgEeq6gmAJBuA06rqo8AZ/bXXAa9uIZskSZIkdU4b1/g9BpySZFmSAGcDDyY5CiDJdwC/DHyghWySJEmS1DkDL35VdTezp3beA9zXz7AWuCrJg8BngE9W1R2DziZJkiRJXdTKs/Kqag2wZqflq/o/kiRJkqQF1NbjHCRJkiRJA2LxkyRJkqSOs/hJkiRJUsdZ/CRJkiSp4yx+kiRJktRxFj9JkiRJ6jiLnyRJkiR1nMVPkiRJkjrO4idJkiRJHddK8UtyZZL7k2xOsi7J4UnOTnJPknuT/FWSV7WRTZIkSZK6ZuDFL8kK4HKgV1XHAUuAC4HfAy6qqhOA/wr8yqCzSZIkSVIXtXWq51JgKMlSYBmwFSjgxf33X9JfkyRJkiQdoFTV4HearAauBmaAW6vqoiRnADf1174GnFJVX9vFZ8eBcYCRkZFVExMTgwu+D6anpxkeHm47hjrK+VKTnC81yflSk5wvNWUxz9bY2NjGqurtbbuBF78kRwDrgbcA24DrgRuAC4B3V9XdSa4CVlbV2/f0Xb1er6amppqOvF8mJycZHR1tO4Y6yvlSk5wvNcn5UpOcLzVlMc9WknkVvzZO9TwHeKSqnqiq7cAG4HTg+Kq6u7/Nx4HTWsgmSZIkSZ3TRvF7DDglybIkAc4GHgBekuTV/W1+CHiwhWySJEmS1DlLB73D/qmcNwD3AM8Cm4C1wOPA+iTPA08BPzfobJIkSZLURQMvfgBVtQZYs9Pyjf0fSZIkSdICautxDpIkSZKkAbH4SZIkSVLHWfwkSZIkqeMsfpIkSZLUcRY/SZIkSeo4i58kSZIkdZzFT5IkSZI6zuInSZIkSR1n8ZMkSZKkjmul+CW5Msn9STYnWZfk8CR/meTe/s/WJDe1kU2SJEmSumbpoHeYZAVwOfD9VTWT5BPAhVV1xpxt1gM3DzqbJEmSJHVRW6d6LgWGkiwFlgFbd7yR5EXAWYBH/CRJkiRpAaSqBr/TZDVwNTAD3FpVF81576eBN1bVv9vNZ8eBcYCRkZFVExMTA0i876anpxkeHm47hjrK+VKTnC81yflSk5wvNWUxz9bY2NjGqurtbbuBF78kRwDrgbcA24DrgRuq6qP99/8c+GBVrd/bd/V6vZqammoy7n6bnJxkdHS07RjqKOdLTXK+1CTnS01yvtSUxTxbSeZV/No41fMc4JGqeqKqtgMbgNMAkhwJnAz8WQu5JEmSJKmT2ih+jwGnJFmWJMDZwIP9934M+NOq+mYLuSRJkiSpkwZe/KrqbuAG4B7gvn6Gtf23LwTWDTqTJEmSJHXZwB/nAFBVa4A1u1gfHXwaSZIkSeq2th7nIEmSJEkaEIufJEmSJHWcxU+SJEmSOm7exS/JKUnuSPLXSc5rMpQkSZIkaeHs9uYuSb67qr40Z+kdwBuBAP8DuKnhbJIkSZKkBbCnu3p+IMlG4Nr+c/W2AT8BPA98bRDhJEmSJEkHbrenelbVecC9wJ8m+SngCmZL3zLAUz0lSZIk6SCxx2v8quqTwLnAcmAD8FBVva+qnhhEOEmSJEnSgdtt8UvyxiR/BdwBbAYuBM5Psi7J/z6ogJIkSZKkA7OnI37/N7NH+94MvLuqtlXVO4BfA64+kJ0muTLJ/Uk294vk4Zl1dZLPJXkwyeUHsg9JkiRJ0qw93dzlaWaP8g0BX9mxWFUP99f3S5IVwOXA91fVTJJP9L8vwNHA91XV80mO2t99SJIkSZL+2Z6O+J3P7I1cnmX2bp4LaSkwlGRpfx9bgf8A/EZVPQ9QVV/Zw+clSZIkSfOUqhr8TpPVzJ4uOgPcWlUXJflH4LeZLZxPAJf3jy7u/NlxYBxgZGRk1cTExOCC74Pp6WmGh4fbjqGOcr7UJOdLTXK+1CTnS01ZzLM1Nja2sap6e9tuT6d6NiLJEcCbgFcy+2zA65P8JPAdwDerqpfkAuBDwBk7f76q1gJrAXq9Xo2Ojg4q+j6ZnJxksWbTwc/5UpOcLzXJ+VKTnC81pQuztcfHOTTkHOCRqnqiqrYz+5iI04DHgfX9bW4EXttCNkmSJEnqnL0WvyS/0D9Kt1AeA05JsixJgLOBB4GbgLP62/wg8LkF3KckSZIkHbLmc6rndwN/l+QeZk+/vKUO4MLAqro7yQ3APczeOGYTs6duDgEfS3IlMA28fX/3IUmSJEn6Z3stflX1K0l+FXgd8LPA+/uPYLiuqr6wPzutqjXAmp2WvwX86P58nyRJkiRp9+Z1jV//CN+X+j/PAkcANyR5T4PZJEmSJEkLYK9H/JJcDlwMfBX4IHBVVW1P8gLgYeA/NhtRkiRJknQg5nON33cCF1TVo3MXq+r5JG9oJpYkSZIkaaHM51TP/wY8ueNFkhcl+TcAVfVgU8EkSZIkSQtjPsXv95i9y+YO3+ivSZIkSZIOAvMpfpn7+Iaqep75nSIqSZIkSVoE5lP8/j7J5UkO6/+sBv6+6WCSJEmSpIUxn+L388BpwBbgceDfAONNhpIkSZIkLZz5PMD9K8CFA8giSZIkSWrAfJ7jdzjwNuAHgMN3rFfVz+3vTpNcCbwdKOA+4GeBDwA/CDzd3+xnqure/d2HJEmSJGnWfE71/GPgu4FzgTuBlwFf398dJlkBXA70quo4YAn/fETxqqo6of9j6ZMkSZKkBTCf4veqqvpV4BtV9RHgR4HXHOB+lwJDSZYCy4CtB/h9kiRJkqTdyJwnNex6g+Rvq+rkJJ8C/g/gS8DfVtX37vdOZ+8MejUwA9xaVRcl+UPgVOBbwO3AO6vqW7v47Dj9m8uMjIysmpiY2N8YjZqenmZ4eLjtGOoo50tNcr7UJOdLTXK+1JTFPFtjY2Mbq6q3t+3mU/zeDqxn9ijfHwLDwK9W1e/vT7AkR/S/7y3ANuB64AZmy96XgBcCa4EvVNVv7Ol0B7g9AAAfpklEQVS7er1eTU1N7U+Mxk1OTjI6Otp2DHWU86UmOV9qkvOlJjlfaspinq0k8yp+e7y5S5IXAF+rqqeATwH7fZRvjnOAR6rqif4+NgCnVdVH++9/K8mHgV9agH1JkiRJ0iFvj9f4VdXzwC8s8D4fA05JsixJgLOBB5N8D0B/7Txg8wLvV5IkSZIOSXt9nANwW5JfAj4OfGPHYlU9uT87rKq7k9wA3AM8C2xi9tTOP0/yXUCAe5l9cLwkSZIk6QDNp/jteF7fpXPWigM47bOq1gBrdlo+a3+/T5IkSZK0e3stflX1ykEEkSRJkiQ1Y6/FL8lP72q9qv5o4eNIkiRJkhbafE71/Ndz/j6c2Zux3ANY/CRJkiTpIDCfUz0vm/s6yUuAP24skSRJkiRpQe3xcQ678QxwzEIHkSRJkiQ1Yz7X+H2S2bt4wmxR/H7gE02GkiRJkiQtnPlc4/ef5/z9LPBoVT3eUB5JkiRJ0gKbT/F7DPiHqvomQJKhJK+oqi82mkySJEmStCDmc43f9cDzc14/11+TJEmSJB0E5lP8llbVP+140f/7hQey0yRXJrk/yeYk65IcPue9300yfSDfL0mSJEn6Z/Mpfk8keeOOF0neBHx1f3eYZAVwOdCrquOAJcCF/fd6wPL9/W5JkiRJ0r80n2v8fh74WJL3918/Dvz0Aux3KMl2YBmwNckS4FrgJ4DzD/D7JUmSJEl9qaq9bwUkGe5v//UD3mmyGrgamAFuraqL+msvqKrfSTJdVcO7+ew4MA4wMjKyamJi4kDjNGJ6eprh4V3+T5AOmPOlJjlfapLzpSY5X2rKYp6tsbGxjVXV29t2ey1+Sf4f4D1Vta3/+gjgF6vqV/YnWP/z64G3ANuYvVHMBmbL3GhVPbun4jdXr9erqamp/YnRuMnJSUZHR9uOoY5yvtQk50tNcr7UJOdLTVnMs5VkXsVvPtf4vX5H6QOoqqeAHzmAbOcAj1TVE1W1ndnS9+vAq4DPJ/kisCzJ5w9gH5IkSZKkvvkUvyVJvmPHiyRDwHfsYfu9eQw4JcmyJAHOBn67qr67ql5RVa8AnqmqVx3APiRJkiRJffO5uctHgduTfBgo4OeAP9rfHVbV3UluAO4BngU2AWv39/skSZIkSXu21+JXVe9J8hlmT9EM8JtVdcuB7LSq1gBr9vD+4rxyUpIkSZIOQvM54kdV/QXwFwBJTk/y/1bVpY0mkyRJkiQtiHkVvyQnAG9l9k6cjzB7QxZJkiRJ0kFgt8UvyauBC5ktfP8IfJzZxz+MDSibJEmSJGkB7OmI32eBvwT+bVV9HiDJlQNJJUmSJElaMHt6nMObgS8B/z3JHyQ5m9mbu0iSJEmSDiK7LX5VdWNVvQX4PmASuBIYSfJ7SV43oHySJEmSpAO01we4V9U3qupjVfUG4GXAvcA7G08mSZIkSVoQey1+c1XVk1X1+1V1VlOBJEmSJEkLa5+KnyRJkiTp4NNK8UtyZZL7k2xOsi7J4UmuS/LpJJ9JckOS4TaySZIkSVLXDLz4JVkBXA70quo4YAmzzwu8sqqOr6rXAo8BvzDobJIkSZLURW2d6rkUGEqyFFgGbK2qrwEkCTAEVEvZJEmSJKlTUjX4fpVkNXA1MAPcWlUX9dc/DPwI8ADwo1X1zC4+Ow6MA4yMjKyamJgYWO59MT09zfCwZ6uqGc6XmuR8qUnOl5rkfKkpi3m2xsbGNlZVb2/bDbz4JTkCWA+8BdgGXA/cUFUf7b+/BPhd4O+q6sN7+q5er1dTU1MNJ94/k5OTjI6Oth1DHeV8qUnOl5rkfKlJzpeasphnK8m8il8bp3qeAzxSVU9U1XZgA3Dajjer6jng48CbW8gmSZIkSZ3TRvF7DDglybL+9XxnAw8meRX8r2v8/i3w2RaySZIkSVLnLB30Dqvq7iQ3APcAzwKbgLXAHUleDAT4NPAfBp1NkiRJkrpo4MUPoKrWAGt2Wj69jSySJEmS1HVtPc5BkiRJkjQgFj9JkiRJ6jiLnyRJkiR1nMVPkiRJkjrO4idJkiRJHWfxkyRJkqSOs/hJkiRJUsdZ/CRJkiSp4yx+kiRJktRxrRS/JFcmuT/J5iTrkhye5GNJHuqvfSjJYW1kkyRJkqSuGXjxS7ICuBzoVdVxwBLgQuBjwPcBrwGGgLcPOpskSZIkddHSFvc7lGQ7sAzYWlW37ngzyd8CL2spmyRJkiR1Sqpq8DtNVgNXAzPArVV10Zz3DgPuBlZX1V/u4rPjwDjAyMjIqomJicGE3kfT09MMDw+3HUMd5XypSc6XmuR8qUnOl5qymGdrbGxsY1X19rbdwItfkiOA9cBbgG3A9cANVfXR/vt/AHyjqq7Y23f1er2amppqMu5+m5ycZHR0tO0Y6ijnS01yvtQk50tNcr7UlMU8W0nmVfzauLnLOcAjVfVEVW0HNgCnASRZA3wX8I4WckmSJElSJ7Vxjd9jwClJljF7qufZwFSStwPnAmdX1fMt5JIkSZKkThp48auqu5PcANwDPAtsAtYC3wAeBf4mCcCGqvqNQeeTJEmSpK5p5a6eVbUGWLMYskiSJElS17XyAHdJkiRJ0uBY/CRJkiSp4yx+kiRJktRxFj9JkiRJ6jiLnyRJkiR1nMVPkiRJkjrO4idJkiRJHWfxkyRJkqSOs/hJkiRJUse1UvySXJnk/iSbk6xLcniSX0jy+SSV5DvbyCVJkiRJXTTw4pdkBXA50Kuq44AlwIXAXwPnAI8OOpMkSZIkddnSFvc7lGQ7sAzYWlWbAJK0FEmSJEmSuilVNfidJquBq4EZ4NaqumjOe19k9mjgV3fz2XFgHGBkZGTVxMRE84H3w/T0NMPDw23HUEc5X2qS86UmOV9qkvOlpizm2RobG9tYVb29bTfwI35JjgDeBLwS2AZcn+Qnq+qj8/l8Va0F1gL0er0aHR1tKuoBmZycZLFm08HP+VKTnC81yflSk5wvNaULs9XGzV3OAR6pqieqajuwATithRySJEmSdEhoo/g9BpySZFlmL+g7G3iwhRySJEmSdEgYePGrqruBG4B7gPv6GdYmuTzJ48DLgM8k+eCgs0mSJElSF7VyV8+qWgOs2Wn5ff0fSZIkSdICauUB7pIkSZKkwbH4SZIkSVLHWfwkSZIkqeMsfpIkSZLUcRY/SZIkSeo4i58kSZIkdZzFT5IkSZI6zuInSZIkSR1n8ZMkSZKkjrP4SZIkSVLHtVL8klyZ5P4km5OsS3J4klcmuTvJw0k+nuSFbWSTJEmSpK4ZePFLsgK4HOhV1XHAEuBC4N3A71TVMcBTwNsGnU2SJEmSuqitUz2XAkNJlgLLgH8AzgJu6L//EeC8lrJJkiRJUqekqga/02Q1cDUwA9wKrAbuqqpX9d8/Gvjz/hHBnT87DowDjIyMrJqYmBhY7n0xPT3N8PBw2zHUUc6XmuR8qUnOl5rkfKkpi3m2xsbGNlZVb2/bLR1EmLmSHAG8CXglsA24Hnj9LjbdZSOtqrXAWoBer1ejo6PNBD1Ak5OTLNZsOvg5X2qS86UmOV9qkvOlpnRhtto41fMc4JGqeqKqtgMbgNOA5f1TPwFeBmxtIZskSZIkdU4bxe8x4JQky5IEOBt4APjvwL/rb3MxcHML2SRJkiSpcwZe/KrqbmZv4nIPcF8/w1rgl4F3JPk8cCRw3aCzSZIkSVIXDfwaP4CqWgOs2Wn574GTW4gjSZIkSZ3W1uMcJEmSJEkDYvGTJEmSpI6z+EmSJElSx1n8JEmSJKnjLH6SJEmS1HEWP0mSJEnqOIufJEmSJHWcxU+SJEmSOs7iJ0mSJEkdN/Dil2Rlknvn/HwtyRVJjk/yN0nuS/LJJC8edDZJkiRJ6qKBF7+qeqiqTqiqE4BVwDPAjcAHgXdW1Wv6r68adDZJkiRJ6qK2T/U8G/hCVT0KrAQ+1V+/DXhza6kkSZIkqUNSVe3tPPkQcE9VvT/J/wDeXVU3J3kH8OtV9aJdfGYcGAcYGRlZNTExMdjQ8zQ9Pc3w8HDbMdRRzpea5HypSc6XmuR8qSmLebbGxsY2VlVvb9u1VvySvBDYCvxAVX05yfcB7wOOBP4EuLyqjtzTd/R6vZqammo+7H6YnJxkdHS07RjqKOdLTXK+1CTnS01yvtSUxTxbSeZV/JYOIsxuvJ7Zo31fBqiqzwKvA0jyauBHW8wmSZIkSZ3R5jV+bwXW7XiR5Kj+7xcAvwJ8oKVckiRJktQprRS/JMuAHwI2zFl+a5LPAZ9l9hTQD7eRTZIkSZK6ppVTPavqGWav5Zu79l7gvW3kkSRJkqQua/txDpIkSZKkhln8JEmSJKnjLH6SJEmS1HEWP0mSJEnqOIufJEmSJHWcxU+SJEmSOs7iJ0mSJEkdZ/GTJEmSpI6z+EmSJElSxw28+CVZmeTeOT9fS3JFkhOS3NVfm0py8qCzSZIkSVIXLR30DqvqIeAEgCRLgC3AjcAfAL9eVX+e5EeA9wCjg84nSZIkSV3T9qmeZwNfqKpHgQJe3F9/CbC1tVSSJEmS1CGpqvZ2nnwIuKeq3p/kWOAWIMwW0tP6hXDnz4wD4wAjIyOrJiYmBhl53qanpxkeHm47hjrK+VKTnC81yflSk5wvNWUxz9bY2NjGqurtbbvWil+SFzJ7VO8HqurLSd4H3FlV65P8ODBeVefs6Tt6vV5NTU0NIu4+m5ycZHR0tO0Y6ijnS01yvtQk50tNcr7UlMU8W0nmVfzaPNXz9cwe7fty//XFwIb+39cD3txFkiRJkhZAm8XvrcC6Oa+3Aj/Y//ss4OGBJ5IkSZKkDhr4XT0BkiwDfgj493OWLwHem2Qp8E361/FJkiRJkg5MK8Wvqp4Bjtxp7a+AVW3kkSRJkqQua/txDpIkSZKkhln8JEmSJKnjLH6SJEmS1HEWP0mSJEnqOIufJEmSJHWcxU+SJEmSOs7iJ0mSJEkdZ/GTJEmSpI6z+EmSJElSxy0d9A6TrAQ+Pmfpe4FfA04FVvbXlgPbquqEAceTJEmSpM4ZePGrqoeAEwCSLAG2ADdW1X/ZsU2S3wKeHnQ2SZIkSeqigRe/nZwNfKGqHt2xkCTAjwNntZZKkiRJkjokVdXezpMPAfdU1fvnrJ0J/HZV9XbzmXFgHGBkZGTVxMTEQLLuq+npaYaHh9uOoY5yvtQk50tNcr7UJOdLTVnMszU2NrZxd91prtaKX5IXAluBH6iqL89Z/z3g81X1W3v7jl6vV1NTUw2m3H+Tk5OMjo62HUMd5XypSc6XmuR8qUnOl5qymGcrybyKX5uner6e2aN9c0vfUuACYFVrqSRJkiSpY9p8nMNbgXU7rZ0DfLaqHm8hjyRJkiR1UivFL8ky4IeADTu9dSH/sgxKkiRJkg5AK6d6VtUzwJG7WP+ZwaeRJEmSpG5r81RPSZIkSdIAWPwkSZIkqeMsfpIkSZLUcRY/SZIkSeo4i98Cu2nTFk6/5g7u2/I0p19zBzdt2tJ2JEmSJEmHuDYf4N45N23awrs23MfM9ufgaNiybYZ3bbgPgPNOXNFyOkmSJEmHKo/4LaBrb3lotvTNMbP9Oa695aGWEkmSJEmSxW9Bbd02s0/rkiRJkjQIFr8F9NLlQ/u0LkmSJEmDMPDil2Rlknvn/HwtyRX99y5L8lCS+5O8Z9DZDtRV565k6LAl37Y2dNgSrjp3ZUuJJEmSJKmFm7tU1UPACQBJlgBbgBuTjAFvAl5bVd9KctSgsx2oHTdwmb2m7+usWD7EVeeu9MYu0v/f3v3HWHaXZQB/3rQ0pAwRbe1ICgpVbLTELHQkGEmccVPCGqWAIbFBKZi4EI1AAlHACCaGhKgBYzYRMTZUhZ2a2EL9kUhTmeg/KNNNlRKoWNNFWLKFCC3jqgT6+sfcjdfNdndnds49s4fPJ5nMnO+9d+/7x5M5+8w59xwAAEY19lU9DyZ5qLuPV9VvJ3l3d/9PknT3I+OOtjsve961ednzrs3GxkZ++VWrY48DAACQ6u7x3rzqtiTHuvtIVd2f5CNJXpLkv5O8pbs/cZbXHE5yOEmWl5dvXF9fX+TIF2xraytLS0tjj8FEyRdDki+GJF8MSb4Yyn7O1tra2n3dvXK+541W/KrqiiQnktzQ3Ser6oEkf5vkjUl+OMkdSa7rcwy4srLSm5ubC5l3pzY2NrK6ujr2GEyUfDEk+WJI8sWQ5Iuh7OdsVdUFFb8xr+p5KNtH+07Otj+f5M7e9o9JHk9y9WjTAQAATMSYxe+WJEfntj+c5MeTpKq+P8kVSb48wlwAAACTMkrxq6ork9yU5M655duSXDc75XM9ya3nOs0TAACACzPKVT27+1SSq85Y+3qSnx1jHgAAgCkb81RPAAAAFkDxAwAAmDjFDwAAYOIUPwAAgIlT/AAAACZO8QMAAJg4xQ8AAGDiFD8AAICJU/wAAAAm7vJFv2FVXZ/kjrml65K8I8nTkvxCki/N1t/e3X+94PEAAAAmZ+HFr7sfTHIgSarqsiRfSHJXktcmeW93/86iZwIAAJiysU/1PJjkoe4+PvIcAAAAk1XdPd6bV92W5Fh3H6mq30jymiSPJdlM8ubu/spZXnM4yeEkWV5evnF9fX1xA+/A1tZWlpaWxh6DiZIvhiRfDEm+GJJ8MZT9nK21tbX7unvlfM8brfhV1RVJTiS5obtPVtVyki8n6SS/meTp3f3z5/o3VlZWenNzc/hhd2FjYyOrq6tjj8FEyRdDki+GJF8MSb4Yyn7OVlVdUPEb81TPQ9k+2ncySbr7ZHd/s7sfT/KHSV4w4mwAAACTMWbxuyXJ0dMbVfX0ucdenuSBhU8EAAAwQQu/qmeSVNWVSW5K8rq55d+qqgPZPtXz4TMeAwAAYJdGKX7dfSrJVWes/dwYswAAAEzd2LdzAAAAYGCKHwAAwMQpfgAAABOn+AEAAEyc4gcAADBxih8AAMDEKX4AAAATp/gBAABMnOIHAAAwcQsvflV1fVXdP/f1WFW9ae7xt1RVV9XVi54NAABgii5f9Bt294NJDiRJVV2W5AtJ7pptPzPJTUk+t+i5AAAApmrsUz0PJnmou4/Ptt+b5FeS9HgjAQAATEt1j9exquq2JMe6+0hVvTTJwe5+Y1U9nGSlu798ltccTnI4SZaXl29cX19f6MwXamtrK0tLS2OPwUTJF0OSL4YkXwxJvhjKfs7W2trafd29cr7njVb8quqKJCeS3JDka0k+luTF3f3ouYrfvJWVld7c3Bx81t3Y2NjI6urq2GMwUfLFkOSLIckXQ5IvhrKfs1VVF1T8xjzV81C2j/adTPK9SZ6d5J9mpe8ZSY5V1XeNOB8AAMAkLPziLnNuSXI0Sbr7k0muOf3AhR7xAwAA4PxGOeJXVVdm++qdd47x/gAAAN9KRjni192nklx1jseftbhpAAAApm3s2zkAAAAwMMUPAABg4hQ/AACAiVP8AAAAJk7xAwAAmDjFDwAAYOIUPwAAgIlT/AAAACZO8QMAAJi4yxf9hlV1fZI75pauS/KOJFcluTnJ40keSfKa7j6x6PkAAACmZuHFr7sfTHIgSarqsiRfSHJXkq9096/P1t+Q7TL4+kXPBwAAMDULL35nOJjkoe4+fsb6U5L0CPMAAABMTnWP16+q6rYkx7r7yGz7XUleneTRJGvd/aWzvOZwksNJsry8fOP6+voCJ75wW1tbWVpaGnsMJkq+GJJ8MST5YkjyxVD2c7bW1tbu6+6V8z1vtOJXVVckOZHkhu4+ecZjb0vy5O5+57n+jZWVld7c3Bxwyt3b2NjI6urq2GMwUfLFkOSLIckXQ5IvhrKfs1VVF1T8xryq56FsH+07eZbHPpTkpxc8DwAAwCSNWfxuSXL09EZVPWfusZcm+czCJwIAAJigUS7uUlVXJrkpyevmlt89u9XD40mOxxU9AQAA9sQoxa+7T2X7vn3za07tBAAAGMCYp3oCAACwAIofAADAxCl+AAAAE6f4AQAATJziBwAAMHGKHwAAwMQpfgAAABOn+AEAAEyc4gcAADBxih8AAMDEXb7oN6yq65PcMbd0XZJ3JLk2yU8l+XqSh5K8tru/uuj5AAAApmbhR/y6+8HuPtDdB5LcmORUkruS3JPkud39Q0n+JcnbFj0bAADAFI19qufBJA919/Hu/mh3f2O2/vEkzxhxLgAAgMmo7h7vzatuS3Ksu4+csf4XSe7o7j89y2sOJzmcJMvLyzeur68vZNad2traytLS0thjMFHyxZDkiyHJF0OSL4ayn7O1trZ2X3evnO95oxW/qroiyYkkN3T3ybn1X0uykuQVfZ7hVlZWenNzc9hBd2ljYyOrq6tjj8FEyRdDki+GJF8MSb4Yyn7OVlVdUPFb+MVd5hzK9tG++dJ3a5KfTHLwfKUPAACACzNm8bslydHTG1X1kiS/muTHuvvUaFMBAABMzCgXd6mqK5PclOTOueUjSZ6a5J6qur+q3jfGbAAAAFMzyhG/2RG9q85Y+74xZgEAAJi6sW/nAAAAwMAUPwAAgIlT/AAAACZO8QMAAJg4xQ8AAGDiFD8AAICJU/wAAAAmTvEDAACYOMUPAABg4hZe/Krq+qq6f+7rsap6U1W9sqo+VVWPV9XKoucCAACYqssX/Ybd/WCSA0lSVZcl+UKSu5JcmeQVSf5g0TMBAABM2cKL3xkOJnmou4+fXqiqEccBAACYnuru8d686rYkx7r7yNzaRpK3dPfmE7zmcJLDSbK8vHzj+vr6Ikbdsa2trSwtLY09BhMlXwxJvhiSfDEk+WIo+zlba2tr93X3eT8qN9oRv6q6IslLk7xtJ6/r7vcneX+SrKys9Orq6t4Ptwc2NjayX2fj0idfDEm+GJJ8MST5YihTyNaYV/U8lO2jfSdHnAEAAGDyxix+tyQ5OuL7AwAAfEsYpfhV1ZVJbkpy59zay6vq80l+JMlfVdXfjDEbAADA1IzyGb/uPpXkqjPW7sr2bR0AAADYQ2Oe6gkAAMACKH4AAAATp/gBAABMnOIHAAAwcYofAADAxCl+AAAAE6f4AQAATJziBwAAMHGKHwAAwMQtvPhV1fVVdf/c12NV9aaq+o6quqeqPjv7/u2Lng0AAGCKFl78uvvB7j7Q3QeS3JjkVJK7krw1yb3d/Zwk9862AQAAuEhjn+p5MMlD3X08yc1Jbp+t357kZaNNBQAAMCGXj/z+P5Pk6Ozn5e7+YpJ09xer6pqzvaCqDic5PNvcqqoHhx9zV65O8uWxh2Cy5IshyRdDki+GJF8MZT9n63su5EnV3UMPcvY3rroiyYkkN3T3yar6anc/be7xr3T3Jfs5v6ra7O6VsedgmuSLIckXQ5IvhiRfDGUK2RrzVM9DSY5198nZ9smqenqSzL4/MtpkAAAAEzJm8bsl/3eaZ5LcneTW2c+3JvnIwicCAACYoFGKX1VdmeSmJHfOLb87yU1V9dnZY+8eY7Y99P6xB2DS5IshyRdDki+GJF8M5ZLP1mif8QMAAGAxxr6dAwAAAANT/AAAACZO8btIVXVbVT1SVQ/MrX1HVd1TVZ+dfb9kb0vBuKrqmVX1sar6dFV9qqreOFuXMfZEVT1cVZ+sqvuranO2Jl/syk72ibXt96rqX6vqn6vq+eNNzqVgp/tEGWM3quoNs4x98BzP2Zp9f9b877v9TvG7eB9I8pIz1t6a5N7ufk6Se2fbsBvfSPLm7v6BJC9M8ktV9YORMfbWWncfmLs/kXyxWx/Ihe8TDyV5zuzrcJLfX9CMXLp2uk+UMXbjF5P8RHe/auxB9prid5G6+++S/McZyzcnuX328+1JXrbQoZiM7v5idx+b/fy1JJ9Ocm1kjGHJF7uyw33izUn+uLd9PMnTTt/PF85mF/tEGWNHqup9Sa5LcndVPVpVb5l77IGqetZYs+0FxW8Yy939xWT7l1SSa0aehwmY/bJ5XpJ/iIyxdzrJR6vqvqo6PFuTL/bSE+Xp2iT/Pve8z8/W4LwucJ8oY+xId78+yYkka0neO/I4e+7ysQcAzq+qlpL8eZI3dfdjVTX2SEzHj3b3iaq6Jsk9VfWZsQfiW8bZfpG5xxTntYN9oozBHEf8hnHy9KkEs++PjDwPl7CqelK2d3Af7O47Z8syxp7o7hOz748kuSvJCyJf7K0nytPnkzxz7nnPyPZf2uEJ7XCfKGNcjG/k/3elJ481yF5R/IZxd5JbZz/fmuQjI87CJay2/4z5R0k+3d3vmXtIxrhoVfWUqnrq6Z+TvDjJA5Ev9tYT5enuJK+eXXnxhUkePX26HpzNLvaJMsbFeDjJ85NkdkXYZ486zR6obke8L0ZVHU2ymuTqJCeTvDPJh5P8WZLvTvK5JK/s7jM/7A7nVVUvSvL3ST6Z5PHZ8tuz/ZkGGeOiVNV12T7Kl2yf+v+h7n5XVV0V+WIXdrJPnP0n/ki2rwJ6Kslru3tzjLm5NOx0nyhj7EZVPZxkJcl/ZvuPCNck+USSFyU51N0PV9VWdy/NPmv6l9393JHG3RHFDwAAYOKc6gkAADBxih8AAMDEKX4AAAATp/gBAABMnOIHAAAwcZePPQAAXGqq6pvZvqT8k7J9k9/bk/xudz9+zhcCwEgUPwDYuf/q7gNJUlXXJPlQkm/L9n3rAGDfcR8/ANih0zfvndu+Lts3+L267VgB2Id8xg8ALlJ3/1u296nXjD0LAJyN4gcAe6PGHgAAnojiBwAXaXaq5zeTPDL2LABwNoofAFyEqvrOJO9LcsTn+wDYr1zcBQB26Cy3c/iTJO9xOwcA9ivFDwAAYOKc6gkAADBxih8AAMDEKX4AAAATp/gBAABMnOIHAAAwcYofAADAxCl+AAAAE/e/JzBkyMWMB/QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f936abb0fd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plotting accuracies for all values of D\n",
    "\n",
    "plt.figure(figsize=(15, 10))\n",
    "plt.scatter(np.arange(6), accuracies)\n",
    "plt.xticks(np.arange(6), [str(val) for val in d[:-1]]+['full'])\n",
    "plt.yticks(np.arange(70, 101, 1))\n",
    "plt.xlabel('D')\n",
    "plt.ylabel('Accuracy %')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The plot above shows the change in accuracies as the singular values D vary. It can be seen that the accuracy increases with the increase in D. The accuracy when D=full is as good as the baseline while D=10 gives the worst performance among the others. Although the accuracy is low for lower values of D, they reduce the amount of computation during feedforward. These values can be thus be used to initialize weights for a compressed network, which can be finetuned to get better accuracy.   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# path to save/load new model \n",
    "svd_20_path = \"models/mnist-shallow/svd_20.ckpt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# helper to generate keys for weights/biases dict\n",
    "\n",
    "keys = []\n",
    "\n",
    "for i in range(5):\n",
    "    keys.append('{}_u'.format(i+1))\n",
    "    keys.append('{}_v'.format(i+1))\n",
    "keys.append('y')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# store global variables that have been initialized till now\n",
    "initialized = set(tf.global_variables())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# new weights and biases for D=20\n",
    "\n",
    "weights_20 = {}\n",
    "biases_20 = {}\n",
    " \n",
    "for key in keys[:-1]:\n",
    "    layer = int(key[0])-1\n",
    "    svd_mat = key[-1]\n",
    "    \n",
    "    if svd_mat=='u':\n",
    "        weights_20['h'+key] =  tf.Variable(sess.run(svd['u'][layer][:,:20]))\n",
    "        biases_20['b'+key] = tf.Variable(tf.zeros([20]))\n",
    "    else:\n",
    "        weights_20['h'+key] = tf.Variable(sess.run(tf.matmul(\n",
    "            tf.diag(svd['s'][layer])[:20,:20], \n",
    "            tf.transpose(svd['v'][layer][:,:20])\n",
    "        )))\n",
    "        biases_20['b'+key] = tf.Variable(tf.zeros([hidden_units]))\n",
    "        \n",
    "weights_20['y'] = tf.Variable(sess.run(weights['y']))\n",
    "biases_20['y'] = tf.Variable(tf.zeros([10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# new n/w architecture\n",
    "\n",
    "hidden1_u = tf.nn.relu(tf.matmul(x, weights_20['h1_u']) + biases_20['b1_u'])\n",
    "hidden1_v = tf.nn.relu(tf.matmul(hidden1_u, weights_20['h1_v']) + biases_20['b1_v'])\n",
    "\n",
    "hidden2_u = tf.nn.relu(tf.matmul(hidden1_v, weights_20['h2_u']) + biases_20['b2_u'])\n",
    "hidden2_v = tf.nn.relu(tf.matmul(hidden2_u, weights_20['h2_v']) + biases_20['b2_v'])\n",
    "\n",
    "hidden3_u = tf.nn.relu(tf.matmul(hidden2_v, weights_20['h3_u']) + biases_20['b3_u'])\n",
    "hidden3_v = tf.nn.relu(tf.matmul(hidden3_u, weights_20['h3_v']) + biases_20['b3_v'])\n",
    "\n",
    "hidden4_u = tf.nn.relu(tf.matmul(hidden3_v, weights_20['h4_u']) + biases_20['b4_u'])\n",
    "hidden4_v = tf.nn.relu(tf.matmul(hidden4_u, weights_20['h4_v']) + biases_20['b4_v'])\n",
    "\n",
    "hidden5_u = tf.nn.relu(tf.matmul(hidden4_v, weights_20['h5_u']) + biases_20['b5_u'])\n",
    "hidden5_v = tf.nn.relu(tf.matmul(hidden5_u, weights_20['h5_v']) + biases_20['b5_v'])\n",
    "\n",
    "y_20 = tf.matmul(hidden5_v, weights_20['y']) + biases_20['y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cross_entropy_20 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=y_20, labels=y_))\n",
    "train_step_20 = tf.train.AdamOptimizer().minimize(cross_entropy_20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# init only new variables \n",
    "sess.run(tf.variables_initializer(set(tf.global_variables()) - initialized))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 \tcost=0.025180072\n",
      "Epoch: 02 \tcost=0.021299959\n",
      "Epoch: 03 \tcost=0.020269438\n",
      "Epoch: 04 \tcost=0.021382123\n",
      "Epoch: 05 \tcost=0.015896063\n",
      "Epoch: 06 \tcost=0.020531045\n",
      "Epoch: 07 \tcost=0.024283398\n",
      "Epoch: 08 \tcost=0.021523427\n",
      "Epoch: 09 \tcost=0.023597770\n",
      "Epoch: 10 \tcost=0.019215811\n",
      "Epoch: 11 \tcost=0.022588605\n",
      "Epoch: 12 \tcost=0.019381890\n",
      "Epoch: 13 \tcost=0.020448377\n",
      "Epoch: 14 \tcost=0.024595842\n",
      "Epoch: 15 \tcost=0.017237076\n",
      "Epoch: 16 \tcost=0.015975158\n",
      "Epoch: 17 \tcost=0.018712164\n",
      "Epoch: 18 \tcost=0.019698384\n",
      "Epoch: 19 \tcost=0.022255920\n",
      "Epoch: 20 \tcost=0.023224746\n",
      "Epoch: 21 \tcost=0.017358900\n",
      "Epoch: 22 \tcost=0.017553471\n",
      "Epoch: 23 \tcost=0.028187710\n",
      "Epoch: 24 \tcost=0.014870938\n",
      "Epoch: 25 \tcost=0.016509837\n",
      "Epoch: 26 \tcost=0.015014031\n",
      "Epoch: 27 \tcost=0.022046055\n",
      "Epoch: 28 \tcost=0.015916045\n",
      "Epoch: 29 \tcost=0.016236033\n",
      "Epoch: 30 \tcost=0.021404265\n",
      "Epoch: 31 \tcost=0.013881243\n",
      "Epoch: 32 \tcost=0.013140946\n",
      "Epoch: 33 \tcost=0.018450375\n",
      "Epoch: 34 \tcost=0.019727645\n",
      "Epoch: 35 \tcost=0.023245887\n",
      "Epoch: 36 \tcost=0.016114210\n",
      "Epoch: 37 \tcost=0.017982884\n",
      "Epoch: 38 \tcost=0.018814561\n",
      "Epoch: 39 \tcost=0.019952245\n",
      "Epoch: 40 \tcost=0.011512057\n",
      "Epoch: 41 \tcost=0.013827032\n",
      "Epoch: 42 \tcost=0.021954334\n",
      "Epoch: 43 \tcost=0.015700458\n",
      "Epoch: 44 \tcost=0.022254788\n",
      "Epoch: 45 \tcost=0.018892320\n",
      "Epoch: 46 \tcost=0.016871961\n",
      "Epoch: 47 \tcost=0.017156099\n",
      "Epoch: 48 \tcost=0.018479511\n",
      "Epoch: 49 \tcost=0.018366672\n",
      "Epoch: 50 \tcost=0.014424528\n"
     ]
    }
   ],
   "source": [
    "train(mnist, sess, train_step_20, cross_entropy_20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save new model\n",
    "save_path_20 = saver.save(sess, svd_20_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from models/mnist-shallow/svd_20.ckpt\n"
     ]
    }
   ],
   "source": [
    "# restore svd model\n",
    "saver.restore(sess, svd_20_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 97.19%\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {:.2f}%'.format(test(y_20, y_, mnist)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}