DNC Test Area.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('./IMDb-data/positive-samples.pickle', 'rb')\n",
    "train_pos = pickle.load(file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('./IMDb-data/pos-train.pickle', 'rb')\n",
    "train_pos_labels = pickle.load(file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('./IMDb-data/negative-samples.pickle', 'rb')\n",
    "train_neg = pickle.load(file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('./IMDb-data/neg-train.pickle', 'rb')\n",
    "train_neg_labels = pickle.load(file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train_pos + train_neg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = train_pos_labels + train_neg_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels[12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_binput(text):\n",
    "    return text.decode().replace('<br />', '').replace('.', ' ').replace('(', '').replace(')', '').replace(',', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def numerify(ins, word_list):\n",
    "    vectorized = []\n",
    "    for word in ins:\n",
    "        if word in word_list:\n",
    "            vectorized.append(word_list.index(word)+1)\n",
    "    return vectorized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_label(label_fname):\n",
    "     return int(label_fname.split('_')[-1].split('.')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset = []\n",
    "for i in train:\n",
    "    x = clean_binput(i).split()\n",
    "    trainset.append(numerify(x, words))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainlabels = []\n",
    "for i in train_labels:\n",
    "    trainlabels.append(extract_label(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(trainset)):\n",
    "    if len(trainset[i]) < 2000:\n",
    "        while len(trainset[i]) < 2000:\n",
    "            trainset[i].insert(0, 0)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = T.tensor(trainset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = inputs.reshape((25000, 80, 25))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = inputs.to(T.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = trainlabels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = h5py.File('train.h5py', 'w')\n",
    "f.create_dataset('train_data', data=inputs)\n",
    "f.create_dataset('train_labels', data=labels)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('trainset', 'wb')\n",
    "pickle.dump(trainset, file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('train', 'wb')\n",
    "pickle.dump(train, file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('trainset', 'rb')\n",
    "trainset = pickle.load(file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('trainlabels', 'rb')\n",
    "trainlabels = pickle.load(file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(maxItem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = list(zip(*train)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_format = list(splits[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_format[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_format[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(test_format)):\n",
    "    test_format[i] = test_format[i].reshape((80,25))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(splits[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[0][0].reshape((2000,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedder(T.tensor(train[0][0]).to(T.long))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "memory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data and start from here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch as T\n",
    "from dnc import DNC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('./IMDb-data/imdbwords.pickle', 'rb')\n",
    "words = pickle.load(file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = h5py.File('train.h5py', 'r+')\n",
    "trains = f.get('train_data')\n",
    "trainlabels = f.get('train_labels')\n",
    "for i in range(len(trainlabels)):\n",
    "    trainlabels[i] = 1 if i < 12500 else 0\n",
    "trainz = zip(trains, trainlabels)\n",
    "train = list(trainz)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = h5py.File('test.h5py', 'r+')\n",
    "tests = f.get('test_data')\n",
    "testlabels = f.get('test_labels')\n",
    "for i in range(len(testlabels)):\n",
    "    testlabels[i] = 1 if i < 12500 else 0\n",
    "testz = zip(tests, testlabels)\n",
    "test = list(testz)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "in_size =  25 #5 with embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "diffy = DNC(input_size=in_size, hidden_size=28, num_layers=2, independent_linears=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedder = T.nn.Embedding(len(words)+1, in_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = T.nn.MSELoss() #T.nn.MSELoss() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "first epoch lr 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "params = diffy.parameters() #[embedder.parameters(), diffy.parameters()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = T.optim.Adam(diffy.parameters(), lr=1e-5, eps=1e-9, betas=[0.9, 0.98]) #itertools.chain(*params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "(controller_hidden, memory, read_vectors) = (None, None, None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 50\n",
    "train_data_loader = T.utils.data.DataLoader(dataset=train, batch_size=batch_size, shuffle=True)\n",
    "trainload = iter(train_data_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranges = len(trainload)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 1, Loss: 0.2978907823562622\n",
      "Step: 2, Loss: 0.29356178641319275\n",
      "Step: 3, Loss: 0.287936270236969\n",
      "Step: 4, Loss: 0.28123942017555237\n",
      "Step: 5, Loss: 0.268457293510437\n",
      "Step: 6, Loss: 0.2568327784538269\n",
      "Step: 7, Loss: 0.2680361866950989\n",
      "Step: 8, Loss: 0.2844295799732208\n",
      "Step: 9, Loss: 0.30456510186195374\n",
      "Step: 10, Loss: 0.2801378667354584\n",
      "Step: 11, Loss: 0.29828131198883057\n",
      "Step: 12, Loss: 0.2962247133255005\n",
      "Step: 13, Loss: 0.30923017859458923\n",
      "Step: 14, Loss: 0.31091827154159546\n",
      "Step: 15, Loss: 0.27789679169654846\n",
      "Step: 16, Loss: 0.2624688446521759\n",
      "Step: 17, Loss: 0.22511954605579376\n",
      "Step: 18, Loss: 0.280689001083374\n",
      "Step: 19, Loss: 0.27189648151397705\n",
      "Step: 20, Loss: 0.24390874803066254\n",
      "Step: 21, Loss: 0.2661014497280121\n",
      "Step: 22, Loss: 0.29457494616508484\n",
      "Step: 23, Loss: 0.25290507078170776\n",
      "Step: 24, Loss: 0.3124399483203888\n",
      "Step: 25, Loss: 0.23791244626045227\n",
      "Step: 26, Loss: 0.28939586877822876\n",
      "Step: 27, Loss: 0.3050451874732971\n",
      "Step: 28, Loss: 0.23892009258270264\n",
      "Step: 29, Loss: 0.28415411710739136\n",
      "Step: 30, Loss: 0.24748684465885162\n",
      "Step: 31, Loss: 0.25426000356674194\n",
      "Step: 32, Loss: 0.2565612196922302\n",
      "Step: 33, Loss: 0.31153830885887146\n",
      "Step: 34, Loss: 0.25231972336769104\n",
      "Step: 35, Loss: 0.261467844247818\n",
      "Step: 36, Loss: 0.3358040750026703\n",
      "Step: 37, Loss: 0.26735973358154297\n",
      "Step: 38, Loss: 0.27269741892814636\n",
      "Step: 39, Loss: 0.2214057743549347\n",
      "Step: 40, Loss: 0.2727244198322296\n",
      "Step: 41, Loss: 0.26214462518692017\n",
      "Step: 42, Loss: 0.23453934490680695\n",
      "Step: 43, Loss: 0.26734185218811035\n",
      "Step: 44, Loss: 0.3184107542037964\n",
      "Step: 45, Loss: 0.28150978684425354\n",
      "Step: 46, Loss: 0.27907463908195496\n",
      "Step: 47, Loss: 0.25934407114982605\n",
      "Step: 48, Loss: 0.29265862703323364\n",
      "Step: 49, Loss: 0.2765781581401825\n",
      "Step: 50, Loss: 0.27506712079048157\n",
      "Step: 51, Loss: 0.300619900226593\n",
      "Step: 52, Loss: 0.24754850566387177\n",
      "Step: 53, Loss: 0.30915701389312744\n",
      "Step: 54, Loss: 0.23356984555721283\n",
      "Step: 55, Loss: 0.3041939437389374\n",
      "Step: 56, Loss: 0.2542807459831238\n",
      "Step: 57, Loss: 0.2890031635761261\n",
      "Step: 58, Loss: 0.293296217918396\n",
      "Step: 59, Loss: 0.2382289171218872\n",
      "Step: 60, Loss: 0.23992705345153809\n",
      "Step: 61, Loss: 0.31070470809936523\n",
      "Step: 62, Loss: 0.26553404331207275\n",
      "Step: 63, Loss: 0.2318907529115677\n",
      "Step: 64, Loss: 0.26551446318626404\n",
      "Step: 65, Loss: 0.2766267955303192\n",
      "Step: 66, Loss: 0.24608950316905975\n",
      "Step: 67, Loss: 0.2382132112979889\n",
      "Step: 68, Loss: 0.2542021572589874\n",
      "Step: 69, Loss: 0.3008459508419037\n",
      "Step: 70, Loss: 0.27822986245155334\n",
      "Step: 71, Loss: 0.271885484457016\n",
      "Step: 72, Loss: 0.3109878599643707\n",
      "Step: 73, Loss: 0.23395195603370667\n",
      "Step: 74, Loss: 0.3015614449977875\n",
      "Step: 75, Loss: 0.2878125011920929\n",
      "Step: 76, Loss: 0.2638128995895386\n",
      "Step: 77, Loss: 0.24645505845546722\n",
      "Step: 78, Loss: 0.2624821662902832\n",
      "Step: 79, Loss: 0.2756505310535431\n",
      "Step: 80, Loss: 0.3236788809299469\n",
      "Step: 81, Loss: 0.34439966082572937\n",
      "Step: 82, Loss: 0.2590606212615967\n",
      "Step: 83, Loss: 0.2739482522010803\n",
      "Step: 84, Loss: 0.25933659076690674\n",
      "Step: 85, Loss: 0.32190677523612976\n",
      "Step: 86, Loss: 0.2831573188304901\n",
      "Step: 87, Loss: 0.30831655859947205\n",
      "Step: 88, Loss: 0.297978013753891\n",
      "Step: 89, Loss: 0.29120203852653503\n",
      "Step: 90, Loss: 0.26369038224220276\n",
      "Step: 91, Loss: 0.3144347071647644\n",
      "Step: 92, Loss: 0.31795021891593933\n",
      "Step: 93, Loss: 0.24618715047836304\n",
      "Step: 94, Loss: 0.2574056386947632\n",
      "Step: 95, Loss: 0.25559788942337036\n",
      "Step: 96, Loss: 0.24186304211616516\n",
      "Step: 97, Loss: 0.2908432185649872\n",
      "Step: 98, Loss: 0.29374343156814575\n",
      "Step: 99, Loss: 0.25123873353004456\n",
      "Step: 100, Loss: 0.2644455134868622\n",
      "Step: 101, Loss: 0.2672653794288635\n",
      "Step: 102, Loss: 0.29694411158561707\n",
      "Step: 103, Loss: 0.28013238310813904\n",
      "Step: 104, Loss: 0.2648434638977051\n",
      "Step: 105, Loss: 0.2540492117404938\n",
      "Step: 106, Loss: 0.25839000940322876\n",
      "Step: 107, Loss: 0.3405538499355316\n",
      "Step: 108, Loss: 0.2741038501262665\n",
      "Step: 109, Loss: 0.2446860373020172\n",
      "Step: 110, Loss: 0.2547439634799957\n",
      "Step: 111, Loss: 0.2786194980144501\n",
      "Step: 112, Loss: 0.2848895192146301\n",
      "Step: 113, Loss: 0.263119101524353\n",
      "Step: 114, Loss: 0.2897544801235199\n",
      "Step: 115, Loss: 0.2900618612766266\n",
      "Step: 116, Loss: 0.3184790015220642\n",
      "Step: 117, Loss: 0.2764858901500702\n",
      "Step: 118, Loss: 0.26145103573799133\n",
      "Step: 119, Loss: 0.29988470673561096\n",
      "Step: 120, Loss: 0.26049479842185974\n",
      "Step: 121, Loss: 0.28616857528686523\n",
      "Step: 122, Loss: 0.30575031042099\n",
      "Step: 123, Loss: 0.2325785905122757\n",
      "Step: 124, Loss: 0.274896502494812\n",
      "Step: 125, Loss: 0.28119295835494995\n",
      "Step: 126, Loss: 0.28007373213768005\n",
      "Step: 127, Loss: 0.278195858001709\n",
      "Step: 128, Loss: 0.28134968876838684\n",
      "Step: 129, Loss: 0.2877468466758728\n",
      "Step: 130, Loss: 0.32558172941207886\n",
      "Step: 131, Loss: 0.25900503993034363\n",
      "Step: 132, Loss: 0.32323572039604187\n",
      "Step: 133, Loss: 0.25288259983062744\n",
      "Step: 134, Loss: 0.25021567940711975\n",
      "Step: 135, Loss: 0.2921394109725952\n",
      "Step: 136, Loss: 0.29653751850128174\n",
      "Step: 137, Loss: 0.2862666845321655\n",
      "Step: 138, Loss: 0.2774815559387207\n",
      "Step: 139, Loss: 0.2756232023239136\n",
      "Step: 140, Loss: 0.25889429450035095\n",
      "Step: 141, Loss: 0.3233344256877899\n",
      "Step: 142, Loss: 0.2823447287082672\n",
      "Step: 143, Loss: 0.3011782765388489\n",
      "Step: 144, Loss: 0.30644792318344116\n",
      "Step: 145, Loss: 0.30394434928894043\n",
      "Step: 146, Loss: 0.3211207687854767\n",
      "Step: 147, Loss: 0.27526071667671204\n",
      "Step: 148, Loss: 0.2490907460451126\n",
      "Step: 149, Loss: 0.29234665632247925\n",
      "Step: 150, Loss: 0.29736635088920593\n",
      "Step: 151, Loss: 0.2696419656276703\n",
      "Step: 152, Loss: 0.22702449560165405\n",
      "Step: 153, Loss: 0.31723925471305847\n",
      "Step: 154, Loss: 0.3253610134124756\n",
      "Step: 155, Loss: 0.2952568233013153\n",
      "Step: 156, Loss: 0.27593690156936646\n",
      "Step: 157, Loss: 0.2601201832294464\n",
      "Step: 158, Loss: 0.34046825766563416\n",
      "Step: 159, Loss: 0.3154149353504181\n",
      "Step: 160, Loss: 0.28051626682281494\n",
      "Step: 161, Loss: 0.27361980080604553\n",
      "Step: 162, Loss: 0.25606656074523926\n",
      "Step: 163, Loss: 0.29154205322265625\n",
      "Step: 164, Loss: 0.2815250754356384\n",
      "Step: 165, Loss: 0.271587073802948\n",
      "Step: 166, Loss: 0.26926013827323914\n",
      "Step: 167, Loss: 0.28512904047966003\n",
      "Step: 168, Loss: 0.28121626377105713\n",
      "Step: 169, Loss: 0.3115554749965668\n",
      "Step: 170, Loss: 0.26954901218414307\n",
      "Step: 171, Loss: 0.27609923481941223\n",
      "Step: 172, Loss: 0.27147600054740906\n",
      "Step: 173, Loss: 0.28645044565200806\n",
      "Step: 174, Loss: 0.2929515838623047\n",
      "Step: 175, Loss: 0.25728827714920044\n",
      "Step: 176, Loss: 0.2665253281593323\n",
      "Step: 177, Loss: 0.26819077134132385\n",
      "Step: 178, Loss: 0.3024134337902069\n",
      "Step: 179, Loss: 0.308459997177124\n",
      "Step: 180, Loss: 0.2519315183162689\n",
      "Step: 181, Loss: 0.26776123046875\n",
      "Step: 182, Loss: 0.28639352321624756\n",
      "Step: 183, Loss: 0.25597256422042847\n",
      "Step: 184, Loss: 0.29394224286079407\n",
      "Step: 185, Loss: 0.2612599730491638\n",
      "Step: 186, Loss: 0.2965462803840637\n",
      "Step: 187, Loss: 0.290078341960907\n",
      "Step: 188, Loss: 0.2662857472896576\n",
      "Step: 189, Loss: 0.29641443490982056\n",
      "Step: 190, Loss: 0.2644363343715668\n",
      "Step: 191, Loss: 0.29310423135757446\n",
      "Step: 192, Loss: 0.2611940801143646\n",
      "Step: 193, Loss: 0.29647135734558105\n",
      "Step: 194, Loss: 0.28036874532699585\n",
      "Step: 195, Loss: 0.2495017647743225\n",
      "Step: 196, Loss: 0.26362738013267517\n",
      "Step: 197, Loss: 0.31500551104545593\n",
      "Step: 198, Loss: 0.26282423734664917\n",
      "Step: 199, Loss: 0.24318791925907135\n",
      "Step: 200, Loss: 0.2841073274612427\n",
      "Step: 201, Loss: 0.2909313142299652\n",
      "Step: 202, Loss: 0.33746638894081116\n",
      "Step: 203, Loss: 0.29827699065208435\n",
      "Step: 204, Loss: 0.29892563819885254\n",
      "Step: 205, Loss: 0.26378095149993896\n",
      "Step: 206, Loss: 0.2404516339302063\n",
      "Step: 207, Loss: 0.2950161397457123\n",
      "Step: 208, Loss: 0.25320303440093994\n",
      "Step: 209, Loss: 0.3071131408214569\n",
      "Step: 210, Loss: 0.25151607394218445\n",
      "Step: 211, Loss: 0.2705516517162323\n",
      "Step: 212, Loss: 0.2546499967575073\n",
      "Step: 213, Loss: 0.2500455379486084\n",
      "Step: 214, Loss: 0.2533475458621979\n",
      "Step: 215, Loss: 0.22421246767044067\n",
      "Step: 216, Loss: 0.3026421368122101\n",
      "Step: 217, Loss: 0.28238293528556824\n",
      "Step: 218, Loss: 0.2760915160179138\n",
      "Step: 219, Loss: 0.28172746300697327\n",
      "Step: 220, Loss: 0.26820337772369385\n",
      "Step: 221, Loss: 0.24966207146644592\n",
      "Step: 222, Loss: 0.27158457040786743\n",
      "Step: 223, Loss: 0.2599634826183319\n",
      "Step: 224, Loss: 0.30421844124794006\n",
      "Step: 225, Loss: 0.2778499722480774\n",
      "Step: 226, Loss: 0.27382779121398926\n",
      "Step: 227, Loss: 0.30403366684913635\n",
      "Step: 228, Loss: 0.2792084217071533\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 229, Loss: 0.27217331528663635\n",
      "Step: 230, Loss: 0.3318787217140198\n",
      "Step: 231, Loss: 0.26237544417381287\n",
      "Step: 232, Loss: 0.27074697613716125\n",
      "Step: 233, Loss: 0.27756467461586\n",
      "Step: 234, Loss: 0.2709818482398987\n",
      "Step: 235, Loss: 0.28040847182273865\n",
      "Step: 236, Loss: 0.2440173178911209\n",
      "Step: 237, Loss: 0.26873642206192017\n",
      "Step: 238, Loss: 0.2682378590106964\n",
      "Step: 239, Loss: 0.298973947763443\n",
      "Step: 240, Loss: 0.3035852313041687\n",
      "Step: 241, Loss: 0.2427515834569931\n",
      "Step: 242, Loss: 0.26286453008651733\n",
      "Step: 243, Loss: 0.24577447772026062\n",
      "Step: 244, Loss: 0.2848333716392517\n",
      "Step: 245, Loss: 0.25182920694351196\n",
      "Step: 246, Loss: 0.30272576212882996\n",
      "Step: 247, Loss: 0.21794329583644867\n",
      "Step: 248, Loss: 0.2515225410461426\n",
      "Step: 249, Loss: 0.24018867313861847\n",
      "Step: 250, Loss: 0.24686479568481445\n",
      "Step: 251, Loss: 0.30571597814559937\n",
      "Step: 252, Loss: 0.29297637939453125\n",
      "Step: 253, Loss: 0.2636632025241852\n",
      "Step: 254, Loss: 0.2841168940067291\n",
      "Step: 255, Loss: 0.26971930265426636\n",
      "Step: 256, Loss: 0.29820477962493896\n",
      "Step: 257, Loss: 0.275105357170105\n",
      "Step: 258, Loss: 0.2749359905719757\n",
      "Step: 259, Loss: 0.24921894073486328\n",
      "Step: 260, Loss: 0.24547672271728516\n",
      "Step: 261, Loss: 0.3133547008037567\n",
      "Step: 262, Loss: 0.27580389380455017\n",
      "Step: 263, Loss: 0.2870301306247711\n",
      "Step: 264, Loss: 0.28674110770225525\n",
      "Step: 265, Loss: 0.27341726422309875\n",
      "Step: 266, Loss: 0.25806161761283875\n",
      "Step: 267, Loss: 0.26222339272499084\n",
      "Step: 268, Loss: 0.27551528811454773\n",
      "Step: 269, Loss: 0.2658045291900635\n",
      "Step: 270, Loss: 0.25783130526542664\n",
      "Step: 271, Loss: 0.24960212409496307\n",
      "Step: 272, Loss: 0.3092677891254425\n",
      "Step: 273, Loss: 0.2530820965766907\n",
      "Step: 274, Loss: 0.23672010004520416\n",
      "Step: 275, Loss: 0.32093390822410583\n",
      "Step: 276, Loss: 0.26772668957710266\n",
      "Step: 277, Loss: 0.26141759753227234\n",
      "Step: 278, Loss: 0.3030901253223419\n",
      "Step: 279, Loss: 0.29269757866859436\n",
      "Step: 280, Loss: 0.2696848511695862\n",
      "Step: 281, Loss: 0.2683078646659851\n",
      "Step: 282, Loss: 0.24342842400074005\n",
      "Step: 283, Loss: 0.30055639147758484\n",
      "Step: 284, Loss: 0.2615269124507904\n",
      "Step: 285, Loss: 0.2518995404243469\n",
      "Step: 286, Loss: 0.2886759638786316\n",
      "Step: 287, Loss: 0.2864408791065216\n",
      "Step: 288, Loss: 0.27481675148010254\n",
      "Step: 289, Loss: 0.2775943875312805\n",
      "Step: 290, Loss: 0.2667541205883026\n",
      "Step: 291, Loss: 0.3024691641330719\n",
      "Step: 292, Loss: 0.29333415627479553\n",
      "Step: 293, Loss: 0.24571137130260468\n",
      "Step: 294, Loss: 0.28553441166877747\n",
      "Step: 295, Loss: 0.26939037442207336\n",
      "Step: 296, Loss: 0.26416027545928955\n",
      "Step: 297, Loss: 0.2983972132205963\n",
      "Step: 298, Loss: 0.24074645340442657\n",
      "Step: 299, Loss: 0.27964261174201965\n",
      "Step: 300, Loss: 0.29225659370422363\n",
      "Step: 301, Loss: 0.2964559495449066\n",
      "Step: 302, Loss: 0.3133852481842041\n",
      "Step: 303, Loss: 0.26109579205513\n",
      "Step: 304, Loss: 0.28608351945877075\n",
      "Step: 305, Loss: 0.30177977681159973\n",
      "Step: 306, Loss: 0.30181318521499634\n",
      "Step: 307, Loss: 0.33058735728263855\n",
      "Step: 308, Loss: 0.3043132722377777\n",
      "Step: 309, Loss: 0.2851009666919708\n",
      "Step: 310, Loss: 0.27425435185432434\n",
      "Step: 311, Loss: 0.2779446542263031\n",
      "Step: 312, Loss: 0.22528740763664246\n",
      "Step: 313, Loss: 0.27701327204704285\n",
      "Step: 314, Loss: 0.28647369146347046\n",
      "Step: 315, Loss: 0.2844030261039734\n",
      "Step: 316, Loss: 0.23117145895957947\n",
      "Step: 317, Loss: 0.23640070855617523\n",
      "Step: 318, Loss: 0.31304794549942017\n",
      "Step: 319, Loss: 0.29332295060157776\n",
      "Step: 320, Loss: 0.2587848901748657\n",
      "Step: 321, Loss: 0.2553841471672058\n",
      "Step: 322, Loss: 0.2627686560153961\n",
      "Step: 323, Loss: 0.266415536403656\n",
      "Step: 324, Loss: 0.28139787912368774\n",
      "Step: 325, Loss: 0.2492108941078186\n",
      "Step: 326, Loss: 0.25996410846710205\n",
      "Step: 327, Loss: 0.2649913430213928\n",
      "Step: 328, Loss: 0.2836049497127533\n",
      "Step: 329, Loss: 0.3043428063392639\n",
      "Step: 330, Loss: 0.2732861638069153\n",
      "Step: 331, Loss: 0.3073119521141052\n",
      "Step: 332, Loss: 0.3290623724460602\n",
      "Step: 333, Loss: 0.3015859127044678\n",
      "Step: 334, Loss: 0.337896466255188\n",
      "Step: 335, Loss: 0.2837485373020172\n",
      "Step: 336, Loss: 0.30706194043159485\n",
      "Step: 337, Loss: 0.22760836780071259\n",
      "Step: 338, Loss: 0.23271647095680237\n",
      "Step: 339, Loss: 0.306071400642395\n",
      "Step: 340, Loss: 0.2604832649230957\n",
      "Step: 341, Loss: 0.28133878111839294\n",
      "Step: 342, Loss: 0.25581273436546326\n",
      "Step: 343, Loss: 0.2809315621852875\n",
      "Step: 344, Loss: 0.25875094532966614\n",
      "Step: 345, Loss: 0.2951660454273224\n",
      "Step: 346, Loss: 0.26129987835884094\n",
      "Step: 347, Loss: 0.2362499088048935\n",
      "Step: 348, Loss: 0.28644832968711853\n",
      "Step: 349, Loss: 0.31089574098587036\n",
      "Step: 350, Loss: 0.2579699754714966\n",
      "Step: 351, Loss: 0.2693025767803192\n",
      "Step: 352, Loss: 0.28529593348503113\n",
      "Step: 353, Loss: 0.23059019446372986\n",
      "Step: 354, Loss: 0.2254391759634018\n",
      "Step: 355, Loss: 0.268595427274704\n",
      "Step: 356, Loss: 0.24946638941764832\n",
      "Step: 357, Loss: 0.2941683828830719\n",
      "Step: 358, Loss: 0.3135281205177307\n",
      "Step: 359, Loss: 0.23253071308135986\n",
      "Step: 360, Loss: 0.2612003684043884\n",
      "Step: 361, Loss: 0.29927632212638855\n",
      "Step: 362, Loss: 0.2953588664531708\n",
      "Step: 363, Loss: 0.27065548300743103\n",
      "Step: 364, Loss: 0.28075873851776123\n",
      "Step: 365, Loss: 0.26871415972709656\n",
      "Step: 366, Loss: 0.26681604981422424\n",
      "Step: 367, Loss: 0.2774589955806732\n",
      "Step: 368, Loss: 0.27395662665367126\n",
      "Step: 369, Loss: 0.25501370429992676\n",
      "Step: 370, Loss: 0.24540933966636658\n",
      "Step: 371, Loss: 0.2632271945476532\n",
      "Step: 372, Loss: 0.2938196063041687\n"
     ]
    }
   ],
   "source": [
    "diffy.train()\n",
    "#embedder.train()\n",
    "\n",
    "for it in range(ranges):\n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "    ins, tru = next(trainload)\n",
    "    \n",
    "    #Forward pass\n",
    "    #embed = embedder(ins.reshape(batch_size, 2000).to(T.long))\n",
    "    output, (controller_hidden, memory, read_vectors) = diffy(ins, (None, memory, None), reset_experience=False)\n",
    "    \n",
    "    #output_2 = output[:,-1]\n",
    "    final_out = T.sum(output, (1, 2), keepdim=True) #outer(mid_out)\n",
    "    #final_out = T.sum(T.sum(output, dim=1, keepdim=True), dim=2, keepdim=True)\n",
    "    \n",
    "    #final_out = T.nn.functional.sigmoid(final_out)\n",
    "\n",
    "    loss = loss_fn(final_out, tru.to(T.float).reshape((batch_size,1,1)))\n",
    "    \n",
    "    loss.backward()\n",
    "    T.nn.utils.clip_grad_value_(diffy.parameters(), 10)\n",
    "    optimizer.step()\n",
    "    \n",
    "    memory = {k : (v.detach() if isinstance(v, T.autograd.Variable) else v) for k, v in memory.items()}\n",
    "    \n",
    "    print('Step: {}, Loss: {}'.format(it+1, loss.item()))\n",
    "            \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_loader = T.utils.data.DataLoader(dataset=test, batch_size=batch_size, shuffle=True)\n",
    "testload = iter(test_data_loader)\n",
    "testlen = len(testload)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "correct = 0\n",
    "diffy.eval()\n",
    "#embedder.eval()\n",
    "for i in range(testlen):\n",
    "    with T.no_grad():\n",
    "        ins, tru = next(testload)\n",
    "        \n",
    "        #embed = embedder(ins.reshape(batch_size, 2000).to(T.long))\n",
    "        output, (c, m, r) = diffy(ins, (None, memory, None), reset_experience=False)\n",
    "        m = { k : (v.detach() if isinstance(v, T.autograd.Variable) else v) for k, v in m.items()}\n",
    "        (c, m, r) = (None, None, None)\n",
    "        #output_2 = output[:,-1]\n",
    "        y_pred = T.sum(output, (1,2), keepdim=True).round()\n",
    "        \n",
    "        for b in range(batch_size):\n",
    "            if(y_pred[b] == tru[b]):\n",
    "                correct = correct+1\n",
    "                \n",
    "        print('Step: {}, Accuracy: {}'.format(i+1, correct/((i+1)*batch_size)) )\n",
    "            \n",
    "print('Final Accuracy: {}'.format(correct/(testlen * batch_size)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the model and memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('diffy-sentiment', 'wb')\n",
    "pickle.dump(diffy, file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('memory-sentiment', 'wb')\n",
    "pickle.dump(memory, file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('embed-sentiment', 'wb')\n",
    "pickle.dump(embedder, file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('diffy-sentiment', 'wb')\n",
    "diffy = pickle.load(file)\n",
    "file.close()\n",
    "\n",
    "file = open('memory-sentiment', 'wb')\n",
    "memory = pickle.load(file)\n",
    "file.close()\n",
    "\n",
    "file = open('embed-sentiment', 'wb')\n",
    "embedder = pickle.load(file)\n",
    "file.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}