data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "from __future__ import unicode_literals\n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "USE_CUDA = torch.cuda.is_available()\n",
    "device = torch.device(\"cuda\" if USE_CUDA else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class Vocab(object):\n",
    "    \n",
    "    def __init__(self, filename):\n",
    "        self.idx_to_word = {}\n",
    "        self.word_to_idx = {}\n",
    "        self.filename = filename\n",
    "#         self.num_words = num_words\n",
    "        self.unk_vec = None\n",
    "        self.dim = None\n",
    "        \n",
    "        USE_CUDA = torch.cuda.is_available()\n",
    "        self.device = torch.device(\"cuda\" if USE_CUDA else \"cpu\")\n",
    "        with open(filename) as f:\n",
    "            idx = 0\n",
    "            for line in f:\n",
    "                line = line.split()\n",
    "                self.idx_to_word[idx] = line[0]\n",
    "                self.word_to_idx[line[0]] = idx\n",
    "                if not self.dim:\n",
    "                    self.dim = len(line[1:])\n",
    "                idx += 1\n",
    "        \n",
    "        self.embedding_matrix = torch.zeros(len(self.idx_to_word)+2, self.dim, device=self.device)\n",
    "        \n",
    "        with open(filename) as f:\n",
    "            idx = 1;\n",
    "            for line in f:\n",
    "                line = line.split()\n",
    "                self.embedding_matrix[idx] = torch.tensor(list(map(float, line[1:])), device=self.device)\n",
    "                idx += 1\n",
    "            self.unk_vec = torch.sum(self.embedding_matrix, 0)/(len(self.idx_to_word))\n",
    "            self.embedding_matrix[len(self.idx_to_word)+1] = self.unk_vec\n",
    "        \n",
    "    def embedding(self, input_seq):\n",
    "        MAX_LEN = input_seq.size()[0]\n",
    "        batch_size = input_seq.size()[1]\n",
    "        embedded = torch.zeros(MAX_LEN, batch_size, self.dim, device=self.device)\n",
    "        for i in range(MAX_LEN):\n",
    "            for j in range(batch_size):\n",
    "                embedded[i,j,:] = self.embedding_matrix[input_seq[i, j]]\n",
    "        return embedded\n",
    "    \n",
    "    def encode(self, sentence):\n",
    "        encoded = torch.zeros(len(sentence), dtype=torch.long, device=self.device)\n",
    "        idx=0\n",
    "        for word in sentence:\n",
    "            if word in self.word_to_idx:\n",
    "                encoded[idx] = self.word_to_idx[word]\n",
    "            else:\n",
    "                encoded[idx] = len(self.word_to_idx)+1\n",
    "            idx += 1\n",
    "        \n",
    "        return encoded\n",
    "    \n",
    "    def decode(self, sentence):\n",
    "        decoded = torch.zeros(*sentence.size(), device=self.device)\n",
    "        idx = 0\n",
    "        for word in sentence:\n",
    "            decoded[idx] = self.idx_to_word[word]\n",
    "        \n",
    "        return decoded\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def batcher(list_sentence, MAX_LEN, batch_size, test_ratio=0.2):\n",
    "    \n",
    "    USE_CUDA = torch.cuda.is_available()\n",
    "    device = torch.device(\"cuda\" if USE_CUDA else \"cpu\")\n",
    "    total = len(list_sentence)\n",
    "    train_len = total*(1-test_ratio)\n",
    "    test_len = total - train_len\n",
    "    train_sentence = list_sentence[:int(train_len)]\n",
    "    test_sentence = list_sentence[int(train_len):]\n",
    "    train = []\n",
    "    test = []\n",
    "    idx = int(0)\n",
    "    \n",
    "    while True:\n",
    "        pos = int(idx)\n",
    "        if pos == train_len:\n",
    "            break;\n",
    "        if (idx + batch_size <= train_len):\n",
    "            next_pos = int(idx+batch_size)\n",
    "        else:\n",
    "            next_pos = train_len\n",
    "        t_batch = torch.zeros(MAX_LEN, int(next_pos - pos), dtype=torch.long, device=device)\n",
    "        tmp_batch = train_sentence[int(pos):int(next_pos)]\n",
    "        target = torch.zeros(int(next_pos - pos), dtype=torch.long, device=device)\n",
    "        length = []\n",
    "        mask = torch.zeros(MAX_LEN, int(next_pos-pos), dtype=torch.long, device=device)\n",
    "        for batch_n in range(int(next_pos - pos)):\n",
    "            l = MAX_LEN if (len(tmp_batch[batch_n])-1) > MAX_LEN else (len(tmp_batch[batch_n])-1)\n",
    "            length.append(l)\n",
    "            target[batch_n] = tmp_batch[batch_n][-1]\n",
    "            t_batch[:l, batch_n] = tmp_batch[batch_n][:l]\n",
    "            mask[:l, batch_n] = torch.ones(l)\n",
    "        train.append((t_batch, target, mask, length))\n",
    "        idx = next_pos\n",
    "        \n",
    "    idx = int(0)\n",
    "    \n",
    "    while True:\n",
    "        pos = idx\n",
    "        if pos == test_len:\n",
    "            break;\n",
    "        if (idx + batch_size <= test_len):\n",
    "            next_pos = idx+batch_size\n",
    "        else:\n",
    "            next_pos = test_len\n",
    "        t_batch = torch.zeros(MAX_LEN, int(next_pos - pos), dtype=torch.long, device=device)\n",
    "        tmp_batch = test_sentence[int(pos):int(next_pos)]\n",
    "        target = torch.zeros(int(next_pos - pos), dtype=torch.long, device=device)\n",
    "        length = []\n",
    "        mask = torch.zeros(MAX_LEN, int(next_pos-pos), dtype=torch.long, device=device)\n",
    "        for batch_n in range(int(next_pos - pos)):\n",
    "            l = MAX_LEN if (len(tmp_batch[batch_n])-1) > MAX_LEN else (len(tmp_batch[batch_n])-1)\n",
    "            length.append(l)\n",
    "            target[batch_n] = tmp_batch[batch_n][-1]\n",
    "            t_batch[:l, batch_n] = tmp_batch[batch_n][:l]\n",
    "            mask[:l, batch_n] = torch.ones(l)\n",
    "        test.append((t_batch, target, mask, length))\n",
    "        idx = next_pos\n",
    "        \n",
    "    return train, test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data_file = 'sentiment/imdb_labelled.txt'\n",
    "embedding_file = 'glove.6B.50d.txt'\n",
    "\n",
    "X = []\n",
    "vocab = Vocab(embedding_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a very, very, very slow-moving, aimless movie about a distressed, drifting young man.  \t0\n",
      "\n",
      "a very very very slow-moving aimless movie about a distressed drifting young man  \t0\n",
      "\n",
      "it's practically perfect in all of them  a true masterpiece in a sea of faux \"masterpieces.  \t1\n",
      "\n",
      "it's practically perfect in all of them  a true masterpiece in a sea of faux masterpieces  \t1\n",
      "\n",
      "this game rocks.  \t1\n",
      "\n",
      "this game rocks  \t1\n",
      "\n",
      "long, whiny and pointless.  \t0\n",
      "\n",
      "long whiny and pointless  \t0\n",
      "\n",
      "it crackles with an unpredictable, youthful energy - but honestly, i found it hard to follow and concentrate on it meanders so badly.  \t0\n",
      "\n",
      "it crackles with an unpredictable youthful energy - but honestly i found it hard to follow and concentrate on it meanders so badly  \t0\n",
      "\n",
      "definitely worth checking out.  \t1\n",
      "\n",
      "definitely worth checking out  \t1\n",
      "\n",
      "an hour and a half i wish i could bring back.  \t0\n",
      "\n",
      "an hour and a half i wish i could bring back  \t0\n",
      "\n",
      "i liked this movie way too much.  \t1\n",
      "\n",
      "i liked this movie way too much  \t1\n",
      "\n",
      "this is definitely a cult classic well worth viewing and sharing with others.  \t1\n",
      "\n",
      "this is definitely a cult classic well worth viewing and sharing with others  \t1\n",
      "\n",
      "awful.  \t0\n",
      "\n",
      "awful  \t0\n",
      "\n",
      "the lead man is charisma-free.  \t0\n",
      "\n",
      "the lead man is charisma-free  \t0\n",
      "\n",
      "so bad.  \t0\n",
      "\n",
      "so bad  \t0\n",
      "\n",
      "ironically i mostly find his films a total waste of time to watch.  \t0\n",
      "\n",
      "ironically i mostly find his films a total waste of time to watch  \t0\n",
      "\n",
      "do not waste your time.  \t0\n",
      "\n",
      "do not waste your time  \t0\n",
      "\n",
      "the soundtrack sucked.  \t0\n",
      "\n",
      "the soundtrack sucked  \t0\n",
      "\n",
      "it is an excellent drama, an excellent thriller, and an excellent film.  \t1\n",
      "\n",
      "it is an excellent drama an excellent thriller and an excellent film  \t1\n",
      "\n",
      "really bad.  \t0\n",
      "\n",
      "really bad  \t0\n",
      "\n",
      "in a most wonderful location lies a story of contrast.  \t1\n",
      "\n",
      "in a most wonderful location lies a story of contrast  \t1\n",
      "\n",
      "i am so thrilled after seeing a movie like this.  \t1\n",
      "\n",
      "i am so thrilled after seeing a movie like this  \t1\n",
      "\n",
      "it is very educational for children 1 to 8 years old.  \t1\n",
      "\n",
      "it is very educational for children 1 to 8 years old  \t1\n",
      "\n",
      "this is one of the best movies out there that shows such strong sibling bond for each other.  \t1\n",
      "\n",
      "this is one of the best movies out there that shows such strong sibling bond for each other  \t1\n",
      "\n",
      "this totally unfunny movie is so over the top and pathetic and unrealistic that throughout the whole 90 minutes of utter torture i probably looked at my watch about 70000 times!  \t0\n",
      "\n",
      "this totally unfunny movie is so over the top and pathetic and unrealistic that throughout the whole 90 minutes of utter torture i probably looked at my watch about 70000 times  \t0\n",
      "\n",
      "the most annoying thing about 'cover girl' is the way in which rita hayworth is put up on a pedestal.  \t0\n",
      "\n",
      "the most annoying thing about 'cover girl' is the way in which rita hayworth is put up on a pedestal  \t0\n",
      "\n",
      "i would have casted her in that role after ready the script.  \t1\n",
      "\n",
      "i would have casted her in that role after ready the script  \t1\n",
      "\n",
      "in short - this was a monumental waste of time and energy and i would not recommend anyone to ever see this film.  \t0\n",
      "\n",
      "in short - this was a monumental waste of time and energy and i would not recommend anyone to ever see this film  \t0\n",
      "\n",
      "wow, what a bad film.  \t0\n",
      "\n",
      "wow what a bad film  \t0\n",
      "\n",
      "i knew when i saw the film that more great things were to come from this gifted actor.  \t1\n",
      "\n",
      "i knew when i saw the film that more great things were to come from this gifted actor  \t1\n",
      "\n",
      "overall i rate this movie a 10 out of a 1-10 scale.  \t1\n",
      "\n",
      "overall i rate this movie a 10 out of a 1-10 scale  \t1\n",
      "\n",
      "cinematography noteworthy including fine views of barcelona and its famed gaudi towers.  \t1\n",
      "\n",
      "cinematography noteworthy including fine views of barcelona and its famed gaudi towers  \t1\n",
      "\n",
      "rating: 0/10 (grade: z) note: the show is so bad that even mother of the cast pull her daughter out of the show.  \t0\n",
      "\n",
      "rating 010 grade z note the show is so bad that even mother of the cast pull her daughter out of the show  \t0\n",
      "\n",
      "steamboat willie is an amazingly important film to our cinema history.  \t1\n",
      "\n",
      "steamboat willie is an amazingly important film to our cinema history  \t1\n",
      "\n",
      "the script is a big flawed mess.  \t0\n",
      "\n",
      "the script is a big flawed mess  \t0\n",
      "\n",
      "lewis black's considerable talent is wasted here too, as he is at his most incendiary when he is unrestrained, which the pg-13 rating certainly won't allow.  \t0\n",
      "\n",
      "lewis black's considerable talent is wasted here too as he is at his most incendiary when he is unrestrained which the pg-13 rating certainly won't allow  \t0\n",
      "\n",
      "the memories are murky but i can only say that i enjoyed every single episode and product related to the show.  \t1\n",
      "\n",
      "the memories are murky but i can only say that i enjoyed every single episode and product related to the show  \t1\n",
      "\n",
      "the characters are interesting and you want to find out more about them the longer the movie goes on, and i think people will be surprised by who does and doesn't make it.  \t1\n",
      "\n",
      "the characters are interesting and you want to find out more about them the longer the movie goes on and i think people will be surprised by who does and doesn't make it  \t1\n",
      "\n",
      "i had always known that errol flynn was a brilliant actor as he was my dads favourite actor, and i grew up watching his films as a child.  \t1\n",
      "\n",
      "i had always known that errol flynn was a brilliant actor as he was my dads favourite actor and i grew up watching his films as a child  \t1\n",
      "\n",
      "this movie does an excellent job of revealing the complexity of the task and the incredible challenges facing south africa.  \t1\n",
      "\n",
      "this movie does an excellent job of revealing the complexity of the task and the incredible challenges facing south africa  \t1\n",
      "\n",
      "if you do watch it, however, there are small consolations: the actresses playing anne's sisters each do a wonderful job with their roles.  \t1\n",
      "\n",
      "if you do watch it however there are small consolations the actresses playing anne's sisters each do a wonderful job with their roles  \t1\n",
      "\n",
      "this is definitely one of the better documentaries i have seen looking at family relationships and marriage.  \t1\n",
      "\n",
      "this is definitely one of the better documentaries i have seen looking at family relationships and marriage  \t1\n",
      "\n",
      "but she is still a bad actress, repeating her robotic face moves in each of her pictures.  \t0\n",
      "\n",
      "but she is still a bad actress repeating her robotic face moves in each of her pictures  \t0\n",
      "\n",
      "that's how i'd describe this painfully dreary time-waster of a film.  \t0\n",
      "\n",
      "that's how i'd describe this painfully dreary time-waster of a film  \t0\n",
      "\n",
      "the plot has more holes than a pair of fishnet stockings and the direction and editing is astonishingly ham fisted.  \t0\n",
      "\n",
      "the plot has more holes than a pair of fishnet stockings and the direction and editing is astonishingly ham fisted  \t0\n",
      "\n",
      "this early film from future goremeister lucio fulci is a very good addition to the giallo sub-genre.  \t1\n",
      "\n",
      "this early film from future goremeister lucio fulci is a very good addition to the giallo sub-genre  \t1\n",
      "\n",
      "also the music by mark snow is possibly the best score i've ever heard.  \t1\n",
      "\n",
      "also the music by mark snow is possibly the best score i've ever heard  \t1\n",
      "\n",
      "the film is well paced, understated and one of the best courtroom documentaries i've seen.  \t1\n",
      "\n",
      "the film is well paced understated and one of the best courtroom documentaries i've seen  \t1\n",
      "\n",
      "of course the footage from the 70s was grainy, but that only enhanced the film.  \t1\n",
      "\n",
      "of course the footage from the 70s was grainy but that only enhanced the film  \t1\n",
      "\n",
      "juano hernandez (an exceptional actor who played supporting roles in many films of the era) is a proud black man who is accused of murdering a white man in the south.  \t1\n",
      "\n",
      "juano hernandez an exceptional actor who played supporting roles in many films of the era is a proud black man who is accused of murdering a white man in the south  \t1\n",
      "\n",
      "great character actors telly savalas and peter boyle.  \t1\n",
      "\n",
      "great character actors telly savalas and peter boyle  \t1\n",
      "\n",
      "the scripting of the subtle comedy is unmatched by any movie in recent years.  \t1\n",
      "\n",
      "the scripting of the subtle comedy is unmatched by any movie in recent years  \t1\n",
      "\n",
      "the acting sucks, the music sucks, the script sucks, the pacing sucks, the special fx suck, the directing sucks... basically, this movie sucks.  \t0\n",
      "\n",
      "the acting sucks the music sucks the script sucks the pacing sucks the special fx suck the directing sucks basically this movie sucks  \t0\n",
      "\n",
      "otherwise, don't even waste your time on this.  \t0\n",
      "\n",
      "otherwise don't even waste your time on this  \t0\n",
      "\n",
      "i agree with jessica, this movie is pretty bad.  \t0\n",
      "\n",
      "i agree with jessica this movie is pretty bad  \t0\n",
      "\n",
      "the two main characters may be two of the most believable children i ever saw put on screen.  \t1\n",
      "\n",
      "the two main characters may be two of the most believable children i ever saw put on screen  \t1\n",
      "\n",
      "it's too bad that everyone else involved didn't share crowe's level of dedication to quality, for if they did, we'd have a far better film on our hands than this sub-par mess.  \t0\n",
      "\n",
      "it's too bad that everyone else involved didn't share crowe's level of dedication to quality for if they did we'd have a far better film on our hands than this sub-par mess  \t0\n",
      "\n",
      "if you act in such a film, you should be glad that you're gonna drift away from earth as far as possible!  \t0\n",
      "\n",
      "if you act in such a film you should be glad that you're gonna drift away from earth as far as possible  \t0\n",
      "\n",
      "the opening sequence of this gem is a classic, and the cat n mouse games that follow are a delight to watch.  \t1\n",
      "\n",
      "the opening sequence of this gem is a classic and the cat n mouse games that follow are a delight to watch  \t1\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import io\n",
    "X = []\n",
    "with io.open(data_file, encoding='utf-8', errors='ignore') as f:\n",
    "    idx = 0\n",
    "    for line in f:\n",
    "        line = line.lower()\n",
    "        if idx%18 == 0:\n",
    "            print(line)\n",
    "        line = re.sub(r\"[()\\\"#/@;:<>{}`+=~|.!?,]\", \"\", line)\n",
    "        if idx%18 == 0:\n",
    "            print(line)\n",
    "        new_line = line.split()\n",
    "        new_line = [word.strip() for word in new_line if word != '']\n",
    "        tokenized_line = vocab.encode(new_line[:-1])\n",
    "#         print(new_line[-1])\n",
    "        X.append(torch.cat((tokenized_line, torch.tensor([float(new_line[-1])], dtype=torch.long, device=device)), dim=0))\n",
    "        idx += 1\n",
    "# train, test = batcher(X, 15, 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "train, test = batcher(X, 15, 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([    7,   191,   191,   191, 81023, 57837,  1005,    59,     7, 16108,\n",
       "        17674,   461,   300,     0], device='cuda:0')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[     7,     36,   4997],\n",
      "        [   191,   1085, 307248],\n",
      "        [   191,     38,     17],\n",
      "        [   191,     15,    521],\n",
      "        [ 81023,     56,   1161],\n",
      "        [ 57837,    402,    298],\n",
      "        [  1005,     11,      5],\n",
      "        [    59,      0,  11114],\n",
      "        [     7,   2983,   3534],\n",
      "        [ 16108,   2153,  13352],\n",
      "        [ 17674,     46,      0],\n",
      "        [   461,      0,   1005],\n",
      "        [   300,   2052,   4254],\n",
      "        [     0,    573,     11],\n",
      "        [     0,    343,    302]], device='cuda:0') tensor([0, 0, 0], device='cuda:0') tensor([[1, 1, 1],\n",
      "        [1, 1, 1],\n",
      "        [1, 1, 1],\n",
      "        [1, 1, 1],\n",
      "        [1, 1, 1],\n",
      "        [1, 1, 1],\n",
      "        [1, 1, 1],\n",
      "        [1, 1, 1],\n",
      "        [1, 1, 1],\n",
      "        [1, 1, 1],\n",
      "        [1, 1, 1],\n",
      "        [1, 1, 1],\n",
      "        [1, 1, 1],\n",
      "        [0, 1, 1],\n",
      "        [0, 1, 1]], device='cuda:0') [13, 15, 15]\n"
     ]
    }
   ],
   "source": [
    "print(train[0][0][:,:3], train[0][1][:3], train[0][2][:,:3], train[0][3][:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from classes import ELMo, biLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "model_ELMo = ELMo(50, vocab.embedding, n_layers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class Task(nn.Module):\n",
    "    def __init__(self, input_size, output_size):\n",
    "        super(Task, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.output_size = output_size\n",
    "        self.layer = nn.LSTM(input_size, output_size)\n",
    "        \n",
    "    def forward(self, input_seq, input_lengths):\n",
    "        \n",
    "        batch_size = input_seq.size()[1]\n",
    "        outputs = torch.zeros(batch_size, self.output_size)\n",
    "        for i in range(batch_size):\n",
    "#             hidden = (torch.zeros(), None)\n",
    "            output, _ = self.layer(input_seq[:input_lengths[i], i, :].unsqueeze(0))\n",
    "            outputs[i] = output[-1,0,:]\n",
    "        return outputs\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "model_task = Task(50, 2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "model_ELMo = model_ELMo.to(device)\n",
    "model_task = model_task.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "clip = 50.0\n",
    "learning_rate = 0.002\n",
    "n_iteration = 30\n",
    "print_every = 1\n",
    "\n",
    "model_ELMo.train()\n",
    "model_task.train()\n",
    "\n",
    "ELMo_opt = optim.Adam(model_ELMo.parameters(), lr=learning_rate)\n",
    "task_opt = optim.Adam(model_task.parameters(), lr=learning_rate)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "list_loss = []\n",
    "accuracy = []\n",
    "\n",
    "def make_confusion_matrix(true, pred):\n",
    "    K = len(np.unique(true)) # Number of classes \n",
    "    result = np.zeros((K, K))\n",
    "\n",
    "    for i in range(len(true)):\n",
    "        result[true[i]][pred[i]] += 1\n",
    "\n",
    "    return result\n",
    "train_pred = []\n",
    "train_true = []\n",
    "test_pred = []\n",
    "test_true = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss for iteration =  0 is  0.695381687238 and accuracy is =  0.52125\n",
      "Loss for iteration =  1 is  0.690268745789 and accuracy is =  0.5425\n",
      "Loss for iteration =  2 is  0.687534213066 and accuracy is =  0.5625\n",
      "Loss for iteration =  3 is  0.684150232719 and accuracy is =  0.575\n",
      "Loss for iteration =  4 is  0.678868206648 and accuracy is =  0.5875\n",
      "Loss for iteration =  5 is  0.669945693933 and accuracy is =  0.6275\n",
      "Loss for iteration =  6 is  0.659065150298 and accuracy is =  0.635\n",
      "Loss for iteration =  7 is  0.631867903929 and accuracy is =  0.68125\n",
      "Loss for iteration =  8 is  0.604190033216 and accuracy is =  0.70875\n",
      "Loss for iteration =  9 is  0.569348660799 and accuracy is =  0.74125\n"
     ]
    }
   ],
   "source": [
    "n_iteration = 10\n",
    "for iteration in range(n_iteration):\n",
    "    for batch in train:\n",
    "        ELMo_opt.zero_grad()\n",
    "        task_opt.zero_grad()\n",
    "        \n",
    "        train_X, train_Y, mask, lengths = batch\n",
    "        train_X = train_X.to(device)\n",
    "        train_Y = train_Y.to(device)\n",
    "        mask = mask.to(device)\n",
    "        lengths = torch.tensor(lengths)\n",
    "        lengths = lengths.to(device)\n",
    "        \n",
    "        ELMo_embedding, _ = model_ELMo(train_X, lengths, mask)\n",
    "        predictions = model_task(ELMo_embedding, lengths)\n",
    "        predictions = predictions.to(device)\n",
    "        lossF = nn.CrossEntropyLoss()\n",
    "        loss = lossF(predictions, train_Y)\n",
    "        loss = loss.to(device)\n",
    "        pred_class = torch.argmax(predictions, dim=1)\n",
    "        correct_guess = sum([1 if pred == act else 0 for pred, act in zip(pred_class, train_Y)])\n",
    "        accuracy.append((correct_guess, lengths.size()[0]))\n",
    "        loss.backward(retain_graph=True)\n",
    "        _ = nn.utils.clip_grad_norm_(model_ELMo.parameters(), clip)\n",
    "        _ = nn.utils.clip_grad_norm_(model_task.parameters(), clip)\n",
    "        train_pred += pred_class.tolist()\n",
    "        train_true += train_Y.tolist()\n",
    "        ELMo_opt.step()\n",
    "        task_opt.step()\n",
    "        list_loss.append(loss.item())\n",
    "    total_correct = sum([tp[0] for tp in accuracy[-(len(train)):]])\n",
    "    total_preds = sum([tp[1] for tp in accuracy[-(len(train)):]])\n",
    "    cur_acc = total_correct/total_preds\n",
    "    loss_over_data = sum(list_loss[-(len(train)):])/len(train)\n",
    "    print(\"Loss for iteration = \", iteration, \"is \", loss_over_data, \"and accuracy is = \", cur_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_mat = make_confusion_matrix(train_true[-800:], train_pred[-800:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "304.0 110.0\n",
      "97.0 289.0\n"
     ]
    }
   ],
   "source": [
    "for i in range(2):\n",
    "    print(train_mat[i][0], train_mat[i][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test accuracy =  0.57\n"
     ]
    }
   ],
   "source": [
    "test_accuracy = []\n",
    "for batch_n in range(len(test)):\n",
    "    test_X, test_Y, mask, lengths = test[batch_n]\n",
    "    test_X = test_X.to(device)\n",
    "    test_Y = test_Y.to(device)\n",
    "    mask = mask.to(device)\n",
    "    lengths = torch.tensor(lengths, device=device)\n",
    "    \n",
    "    ELMo_embedding, _ = model_ELMo(test_X, lengths, mask)\n",
    "    predictions = model_task(ELMo_embedding, lengths)\n",
    "    predictions = predictions.to(device)\n",
    "    pred_class = torch.argmax(predictions, dim=1)\n",
    "    test_true += test_Y.tolist()\n",
    "    test_pred += pred_class.tolist()\n",
    "    correct_guess = sum([1 if pred == act else 0 for pred, act in zip(pred_class, test_Y)])\n",
    "    test_accuracy.append((correct_guess, lengths.size()[0]))\n",
    "total_correct = sum([tp[0] for tp in test_accuracy[-(len(test)):]])\n",
    "total_preds = sum([tp[1] for tp in test_accuracy[-(len(test)):]])\n",
    "\n",
    "print(\"test accuracy = \", total_correct/total_preds)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}