sentiment_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import pickle\n",
    "from tqdm import tqdm\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = set()\n",
    "labels = []\n",
    "raw_sents = []\n",
    "for line in open('yelp_labelled.txt'):\n",
    "    line = line.strip()\n",
    "    sent, label = line.split('\\t')\n",
    "    words = sent.split(' ')\n",
    "    raw_sents.append(words)\n",
    "    labels.append(int(label))\n",
    "    for word in words:\n",
    "        vocab.add(word.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(raw_sents)):\n",
    "    for j in range(len(raw_sents[i])):\n",
    "        raw_sents[i][j]=raw_sents[i][j].lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['wow...', 'loved', 'this', 'place.'],\n",
       " ['crust', 'is', 'not', 'good.'],\n",
       " ['not', 'tasty', 'and', 'the', 'texture', 'was', 'just', 'nasty.'],\n",
       " ['stopped',\n",
       "  'by',\n",
       "  'during',\n",
       "  'the',\n",
       "  'late',\n",
       "  'may',\n",
       "  'bank',\n",
       "  'holiday',\n",
       "  'off',\n",
       "  'rick',\n",
       "  'steve',\n",
       "  'recommendation',\n",
       "  'and',\n",
       "  'loved',\n",
       "  'it.'],\n",
       " ['the',\n",
       "  'selection',\n",
       "  'on',\n",
       "  'the',\n",
       "  'menu',\n",
       "  'was',\n",
       "  'great',\n",
       "  'and',\n",
       "  'so',\n",
       "  'were',\n",
       "  'the',\n",
       "  'prices.']]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_sents[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['acknowledged,',\n",
       " 'acknowledged.',\n",
       " 'across.',\n",
       " 'actual',\n",
       " 'actually',\n",
       " 'actually.',\n",
       " 'added',\n",
       " 'affordable',\n",
       " 'after',\n",
       " 'afternoon!']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = ['<PAD>'] + sorted(list(vocab))\n",
    "vocab[100:110]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert words to their index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "indexed_sents = []\n",
    "for cur_raw_sent in raw_sents:\n",
    "    cur_indexed_sent = []\n",
    "    for word in cur_raw_sent:\n",
    "        cur_indexed_sent.append(vocab.index(word))\n",
    "    indexed_sents.append(cur_indexed_sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[2697, 1415, 2418, 1805],\n",
       " [594, 1286, 1613, 1054],\n",
       " [1613, 2355, 151, 2389, 2378, 2602, 1320, 1576],\n",
       " [2273,\n",
       "  414,\n",
       "  769,\n",
       "  2389,\n",
       "  1349,\n",
       "  1466,\n",
       "  251,\n",
       "  1182,\n",
       "  1632,\n",
       "  1995,\n",
       "  2263,\n",
       "  1941,\n",
       "  151,\n",
       "  1415,\n",
       "  1293],\n",
       " [2389, 2094, 1649, 2389, 1502, 2602, 1068, 151, 2184, 2637, 2389, 1866]]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indexed_sents[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find max length / Assign max length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_len = -1\n",
    "for cur_indexed_sent in indexed_sents:\n",
    "    max_len = max(max_len, len(cur_indexed_sent))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_len"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# padded_sents = []\n",
    "# for cur_indexed_sent in indexed_sents:\n",
    "#     cur_padded_sent = cur_indexed_sent\n",
    "#     cur_sent_len = len(cur_indexed_sent)\n",
    "#     for i in range(cur_sent_len, max_len):\n",
    "#         cur_padded_sent.append(0)\n",
    "#     padded_sents.append(cur_padded_sent)\n",
    "    \n",
    "padded_sents = []\n",
    "for cur_indexed_sent in indexed_sents:\n",
    "    cur_padded_sent=[]\n",
    "    for x in range(max_len-len(cur_indexed_sent)):\n",
    "        cur_padded_sent.append(0)\n",
    "    for words in cur_indexed_sent:\n",
    "        cur_padded_sent.append(words)\n",
    "    padded_sents.append(cur_padded_sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  2697,\n",
       "  1415,\n",
       "  2418,\n",
       "  1805],\n",
       " [0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  594,\n",
       "  1286,\n",
       "  1613,\n",
       "  1054],\n",
       " [0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  1613,\n",
       "  2355,\n",
       "  151,\n",
       "  2389,\n",
       "  2378,\n",
       "  2602,\n",
       "  1320,\n",
       "  1576],\n",
       " [0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  2273,\n",
       "  414,\n",
       "  769,\n",
       "  2389,\n",
       "  1349,\n",
       "  1466,\n",
       "  251,\n",
       "  1182,\n",
       "  1632,\n",
       "  1995,\n",
       "  2263,\n",
       "  1941,\n",
       "  151,\n",
       "  1415,\n",
       "  1293],\n",
       " [0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  2389,\n",
       "  2094,\n",
       "  1649,\n",
       "  2389,\n",
       "  1502,\n",
       "  2602,\n",
       "  1068,\n",
       "  151,\n",
       "  2184,\n",
       "  2637,\n",
       "  2389,\n",
       "  1866]]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "padded_sents[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "32\n"
     ]
    }
   ],
   "source": [
    "print(len(raw_sents[0]))\n",
    "print(len(padded_sents[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert data to numpy and torch tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(padded_sents)\n",
    "Y = np.array(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000,)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = torch.from_numpy(X).cuda(device=1)\n",
    "Y = torch.from_numpy(Y).long().cuda(device=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_size = X.size(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_size = int(data_size * 0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainX = X[:train_size]\n",
    "trainY = Y[:train_size]\n",
    "testX = X[train_size:]\n",
    "testY = Y[train_size:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load embeddings from glove 50d "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import KeyedVectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = KeyedVectors.load_word2vec_format('/home/mtinterndec18/parth/nematus_system/glove.6B/glove_word2vec_50d.txt', binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "unk_vector = embeddings.vectors.mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectors = [np.zeros(embeddings.vector_size)]\n",
    "for word in vocab[1:]:\n",
    "    if word in embeddings.vocab:\n",
    "        vectors.append(embeddings[word])\n",
    "      \n",
    "    else:\n",
    "        vectors.append(unk_vector)\n",
    "\n",
    "vectors = np.array(vectors, dtype=np.float32)\n",
    "vectors = torch.from_numpy(vectors).cuda(device=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2729, 50])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectors.size()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Network(nn.Module):\n",
    "    \n",
    "    def __init__(self, max_seq_len, embed_dim, hidden_dim, output_dim):\n",
    "        super(Network, self).__init__()\n",
    "        self.max_seq_len = max_seq_len\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.embed_dim = embed_dim\n",
    "        self.output_dim = output_dim\n",
    "        self.embedder = nn.Embedding(len(vocab), self.embed_dim)\n",
    "        self.embedder.weight.data.copy_(vectors)\n",
    "        self.fc1 = nn.Linear(self.max_seq_len * self.embed_dim, self.hidden_dim)\n",
    "        self.relu1 = nn.ReLU()\n",
    "        self.fc2 = nn.Linear(self.hidden_dim, self.output_dim)\n",
    "        \n",
    "    def embed(self, x):\n",
    "        return self.embedder(x)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.embedder(x)\n",
    "#         print(x.size())\n",
    "        x = x.view(-1, self.max_seq_len * self.embed_dim)\n",
    "#         print(x.size())\n",
    "        x = self.fc1(x)\n",
    "        x = self.relu1(x)\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Network(\n",
       "  (embedder): Embedding(2729, 50)\n",
       "  (fc1): Linear(in_features=1600, out_features=200, bias=True)\n",
       "  (relu1): ReLU()\n",
       "  (fc2): Linear(in_features=200, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Network(max_len, 50, 200, 2)\n",
    "model.cuda(device=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  0  loss =  0.696826696395874\n",
      "Epoch  10  loss =  0.4058898091316223\n",
      "Epoch  20  loss =  0.20926892757415771\n",
      "Epoch  30  loss =  0.08637973666191101\n",
      "Epoch  40  loss =  0.02816195599734783\n",
      "Epoch  50  loss =  0.008785173296928406\n",
      "Epoch  60  loss =  0.003401532769203186\n",
      "Epoch  70  loss =  0.0018086537020280957\n",
      "Epoch  80  loss =  0.0011982298456132412\n",
      "Epoch  90  loss =  0.0009034023969434202\n",
      "Epoch  100  loss =  0.0007315491093322635\n",
      "Epoch  110  loss =  0.0006160555640235543\n",
      "Epoch  120  loss =  0.0005308306426741183\n",
      "Epoch  130  loss =  0.00046429241774603724\n",
      "Epoch  140  loss =  0.00041044660611078143\n",
      "Epoch  150  loss =  0.0003659244976006448\n",
      "Epoch  160  loss =  0.00032849106355570257\n",
      "Epoch  170  loss =  0.0002966121246572584\n",
      "Epoch  180  loss =  0.00026921884273178875\n",
      "Epoch  190  loss =  0.0002454982604831457\n"
     ]
    }
   ],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "num_epochs = 200\n",
    "for epoch_cntr in range(num_epochs):\n",
    "    optimizer.zero_grad()\n",
    "    trainO = model(trainX)\n",
    "    loss = criterion(trainO, trainY)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    if epoch_cntr % 10 == 0:\n",
    "        print('Epoch ', epoch_cntr, ' loss = ', loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training score:  1.0\n",
      "test score:  0.6966666666666667\n"
     ]
    }
   ],
   "source": [
    "trainO = model(trainX)\n",
    "trainP = torch.argmax(trainO, dim=1)\n",
    "\n",
    "print(\"training score: \",(trainY == trainP).sum().item() / trainY.size(0))\n",
    "\n",
    "testO = model(testX)\n",
    "testP = torch.argmax(testO, dim=1)\n",
    "\n",
    "print(\"test score: \",(testY == testP).sum().item() / testY.size(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_sent(sent):\n",
    "    sent = sent.strip()\n",
    "    words = sent.split(' ')\n",
    "    padded_sents = []\n",
    "    indexed_sent = []\n",
    "    for word in words:\n",
    "        indexed_sent.append(vocab.index(word.lower()))\n",
    "    indexed_len = len(indexed_sent)\n",
    "    for x in range(max_len-len(indexed_sent)):\n",
    "        padded_sents.append(0)\n",
    "    for word in indexed_sent:\n",
    "        padded_sents.append(word)\n",
    "    x = np.array(padded_sents)\n",
    "    x = torch.from_numpy(x).cuda(device=1)\n",
    "    return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sent = encode_sent('food quality is good')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "           0,    0,    0,    0,  957, 1899, 1286, 1050], device='cuda:1')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sent_output = model(test_sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sent_pred = torch.argmax(test_sent_output, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1.2057,  1.3180]], device='cuda:1', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_sent_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1], device='cuda:1')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_sent_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 50])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.embed(test_sent).size()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RecurrentNetwork(nn.Module):\n",
    "    \n",
    "    def __init__(self, max_seq_len, embed_dim, hidden_dim, hidden_state_dim, output_dim):\n",
    "        super(RecurrentNetwork, self).__init__()\n",
    "        self.max_seq_len = max_seq_len\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.hidden_state_dim = hidden_state_dim\n",
    "        self.embed_dim = embed_dim\n",
    "        self.output_dim = output_dim\n",
    "        self.embedder = nn.Embedding(len(vocab), self.embed_dim)\n",
    "        self.embedder.weight.data.copy_(vectors)\n",
    "        \n",
    "        self.U = nn.Linear(self.hidden_state_dim, self.hidden_state_dim)\n",
    "        self.W = nn.Linear(self.embed_dim, self.hidden_state_dim)\n",
    "        self.relu1 = nn.ReLU()\n",
    "        self.relu2 = nn.ReLU()\n",
    "        self.fc1 = nn.Linear(self.hidden_state_dim, self.hidden_dim)\n",
    "        self.fc2 = nn.Linear(self.hidden_dim, self.output_dim)\n",
    "        \n",
    "    def embed(self, x):\n",
    "        return self.embedder(x)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \n",
    "        x = self.embedder(x)\n",
    "        h = torch.zeros(x.size(0), self.hidden_state_dim).cuda(device=1)\n",
    "        print(x.size())\n",
    "        print(h.size())\n",
    "        for i in range(x.size(1)):\n",
    "            h = self.relu1(self.U(h) + self.W(x[:, i, :]))\n",
    "        h = self.fc1(h)\n",
    "        h = self.relu2(h)\n",
    "        h = self.fc2(h)\n",
    "        return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RecurrentNetwork(\n",
       "  (embedder): Embedding(2729, 50)\n",
       "  (U): Linear(in_features=200, out_features=200, bias=True)\n",
       "  (W): Linear(in_features=50, out_features=200, bias=True)\n",
       "  (relu1): ReLU()\n",
       "  (relu2): ReLU()\n",
       "  (fc1): Linear(in_features=200, out_features=100, bias=True)\n",
       "  (fc2): Linear(in_features=100, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rec_model = RecurrentNetwork(max_len, 50, 100, 200, 2)\n",
    "rec_model.cuda(device=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([700, 32, 50])\n",
      "torch.Size([700, 200])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0383, -0.0237],\n",
       "        [-0.0305, -0.0409],\n",
       "        [-0.0476, -0.0326],\n",
       "        ...,\n",
       "        [-0.0678, -0.0436],\n",
       "        [-0.0696, -0.0088],\n",
       "        [-0.0144, -0.0199]], device='cuda:1', grad_fn=<AddmmBackward>)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rec_model(trainX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  0  loss =  0.00022482668282464147\n",
      "Epoch  10  loss =  2.4252960429294035e-05\n",
      "Epoch  20  loss =  5.497251095221145e-06\n",
      "Epoch  30  loss =  2.034051021837513e-06\n",
      "Epoch  40  loss =  1.2949534493600368e-06\n",
      "Epoch  50  loss =  9.768349400474108e-07\n",
      "Epoch  60  loss =  8.181163480003306e-07\n",
      "Epoch  70  loss =  7.172993150561524e-07\n",
      "Epoch  80  loss =  6.437301749429025e-07\n",
      "Epoch  90  loss =  5.742481903325825e-07\n",
      "Epoch  100  loss =  5.211148845774005e-07\n",
      "Epoch  110  loss =  4.7206879116856726e-07\n",
      "Epoch  120  loss =  4.352841926902329e-07\n",
      "Epoch  130  loss =  3.9645604488214303e-07\n",
      "Epoch  140  loss =  3.6852699736300565e-07\n",
      "Epoch  150  loss =  3.3719197745085694e-07\n",
      "Epoch  160  loss =  3.1335014227806823e-07\n",
      "Epoch  170  loss =  2.9359546260820935e-07\n",
      "Epoch  180  loss =  2.711159936552576e-07\n",
      "Epoch  190  loss =  2.5136131398539874e-07\n"
     ]
    }
   ],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "num_epochs = 200\n",
    "for epoch_cntr in range(num_epochs):\n",
    "    optimizer.zero_grad()\n",
    "    trainO = model(trainX)\n",
    "    loss = criterion(trainO, trainY)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    if epoch_cntr % 10 == 0:\n",
    "        print('Epoch ', epoch_cntr, ' loss = ', loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training score:  1.0\n",
      "test score:  0.6866666666666666\n"
     ]
    }
   ],
   "source": [
    "trainO = model(trainX)\n",
    "trainP = torch.argmax(trainO, dim=1)\n",
    "\n",
    "print('training score: ',(trainY == trainP).sum().item() / trainY.size(0))\n",
    "\n",
    "testO = model(testX)\n",
    "testP = torch.argmax(testO, dim=1)\n",
    "\n",
    "print(\"test score: \",(testY == testP).sum().item() / testY.size(0))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}