cnn_lstm_imdb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hZyvmRAkMjyq",
    "outputId": "68d74e41-0ce1-42ea-e443-e9862ae20d43"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/gdrive/; to attempt to forcibly remount, call drive.mount(\"/content/gdrive/\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive/')\n",
    "import sys\n",
    "import os\n",
    "prefix = '/content/gdrive/My Drive/'\n",
    "# modify \"customized_path_to_your_homework\" here to where you uploaded your homework\n",
    "customized_path_to_your_homework = 'IDLSProject-main'\n",
    "sys_path = os.path.join(prefix, customized_path_to_your_homework)\n",
    "sys.path.append(sys_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3dAGpxT0M6b0",
    "outputId": "d9ff80ed-d35c-4b2a-ae1e-f64fd1a03efd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/gdrive/My Drive/IDLSProject-main\n"
     ]
    }
   ],
   "source": [
    "%cd '/content/gdrive/My Drive/IDLSProject-main'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3Ww4RNgRM-z3",
    "outputId": "11d3a213-f60f-4b61-9ff8-171172d8884f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/gdrive/My Drive/IDLSProject-main\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "DERqULIrzfno"
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4D3E9ODhLLBz",
    "outputId": "26602ba2-7524-4950-d431-12bdee5d3762"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10000, 1])\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import pickle\n",
    "import torch\n",
    "import torch.utils.data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "data_dir = './data/pytorch'\n",
    "with open(os.path.join(data_dir, 'word_dict_qrnn.pkl'), \"rb\") as f:\n",
    "    word_dict = pickle.load(f)\n",
    "\n",
    "train = pd.read_csv(os.path.join(data_dir, 'train_qrnn.csv'), header=None, names=None)\n",
    "test_sample = pd.read_csv(os.path.join(data_dir, 'test_qrnn.csv'), header=None, names=None)\n",
    "\n",
    "test, val = train_test_split(test_sample, test_size=0.5)\n",
    "train.shape, test.shape, val.shape\n",
    "\n",
    "\n",
    "# Turn the input pandas dataframe into tensors\n",
    "train_y = torch.from_numpy(train[[0]].values).float()\n",
    "train_X = torch.from_numpy(train.drop([0, 1], axis=1).values).long()\n",
    "\n",
    "# Build the dataset\n",
    "train_ds = torch.utils.data.TensorDataset(train_X, train_y)\n",
    "# Build the dataloader\n",
    "train_dl = torch.utils.data.DataLoader(train_ds, batch_size=50)\n",
    "\n",
    "######val data\n",
    "# Turn the input pandas dataframe into tensors\n",
    "val_y = torch.from_numpy(val[[0]].values).float()\n",
    "val_X = torch.from_numpy(val.drop([0, 1], axis=1).values).long()\n",
    "\n",
    "# Build the dataset\n",
    "val_ds = torch.utils.data.TensorDataset(val_X, val_y)\n",
    "# Build the dataloader\n",
    "val_dl = torch.utils.data.DataLoader(val_ds, batch_size=50)\n",
    "\n",
    "\n",
    "#### Test data\n",
    "# Turn the input pandas dataframe into tensors\n",
    "test_y = torch.from_numpy(test[[0]].values).float()\n",
    "test_X = torch.from_numpy(test.drop([0, 1], axis=1).values).long()\n",
    "\n",
    "# Build the dataset\n",
    "test_ds = torch.utils.data.TensorDataset(test_X, test_y)\n",
    "# Build the dataloader\n",
    "test_dl = torch.utils.data.DataLoader(test_ds, batch_size=50)\n",
    "print(test_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "IsIBQGYpPtRh"
   },
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "epochs = 10\n",
    "batch_size = 50\n",
    "learning_rate = 0.001\n",
    "seq_len = 500\n",
    "dropout = 0.5\n",
    "filter_size = 100\n",
    "vocab_size = 5000\n",
    "embed_dims = 32\n",
    "hidden_size = 100\n",
    "kernel_size = [3,4,5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "2vzS4JGiOAGX"
   },
   "outputs": [],
   "source": [
    "# write to file.\n",
    "filename = \"cnn-lstm-imdb.csv\"\n",
    "def write_to_csv(epochs, train_loss, train_acc, val_loss, val_acc, time_train):\n",
    "    epoch = [i for i in range(epochs)]\n",
    "    df_metrics = pd.DataFrame(list(zip(epoch, train_loss, train_acc, val_loss, val_acc, time_train)), columns =['Epoch', 'train_loss', 'train_acc', 'val_loss', 'val_acc', 'train_time'])\n",
    "    df_metrics.to_csv(filename, index=False)\n",
    "    \n",
    "def append_to_csv(epochs, accuracy):\n",
    "    acc = [accuracy for i in range(epochs)]\n",
    "    df_csv = pd.read_csv(filename)\n",
    "    df_csv['Test_Accuracy']  = acc\n",
    "    df_csv.to_csv(filename, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "Tj3j2VvMO5Yb"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "class Combine(nn.Module):\n",
    "    def __init__(self,vocab_size, embed_size, filter_size, kernel_size, dropout, seq_len,hidden_size):\n",
    "        super(Combine, self).__init__()\n",
    "        self.embedding = torch.nn.Embedding(vocab_size,embed_size)\n",
    "        self.conv1 = torch.nn.Conv2d(1,filter_size,kernel_size=[kernel_size[0],embed_size])\n",
    "        self.conv2 = torch.nn.Conv2d(1,filter_size,kernel_size=[kernel_size[1],embed_size])\n",
    "        self.conv3 = torch.nn.Conv2d(1,filter_size,kernel_size=[kernel_size[2],embed_size])\n",
    "        self.mp1 = torch.nn.MaxPool1d(seq_len+1-kernel_size[0])\n",
    "        self.mp2 = torch.nn.MaxPool1d(seq_len+1-kernel_size[1])\n",
    "        self.mp3 = torch.nn.MaxPool1d(seq_len+1-kernel_size[2])\n",
    "        self.dropout = torch.nn.Dropout(dropout)\n",
    "        #self.cnn = CNN(vocab_size, embed_size, filter_size, kernel_size, dropout, seq_len)\n",
    "        self.lstm = nn.LSTM(\n",
    "            input_size=3*filter_size, \n",
    "            hidden_size=hidden_size, \n",
    "            num_layers=1,\n",
    "            batch_first=True)\n",
    "        self.dense = nn.Linear(hidden_size,1)\n",
    "        self.sig = nn.Sigmoid()\n",
    "        self.word_dict = None\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        \n",
    "        embed_input = self.embedding(x)\n",
    "        embed_input.unsqueeze_(1)\n",
    "        x1 = torch.tanh(self.dropout(self.conv1(embed_input))).squeeze(3)\n",
    "        # print(\"x1\")\n",
    "        # print(x1.shape)\n",
    "        x2 = torch.tanh(self.dropout(self.conv2(embed_input))).squeeze(3)\n",
    "        x3 = torch.tanh(self.dropout(self.conv3(embed_input))).squeeze(3)\n",
    "        f1 = self.mp1(x1).squeeze(2)\n",
    "        # print(\"f1\")\n",
    "        # print(f1.shape)\n",
    "        f2 = self.mp2(x2).squeeze(2)\n",
    "        f3 = self.mp3(x3).squeeze(2)\n",
    "        c_out = torch.cat([f1,f2,f3],dim=1)\n",
    "        r_in = c_out.view(batch_size,1, -1)\n",
    "        lstm_out, _ = self.lstm(r_in)\n",
    "        out = self.dense(lstm_out[:, -1, :])\n",
    "        return self.sig(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "f_cG9-y1X15R"
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np \n",
    "def train(model, train_dl, val_dl, epochs, optimizer, loss_fn, device):\n",
    "    train_loss_epoch = []\n",
    "    train_acc_epoch = []\n",
    "    val_loss_epoch = []\n",
    "    val_accuracy_epoch = []\n",
    "    time_train = []\n",
    "    for epoch in range(epochs):\n",
    "        start = time.time()\n",
    "        model.train()\n",
    "        total_loss = 0\n",
    "        train_acc = 0\n",
    "        total = 0\n",
    "        correct = 0\n",
    "        for batch in train_dl:         \n",
    "            batch_X, batch_y = batch\n",
    "            batch_X = batch_X.to(device)\n",
    "            batch_y = batch_y.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            prediction = model(batch_X)\n",
    "            loss = loss_fn(prediction, batch_y)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            result = np.round(prediction.detach().cpu())\n",
    "            total_loss += loss.data.item()\n",
    "            total += batch_y.size(0)\n",
    "            correct += (result == batch_y.cpu()).sum().item()\n",
    "            train_acc = correct/total\n",
    "        train_loss_epoch.append(np.round(total_loss / len(train_dl), 3))\n",
    "        train_acc_epoch.append(np.round(train_acc*100,3))\n",
    "        print(\"Epoch: {}, BCELoss: {}\".format(epoch, total_loss / len(train_dl)))\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            model.eval()\n",
    "            correct = 0\n",
    "            total = 0\n",
    "            val_loss = []\n",
    "            for inputs, labels in val_dl:\n",
    "                inputs_val, labels_val = inputs.to(device), labels.to(device)\n",
    "                optimizer.zero_grad()\n",
    "                prediction = model(inputs_val)\n",
    "                loss = loss_fn(prediction, labels_val)\n",
    "                val_loss.append(loss.item())\n",
    "                result = np.round(prediction.cpu())\n",
    "                total += labels_val.size(0)\n",
    "                correct += (result == labels_val.cpu()).sum().item()\n",
    "            val_accuracy_epoch.append(np.round((correct/total)*100, 3))\n",
    "            val_loss_epoch.append(np.round(np.mean(val_loss),3))\n",
    "            end = time.time() - start\n",
    "            print(\"Val Loss: {:.3f}\".format(np.mean(val_loss)), \"\\tVal Acc: {:.3f}\".format(correct/total))\n",
    "            time_train.append(np.round(end,3))\n",
    "    write_to_csv(epochs, train_loss_epoch, train_acc_epoch, val_loss_epoch, val_accuracy_epoch, time_train)\n",
    "    return model\n",
    "\n",
    "\n",
    "def test(model, test_dl, epochs):\n",
    "    model.eval()\n",
    "    correct = 0\n",
    "    total = 0\n",
    "#     results = []\n",
    "#     labels = []\n",
    "    with torch.no_grad():\n",
    "        for batch in test_dl:         \n",
    "            batch_X, batch_y = batch\n",
    "            batch_X = batch_X.to(device)\n",
    "            prediction = model(batch_X)\n",
    "            result = np.round(prediction.cpu())\n",
    "#             results.extend(list(result.numpy()))\n",
    "#             labels.extend(list(batch_y.numpy()))\n",
    "            total += batch_y.size(0)\n",
    "            correct += (result == batch_y).sum().item()\n",
    "    acc = correct/total\n",
    "    append_to_csv(epochs, np.round(acc*100, 3))\n",
    "    print(\"Accuracy:\", (correct/total)*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yqMnwMrFYZpE",
    "outputId": "000b2d2a-fd06-4ccf-ba0c-efbc3d34ac79"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0, BCELoss: 0.6887181899944941\n",
      "Val Loss: 0.635 \tVal Acc: 0.619\n",
      "Epoch: 1, BCELoss: 0.43255942478775977\n",
      "Val Loss: 0.377 \tVal Acc: 0.835\n",
      "Epoch: 2, BCELoss: 0.3309057215228677\n",
      "Val Loss: 0.346 \tVal Acc: 0.846\n",
      "Epoch: 3, BCELoss: 0.28803768335531155\n",
      "Val Loss: 0.317 \tVal Acc: 0.866\n",
      "Epoch: 4, BCELoss: 0.2618781572704514\n",
      "Val Loss: 0.304 \tVal Acc: 0.875\n",
      "Epoch: 5, BCELoss: 0.23945270324125886\n",
      "Val Loss: 0.293 \tVal Acc: 0.880\n",
      "Epoch: 6, BCELoss: 0.22383364766215286\n",
      "Val Loss: 0.291 \tVal Acc: 0.879\n",
      "Epoch: 7, BCELoss: 0.20844128130624692\n",
      "Val Loss: 0.292 \tVal Acc: 0.882\n",
      "Epoch: 8, BCELoss: 0.19677683241665364\n",
      "Val Loss: 0.296 \tVal Acc: 0.879\n",
      "Epoch: 9, BCELoss: 0.18275199824944138\n",
      "Val Loss: 0.302 \tVal Acc: 0.878\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Combine(\n",
       "  (embedding): Embedding(5000, 32)\n",
       "  (conv1): Conv2d(1, 100, kernel_size=(3, 32), stride=(1, 1))\n",
       "  (conv2): Conv2d(1, 100, kernel_size=(4, 32), stride=(1, 1))\n",
       "  (conv3): Conv2d(1, 100, kernel_size=(5, 32), stride=(1, 1))\n",
       "  (mp1): MaxPool1d(kernel_size=498, stride=498, padding=0, dilation=1, ceil_mode=False)\n",
       "  (mp2): MaxPool1d(kernel_size=497, stride=497, padding=0, dilation=1, ceil_mode=False)\n",
       "  (mp3): MaxPool1d(kernel_size=496, stride=496, padding=0, dilation=1, ceil_mode=False)\n",
       "  (dropout): Dropout(p=0.5, inplace=False)\n",
       "  (lstm): LSTM(300, 100, batch_first=True)\n",
       "  (dense): Linear(in_features=100, out_features=1, bias=True)\n",
       "  (sig): Sigmoid()\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "epochs = 10\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model = Combine(vocab_size, embed_dims, filter_size, kernel_size, dropout, seq_len,hidden_size).to(device).to(device)\n",
    "optimizer = optim.Adam(model.parameters())\n",
    "loss_fn = torch.nn.BCELoss()\n",
    "train(model, train_dl, val_dl, epochs, optimizer, loss_fn, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "zyk4PPMZUO5m"
   },
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(),\"./cnn-lstm-imdb.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "id": "NJo3d8_TPxMd"
   },
   "outputs": [],
   "source": [
    "# def test(model, test_dl, epochs):\n",
    "#     model.eval()\n",
    "#     correct = 0\n",
    "#     total = 0\n",
    "# #     results = []\n",
    "# #     labels = []\n",
    "#     with torch.no_grad():\n",
    "#         for batch in test_dl:         \n",
    "#             batch_X, batch_y = batch\n",
    "#             batch_X = batch_X.to(device)\n",
    "#             prediction = model(batch_X)\n",
    "#             result = np.round(prediction.cpu())\n",
    "# #             results.extend(list(result.numpy()))\n",
    "# #             labels.extend(list(batch_y.numpy()))\n",
    "#             total += batch_y.size(0)\n",
    "#             correct += (result == batch_y).sum().item()\n",
    "#     acc = correct/total\n",
    "#     print(\"Accuracy:\", correct/total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "JsC1Ym2CP4Qd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 87.32\n"
     ]
    }
   ],
   "source": [
    "test(model, test_dl, epochs)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "cnn-lstm-imdb.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "my_env2_gpu",
   "language": "python",
   "name": "my_env2_gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}