cnn_lstm_imdb_hyperband.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cXRRPjTk5gv1",
    "outputId": "5f66c255-f802-4b61-ab21-37645be2d402"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive/')\n",
    "import sys\n",
    "import os\n",
    "prefix = '/content/gdrive/My Drive/'\n",
    "# modify \"customized_path_to_your_homework\" here to where you uploaded your homework\n",
    "customized_path_to_your_homework = 'IDLSProject-main'\n",
    "sys_path = os.path.join(prefix, customized_path_to_your_homework)\n",
    "sys.path.append(sys_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "b8m8aQCL5pFJ",
    "outputId": "32b04df3-2805-45d2-ab70-240978e451f5"
   },
   "outputs": [],
   "source": [
    "%cd '/content/gdrive/My Drive/IDLSProject-main-2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0h8gAUJ_5q-2",
    "outputId": "49a18b80-e747-4520-b710-5b20b41c4c6e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10000, 1])\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import pickle\n",
    "import torch\n",
    "import torch.utils.data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "data_dir = './data/pytorch'\n",
    "with open(os.path.join(data_dir, 'word_dict_qrnn.pkl'), \"rb\") as f:\n",
    "    word_dict = pickle.load(f)\n",
    "\n",
    "train = pd.read_csv(os.path.join(data_dir, 'train_qrnn.csv'), header=None, names=None)\n",
    "test_sample = pd.read_csv(os.path.join(data_dir, 'test_qrnn.csv'), header=None, names=None)\n",
    "\n",
    "test, val = train_test_split(test_sample, test_size=0.5)\n",
    "train.shape, test.shape, val.shape\n",
    "\n",
    "\n",
    "# Turn the input pandas dataframe into tensors\n",
    "train_y = torch.from_numpy(train[[0]].values).float()\n",
    "train_X = torch.from_numpy(train.drop([0, 1], axis=1).values).long()\n",
    "\n",
    "# Build the dataset\n",
    "train_ds = torch.utils.data.TensorDataset(train_X, train_y)\n",
    "# Build the dataloader\n",
    "train_dl = torch.utils.data.DataLoader(train_ds, batch_size=50)\n",
    "\n",
    "######val data\n",
    "# Turn the input pandas dataframe into tensors\n",
    "val_y = torch.from_numpy(val[[0]].values).float()\n",
    "val_X = torch.from_numpy(val.drop([0, 1], axis=1).values).long()\n",
    "\n",
    "# Build the dataset\n",
    "val_ds = torch.utils.data.TensorDataset(val_X, val_y)\n",
    "# Build the dataloader\n",
    "val_dl = torch.utils.data.DataLoader(val_ds, batch_size=50)\n",
    "\n",
    "\n",
    "#### Test data\n",
    "# Turn the input pandas dataframe into tensors\n",
    "test_y = torch.from_numpy(test[[0]].values).float()\n",
    "test_X = torch.from_numpy(test.drop([0, 1], axis=1).values).long()\n",
    "\n",
    "# Build the dataset\n",
    "test_ds = torch.utils.data.TensorDataset(test_X, test_y)\n",
    "# Build the dataloader\n",
    "test_dl = torch.utils.data.DataLoader(test_ds, batch_size=50)\n",
    "print(test_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "DT-bPvic542j"
   },
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "epochs = 10\n",
    "batch_size = 50\n",
    "learning_rate = 0.001\n",
    "seq_len = 500\n",
    "dropout = 0.5\n",
    "filter_size = 100\n",
    "vocab_size = 5000\n",
    "embed_dims = 32\n",
    "hidden_size = 100\n",
    "kernel_size = [3,4,5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "OtBMgWdy58jp"
   },
   "outputs": [],
   "source": [
    "data_dir = './cnn-lstm-imdb-hyperband-trails_15/' # The folder we will use for storing data\n",
    "if not os.path.exists(data_dir): # Make sure that the folder exists\n",
    "    os.makedirs(data_dir)\n",
    "\n",
    "filename = \"\"\n",
    "def write_to_csv(trail_num, epochs, train_loss, train_acc, val_loss, val_acc, time_train):\n",
    "    global filename\n",
    "    filename = \"./cnn-lstm-imdb-hyperband-trails_15/\"+str(trail_num)+\".csv\"\n",
    "    epoch = [i for i in range(epochs)]\n",
    "    df_metrics = pd.DataFrame(list(zip(epoch, train_loss, train_acc, val_loss, val_acc, time_train)), columns =['Epoch', 'train_loss', 'train_acc', 'val_loss', 'val_acc', 'train_time'])\n",
    "    df_metrics.to_csv(filename, index=False)    \n",
    "    \n",
    "def append_to_csv(epochs, accuracy):\n",
    "    acc = [accuracy for i in range(epochs)]\n",
    "    df_csv = pd.read_csv(filename)\n",
    "    df_csv['Test_Accuracy']  = acc\n",
    "    df_csv.to_csv(filename, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "lOi5VnD-6Afi"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "class Combine(nn.Module):\n",
    "    def __init__(self,trial,vocab_size, embed_size, filter_size, kernel_size, dropout, seq_len,hidden_size):\n",
    "        super(Combine, self).__init__()\n",
    "        dropout = trial.suggest_uniform(\"dropout\",0.1, 0.6)\n",
    "        hidden_size = trial.suggest_int(\"hidden_size\",32,256)\n",
    "        embed_size = trial.suggest_int(\"embed_size\",16,128)\n",
    "        filter_size = trial.suggest_int(\"filter_size\",50,200)\n",
    "        self.embedding = torch.nn.Embedding(vocab_size,embed_size)\n",
    "        self.conv1 = torch.nn.Conv2d(1,filter_size,kernel_size=[kernel_size[0],embed_size])\n",
    "        self.conv2 = torch.nn.Conv2d(1,filter_size,kernel_size=[kernel_size[1],embed_size])\n",
    "        self.conv3 = torch.nn.Conv2d(1,filter_size,kernel_size=[kernel_size[2],embed_size])\n",
    "        self.mp1 = torch.nn.MaxPool1d(seq_len+1-kernel_size[0])\n",
    "        self.mp2 = torch.nn.MaxPool1d(seq_len+1-kernel_size[1])\n",
    "        self.mp3 = torch.nn.MaxPool1d(seq_len+1-kernel_size[2])\n",
    "        self.dropout = torch.nn.Dropout(dropout)\n",
    "        #self.cnn = CNN(vocab_size, embed_size, filter_size, kernel_size, dropout, seq_len)\n",
    "        self.lstm = nn.LSTM(\n",
    "            input_size=3*filter_size, \n",
    "            hidden_size=hidden_size, \n",
    "            num_layers=1,\n",
    "            batch_first=True)\n",
    "        self.dense = nn.Linear(hidden_size,1)\n",
    "        self.sig = nn.Sigmoid()\n",
    "        self.word_dict = None\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        \n",
    "        embed_input = self.embedding(x)\n",
    "        embed_input.unsqueeze_(1)\n",
    "        x1 = torch.tanh(self.dropout(self.conv1(embed_input))).squeeze(3)\n",
    "        # print(\"x1\")\n",
    "        # print(x1.shape)\n",
    "        x2 = torch.tanh(self.dropout(self.conv2(embed_input))).squeeze(3)\n",
    "        x3 = torch.tanh(self.dropout(self.conv3(embed_input))).squeeze(3)\n",
    "        f1 = self.mp1(x1).squeeze(2)\n",
    "        # print(\"f1\")\n",
    "        # print(f1.shape)\n",
    "        f2 = self.mp2(x2).squeeze(2)\n",
    "        f3 = self.mp3(x3).squeeze(2)\n",
    "        c_out = torch.cat([f1,f2,f3],dim=1)\n",
    "        r_in = c_out.view(batch_size,1, -1)\n",
    "        lstm_out, _ = self.lstm(r_in)\n",
    "        out = self.dense(lstm_out[:, -1, :])\n",
    "        return self.sig(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "yhUhQgON6Ft4"
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np \n",
    "def objective(trial):\n",
    "    optimizer_name = trial.suggest_categorical(\"optimizer\", [\"Adam\", \"RMSprop\", \"SGD\"])\n",
    "    lr = trial.suggest_loguniform(\"lr\", 1e-5, 1e-0)\n",
    "    #momentum = trial.suggest_uniform(\"momentum\", 0.0, 1.0)\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "    model = Combine(trial,vocab_size, embed_dims, filter_size, kernel_size, dropout, seq_len,hidden_size).to(device).to(device)\n",
    "    trial.set_user_attr(key=\"best_model\", value=model)\n",
    "    optimizer = optim.Adam(model.parameters())\n",
    "    loss_fn = torch.nn.BCELoss()\n",
    "    train_loss_epoch = []\n",
    "    train_acc_epoch = []\n",
    "    val_loss_epoch = []\n",
    "    val_accuracy_epoch = []\n",
    "    time_train = []\n",
    "    val_acc = 0\n",
    "    for epoch in range(epochs):\n",
    "        start = time.time()\n",
    "        model.train()\n",
    "        total_loss = 0\n",
    "        train_acc = 0\n",
    "        total = 0\n",
    "        correct = 0\n",
    "        for batch in train_dl:         \n",
    "            batch_X, batch_y = batch\n",
    "            batch_X = batch_X.to(device)\n",
    "            batch_y = batch_y.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            prediction = model(batch_X)\n",
    "            loss = loss_fn(prediction, batch_y)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            result = np.round(prediction.detach().cpu())\n",
    "            total_loss += loss.data.item()\n",
    "            total += batch_y.size(0)\n",
    "            correct += (result == batch_y.cpu()).sum().item()\n",
    "            train_acc = correct/total\n",
    "        train_loss_epoch.append(np.round(total_loss / len(train_dl), 3))\n",
    "        train_acc_epoch.append(np.round(train_acc*100,3))\n",
    "        print(\"Epoch: {}, BCELoss: {}\".format(epoch, total_loss / len(train_dl)))\n",
    "        with torch.no_grad():\n",
    "            model.eval()\n",
    "            correct = 0\n",
    "            total = 0\n",
    "            val_loss = []\n",
    "            for inputs, labels in val_dl:\n",
    "                inputs_val, labels_val = inputs.to(device), labels.to(device)\n",
    "                optimizer.zero_grad()\n",
    "                prediction = model(inputs_val)\n",
    "                loss = loss_fn(prediction, labels_val)\n",
    "                val_loss.append(loss.item())\n",
    "                result = np.round(prediction.cpu())\n",
    "                total += labels_val.size(0)\n",
    "                correct += (result == labels_val.cpu()).sum().item()\n",
    "            val_accuracy_epoch.append(np.round((correct/total)*100, 3))\n",
    "            val_loss_epoch.append(np.round(np.mean(val_loss),3))\n",
    "            end = time.time() - start\n",
    "            print(\"Val Loss: {:.3f}\".format(np.mean(val_loss)), \"\\tVal Acc: {:.3f}\".format(correct/total))\n",
    "            time_train.append(np.round(end,3))\n",
    "            val_acc = np.round((correct/total)*100, 3)\n",
    "    write_to_csv(trial.number,epochs, train_loss_epoch, train_acc_epoch, val_loss_epoch, val_accuracy_epoch, time_train)\n",
    "    return val_acc\n",
    "\n",
    "\n",
    "def test(model, test_dl, epochs):\n",
    "    model.eval()\n",
    "    correct = 0\n",
    "    total = 0\n",
    "#     results = []\n",
    "#     labels = []\n",
    "    with torch.no_grad():\n",
    "        for batch in test_dl:         \n",
    "            batch_X, batch_y = batch\n",
    "            batch_X = batch_X.to(device)\n",
    "            prediction = model(batch_X)\n",
    "            result = np.round(prediction.cpu())\n",
    "#             results.extend(list(result.numpy()))\n",
    "#             labels.extend(list(batch_y.numpy()))\n",
    "            total += batch_y.size(0)\n",
    "            correct += (result == batch_y).sum().item()\n",
    "    acc = correct/total\n",
    "    print(\"Accuracy:\", (correct/total)*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "P1THPr2-9CNZ"
   },
   "outputs": [],
   "source": [
    "def callback(study, trial):\n",
    "    if study.best_trial.number == trial.number:\n",
    "        study.set_user_attr(key=\"best_model\", value=trial.user_attrs[\"best_model\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "SMRMuN_C_5lR"
   },
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "epochs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "PlVqr1IX9FJn",
    "outputId": "f8cabff3-4400-421b-f1ed-9bf8575d6293"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:10:26,650]\u001b[0m A new study created in memory with name: no-name-20bc19e9-4151-4cde-b250-f2d788660fba\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0, BCELoss: 0.6934993675351143\n",
      "Val Loss: 0.693 \tVal Acc: 0.508\n",
      "Epoch: 1, BCELoss: 0.6933403845628102\n",
      "Val Loss: 0.693 \tVal Acc: 0.508\n",
      "Epoch: 2, BCELoss: 0.5336921380956968\n",
      "Val Loss: 0.409 \tVal Acc: 0.823\n",
      "Epoch: 3, BCELoss: 0.35764983142415685\n",
      "Val Loss: 0.363 \tVal Acc: 0.846\n",
      "Epoch: 4, BCELoss: 0.30623595555623373\n",
      "Val Loss: 0.336 \tVal Acc: 0.859\n",
      "Epoch: 5, BCELoss: 0.27704234308252734\n",
      "Val Loss: 0.328 \tVal Acc: 0.863\n",
      "Epoch: 6, BCELoss: 0.25526158455759285\n",
      "Val Loss: 0.348 \tVal Acc: 0.849\n",
      "Epoch: 7, BCELoss: 0.23996724975605807\n",
      "Val Loss: 0.327 \tVal Acc: 0.861\n",
      "Epoch: 8, BCELoss: 0.2271840187907219\n",
      "Val Loss: 0.353 \tVal Acc: 0.848\n",
      "Epoch: 9, BCELoss: 0.21623181816190482\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:11:46,766]\u001b[0m Trial 0 finished with value: 86.37 and parameters: {'optimizer': 'SGD', 'lr': 0.0004730282867132955, 'dropout': 0.5649000155479039, 'hidden_size': 151, 'embed_size': 16, 'filter_size': 163}. Best is trial 0 with value: 86.37.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.323 \tVal Acc: 0.864\n",
      "Epoch: 0, BCELoss: 0.6929472298423449\n",
      "Val Loss: 0.693 \tVal Acc: 0.493\n",
      "Epoch: 1, BCELoss: 0.46640263686577477\n",
      "Val Loss: 0.378 \tVal Acc: 0.834\n",
      "Epoch: 2, BCELoss: 0.34290798505147296\n",
      "Val Loss: 0.355 \tVal Acc: 0.847\n",
      "Epoch: 3, BCELoss: 0.2991393394768238\n",
      "Val Loss: 0.339 \tVal Acc: 0.853\n",
      "Epoch: 4, BCELoss: 0.26743185809503\n",
      "Val Loss: 0.323 \tVal Acc: 0.862\n",
      "Epoch: 5, BCELoss: 0.2464319201434652\n",
      "Val Loss: 0.316 \tVal Acc: 0.866\n",
      "Epoch: 6, BCELoss: 0.23053405116622647\n",
      "Val Loss: 0.313 \tVal Acc: 0.869\n",
      "Epoch: 7, BCELoss: 0.2150389173688988\n",
      "Val Loss: 0.339 \tVal Acc: 0.864\n",
      "Epoch: 8, BCELoss: 0.2045970657405754\n",
      "Val Loss: 0.338 \tVal Acc: 0.861\n",
      "Epoch: 9, BCELoss: 0.1934785515318314\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:13:18,508]\u001b[0m Trial 1 finished with value: 85.1 and parameters: {'optimizer': 'Adam', 'lr': 0.0017489082807533743, 'dropout': 0.5692791082666415, 'hidden_size': 173, 'embed_size': 47, 'filter_size': 102}. Best is trial 0 with value: 86.37.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.366 \tVal Acc: 0.851\n",
      "Epoch: 0, BCELoss: 0.6423871573309103\n",
      "Val Loss: 0.441 \tVal Acc: 0.798\n",
      "Epoch: 1, BCELoss: 0.3856708044807116\n",
      "Val Loss: 0.357 \tVal Acc: 0.848\n",
      "Epoch: 2, BCELoss: 0.31181598756462336\n",
      "Val Loss: 0.324 \tVal Acc: 0.863\n",
      "Epoch: 3, BCELoss: 0.2695303293565909\n",
      "Val Loss: 0.302 \tVal Acc: 0.873\n",
      "Epoch: 4, BCELoss: 0.24124415327484408\n",
      "Val Loss: 0.301 \tVal Acc: 0.879\n",
      "Epoch: 5, BCELoss: 0.2175229646389683\n",
      "Val Loss: 0.298 \tVal Acc: 0.878\n",
      "Epoch: 6, BCELoss: 0.19672340761249263\n",
      "Val Loss: 0.310 \tVal Acc: 0.879\n",
      "Epoch: 7, BCELoss: 0.18075932644928494\n",
      "Val Loss: 0.312 \tVal Acc: 0.881\n",
      "Epoch: 8, BCELoss: 0.1635255585859219\n",
      "Val Loss: 0.327 \tVal Acc: 0.881\n",
      "Epoch: 9, BCELoss: 0.15523011037458975\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:18:18,855]\u001b[0m Trial 2 finished with value: 87.81 and parameters: {'optimizer': 'Adam', 'lr': 0.009127263281980543, 'dropout': 0.36742034519123545, 'hidden_size': 117, 'embed_size': 31, 'filter_size': 103}. Best is trial 2 with value: 87.81.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.337 \tVal Acc: 0.878\n",
      "Epoch: 0, BCELoss: 0.6519675993919373\n",
      "Val Loss: 0.484 \tVal Acc: 0.786\n",
      "Epoch: 1, BCELoss: 0.40488154396414755\n",
      "Val Loss: 0.388 \tVal Acc: 0.838\n",
      "Epoch: 2, BCELoss: 0.3135503373046716\n",
      "Val Loss: 0.368 \tVal Acc: 0.850\n",
      "Epoch: 3, BCELoss: 0.2661475012203058\n",
      "Val Loss: 0.373 \tVal Acc: 0.855\n",
      "Epoch: 4, BCELoss: 0.23478999187548955\n",
      "Val Loss: 0.380 \tVal Acc: 0.857\n",
      "Epoch: 5, BCELoss: 0.20473822419221202\n",
      "Val Loss: 0.392 \tVal Acc: 0.858\n",
      "Epoch: 6, BCELoss: 0.18492623388767243\n",
      "Val Loss: 0.404 \tVal Acc: 0.865\n",
      "Epoch: 7, BCELoss: 0.17363935296113292\n",
      "Val Loss: 0.425 \tVal Acc: 0.865\n",
      "Epoch: 8, BCELoss: 0.16405669530853628\n",
      "Val Loss: 0.427 \tVal Acc: 0.861\n",
      "Epoch: 9, BCELoss: 0.14648745156514148\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:20:04,936]\u001b[0m Trial 3 finished with value: 85.93 and parameters: {'optimizer': 'RMSprop', 'lr': 2.6139452820296635e-05, 'dropout': 0.5287199368418258, 'hidden_size': 112, 'embed_size': 120, 'filter_size': 141}. Best is trial 2 with value: 87.81.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.464 \tVal Acc: 0.859\n",
      "Epoch: 0, BCELoss: 0.5477635850012302\n",
      "Val Loss: 0.414 \tVal Acc: 0.814\n",
      "Epoch: 1, BCELoss: 0.34176646071175737\n",
      "Val Loss: 0.372 \tVal Acc: 0.839\n",
      "Epoch: 2, BCELoss: 0.2649151468152801\n",
      "Val Loss: 0.379 \tVal Acc: 0.844\n",
      "Epoch: 3, BCELoss: 0.21663830572118362\n",
      "Val Loss: 0.370 \tVal Acc: 0.859\n",
      "Epoch: 4, BCELoss: 0.17448503208036223\n",
      "Val Loss: 0.384 \tVal Acc: 0.862\n",
      "Epoch: 5, BCELoss: 0.1506433240324259\n",
      "Val Loss: 0.410 \tVal Acc: 0.868\n",
      "Epoch: 6, BCELoss: 0.12707636578008533\n",
      "Val Loss: 0.467 \tVal Acc: 0.847\n",
      "Epoch: 7, BCELoss: 0.11067130382638425\n",
      "Val Loss: 0.473 \tVal Acc: 0.862\n",
      "Epoch: 8, BCELoss: 0.08838216179826608\n",
      "Val Loss: 0.503 \tVal Acc: 0.857\n",
      "Epoch: 9, BCELoss: 0.07678598066636672\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:22:26,835]\u001b[0m Trial 4 finished with value: 82.81 and parameters: {'optimizer': 'RMSprop', 'lr': 8.41275450747703e-05, 'dropout': 0.252518947861728, 'hidden_size': 52, 'embed_size': 106, 'filter_size': 113}. Best is trial 2 with value: 87.81.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.600 \tVal Acc: 0.828\n",
      "Epoch: 0, BCELoss: 0.6084987659752369\n",
      "Val Loss: 0.438 \tVal Acc: 0.796\n",
      "Epoch: 1, BCELoss: 0.35423017395039397\n",
      "Val Loss: 0.394 \tVal Acc: 0.829\n",
      "Epoch: 2, BCELoss: 0.2712797071660558\n",
      "Val Loss: 0.354 \tVal Acc: 0.862\n",
      "Epoch: 3, BCELoss: 0.21313809288044772\n",
      "Val Loss: 0.408 \tVal Acc: 0.845\n",
      "Epoch: 4, BCELoss: 0.17231450129921239\n",
      "Val Loss: 0.400 \tVal Acc: 0.859\n",
      "Epoch: 5, BCELoss: 0.14537557010538876\n",
      "Val Loss: 0.546 \tVal Acc: 0.828\n",
      "Epoch: 6, BCELoss: 0.13726761873811483\n",
      "Val Loss: 0.403 \tVal Acc: 0.870\n",
      "Epoch: 7, BCELoss: 0.11170149903744458\n",
      "Val Loss: 0.539 \tVal Acc: 0.840\n",
      "Epoch: 8, BCELoss: 0.082712739406464\n",
      "Val Loss: 0.588 \tVal Acc: 0.836\n",
      "Epoch: 9, BCELoss: 0.07842132059081147\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:24:29,600]\u001b[0m Trial 5 finished with value: 84.02 and parameters: {'optimizer': 'SGD', 'lr': 0.7226810084045228, 'dropout': 0.3371685705906222, 'hidden_size': 203, 'embed_size': 94, 'filter_size': 167}. Best is trial 2 with value: 87.81.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.544 \tVal Acc: 0.840\n",
      "Epoch: 0, BCELoss: 0.5784942804276944\n",
      "Val Loss: 0.429 \tVal Acc: 0.811\n",
      "Epoch: 1, BCELoss: 0.3713804358989\n",
      "Val Loss: 0.354 \tVal Acc: 0.849\n",
      "Epoch: 2, BCELoss: 0.2976862806951006\n",
      "Val Loss: 0.335 \tVal Acc: 0.866\n",
      "Epoch: 3, BCELoss: 0.2524193857982755\n",
      "Val Loss: 0.333 \tVal Acc: 0.868\n",
      "Epoch: 4, BCELoss: 0.2182575473934412\n",
      "Val Loss: 0.353 \tVal Acc: 0.868\n",
      "Epoch: 5, BCELoss: 0.188166968151927\n",
      "Val Loss: 0.363 \tVal Acc: 0.869\n",
      "Epoch: 6, BCELoss: 0.1657237565362205\n",
      "Val Loss: 0.399 \tVal Acc: 0.867\n",
      "Epoch: 7, BCELoss: 0.14861822776806852\n",
      "Val Loss: 0.406 \tVal Acc: 0.867\n",
      "Epoch: 8, BCELoss: 0.13527676592736196\n",
      "Val Loss: 0.429 \tVal Acc: 0.867\n",
      "Epoch: 9, BCELoss: 0.12220570812467485\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:26:16,068]\u001b[0m Trial 6 finished with value: 85.09 and parameters: {'optimizer': 'RMSprop', 'lr': 1.2085866271858766e-05, 'dropout': 0.3126621747226628, 'hidden_size': 54, 'embed_size': 97, 'filter_size': 65}. Best is trial 2 with value: 87.81.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.484 \tVal Acc: 0.851\n",
      "Epoch: 0, BCELoss: 0.6474696617325147\n",
      "Val Loss: 0.463 \tVal Acc: 0.788\n",
      "Epoch: 1, BCELoss: 0.36826086781919004\n",
      "Val Loss: 0.342 \tVal Acc: 0.862\n",
      "Epoch: 2, BCELoss: 0.2718229848891497\n",
      "Val Loss: 0.334 \tVal Acc: 0.871\n",
      "Epoch: 3, BCELoss: 0.20525363827124238\n",
      "Val Loss: 0.354 \tVal Acc: 0.868\n",
      "Epoch: 4, BCELoss: 0.15920619444611173\n",
      "Val Loss: 0.397 \tVal Acc: 0.865\n",
      "Epoch: 5, BCELoss: 0.13081281774056455\n",
      "Val Loss: 0.498 \tVal Acc: 0.839\n",
      "Epoch: 6, BCELoss: 0.10680150277291735\n",
      "Val Loss: 0.475 \tVal Acc: 0.860\n",
      "Epoch: 7, BCELoss: 0.0920930292736739\n",
      "Val Loss: 0.664 \tVal Acc: 0.815\n",
      "Epoch: 8, BCELoss: 0.07151701068738475\n",
      "Val Loss: 0.643 \tVal Acc: 0.829\n",
      "Epoch: 9, BCELoss: 0.06286663261785483\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:28:37,894]\u001b[0m Trial 7 finished with value: 83.68 and parameters: {'optimizer': 'SGD', 'lr': 0.003782240272204917, 'dropout': 0.2163416157176201, 'hidden_size': 246, 'embed_size': 85, 'filter_size': 198}. Best is trial 2 with value: 87.81.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.635 \tVal Acc: 0.837\n",
      "Epoch: 0, BCELoss: 0.6566002339124679\n",
      "Val Loss: 0.513 \tVal Acc: 0.756\n",
      "Epoch: 1, BCELoss: 0.42424306536714235\n",
      "Val Loss: 0.385 \tVal Acc: 0.836\n",
      "Epoch: 2, BCELoss: 0.3443436750769615\n",
      "Val Loss: 0.349 \tVal Acc: 0.854\n",
      "Epoch: 3, BCELoss: 0.3055255988612771\n",
      "Val Loss: 0.345 \tVal Acc: 0.857\n",
      "Epoch: 4, BCELoss: 0.2740861959507068\n",
      "Val Loss: 0.326 \tVal Acc: 0.863\n",
      "Epoch: 5, BCELoss: 0.24945937596261503\n",
      "Val Loss: 0.326 \tVal Acc: 0.865\n",
      "Epoch: 6, BCELoss: 0.2349550919731458\n",
      "Val Loss: 0.343 \tVal Acc: 0.856\n",
      "Epoch: 7, BCELoss: 0.2194499227652947\n",
      "Val Loss: 0.338 \tVal Acc: 0.862\n",
      "Epoch: 8, BCELoss: 0.20341794178510705\n",
      "Val Loss: 0.334 \tVal Acc: 0.862\n",
      "Epoch: 9, BCELoss: 0.1924577478443583\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:30:13,285]\u001b[0m Trial 8 finished with value: 85.1 and parameters: {'optimizer': 'SGD', 'lr': 2.7614870707656176e-05, 'dropout': 0.5043142335837191, 'hidden_size': 219, 'embed_size': 68, 'filter_size': 50}. Best is trial 2 with value: 87.81.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.367 \tVal Acc: 0.851\n",
      "Epoch: 0, BCELoss: 0.5941965018709501\n",
      "Val Loss: 0.424 \tVal Acc: 0.805\n",
      "Epoch: 1, BCELoss: 0.36462262568374476\n",
      "Val Loss: 0.363 \tVal Acc: 0.844\n",
      "Epoch: 2, BCELoss: 0.2938817800333103\n",
      "Val Loss: 0.345 \tVal Acc: 0.855\n",
      "Epoch: 3, BCELoss: 0.2545235606655478\n",
      "Val Loss: 0.348 \tVal Acc: 0.853\n",
      "Epoch: 4, BCELoss: 0.22262927843257785\n",
      "Val Loss: 0.319 \tVal Acc: 0.875\n",
      "Epoch: 5, BCELoss: 0.18919878158097467\n",
      "Val Loss: 0.340 \tVal Acc: 0.871\n",
      "Epoch: 6, BCELoss: 0.16371581323755285\n",
      "Val Loss: 0.367 \tVal Acc: 0.866\n",
      "Epoch: 7, BCELoss: 0.15654770663628975\n",
      "Val Loss: 0.379 \tVal Acc: 0.865\n",
      "Epoch: 8, BCELoss: 0.1326734842484196\n",
      "Val Loss: 0.456 \tVal Acc: 0.848\n",
      "Epoch: 9, BCELoss: 0.11000216244099041\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:31:35,966]\u001b[0m Trial 9 finished with value: 85.98 and parameters: {'optimizer': 'Adam', 'lr': 0.0004248421270298003, 'dropout': 0.24398457510267588, 'hidden_size': 145, 'embed_size': 44, 'filter_size': 108}. Best is trial 2 with value: 87.81.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.449 \tVal Acc: 0.860\n",
      "Epoch: 0, BCELoss: 0.6831641760468483\n",
      "Val Loss: 0.572 \tVal Acc: 0.699\n",
      "Epoch: 1, BCELoss: 0.4191985540340344\n",
      "Val Loss: 0.378 \tVal Acc: 0.836\n",
      "Epoch: 2, BCELoss: 0.31858775465438766\n",
      "Val Loss: 0.349 \tVal Acc: 0.855\n",
      "Epoch: 3, BCELoss: 0.2698600031559666\n",
      "Val Loss: 0.338 \tVal Acc: 0.860\n",
      "Epoch: 4, BCELoss: 0.23388809517025946\n",
      "Val Loss: 0.333 \tVal Acc: 0.865\n",
      "Epoch: 5, BCELoss: 0.2020425265158216\n",
      "Val Loss: 0.339 \tVal Acc: 0.869\n",
      "Epoch: 6, BCELoss: 0.17776443293939034\n",
      "Val Loss: 0.362 \tVal Acc: 0.868\n",
      "Epoch: 7, BCELoss: 0.1560098559036851\n",
      "Val Loss: 0.376 \tVal Acc: 0.869\n",
      "Epoch: 8, BCELoss: 0.14198901900090277\n",
      "Val Loss: 0.403 \tVal Acc: 0.864\n",
      "Epoch: 9, BCELoss: 0.15032231161991755\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:32:35,591]\u001b[0m Trial 10 finished with value: 86.96 and parameters: {'optimizer': 'Adam', 'lr': 0.06056242236280045, 'dropout': 0.10289239400838077, 'hidden_size': 103, 'embed_size': 17, 'filter_size': 76}. Best is trial 2 with value: 87.81.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.389 \tVal Acc: 0.870\n",
      "Epoch: 0, BCELoss: 0.6302971597015857\n",
      "Val Loss: 0.494 \tVal Acc: 0.767\n",
      "Epoch: 1, BCELoss: 0.39712020417054494\n",
      "Val Loss: 0.388 \tVal Acc: 0.833\n",
      "Epoch: 2, BCELoss: 0.31066420901566744\n",
      "Val Loss: 0.359 \tVal Acc: 0.850\n",
      "Epoch: 3, BCELoss: 0.2631493150567015\n",
      "Val Loss: 0.343 \tVal Acc: 0.860\n",
      "Epoch: 4, BCELoss: 0.22984447243312994\n",
      "Val Loss: 0.329 \tVal Acc: 0.869\n",
      "Epoch: 5, BCELoss: 0.19789459723358352\n",
      "Val Loss: 0.346 \tVal Acc: 0.864\n",
      "Epoch: 6, BCELoss: 0.16875217822069924\n",
      "Val Loss: 0.351 \tVal Acc: 0.871\n",
      "Epoch: 7, BCELoss: 0.143743659115086\n",
      "Val Loss: 0.374 \tVal Acc: 0.869\n",
      "Epoch: 8, BCELoss: 0.13020557495454946\n",
      "Val Loss: 0.383 \tVal Acc: 0.874\n",
      "Epoch: 9, BCELoss: 0.13724379069171846\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:33:37,232]\u001b[0m Trial 11 finished with value: 86.08 and parameters: {'optimizer': 'Adam', 'lr': 0.053173530377097634, 'dropout': 0.10027162191174084, 'hidden_size': 105, 'embed_size': 17, 'filter_size': 87}. Best is trial 2 with value: 87.81.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.403 \tVal Acc: 0.861\n",
      "Epoch: 0, BCELoss: 0.6548438849548499\n",
      "Val Loss: 0.507 \tVal Acc: 0.746\n",
      "Epoch: 1, BCELoss: 0.4148750760157903\n",
      "Val Loss: 0.359 \tVal Acc: 0.848\n",
      "Epoch: 2, BCELoss: 0.3252248259757956\n",
      "Val Loss: 0.327 \tVal Acc: 0.860\n",
      "Epoch: 3, BCELoss: 0.28430973255385955\n",
      "Val Loss: 0.308 \tVal Acc: 0.869\n",
      "Epoch: 4, BCELoss: 0.2578832140316566\n",
      "Val Loss: 0.302 \tVal Acc: 0.872\n",
      "Epoch: 5, BCELoss: 0.2317591834316651\n",
      "Val Loss: 0.309 \tVal Acc: 0.870\n",
      "Epoch: 6, BCELoss: 0.21804330409814915\n",
      "Val Loss: 0.309 \tVal Acc: 0.872\n",
      "Epoch: 7, BCELoss: 0.1995214283838868\n",
      "Val Loss: 0.311 \tVal Acc: 0.875\n",
      "Epoch: 8, BCELoss: 0.1869710038540264\n",
      "Val Loss: 0.309 \tVal Acc: 0.878\n",
      "Epoch: 9, BCELoss: 0.1785888385710617\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:35:01,713]\u001b[0m Trial 12 finished with value: 87.58 and parameters: {'optimizer': 'Adam', 'lr': 0.05536647392481946, 'dropout': 0.4270350353134697, 'hidden_size': 97, 'embed_size': 37, 'filter_size': 85}. Best is trial 2 with value: 87.81.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.322 \tVal Acc: 0.876\n",
      "Epoch: 0, BCELoss: 0.6600690307716529\n",
      "Val Loss: 0.483 \tVal Acc: 0.770\n",
      "Epoch: 1, BCELoss: 0.38337157234549524\n",
      "Val Loss: 0.337 \tVal Acc: 0.857\n",
      "Epoch: 2, BCELoss: 0.29947164477159577\n",
      "Val Loss: 0.310 \tVal Acc: 0.871\n",
      "Epoch: 3, BCELoss: 0.261622343386213\n",
      "Val Loss: 0.304 \tVal Acc: 0.875\n",
      "Epoch: 4, BCELoss: 0.23285146425788603\n",
      "Val Loss: 0.308 \tVal Acc: 0.876\n",
      "Epoch: 5, BCELoss: 0.212929193880409\n",
      "Val Loss: 0.309 \tVal Acc: 0.878\n",
      "Epoch: 6, BCELoss: 0.1970222249130408\n",
      "Val Loss: 0.310 \tVal Acc: 0.877\n",
      "Epoch: 7, BCELoss: 0.18159213591056567\n",
      "Val Loss: 0.329 \tVal Acc: 0.875\n",
      "Epoch: 8, BCELoss: 0.1641764525230974\n",
      "Val Loss: 0.331 \tVal Acc: 0.876\n",
      "Epoch: 9, BCELoss: 0.15233335985802113\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:36:29,114]\u001b[0m Trial 13 finished with value: 87.7 and parameters: {'optimizer': 'Adam', 'lr': 0.023971812037957868, 'dropout': 0.43852645869169377, 'hidden_size': 84, 'embed_size': 44, 'filter_size': 131}. Best is trial 2 with value: 87.81.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.354 \tVal Acc: 0.877\n",
      "Epoch: 0, BCELoss: 0.6513309746980667\n",
      "Val Loss: 0.469 \tVal Acc: 0.778\n",
      "Epoch: 1, BCELoss: 0.380674097041289\n",
      "Val Loss: 0.355 \tVal Acc: 0.849\n",
      "Epoch: 2, BCELoss: 0.2999256575728456\n",
      "Val Loss: 0.322 \tVal Acc: 0.866\n",
      "Epoch: 3, BCELoss: 0.2549949782838424\n",
      "Val Loss: 0.318 \tVal Acc: 0.869\n",
      "Epoch: 4, BCELoss: 0.22609017677605153\n",
      "Val Loss: 0.326 \tVal Acc: 0.873\n",
      "Epoch: 5, BCELoss: 0.2017523917555809\n",
      "Val Loss: 0.338 \tVal Acc: 0.868\n",
      "Epoch: 6, BCELoss: 0.18482356718430917\n",
      "Val Loss: 0.350 \tVal Acc: 0.871\n",
      "Epoch: 7, BCELoss: 0.1674177346068124\n",
      "Val Loss: 0.368 \tVal Acc: 0.868\n",
      "Epoch: 8, BCELoss: 0.1564461703443279\n",
      "Val Loss: 0.391 \tVal Acc: 0.861\n",
      "Epoch: 9, BCELoss: 0.142950885038202\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 21:38:07,263]\u001b[0m Trial 14 finished with value: 84.07 and parameters: {'optimizer': 'Adam', 'lr': 0.010157904172026881, 'dropout': 0.44086722785937166, 'hidden_size': 71, 'embed_size': 62, 'filter_size': 143}. Best is trial 2 with value: 87.81.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.485 \tVal Acc: 0.841\n",
      "Study statistics: \n",
      "  Number of finished trials:  15\n",
      "  Number of complete trials:  15\n",
      "Best trial:\n",
      "  Value:  87.81\n",
      "  Params: \n",
      "    optimizer: Adam\n",
      "    lr: 0.009127263281980543\n",
      "    dropout: 0.36742034519123545\n",
      "    hidden_size: 117\n",
      "    embed_size: 31\n",
      "    filter_size: 103\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "from optuna.trial import TrialState\n",
    "study = optuna.create_study(direction=\"maximize\",pruner=optuna.pruners.HyperbandPruner(\n",
    "        min_resource=1, max_resource=epochs, reduction_factor=3\n",
    "    ),)\n",
    "study.optimize(objective, n_trials=15,callbacks=[callback])\n",
    "complete_trials = study.get_trials(deepcopy=False, states=[TrialState.COMPLETE])\n",
    "print(\"Study statistics: \")\n",
    "print(\"  Number of finished trials: \", len(study.trials))\n",
    "print(\"  Number of complete trials: \", len(complete_trials))\n",
    "print(\"Best trial:\")\n",
    "trial = study.best_trial\n",
    "best_model=study.user_attrs[\"best_model\"]\n",
    "print(\"  Value: \", trial.value)\n",
    "print(\"  Params: \")\n",
    "for key, value in trial.params.items():\n",
    "    print(\"    {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "SczpZ5Ab6Kb_"
   },
   "outputs": [],
   "source": [
    "torch.save(best_model.state_dict(),\"./cnn-lstm-imdb-hyperband.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cSg_6YND9Jus",
    "outputId": "cade58fa-45f9-444f-db90-600592716398"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/ext3/miniconda3/lib/python3.9/site-packages/torch/nn/modules/rnn.py:691: UserWarning: RNN module weights are not part of single contiguous chunk of memory. This means they need to be compacted at every call, possibly greatly increasing memory usage. To compact weights again call flatten_parameters(). (Triggered internally at  ../aten/src/ATen/native/cudnn/RNN.cpp:925.)\n",
      "  result = _VF.lstm(input, hx, self._flat_weights, self.bias, self.num_layers,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 88.03999999999999\n"
     ]
    }
   ],
   "source": [
    "test(best_model, test_dl, epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "cnn-lstm-imdb-hyperband.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "my_env2_gpu",
   "language": "python",
   "name": "my_env2_gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}