lstm_cnn_amazon_hyperband.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive/')\n",
        "import sys\n",
        "import os\n",
        "prefix = '/content/gdrive/My Drive/'\n",
        "# modify \"customized_path_to_your_homework\" here to where you uploaded your homework\n",
        "customized_path_to_your_homework = 'IDLSProject-main-2'\n",
        "sys_path = os.path.join(prefix, customized_path_to_your_homework)\n",
        "sys.path.append(sys_path)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t6q1pWRx5WDR",
        "outputId": "e76512dc-a15c-497c-c447-a215320a8c96"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive/\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd '/content/gdrive/My Drive/IDLSProject-main'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "riUZEOLU5YPw",
        "outputId": "0c970d41-3a62-4c3c-8bbe-fa39480afd53"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/gdrive/My Drive/IDLSProject-main\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hcVl4thz5SDD",
        "outputId": "db48c5e7-d107-4277-d47a-5f5fdeefd74f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/gdrive/My Drive/IDLSProject-main\n"
          ]
        }
      ],
      "source": [
        "!pwd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QJUW-ZwSV9sY",
        "outputId": "fba70bee-df99-4482-c928-3ede6ba51171"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(83000, 502) (21975, 502)\n",
            "torch.Size([10760, 1]) torch.Size([10760, 1])\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import os\n",
        "import pickle\n",
        "import torch\n",
        "import torch.utils.data\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "data_dir = './data/pytorch'\n",
        "\n",
        "with open(os.path.join(data_dir, 'word_dict_amazon.pkl'), \"rb\") as f:\n",
        "    word_dict = pickle.load(f)\n",
        "    \n",
        "train = pd.read_csv(os.path.join(data_dir, 'train_amazon.csv'), header=None, names=None)\n",
        "test_sample = pd.read_csv(os.path.join(data_dir, 'test_amazon.csv'), header=None, names=None)\n",
        "print(train.shape, test_sample.shape)\n",
        "\n",
        "\n",
        "test, val = train_test_split(test_sample, test_size=0.5)\n",
        "train.shape, test.shape, val.shape\n",
        "\n",
        "#drop rows\n",
        "test.drop(test.tail(227).index,inplace = True)\n",
        "val.drop(val.tail(228).index,inplace = True)\n",
        "test.shape, val.shape\n",
        "\n",
        "# Turn the input pandas dataframe into tensors\n",
        "train_y = torch.from_numpy(train[[0]].values).float()\n",
        "train_X = torch.from_numpy(train.drop([0, 1], axis=1).values).long()\n",
        "\n",
        "# Build the dataset\n",
        "train_ds = torch.utils.data.TensorDataset(train_X, train_y)\n",
        "# Build the dataloader\n",
        "train_dl = torch.utils.data.DataLoader(train_ds, batch_size=50)\n",
        "\n",
        "######val data\n",
        "# Turn the input pandas dataframe into tensors\n",
        "val_y = torch.from_numpy(val[[0]].values).float()\n",
        "val_X = torch.from_numpy(val.drop([0, 1], axis=1).values).long()\n",
        "\n",
        "# Build the dataset\n",
        "val_ds = torch.utils.data.TensorDataset(val_X, val_y)\n",
        "# Build the dataloader\n",
        "val_dl = torch.utils.data.DataLoader(val_ds, batch_size=50)\n",
        "\n",
        "\n",
        "#### Test data\n",
        "# Turn the input pandas dataframe into tensors\n",
        "test_y = torch.from_numpy(test[[0]].values).float()\n",
        "test_X = torch.from_numpy(test.drop([0, 1], axis=1).values).long()\n",
        "\n",
        "# Build the dataset\n",
        "test_ds = torch.utils.data.TensorDataset(test_X, test_y)\n",
        "# Build the dataloader\n",
        "test_dl = torch.utils.data.DataLoader(test_ds, batch_size=50)\n",
        "print(test_y.shape, val_y.shape)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "AfhXAHD_WEh8"
      },
      "outputs": [],
      "source": [
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "epochs = 10\n",
        "batch_size = 50\n",
        "learning_rate = 0.001\n",
        "seq_len = 500\n",
        "dropout = 0.5\n",
        "filter_size = 100\n",
        "vocab_size = 10000\n",
        "embed_dims = 32\n",
        "hidden_size = 100\n",
        "kernel_size = [3,4,5]\n",
        "criterion = torch.nn.BCELoss().to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "_Yp0LBkIWHEw"
      },
      "outputs": [],
      "source": [
        "data_dir = './lstm-cnn-amazon-hyperband-trails/' # The folder we will use for storing data\n",
        "if not os.path.exists(data_dir): # Make sure that the folder exists\n",
        "    os.makedirs(data_dir)\n",
        "\n",
        "filename = \"\"\n",
        "def write_to_csv(trail_num, epochs, train_loss, train_acc, val_loss, val_acc, time_train):\n",
        "    global filename\n",
        "    filename = \"./lstm-cnn-amazon-hyperband-trails/\"+str(trail_num)+\".csv\"\n",
        "    epoch = [i for i in range(epochs)]\n",
        "    df_metrics = pd.DataFrame(list(zip(epoch, train_loss, train_acc, val_loss, val_acc, time_train)), columns =['Epoch', 'train_loss', 'train_acc', 'val_loss', 'val_acc', 'train_time'])\n",
        "    df_metrics.to_csv(filename, index=False)    \n",
        "    \n",
        "def append_to_csv(epochs, accuracy):\n",
        "    acc = [accuracy for i in range(epochs)]\n",
        "    df_csv = pd.read_csv(filename)\n",
        "    df_csv['Test_Accuracy']  = acc\n",
        "    df_csv.to_csv(filename, index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "REymrbOqWO-G"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import numpy as np\n",
        "class Combine(nn.Module):\n",
        "    def __init__(self,trial,vocab_size, embed_size, filter_size, kernel_size, dropout, seq_len,hidden_size):\n",
        "        super(Combine, self).__init__()\n",
        "        dropout = trial.suggest_uniform(\"dropout\",0.1, 0.6)\n",
        "        hidden_size = trial.suggest_int(\"hidden_size\",32,256)\n",
        "        embed_size = trial.suggest_int(\"embed_size\",16,128)\n",
        "        filter_size = trial.suggest_int(\"filter_size\",50,200)\n",
        "        self.embedding = nn.Embedding(vocab_size, embed_size, padding_idx=0)\n",
        "        self.lstm = nn.LSTM(input_size = embed_size, hidden_size = hidden_size,num_layers=1,batch_first=True)\n",
        "        self.conv1 = torch.nn.Conv2d(1,filter_size,kernel_size=[kernel_size[0],hidden_size])\n",
        "        self.conv2 = torch.nn.Conv2d(1,filter_size,kernel_size=[kernel_size[1],hidden_size])\n",
        "        self.conv3 = torch.nn.Conv2d(1,filter_size,kernel_size=[kernel_size[2],hidden_size])\n",
        "        self.mp1 = torch.nn.MaxPool1d(seq_len+1-kernel_size[0])\n",
        "        self.mp2 = torch.nn.MaxPool1d(seq_len+1-kernel_size[1])\n",
        "        self.mp3 = torch.nn.MaxPool1d(seq_len+1-kernel_size[2])\n",
        "        self.dropout = torch.nn.Dropout(dropout)\n",
        "        self.dense = nn.Linear(3*filter_size,1)\n",
        "        self.sig = nn.Sigmoid()\n",
        "        self.word_dict = None\n",
        "\n",
        "\n",
        "    def forward(self, x,target):\n",
        "    \n",
        "        embeds = self.embedding(x)\n",
        "        # print('embedds')\n",
        "        # print(embeds.shape)\n",
        "        lstm_out, _ = self.lstm(embeds)\n",
        "        # print('lstm output')\n",
        "        # print(lstm_out.shape)\n",
        "        lstm_out = lstm_out.clone()\n",
        "        lstm_out.unsqueeze_(1)\n",
        "        # print('lstm output after')\n",
        "        # print(lstm_out.shape)\n",
        "        x1 = torch.tanh(self.dropout(self.conv1(lstm_out))).squeeze(3)\n",
        "        x2 = torch.tanh(self.dropout(self.conv2(lstm_out))).squeeze(3)\n",
        "        x3 = torch.tanh(self.dropout(self.conv3(lstm_out))).squeeze(3)\n",
        "        # print('x1.shape')\n",
        "        # print(x1.shape)\n",
        "        f1 = self.mp1(x1).squeeze(2)\n",
        "        # print('f1')\n",
        "        # print(f1.shape)\n",
        "        f2 = self.mp2(x2).squeeze(2)\n",
        "        # print('f2')\n",
        "        # print(f2.shape)\n",
        "        f3 = self.mp3(x3).squeeze(2)\n",
        "        # print('f3')\n",
        "        # print(f3.shape)\n",
        "        hidden = torch.cat([f1,f2,f3],dim=1)\n",
        "        logits = self.dense(hidden)\n",
        "        prediction = torch.sigmoid(logits)\n",
        "        target = target.view([-1,1])\n",
        "        correct_pred = torch.eq(torch.round(prediction).type(target.type()),target)\n",
        "        accuracy = torch.sum(correct_pred)\n",
        "        return prediction, accuracy\n",
        "        # return self.sig(out)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "wZ2NQqkBWV_z"
      },
      "outputs": [],
      "source": [
        "import time\n",
        "import numpy as np\n",
        "import torch.optim as optim\n",
        "\n",
        "def objective(trial):\n",
        "  optimizer_name = trial.suggest_categorical(\"optimizer\", [\"Adam\", \"RMSprop\", \"SGD\"])\n",
        "  lr = trial.suggest_loguniform(\"lr\", 1e-5, 1e-0)\n",
        "  #momentum = trial.suggest_uniform(\"momentum\", 0.0, 1.0)\n",
        "  device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "  model = Combine(trial,vocab_size, embed_dims, filter_size, kernel_size, dropout, seq_len,hidden_size).to(device).to(device)\n",
        "  trial.set_user_attr(key=\"best_model\", value=model)\n",
        "  optimizer = optim.Adam(model.parameters())\n",
        "  optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
        "  counter = 0\n",
        "  CNN_acc = []\n",
        "  CNN_valacc = []\n",
        "  train_loss_epoch = []\n",
        "  train_acc_epoch = []\n",
        "  val_loss_epoch = []\n",
        "  val_acc_epoch = []\n",
        "  time_epoch = []\n",
        "  model.train()\n",
        "  final_val_acc = 0\n",
        "  for e in range(epochs):\n",
        "    start_time = time.time()\n",
        "    train_loss = []\n",
        "    train_acc = []\n",
        "    for inputs, labels in train_dl:\n",
        "        inputs, labels = inputs.cuda(), labels.cuda()\n",
        "        model.zero_grad()\n",
        "        logits, accuracy = model(inputs,labels)\n",
        "        loss = criterion(logits,labels.float())\n",
        "        loss.backward()\n",
        "        nn.utils.clip_grad_norm_(model.parameters(), 5)\n",
        "        optimizer.step()\n",
        "        train_loss.append(loss.item())\n",
        "        train_acc.append(accuracy.item()*100/batch_size)\n",
        "        if counter%100==0:\n",
        "            print(\"Epoch: {}/{}\".format(e,epochs),\n",
        "                         \"\\tIteration: {}\".format(counter),\n",
        "                         \"\\t\\tTrain Loss: {:.3f}\".format(loss.item()),\n",
        "                         \"\\tTrain Accuracy: {:.2f}\".format(accuracy.item()*100/batch_size))\n",
        "            CNN_acc.append(accuracy.item()*100/batch_size)\n",
        "        counter += 1\n",
        "    train_loss_epoch.append(np.round(np.mean(train_loss), 3))\n",
        "    train_acc_epoch.append(np.round(np.mean(train_acc), 3))\n",
        "    print(\"\\tTrain Loss: {:.3f}\".format(np.mean(train_loss)), \"\\tTrain Acc: {:.3f}\".format(np.mean(train_acc)))\n",
        "\n",
        "    with torch.no_grad():\n",
        "        model.eval()\n",
        "        val_acc = []\n",
        "        val_loss = []\n",
        "        for inputs, labels in val_dl:\n",
        "            inputs_val, labels_val = inputs.cuda(), labels.cuda()\n",
        "            logits_val,accuracy_val = model(inputs_val,labels_val)\n",
        "            loss_val = criterion(logits_val,labels_val.float())\n",
        "            val_acc.append(accuracy_val.item()*100/batch_size)\n",
        "            val_loss.append(loss_val.item())\n",
        "        val_loss_epoch.append(np.round(np.mean(val_loss), 3))\n",
        "        val_acc_epoch.append(np.round(np.mean(val_acc), 3))\n",
        "        final_val_acc = np.round(np.mean(val_acc), 3)\n",
        "        print(\"\\t\\tVal Loss: {:.3f}\".format(np.mean(val_loss)), \"\\t\\tVal Acc: {:.3f}\".format(np.mean(val_acc)))\n",
        "        CNN_valacc.append(np.mean(val_acc))\n",
        "        model.train()\n",
        "        \n",
        "    end_time = time.time()-start_time\n",
        "    print(\"Time to train epoch: {0} s\".format(end_time))\n",
        "    time_epoch.append(np.round(end_time, 3))\n",
        "  write_to_csv(trial.number,epochs, train_loss_epoch, train_acc_epoch, val_loss_epoch, val_acc_epoch, time_epoch)\n",
        "  return final_val_acc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "Wb_YBPvvtgI0"
      },
      "outputs": [],
      "source": [
        "def callback(study, trial):\n",
        "    if study.best_trial.number == trial.number:\n",
        "        study.set_user_attr(key=\"best_model\", value=trial.user_attrs[\"best_model\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uitgSC33tqve",
        "outputId": "df12a0dc-864f-4ee1-b1b5-bf7286296bc3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting optuna\n",
            "  Downloading optuna-2.10.0-py3-none-any.whl (308 kB)\n",
            "\u001b[K     |████████████████████████████████| 308 kB 5.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: sqlalchemy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from optuna) (1.4.36)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from optuna) (1.21.6)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from optuna) (3.13)\n",
            "Collecting cliff\n",
            "  Downloading cliff-3.10.1-py3-none-any.whl (81 kB)\n",
            "\u001b[K     |████████████████████████████████| 81 kB 11.6 MB/s \n",
            "\u001b[?25hCollecting alembic\n",
            "  Downloading alembic-1.7.7-py3-none-any.whl (210 kB)\n",
            "\u001b[K     |████████████████████████████████| 210 kB 72.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from optuna) (21.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from optuna) (4.64.0)\n",
            "Requirement already satisfied: scipy!=1.4.0 in /usr/local/lib/python3.7/dist-packages (from optuna) (1.4.1)\n",
            "Collecting colorlog\n",
            "  Downloading colorlog-6.6.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting cmaes>=0.8.2\n",
            "  Downloading cmaes-0.8.2-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->optuna) (3.0.8)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.7/dist-packages (from sqlalchemy>=1.1.0->optuna) (1.1.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from sqlalchemy>=1.1.0->optuna) (4.11.3)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from alembic->optuna) (5.7.1)\n",
            "Collecting Mako\n",
            "  Downloading Mako-1.2.0-py3-none-any.whl (78 kB)\n",
            "\u001b[K     |████████████████████████████████| 78 kB 7.9 MB/s \n",
            "\u001b[?25hCollecting autopage>=0.4.0\n",
            "  Downloading autopage-0.5.0-py3-none-any.whl (29 kB)\n",
            "Collecting pbr!=2.1.0,>=2.0.0\n",
            "  Downloading pbr-5.9.0-py2.py3-none-any.whl (112 kB)\n",
            "\u001b[K     |████████████████████████████████| 112 kB 74.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: PrettyTable>=0.7.2 in /usr/local/lib/python3.7/dist-packages (from cliff->optuna) (3.2.0)\n",
            "Collecting stevedore>=2.0.1\n",
            "  Downloading stevedore-3.5.0-py3-none-any.whl (49 kB)\n",
            "\u001b[K     |████████████████████████████████| 49 kB 7.8 MB/s \n",
            "\u001b[?25hCollecting cmd2>=1.0.0\n",
            "  Downloading cmd2-2.4.1-py3-none-any.whl (146 kB)\n",
            "\u001b[K     |████████████████████████████████| 146 kB 67.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from cmd2>=1.0.0->cliff->optuna) (4.2.0)\n",
            "Requirement already satisfied: attrs>=16.3.0 in /usr/local/lib/python3.7/dist-packages (from cmd2>=1.0.0->cliff->optuna) (21.4.0)\n",
            "Collecting pyperclip>=1.6\n",
            "  Downloading pyperclip-1.8.2.tar.gz (20 kB)\n",
            "Requirement already satisfied: wcwidth>=0.1.7 in /usr/local/lib/python3.7/dist-packages (from cmd2>=1.0.0->cliff->optuna) (0.2.5)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->sqlalchemy>=1.1.0->optuna) (3.8.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.9.2 in /usr/local/lib/python3.7/dist-packages (from Mako->alembic->optuna) (2.0.1)\n",
            "Building wheels for collected packages: pyperclip\n",
            "  Building wheel for pyperclip (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyperclip: filename=pyperclip-1.8.2-py3-none-any.whl size=11137 sha256=9d095b85212c6fa2a2317e095c3bb5a5ddc9572d1903fb97b0607588a1ec2f0e\n",
            "  Stored in directory: /root/.cache/pip/wheels/9f/18/84/8f69f8b08169c7bae2dde6bd7daf0c19fca8c8e500ee620a28\n",
            "Successfully built pyperclip\n",
            "Installing collected packages: pyperclip, pbr, stevedore, Mako, cmd2, autopage, colorlog, cmaes, cliff, alembic, optuna\n",
            "Successfully installed Mako-1.2.0 alembic-1.7.7 autopage-0.5.0 cliff-3.10.1 cmaes-0.8.2 cmd2-2.4.1 colorlog-6.6.0 optuna-2.10.0 pbr-5.9.0 pyperclip-1.8.2 stevedore-3.5.0\n"
          ]
        }
      ],
      "source": [
        "!pip install optuna"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZpsCU3hEth_l",
        "outputId": "572e35c9-eeeb-4063-f2f2-862c5c2b3245"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2022-05-15 04:07:44,712]\u001b[0m A new study created in memory with name: no-name-26f95c5f-3eeb-4a02-a0fc-0c85ac9c0d2e\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 0/10 \tIteration: 0 \t\tTrain Loss: 0.682 \tTrain Accuracy: 78.00\n",
            "Epoch: 0/10 \tIteration: 100 \t\tTrain Loss: 0.608 \tTrain Accuracy: 74.00\n",
            "Epoch: 0/10 \tIteration: 200 \t\tTrain Loss: 0.404 \tTrain Accuracy: 78.00\n",
            "Epoch: 0/10 \tIteration: 300 \t\tTrain Loss: 0.397 \tTrain Accuracy: 84.00\n",
            "Epoch: 0/10 \tIteration: 400 \t\tTrain Loss: 0.228 \tTrain Accuracy: 96.00\n",
            "Epoch: 0/10 \tIteration: 500 \t\tTrain Loss: 0.291 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 600 \t\tTrain Loss: 0.172 \tTrain Accuracy: 96.00\n",
            "Epoch: 0/10 \tIteration: 700 \t\tTrain Loss: 0.425 \tTrain Accuracy: 82.00\n",
            "Epoch: 0/10 \tIteration: 800 \t\tTrain Loss: 0.318 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 900 \t\tTrain Loss: 0.125 \tTrain Accuracy: 96.00\n",
            "Epoch: 0/10 \tIteration: 1000 \t\tTrain Loss: 0.320 \tTrain Accuracy: 84.00\n",
            "Epoch: 0/10 \tIteration: 1100 \t\tTrain Loss: 0.347 \tTrain Accuracy: 84.00\n",
            "Epoch: 0/10 \tIteration: 1200 \t\tTrain Loss: 0.214 \tTrain Accuracy: 92.00\n",
            "Epoch: 0/10 \tIteration: 1300 \t\tTrain Loss: 0.180 \tTrain Accuracy: 94.00\n",
            "Epoch: 0/10 \tIteration: 1400 \t\tTrain Loss: 0.250 \tTrain Accuracy: 84.00\n",
            "Epoch: 0/10 \tIteration: 1500 \t\tTrain Loss: 0.298 \tTrain Accuracy: 90.00\n",
            "Epoch: 0/10 \tIteration: 1600 \t\tTrain Loss: 0.178 \tTrain Accuracy: 94.00\n",
            "\tTrain Loss: 0.321 \tTrain Acc: 86.120\n",
            "\t\tVal Loss: 0.272 \t\tVal Acc: 87.926\n",
            "Time to train epoch: 125.99290943145752 s\n",
            "Epoch: 1/10 \tIteration: 1700 \t\tTrain Loss: 0.207 \tTrain Accuracy: 94.00\n",
            "Epoch: 1/10 \tIteration: 1800 \t\tTrain Loss: 0.250 \tTrain Accuracy: 88.00\n",
            "Epoch: 1/10 \tIteration: 1900 \t\tTrain Loss: 0.382 \tTrain Accuracy: 80.00\n",
            "Epoch: 1/10 \tIteration: 2000 \t\tTrain Loss: 0.199 \tTrain Accuracy: 92.00\n",
            "Epoch: 1/10 \tIteration: 2100 \t\tTrain Loss: 0.234 \tTrain Accuracy: 92.00\n",
            "Epoch: 1/10 \tIteration: 2200 \t\tTrain Loss: 0.287 \tTrain Accuracy: 88.00\n",
            "Epoch: 1/10 \tIteration: 2300 \t\tTrain Loss: 0.197 \tTrain Accuracy: 88.00\n",
            "Epoch: 1/10 \tIteration: 2400 \t\tTrain Loss: 0.200 \tTrain Accuracy: 90.00\n",
            "Epoch: 1/10 \tIteration: 2500 \t\tTrain Loss: 0.289 \tTrain Accuracy: 88.00\n",
            "Epoch: 1/10 \tIteration: 2600 \t\tTrain Loss: 0.180 \tTrain Accuracy: 94.00\n",
            "Epoch: 1/10 \tIteration: 2700 \t\tTrain Loss: 0.430 \tTrain Accuracy: 76.00\n",
            "Epoch: 1/10 \tIteration: 2800 \t\tTrain Loss: 0.235 \tTrain Accuracy: 92.00\n",
            "Epoch: 1/10 \tIteration: 2900 \t\tTrain Loss: 0.360 \tTrain Accuracy: 90.00\n",
            "Epoch: 1/10 \tIteration: 3000 \t\tTrain Loss: 0.149 \tTrain Accuracy: 94.00\n",
            "Epoch: 1/10 \tIteration: 3100 \t\tTrain Loss: 0.123 \tTrain Accuracy: 96.00\n",
            "Epoch: 1/10 \tIteration: 3200 \t\tTrain Loss: 0.355 \tTrain Accuracy: 80.00\n",
            "Epoch: 1/10 \tIteration: 3300 \t\tTrain Loss: 0.203 \tTrain Accuracy: 90.00\n",
            "\tTrain Loss: 0.245 \tTrain Acc: 89.778\n",
            "\t\tVal Loss: 0.271 \t\tVal Acc: 88.593\n",
            "Time to train epoch: 132.83009243011475 s\n",
            "Epoch: 2/10 \tIteration: 3400 \t\tTrain Loss: 0.211 \tTrain Accuracy: 90.00\n",
            "Epoch: 2/10 \tIteration: 3500 \t\tTrain Loss: 0.216 \tTrain Accuracy: 88.00\n",
            "Epoch: 2/10 \tIteration: 3600 \t\tTrain Loss: 0.150 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 3700 \t\tTrain Loss: 0.114 \tTrain Accuracy: 98.00\n",
            "Epoch: 2/10 \tIteration: 3800 \t\tTrain Loss: 0.173 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 3900 \t\tTrain Loss: 0.206 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 4000 \t\tTrain Loss: 0.225 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 4100 \t\tTrain Loss: 0.152 \tTrain Accuracy: 88.00\n",
            "Epoch: 2/10 \tIteration: 4200 \t\tTrain Loss: 0.280 \tTrain Accuracy: 86.00\n",
            "Epoch: 2/10 \tIteration: 4300 \t\tTrain Loss: 0.079 \tTrain Accuracy: 98.00\n",
            "Epoch: 2/10 \tIteration: 4400 \t\tTrain Loss: 0.146 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 4500 \t\tTrain Loss: 0.231 \tTrain Accuracy: 90.00\n",
            "Epoch: 2/10 \tIteration: 4600 \t\tTrain Loss: 0.092 \tTrain Accuracy: 98.00\n",
            "Epoch: 2/10 \tIteration: 4700 \t\tTrain Loss: 0.213 \tTrain Accuracy: 96.00\n",
            "Epoch: 2/10 \tIteration: 4800 \t\tTrain Loss: 0.273 \tTrain Accuracy: 88.00\n",
            "Epoch: 2/10 \tIteration: 4900 \t\tTrain Loss: 0.245 \tTrain Accuracy: 90.00\n",
            "\tTrain Loss: 0.200 \tTrain Acc: 91.890\n",
            "\t\tVal Loss: 0.307 \t\tVal Acc: 88.769\n",
            "Time to train epoch: 134.4612057209015 s\n",
            "Epoch: 3/10 \tIteration: 5000 \t\tTrain Loss: 0.215 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 5100 \t\tTrain Loss: 0.337 \tTrain Accuracy: 86.00\n",
            "Epoch: 3/10 \tIteration: 5200 \t\tTrain Loss: 0.092 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 5300 \t\tTrain Loss: 0.213 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 5400 \t\tTrain Loss: 0.307 \tTrain Accuracy: 88.00\n",
            "Epoch: 3/10 \tIteration: 5500 \t\tTrain Loss: 0.118 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 5600 \t\tTrain Loss: 0.183 \tTrain Accuracy: 90.00\n",
            "Epoch: 3/10 \tIteration: 5700 \t\tTrain Loss: 0.169 \tTrain Accuracy: 92.00\n",
            "Epoch: 3/10 \tIteration: 5800 \t\tTrain Loss: 0.191 \tTrain Accuracy: 98.00\n",
            "Epoch: 3/10 \tIteration: 5900 \t\tTrain Loss: 0.137 \tTrain Accuracy: 92.00\n",
            "Epoch: 3/10 \tIteration: 6000 \t\tTrain Loss: 0.182 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 6100 \t\tTrain Loss: 0.134 \tTrain Accuracy: 90.00\n",
            "Epoch: 3/10 \tIteration: 6200 \t\tTrain Loss: 0.138 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 6300 \t\tTrain Loss: 0.087 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 6400 \t\tTrain Loss: 0.183 \tTrain Accuracy: 98.00\n",
            "Epoch: 3/10 \tIteration: 6500 \t\tTrain Loss: 0.092 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 6600 \t\tTrain Loss: 0.158 \tTrain Accuracy: 94.00\n",
            "\tTrain Loss: 0.145 \tTrain Acc: 94.519\n",
            "\t\tVal Loss: 0.422 \t\tVal Acc: 88.324\n",
            "Time to train epoch: 134.5983908176422 s\n",
            "Epoch: 4/10 \tIteration: 6700 \t\tTrain Loss: 0.079 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 6800 \t\tTrain Loss: 0.075 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 6900 \t\tTrain Loss: 0.033 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7000 \t\tTrain Loss: 0.134 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 7100 \t\tTrain Loss: 0.081 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 7200 \t\tTrain Loss: 0.026 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7300 \t\tTrain Loss: 0.091 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 7400 \t\tTrain Loss: 0.027 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7500 \t\tTrain Loss: 0.099 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 7600 \t\tTrain Loss: 0.029 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7700 \t\tTrain Loss: 0.127 \tTrain Accuracy: 92.00\n",
            "Epoch: 4/10 \tIteration: 7800 \t\tTrain Loss: 0.242 \tTrain Accuracy: 92.00\n",
            "Epoch: 4/10 \tIteration: 7900 \t\tTrain Loss: 0.090 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 8000 \t\tTrain Loss: 0.016 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 8100 \t\tTrain Loss: 0.028 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 8200 \t\tTrain Loss: 0.164 \tTrain Accuracy: 92.00\n",
            "\tTrain Loss: 0.095 \tTrain Acc: 96.651\n",
            "\t\tVal Loss: 0.625 \t\tVal Acc: 87.713\n",
            "Time to train epoch: 134.68089294433594 s\n",
            "Epoch: 5/10 \tIteration: 8300 \t\tTrain Loss: 0.097 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 8400 \t\tTrain Loss: 0.203 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 8500 \t\tTrain Loss: 0.109 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 8600 \t\tTrain Loss: 0.160 \tTrain Accuracy: 94.00\n",
            "Epoch: 5/10 \tIteration: 8700 \t\tTrain Loss: 0.078 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 8800 \t\tTrain Loss: 0.089 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 8900 \t\tTrain Loss: 0.049 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9000 \t\tTrain Loss: 0.047 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9100 \t\tTrain Loss: 0.024 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9200 \t\tTrain Loss: 0.010 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9300 \t\tTrain Loss: 0.159 \tTrain Accuracy: 92.00\n",
            "Epoch: 5/10 \tIteration: 9400 \t\tTrain Loss: 0.107 \tTrain Accuracy: 94.00\n",
            "Epoch: 5/10 \tIteration: 9500 \t\tTrain Loss: 0.089 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 9600 \t\tTrain Loss: 0.028 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9700 \t\tTrain Loss: 0.038 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9800 \t\tTrain Loss: 0.239 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 9900 \t\tTrain Loss: 0.014 \tTrain Accuracy: 100.00\n",
            "\tTrain Loss: 0.069 \tTrain Acc: 97.539\n",
            "\t\tVal Loss: 0.852 \t\tVal Acc: 87.639\n",
            "Time to train epoch: 134.63597011566162 s\n",
            "Epoch: 6/10 \tIteration: 10000 \t\tTrain Loss: 0.065 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10100 \t\tTrain Loss: 0.061 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 10200 \t\tTrain Loss: 0.227 \tTrain Accuracy: 94.00\n",
            "Epoch: 6/10 \tIteration: 10300 \t\tTrain Loss: 0.107 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 10400 \t\tTrain Loss: 0.018 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 10500 \t\tTrain Loss: 0.032 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 10600 \t\tTrain Loss: 0.039 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10700 \t\tTrain Loss: 0.025 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 10800 \t\tTrain Loss: 0.033 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 10900 \t\tTrain Loss: 0.063 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 11000 \t\tTrain Loss: 0.217 \tTrain Accuracy: 92.00\n",
            "Epoch: 6/10 \tIteration: 11100 \t\tTrain Loss: 0.136 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 11200 \t\tTrain Loss: 0.016 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 11300 \t\tTrain Loss: 0.022 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 11400 \t\tTrain Loss: 0.035 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 11500 \t\tTrain Loss: 0.219 \tTrain Accuracy: 92.00\n",
            "Epoch: 6/10 \tIteration: 11600 \t\tTrain Loss: 0.030 \tTrain Accuracy: 100.00\n",
            "\tTrain Loss: 0.051 \tTrain Acc: 98.167\n",
            "\t\tVal Loss: 1.119 \t\tVal Acc: 88.083\n",
            "Time to train epoch: 134.22503900527954 s\n",
            "Epoch: 7/10 \tIteration: 11700 \t\tTrain Loss: 0.030 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 11800 \t\tTrain Loss: 0.019 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 11900 \t\tTrain Loss: 0.005 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12000 \t\tTrain Loss: 0.029 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12100 \t\tTrain Loss: 0.055 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12200 \t\tTrain Loss: 0.046 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12300 \t\tTrain Loss: 0.065 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 12400 \t\tTrain Loss: 0.078 \tTrain Accuracy: 94.00\n",
            "Epoch: 7/10 \tIteration: 12500 \t\tTrain Loss: 0.096 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12600 \t\tTrain Loss: 0.030 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12700 \t\tTrain Loss: 0.005 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12800 \t\tTrain Loss: 0.024 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12900 \t\tTrain Loss: 0.005 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 13000 \t\tTrain Loss: 0.024 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 13100 \t\tTrain Loss: 0.005 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 13200 \t\tTrain Loss: 0.015 \tTrain Accuracy: 100.00\n",
            "\tTrain Loss: 0.039 \tTrain Acc: 98.645\n",
            "\t\tVal Loss: 1.294 \t\tVal Acc: 87.796\n",
            "Time to train epoch: 134.25688242912292 s\n",
            "Epoch: 8/10 \tIteration: 13300 \t\tTrain Loss: 0.075 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 13400 \t\tTrain Loss: 0.005 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13500 \t\tTrain Loss: 0.011 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13600 \t\tTrain Loss: 0.015 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13700 \t\tTrain Loss: 0.002 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13800 \t\tTrain Loss: 0.036 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 13900 \t\tTrain Loss: 0.026 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14000 \t\tTrain Loss: 0.029 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14100 \t\tTrain Loss: 0.019 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14200 \t\tTrain Loss: 0.039 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14300 \t\tTrain Loss: 0.023 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14400 \t\tTrain Loss: 0.014 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14500 \t\tTrain Loss: 0.075 \tTrain Accuracy: 96.00\n",
            "Epoch: 8/10 \tIteration: 14600 \t\tTrain Loss: 0.036 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14700 \t\tTrain Loss: 0.019 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14800 \t\tTrain Loss: 0.090 \tTrain Accuracy: 96.00\n",
            "Epoch: 8/10 \tIteration: 14900 \t\tTrain Loss: 0.013 \tTrain Accuracy: 100.00\n",
            "\tTrain Loss: 0.031 \tTrain Acc: 98.958\n",
            "\t\tVal Loss: 1.462 \t\tVal Acc: 87.907\n",
            "Time to train epoch: 134.63911318778992 s\n",
            "Epoch: 9/10 \tIteration: 15000 \t\tTrain Loss: 0.016 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15100 \t\tTrain Loss: 0.009 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15200 \t\tTrain Loss: 0.009 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15300 \t\tTrain Loss: 0.117 \tTrain Accuracy: 96.00\n",
            "Epoch: 9/10 \tIteration: 15400 \t\tTrain Loss: 0.022 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15500 \t\tTrain Loss: 0.004 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15600 \t\tTrain Loss: 0.025 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15700 \t\tTrain Loss: 0.025 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15800 \t\tTrain Loss: 0.060 \tTrain Accuracy: 96.00\n",
            "Epoch: 9/10 \tIteration: 15900 \t\tTrain Loss: 0.013 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16000 \t\tTrain Loss: 0.034 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 16100 \t\tTrain Loss: 0.011 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16200 \t\tTrain Loss: 0.015 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16300 \t\tTrain Loss: 0.012 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16400 \t\tTrain Loss: 0.005 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16500 \t\tTrain Loss: 0.074 \tTrain Accuracy: 96.00\n",
            "\tTrain Loss: 0.027 \tTrain Acc: 99.084\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2022-05-15 04:30:14,319]\u001b[0m Trial 0 finished with value: 87.787 and parameters: {'optimizer': 'RMSprop', 'lr': 4.429401815530183e-05, 'dropout': 0.2221586224652677, 'hidden_size': 232, 'embed_size': 66, 'filter_size': 143}. Best is trial 0 with value: 87.787.\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\t\tVal Loss: 1.811 \t\tVal Acc: 87.787\n",
            "Time to train epoch: 134.2358009815216 s\n",
            "Epoch: 0/10 \tIteration: 0 \t\tTrain Loss: 0.662 \tTrain Accuracy: 78.00\n",
            "Epoch: 0/10 \tIteration: 100 \t\tTrain Loss: 0.430 \tTrain Accuracy: 88.00\n",
            "Epoch: 0/10 \tIteration: 200 \t\tTrain Loss: 0.353 \tTrain Accuracy: 84.00\n",
            "Epoch: 0/10 \tIteration: 300 \t\tTrain Loss: 0.393 \tTrain Accuracy: 78.00\n",
            "Epoch: 0/10 \tIteration: 400 \t\tTrain Loss: 0.248 \tTrain Accuracy: 90.00\n",
            "Epoch: 0/10 \tIteration: 500 \t\tTrain Loss: 0.257 \tTrain Accuracy: 88.00\n",
            "Epoch: 0/10 \tIteration: 600 \t\tTrain Loss: 0.193 \tTrain Accuracy: 96.00\n",
            "Epoch: 0/10 \tIteration: 700 \t\tTrain Loss: 0.343 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 800 \t\tTrain Loss: 0.296 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 900 \t\tTrain Loss: 0.087 \tTrain Accuracy: 98.00\n",
            "Epoch: 0/10 \tIteration: 1000 \t\tTrain Loss: 0.365 \tTrain Accuracy: 82.00\n",
            "Epoch: 0/10 \tIteration: 1100 \t\tTrain Loss: 0.307 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 1200 \t\tTrain Loss: 0.182 \tTrain Accuracy: 94.00\n",
            "Epoch: 0/10 \tIteration: 1300 \t\tTrain Loss: 0.171 \tTrain Accuracy: 96.00\n",
            "Epoch: 0/10 \tIteration: 1400 \t\tTrain Loss: 0.312 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 1500 \t\tTrain Loss: 0.310 \tTrain Accuracy: 90.00\n",
            "Epoch: 0/10 \tIteration: 1600 \t\tTrain Loss: 0.215 \tTrain Accuracy: 88.00\n",
            "\tTrain Loss: 0.306 \tTrain Acc: 86.778\n",
            "\t\tVal Loss: 0.267 \t\tVal Acc: 88.704\n",
            "Time to train epoch: 84.56467342376709 s\n",
            "Epoch: 1/10 \tIteration: 1700 \t\tTrain Loss: 0.196 \tTrain Accuracy: 96.00\n",
            "Epoch: 1/10 \tIteration: 1800 \t\tTrain Loss: 0.290 \tTrain Accuracy: 88.00\n",
            "Epoch: 1/10 \tIteration: 1900 \t\tTrain Loss: 0.438 \tTrain Accuracy: 76.00\n",
            "Epoch: 1/10 \tIteration: 2000 \t\tTrain Loss: 0.196 \tTrain Accuracy: 94.00\n",
            "Epoch: 1/10 \tIteration: 2100 \t\tTrain Loss: 0.193 \tTrain Accuracy: 92.00\n",
            "Epoch: 1/10 \tIteration: 2200 \t\tTrain Loss: 0.301 \tTrain Accuracy: 90.00\n",
            "Epoch: 1/10 \tIteration: 2300 \t\tTrain Loss: 0.184 \tTrain Accuracy: 90.00\n",
            "Epoch: 1/10 \tIteration: 2400 \t\tTrain Loss: 0.181 \tTrain Accuracy: 92.00\n",
            "Epoch: 1/10 \tIteration: 2500 \t\tTrain Loss: 0.288 \tTrain Accuracy: 88.00\n",
            "Epoch: 1/10 \tIteration: 2600 \t\tTrain Loss: 0.201 \tTrain Accuracy: 90.00\n",
            "Epoch: 1/10 \tIteration: 2700 \t\tTrain Loss: 0.320 \tTrain Accuracy: 88.00\n",
            "Epoch: 1/10 \tIteration: 2800 \t\tTrain Loss: 0.273 \tTrain Accuracy: 92.00\n",
            "Epoch: 1/10 \tIteration: 2900 \t\tTrain Loss: 0.324 \tTrain Accuracy: 90.00\n",
            "Epoch: 1/10 \tIteration: 3000 \t\tTrain Loss: 0.162 \tTrain Accuracy: 92.00\n",
            "Epoch: 1/10 \tIteration: 3100 \t\tTrain Loss: 0.132 \tTrain Accuracy: 94.00\n",
            "Epoch: 1/10 \tIteration: 3200 \t\tTrain Loss: 0.312 \tTrain Accuracy: 78.00\n",
            "Epoch: 1/10 \tIteration: 3300 \t\tTrain Loss: 0.169 \tTrain Accuracy: 92.00\n",
            "\tTrain Loss: 0.239 \tTrain Acc: 90.113\n",
            "\t\tVal Loss: 0.266 \t\tVal Acc: 89.213\n",
            "Time to train epoch: 84.87624907493591 s\n",
            "Epoch: 2/10 \tIteration: 3400 \t\tTrain Loss: 0.203 \tTrain Accuracy: 90.00\n",
            "Epoch: 2/10 \tIteration: 3500 \t\tTrain Loss: 0.171 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 3600 \t\tTrain Loss: 0.121 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 3700 \t\tTrain Loss: 0.161 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 3800 \t\tTrain Loss: 0.175 \tTrain Accuracy: 90.00\n",
            "Epoch: 2/10 \tIteration: 3900 \t\tTrain Loss: 0.224 \tTrain Accuracy: 88.00\n",
            "Epoch: 2/10 \tIteration: 4000 \t\tTrain Loss: 0.208 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 4100 \t\tTrain Loss: 0.159 \tTrain Accuracy: 88.00\n",
            "Epoch: 2/10 \tIteration: 4200 \t\tTrain Loss: 0.226 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 4300 \t\tTrain Loss: 0.120 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 4400 \t\tTrain Loss: 0.114 \tTrain Accuracy: 98.00\n",
            "Epoch: 2/10 \tIteration: 4500 \t\tTrain Loss: 0.187 \tTrain Accuracy: 90.00\n",
            "Epoch: 2/10 \tIteration: 4600 \t\tTrain Loss: 0.106 \tTrain Accuracy: 96.00\n",
            "Epoch: 2/10 \tIteration: 4700 \t\tTrain Loss: 0.217 \tTrain Accuracy: 90.00\n",
            "Epoch: 2/10 \tIteration: 4800 \t\tTrain Loss: 0.360 \tTrain Accuracy: 86.00\n",
            "Epoch: 2/10 \tIteration: 4900 \t\tTrain Loss: 0.308 \tTrain Accuracy: 84.00\n",
            "\tTrain Loss: 0.197 \tTrain Acc: 92.186\n",
            "\t\tVal Loss: 0.307 \t\tVal Acc: 88.815\n",
            "Time to train epoch: 84.68143010139465 s\n",
            "Epoch: 3/10 \tIteration: 5000 \t\tTrain Loss: 0.133 \tTrain Accuracy: 98.00\n",
            "Epoch: 3/10 \tIteration: 5100 \t\tTrain Loss: 0.276 \tTrain Accuracy: 92.00\n",
            "Epoch: 3/10 \tIteration: 5200 \t\tTrain Loss: 0.064 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 5300 \t\tTrain Loss: 0.128 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 5400 \t\tTrain Loss: 0.350 \tTrain Accuracy: 86.00\n",
            "Epoch: 3/10 \tIteration: 5500 \t\tTrain Loss: 0.108 \tTrain Accuracy: 92.00\n",
            "Epoch: 3/10 \tIteration: 5600 \t\tTrain Loss: 0.231 \tTrain Accuracy: 92.00\n",
            "Epoch: 3/10 \tIteration: 5700 \t\tTrain Loss: 0.175 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 5800 \t\tTrain Loss: 0.154 \tTrain Accuracy: 98.00\n",
            "Epoch: 3/10 \tIteration: 5900 \t\tTrain Loss: 0.110 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 6000 \t\tTrain Loss: 0.242 \tTrain Accuracy: 92.00\n",
            "Epoch: 3/10 \tIteration: 6100 \t\tTrain Loss: 0.091 \tTrain Accuracy: 98.00\n",
            "Epoch: 3/10 \tIteration: 6200 \t\tTrain Loss: 0.122 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 6300 \t\tTrain Loss: 0.108 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 6400 \t\tTrain Loss: 0.173 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 6500 \t\tTrain Loss: 0.077 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 6600 \t\tTrain Loss: 0.168 \tTrain Accuracy: 94.00\n",
            "\tTrain Loss: 0.150 \tTrain Acc: 94.298\n",
            "\t\tVal Loss: 0.387 \t\tVal Acc: 88.315\n",
            "Time to train epoch: 84.51785063743591 s\n",
            "Epoch: 4/10 \tIteration: 6700 \t\tTrain Loss: 0.059 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 6800 \t\tTrain Loss: 0.109 \tTrain Accuracy: 92.00\n",
            "Epoch: 4/10 \tIteration: 6900 \t\tTrain Loss: 0.071 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 7000 \t\tTrain Loss: 0.091 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 7100 \t\tTrain Loss: 0.132 \tTrain Accuracy: 94.00\n",
            "Epoch: 4/10 \tIteration: 7200 \t\tTrain Loss: 0.031 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7300 \t\tTrain Loss: 0.134 \tTrain Accuracy: 94.00\n",
            "Epoch: 4/10 \tIteration: 7400 \t\tTrain Loss: 0.032 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7500 \t\tTrain Loss: 0.088 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 7600 \t\tTrain Loss: 0.016 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7700 \t\tTrain Loss: 0.128 \tTrain Accuracy: 94.00\n",
            "Epoch: 4/10 \tIteration: 7800 \t\tTrain Loss: 0.289 \tTrain Accuracy: 94.00\n",
            "Epoch: 4/10 \tIteration: 7900 \t\tTrain Loss: 0.054 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 8000 \t\tTrain Loss: 0.079 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 8100 \t\tTrain Loss: 0.030 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 8200 \t\tTrain Loss: 0.109 \tTrain Accuracy: 98.00\n",
            "\tTrain Loss: 0.105 \tTrain Acc: 96.252\n",
            "\t\tVal Loss: 0.524 \t\tVal Acc: 88.046\n",
            "Time to train epoch: 84.52167582511902 s\n",
            "Epoch: 5/10 \tIteration: 8300 \t\tTrain Loss: 0.056 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 8400 \t\tTrain Loss: 0.264 \tTrain Accuracy: 94.00\n",
            "Epoch: 5/10 \tIteration: 8500 \t\tTrain Loss: 0.062 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 8600 \t\tTrain Loss: 0.131 \tTrain Accuracy: 94.00\n",
            "Epoch: 5/10 \tIteration: 8700 \t\tTrain Loss: 0.055 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 8800 \t\tTrain Loss: 0.018 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 8900 \t\tTrain Loss: 0.062 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9000 \t\tTrain Loss: 0.048 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9100 \t\tTrain Loss: 0.061 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9200 \t\tTrain Loss: 0.009 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9300 \t\tTrain Loss: 0.117 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 9400 \t\tTrain Loss: 0.030 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9500 \t\tTrain Loss: 0.057 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9600 \t\tTrain Loss: 0.060 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9700 \t\tTrain Loss: 0.035 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9800 \t\tTrain Loss: 0.253 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9900 \t\tTrain Loss: 0.069 \tTrain Accuracy: 96.00\n",
            "\tTrain Loss: 0.077 \tTrain Acc: 97.289\n",
            "\t\tVal Loss: 0.743 \t\tVal Acc: 87.926\n",
            "Time to train epoch: 84.75972104072571 s\n",
            "Epoch: 6/10 \tIteration: 10000 \t\tTrain Loss: 0.036 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10100 \t\tTrain Loss: 0.066 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10200 \t\tTrain Loss: 0.043 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 10300 \t\tTrain Loss: 0.032 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10400 \t\tTrain Loss: 0.048 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 10500 \t\tTrain Loss: 0.090 \tTrain Accuracy: 94.00\n",
            "Epoch: 6/10 \tIteration: 10600 \t\tTrain Loss: 0.038 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10700 \t\tTrain Loss: 0.021 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10800 \t\tTrain Loss: 0.129 \tTrain Accuracy: 94.00\n",
            "Epoch: 6/10 \tIteration: 10900 \t\tTrain Loss: 0.064 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 11000 \t\tTrain Loss: 0.118 \tTrain Accuracy: 94.00\n",
            "Epoch: 6/10 \tIteration: 11100 \t\tTrain Loss: 0.147 \tTrain Accuracy: 94.00\n",
            "Epoch: 6/10 \tIteration: 11200 \t\tTrain Loss: 0.040 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 11300 \t\tTrain Loss: 0.032 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 11400 \t\tTrain Loss: 0.020 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 11500 \t\tTrain Loss: 0.085 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 11600 \t\tTrain Loss: 0.057 \tTrain Accuracy: 96.00\n",
            "\tTrain Loss: 0.058 \tTrain Acc: 98.027\n",
            "\t\tVal Loss: 1.381 \t\tVal Acc: 87.935\n",
            "Time to train epoch: 84.85952520370483 s\n",
            "Epoch: 7/10 \tIteration: 11700 \t\tTrain Loss: 0.035 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 11800 \t\tTrain Loss: 0.091 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 11900 \t\tTrain Loss: 0.012 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12000 \t\tTrain Loss: 0.012 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12100 \t\tTrain Loss: 0.068 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 12200 \t\tTrain Loss: 0.047 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12300 \t\tTrain Loss: 0.140 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 12400 \t\tTrain Loss: 0.057 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 12500 \t\tTrain Loss: 0.104 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12600 \t\tTrain Loss: 0.007 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12700 \t\tTrain Loss: 0.040 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12800 \t\tTrain Loss: 0.035 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12900 \t\tTrain Loss: 0.020 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 13000 \t\tTrain Loss: 0.012 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 13100 \t\tTrain Loss: 0.064 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 13200 \t\tTrain Loss: 0.013 \tTrain Accuracy: 100.00\n",
            "\tTrain Loss: 0.047 \tTrain Acc: 98.357\n",
            "\t\tVal Loss: 1.572 \t\tVal Acc: 87.991\n",
            "Time to train epoch: 84.75507497787476 s\n",
            "Epoch: 8/10 \tIteration: 13300 \t\tTrain Loss: 0.056 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 13400 \t\tTrain Loss: 0.044 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 13500 \t\tTrain Loss: 0.003 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13600 \t\tTrain Loss: 0.029 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13700 \t\tTrain Loss: 0.027 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13800 \t\tTrain Loss: 0.023 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 13900 \t\tTrain Loss: 0.066 \tTrain Accuracy: 96.00\n",
            "Epoch: 8/10 \tIteration: 14000 \t\tTrain Loss: 0.142 \tTrain Accuracy: 96.00\n",
            "Epoch: 8/10 \tIteration: 14100 \t\tTrain Loss: 0.006 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14200 \t\tTrain Loss: 0.025 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14300 \t\tTrain Loss: 0.036 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14400 \t\tTrain Loss: 0.021 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14500 \t\tTrain Loss: 0.013 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14600 \t\tTrain Loss: 0.013 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14700 \t\tTrain Loss: 0.084 \tTrain Accuracy: 96.00\n",
            "Epoch: 8/10 \tIteration: 14800 \t\tTrain Loss: 0.026 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14900 \t\tTrain Loss: 0.020 \tTrain Accuracy: 100.00\n",
            "\tTrain Loss: 0.039 \tTrain Acc: 98.593\n",
            "\t\tVal Loss: 1.912 \t\tVal Acc: 87.907\n",
            "Time to train epoch: 84.68881964683533 s\n",
            "Epoch: 9/10 \tIteration: 15000 \t\tTrain Loss: 0.021 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15100 \t\tTrain Loss: 0.017 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15200 \t\tTrain Loss: 0.039 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15300 \t\tTrain Loss: 0.025 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15400 \t\tTrain Loss: 0.018 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15500 \t\tTrain Loss: 0.007 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15600 \t\tTrain Loss: 0.040 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15700 \t\tTrain Loss: 0.013 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15800 \t\tTrain Loss: 0.077 \tTrain Accuracy: 96.00\n",
            "Epoch: 9/10 \tIteration: 15900 \t\tTrain Loss: 0.026 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 16000 \t\tTrain Loss: 0.041 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 16100 \t\tTrain Loss: 0.119 \tTrain Accuracy: 96.00\n",
            "Epoch: 9/10 \tIteration: 16200 \t\tTrain Loss: 0.037 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 16300 \t\tTrain Loss: 0.003 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16400 \t\tTrain Loss: 0.023 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16500 \t\tTrain Loss: 0.053 \tTrain Accuracy: 98.00\n",
            "\tTrain Loss: 0.035 \tTrain Acc: 98.802\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2022-05-15 04:44:21,196]\u001b[0m Trial 1 finished with value: 87.713 and parameters: {'optimizer': 'SGD', 'lr': 0.970647313508467, 'dropout': 0.45895112616140743, 'hidden_size': 167, 'embed_size': 118, 'filter_size': 63}. Best is trial 0 with value: 87.787.\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\t\tVal Loss: 1.884 \t\tVal Acc: 87.713\n",
            "Time to train epoch: 84.6158607006073 s\n",
            "Epoch: 0/10 \tIteration: 0 \t\tTrain Loss: 0.702 \tTrain Accuracy: 22.00\n",
            "Epoch: 0/10 \tIteration: 100 \t\tTrain Loss: 0.478 \tTrain Accuracy: 84.00\n",
            "Epoch: 0/10 \tIteration: 200 \t\tTrain Loss: 0.390 \tTrain Accuracy: 80.00\n",
            "Epoch: 0/10 \tIteration: 300 \t\tTrain Loss: 0.373 \tTrain Accuracy: 78.00\n",
            "Epoch: 0/10 \tIteration: 400 \t\tTrain Loss: 0.258 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 500 \t\tTrain Loss: 0.312 \tTrain Accuracy: 88.00\n",
            "Epoch: 0/10 \tIteration: 600 \t\tTrain Loss: 0.193 \tTrain Accuracy: 92.00\n",
            "Epoch: 0/10 \tIteration: 700 \t\tTrain Loss: 0.446 \tTrain Accuracy: 82.00\n",
            "Epoch: 0/10 \tIteration: 800 \t\tTrain Loss: 0.300 \tTrain Accuracy: 82.00\n",
            "Epoch: 0/10 \tIteration: 900 \t\tTrain Loss: 0.156 \tTrain Accuracy: 94.00\n",
            "Epoch: 0/10 \tIteration: 1000 \t\tTrain Loss: 0.341 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 1100 \t\tTrain Loss: 0.306 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 1200 \t\tTrain Loss: 0.219 \tTrain Accuracy: 92.00\n",
            "Epoch: 0/10 \tIteration: 1300 \t\tTrain Loss: 0.135 \tTrain Accuracy: 96.00\n",
            "Epoch: 0/10 \tIteration: 1400 \t\tTrain Loss: 0.287 \tTrain Accuracy: 88.00\n",
            "Epoch: 0/10 \tIteration: 1500 \t\tTrain Loss: 0.272 \tTrain Accuracy: 90.00\n",
            "Epoch: 0/10 \tIteration: 1600 \t\tTrain Loss: 0.200 \tTrain Accuracy: 90.00\n",
            "\tTrain Loss: 0.336 \tTrain Acc: 85.136\n",
            "\t\tVal Loss: 0.282 \t\tVal Acc: 87.926\n",
            "Time to train epoch: 91.01955890655518 s\n",
            "Epoch: 1/10 \tIteration: 1700 \t\tTrain Loss: 0.183 \tTrain Accuracy: 96.00\n",
            "Epoch: 1/10 \tIteration: 1800 \t\tTrain Loss: 0.252 \tTrain Accuracy: 92.00\n",
            "Epoch: 1/10 \tIteration: 1900 \t\tTrain Loss: 0.400 \tTrain Accuracy: 78.00\n",
            "Epoch: 1/10 \tIteration: 2000 \t\tTrain Loss: 0.172 \tTrain Accuracy: 98.00\n",
            "Epoch: 1/10 \tIteration: 2100 \t\tTrain Loss: 0.280 \tTrain Accuracy: 86.00\n",
            "Epoch: 1/10 \tIteration: 2200 \t\tTrain Loss: 0.313 \tTrain Accuracy: 86.00\n",
            "Epoch: 1/10 \tIteration: 2300 \t\tTrain Loss: 0.208 \tTrain Accuracy: 92.00\n",
            "Epoch: 1/10 \tIteration: 2400 \t\tTrain Loss: 0.183 \tTrain Accuracy: 94.00\n",
            "Epoch: 1/10 \tIteration: 2500 \t\tTrain Loss: 0.454 \tTrain Accuracy: 78.00\n",
            "Epoch: 1/10 \tIteration: 2600 \t\tTrain Loss: 0.218 \tTrain Accuracy: 90.00\n",
            "Epoch: 1/10 \tIteration: 2700 \t\tTrain Loss: 0.355 \tTrain Accuracy: 84.00\n",
            "Epoch: 1/10 \tIteration: 2800 \t\tTrain Loss: 0.306 \tTrain Accuracy: 90.00\n",
            "Epoch: 1/10 \tIteration: 2900 \t\tTrain Loss: 0.378 \tTrain Accuracy: 88.00\n",
            "Epoch: 1/10 \tIteration: 3000 \t\tTrain Loss: 0.139 \tTrain Accuracy: 96.00\n",
            "Epoch: 1/10 \tIteration: 3100 \t\tTrain Loss: 0.174 \tTrain Accuracy: 94.00\n",
            "Epoch: 1/10 \tIteration: 3200 \t\tTrain Loss: 0.311 \tTrain Accuracy: 80.00\n",
            "Epoch: 1/10 \tIteration: 3300 \t\tTrain Loss: 0.202 \tTrain Accuracy: 92.00\n",
            "\tTrain Loss: 0.260 \tTrain Acc: 89.088\n",
            "\t\tVal Loss: 0.270 \t\tVal Acc: 88.750\n",
            "Time to train epoch: 91.12805414199829 s\n",
            "Epoch: 2/10 \tIteration: 3400 \t\tTrain Loss: 0.170 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 3500 \t\tTrain Loss: 0.273 \tTrain Accuracy: 84.00\n",
            "Epoch: 2/10 \tIteration: 3600 \t\tTrain Loss: 0.166 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 3700 \t\tTrain Loss: 0.154 \tTrain Accuracy: 98.00\n",
            "Epoch: 2/10 \tIteration: 3800 \t\tTrain Loss: 0.208 \tTrain Accuracy: 90.00\n",
            "Epoch: 2/10 \tIteration: 3900 \t\tTrain Loss: 0.252 \tTrain Accuracy: 88.00\n",
            "Epoch: 2/10 \tIteration: 4000 \t\tTrain Loss: 0.164 \tTrain Accuracy: 96.00\n",
            "Epoch: 2/10 \tIteration: 4100 \t\tTrain Loss: 0.295 \tTrain Accuracy: 86.00\n",
            "Epoch: 2/10 \tIteration: 4200 \t\tTrain Loss: 0.261 \tTrain Accuracy: 88.00\n",
            "Epoch: 2/10 \tIteration: 4300 \t\tTrain Loss: 0.170 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 4400 \t\tTrain Loss: 0.173 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 4500 \t\tTrain Loss: 0.191 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 4600 \t\tTrain Loss: 0.179 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 4700 \t\tTrain Loss: 0.222 \tTrain Accuracy: 90.00\n",
            "Epoch: 2/10 \tIteration: 4800 \t\tTrain Loss: 0.441 \tTrain Accuracy: 84.00\n",
            "Epoch: 2/10 \tIteration: 4900 \t\tTrain Loss: 0.288 \tTrain Accuracy: 86.00\n",
            "\tTrain Loss: 0.221 \tTrain Acc: 90.990\n",
            "\t\tVal Loss: 0.285 \t\tVal Acc: 88.824\n",
            "Time to train epoch: 91.13715863227844 s\n",
            "Epoch: 3/10 \tIteration: 5000 \t\tTrain Loss: 0.187 \tTrain Accuracy: 92.00\n",
            "Epoch: 3/10 \tIteration: 5100 \t\tTrain Loss: 0.363 \tTrain Accuracy: 88.00\n",
            "Epoch: 3/10 \tIteration: 5200 \t\tTrain Loss: 0.071 \tTrain Accuracy: 98.00\n",
            "Epoch: 3/10 \tIteration: 5300 \t\tTrain Loss: 0.164 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 5400 \t\tTrain Loss: 0.330 \tTrain Accuracy: 84.00\n",
            "Epoch: 3/10 \tIteration: 5500 \t\tTrain Loss: 0.145 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 5600 \t\tTrain Loss: 0.285 \tTrain Accuracy: 88.00\n",
            "Epoch: 3/10 \tIteration: 5700 \t\tTrain Loss: 0.191 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 5800 \t\tTrain Loss: 0.164 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 5900 \t\tTrain Loss: 0.106 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 6000 \t\tTrain Loss: 0.306 \tTrain Accuracy: 86.00\n",
            "Epoch: 3/10 \tIteration: 6100 \t\tTrain Loss: 0.205 \tTrain Accuracy: 90.00\n",
            "Epoch: 3/10 \tIteration: 6200 \t\tTrain Loss: 0.113 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 6300 \t\tTrain Loss: 0.152 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 6400 \t\tTrain Loss: 0.230 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 6500 \t\tTrain Loss: 0.094 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 6600 \t\tTrain Loss: 0.088 \tTrain Accuracy: 96.00\n",
            "\tTrain Loss: 0.179 \tTrain Acc: 93.098\n",
            "\t\tVal Loss: 0.333 \t\tVal Acc: 88.602\n",
            "Time to train epoch: 91.28622221946716 s\n",
            "Epoch: 4/10 \tIteration: 6700 \t\tTrain Loss: 0.090 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 6800 \t\tTrain Loss: 0.141 \tTrain Accuracy: 92.00\n",
            "Epoch: 4/10 \tIteration: 6900 \t\tTrain Loss: 0.063 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 7000 \t\tTrain Loss: 0.119 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 7100 \t\tTrain Loss: 0.100 \tTrain Accuracy: 94.00\n",
            "Epoch: 4/10 \tIteration: 7200 \t\tTrain Loss: 0.040 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7300 \t\tTrain Loss: 0.195 \tTrain Accuracy: 92.00\n",
            "Epoch: 4/10 \tIteration: 7400 \t\tTrain Loss: 0.051 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7500 \t\tTrain Loss: 0.135 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 7600 \t\tTrain Loss: 0.069 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 7700 \t\tTrain Loss: 0.131 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 7800 \t\tTrain Loss: 0.180 \tTrain Accuracy: 92.00\n",
            "Epoch: 4/10 \tIteration: 7900 \t\tTrain Loss: 0.065 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 8000 \t\tTrain Loss: 0.055 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 8100 \t\tTrain Loss: 0.052 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 8200 \t\tTrain Loss: 0.189 \tTrain Accuracy: 90.00\n",
            "\tTrain Loss: 0.131 \tTrain Acc: 95.290\n",
            "\t\tVal Loss: 0.417 \t\tVal Acc: 88.259\n",
            "Time to train epoch: 91.15287280082703 s\n",
            "Epoch: 5/10 \tIteration: 8300 \t\tTrain Loss: 0.064 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 8400 \t\tTrain Loss: 0.193 \tTrain Accuracy: 94.00\n",
            "Epoch: 5/10 \tIteration: 8500 \t\tTrain Loss: 0.061 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 8600 \t\tTrain Loss: 0.177 \tTrain Accuracy: 90.00\n",
            "Epoch: 5/10 \tIteration: 8700 \t\tTrain Loss: 0.030 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 8800 \t\tTrain Loss: 0.049 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 8900 \t\tTrain Loss: 0.074 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 9000 \t\tTrain Loss: 0.069 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9100 \t\tTrain Loss: 0.040 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9200 \t\tTrain Loss: 0.018 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9300 \t\tTrain Loss: 0.172 \tTrain Accuracy: 90.00\n",
            "Epoch: 5/10 \tIteration: 9400 \t\tTrain Loss: 0.055 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9500 \t\tTrain Loss: 0.053 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9600 \t\tTrain Loss: 0.043 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9700 \t\tTrain Loss: 0.032 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9800 \t\tTrain Loss: 0.210 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 9900 \t\tTrain Loss: 0.012 \tTrain Accuracy: 100.00\n",
            "\tTrain Loss: 0.089 \tTrain Acc: 97.037\n",
            "\t\tVal Loss: 0.555 \t\tVal Acc: 87.889\n",
            "Time to train epoch: 91.08966732025146 s\n",
            "Epoch: 6/10 \tIteration: 10000 \t\tTrain Loss: 0.030 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 10100 \t\tTrain Loss: 0.044 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 10200 \t\tTrain Loss: 0.093 \tTrain Accuracy: 94.00\n",
            "Epoch: 6/10 \tIteration: 10300 \t\tTrain Loss: 0.132 \tTrain Accuracy: 94.00\n",
            "Epoch: 6/10 \tIteration: 10400 \t\tTrain Loss: 0.048 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10500 \t\tTrain Loss: 0.184 \tTrain Accuracy: 94.00\n",
            "Epoch: 6/10 \tIteration: 10600 \t\tTrain Loss: 0.022 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 10700 \t\tTrain Loss: 0.031 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10800 \t\tTrain Loss: 0.084 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 10900 \t\tTrain Loss: 0.052 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 11000 \t\tTrain Loss: 0.064 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 11100 \t\tTrain Loss: 0.121 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 11200 \t\tTrain Loss: 0.127 \tTrain Accuracy: 94.00\n",
            "Epoch: 6/10 \tIteration: 11300 \t\tTrain Loss: 0.044 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 11400 \t\tTrain Loss: 0.009 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 11500 \t\tTrain Loss: 0.160 \tTrain Accuracy: 94.00\n",
            "Epoch: 6/10 \tIteration: 11600 \t\tTrain Loss: 0.018 \tTrain Accuracy: 100.00\n",
            "\tTrain Loss: 0.068 \tTrain Acc: 97.633\n",
            "\t\tVal Loss: 0.979 \t\tVal Acc: 87.806\n",
            "Time to train epoch: 91.05362200737 s\n",
            "Epoch: 7/10 \tIteration: 11700 \t\tTrain Loss: 0.039 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 11800 \t\tTrain Loss: 0.060 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 11900 \t\tTrain Loss: 0.017 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12000 \t\tTrain Loss: 0.066 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12100 \t\tTrain Loss: 0.113 \tTrain Accuracy: 94.00\n",
            "Epoch: 7/10 \tIteration: 12200 \t\tTrain Loss: 0.064 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 12300 \t\tTrain Loss: 0.060 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 12400 \t\tTrain Loss: 0.093 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 12500 \t\tTrain Loss: 0.137 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 12600 \t\tTrain Loss: 0.015 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12700 \t\tTrain Loss: 0.042 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12800 \t\tTrain Loss: 0.061 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12900 \t\tTrain Loss: 0.015 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 13000 \t\tTrain Loss: 0.070 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 13100 \t\tTrain Loss: 0.062 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 13200 \t\tTrain Loss: 0.051 \tTrain Accuracy: 96.00\n",
            "\tTrain Loss: 0.057 \tTrain Acc: 98.052\n",
            "\t\tVal Loss: 1.321 \t\tVal Acc: 87.991\n",
            "Time to train epoch: 91.20092225074768 s\n",
            "Epoch: 8/10 \tIteration: 13300 \t\tTrain Loss: 0.014 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13400 \t\tTrain Loss: 0.051 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 13500 \t\tTrain Loss: 0.004 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13600 \t\tTrain Loss: 0.024 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13700 \t\tTrain Loss: 0.031 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 13800 \t\tTrain Loss: 0.022 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13900 \t\tTrain Loss: 0.030 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14000 \t\tTrain Loss: 0.061 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14100 \t\tTrain Loss: 0.029 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14200 \t\tTrain Loss: 0.032 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14300 \t\tTrain Loss: 0.056 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14400 \t\tTrain Loss: 0.035 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14500 \t\tTrain Loss: 0.060 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14600 \t\tTrain Loss: 0.027 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14700 \t\tTrain Loss: 0.022 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14800 \t\tTrain Loss: 0.006 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14900 \t\tTrain Loss: 0.007 \tTrain Accuracy: 100.00\n",
            "\tTrain Loss: 0.038 \tTrain Acc: 98.770\n",
            "\t\tVal Loss: 1.471 \t\tVal Acc: 88.324\n",
            "Time to train epoch: 91.21876120567322 s\n",
            "Epoch: 9/10 \tIteration: 15000 \t\tTrain Loss: 0.011 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15100 \t\tTrain Loss: 0.011 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15200 \t\tTrain Loss: 0.023 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15300 \t\tTrain Loss: 0.012 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15400 \t\tTrain Loss: 0.029 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15500 \t\tTrain Loss: 0.032 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15600 \t\tTrain Loss: 0.019 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15700 \t\tTrain Loss: 0.042 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15800 \t\tTrain Loss: 0.007 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15900 \t\tTrain Loss: 0.002 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16000 \t\tTrain Loss: 0.023 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16100 \t\tTrain Loss: 0.016 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16200 \t\tTrain Loss: 0.028 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 16300 \t\tTrain Loss: 0.003 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16400 \t\tTrain Loss: 0.005 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16500 \t\tTrain Loss: 0.090 \tTrain Accuracy: 98.00\n",
            "\tTrain Loss: 0.029 \tTrain Acc: 99.104\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2022-05-15 04:59:32,654]\u001b[0m Trial 2 finished with value: 88.139 and parameters: {'optimizer': 'Adam', 'lr': 0.3541538761638133, 'dropout': 0.14804085015911742, 'hidden_size': 188, 'embed_size': 40, 'filter_size': 92}. Best is trial 2 with value: 88.139.\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\t\tVal Loss: 1.478 \t\tVal Acc: 88.139\n",
            "Time to train epoch: 91.14416933059692 s\n",
            "Epoch: 0/10 \tIteration: 0 \t\tTrain Loss: 0.733 \tTrain Accuracy: 20.00\n",
            "Epoch: 0/10 \tIteration: 100 \t\tTrain Loss: 0.597 \tTrain Accuracy: 76.00\n",
            "Epoch: 0/10 \tIteration: 200 \t\tTrain Loss: 0.373 \tTrain Accuracy: 78.00\n",
            "Epoch: 0/10 \tIteration: 300 \t\tTrain Loss: 0.340 \tTrain Accuracy: 84.00\n",
            "Epoch: 0/10 \tIteration: 400 \t\tTrain Loss: 0.288 \tTrain Accuracy: 88.00\n",
            "Epoch: 0/10 \tIteration: 500 \t\tTrain Loss: 0.247 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 600 \t\tTrain Loss: 0.180 \tTrain Accuracy: 94.00\n",
            "Epoch: 0/10 \tIteration: 700 \t\tTrain Loss: 0.370 \tTrain Accuracy: 88.00\n",
            "Epoch: 0/10 \tIteration: 800 \t\tTrain Loss: 0.293 \tTrain Accuracy: 82.00\n",
            "Epoch: 0/10 \tIteration: 900 \t\tTrain Loss: 0.164 \tTrain Accuracy: 94.00\n",
            "Epoch: 0/10 \tIteration: 1000 \t\tTrain Loss: 0.346 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 1100 \t\tTrain Loss: 0.322 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 1200 \t\tTrain Loss: 0.179 \tTrain Accuracy: 92.00\n",
            "Epoch: 0/10 \tIteration: 1300 \t\tTrain Loss: 0.183 \tTrain Accuracy: 90.00\n",
            "Epoch: 0/10 \tIteration: 1400 \t\tTrain Loss: 0.289 \tTrain Accuracy: 84.00\n",
            "Epoch: 0/10 \tIteration: 1500 \t\tTrain Loss: 0.257 \tTrain Accuracy: 94.00\n",
            "Epoch: 0/10 \tIteration: 1600 \t\tTrain Loss: 0.214 \tTrain Accuracy: 90.00\n",
            "\tTrain Loss: 0.317 \tTrain Acc: 86.134\n",
            "\t\tVal Loss: 0.271 \t\tVal Acc: 88.361\n",
            "Time to train epoch: 62.67087531089783 s\n",
            "Epoch: 1/10 \tIteration: 1700 \t\tTrain Loss: 0.196 \tTrain Accuracy: 96.00\n",
            "Epoch: 1/10 \tIteration: 1800 \t\tTrain Loss: 0.273 \tTrain Accuracy: 88.00\n",
            "Epoch: 1/10 \tIteration: 1900 \t\tTrain Loss: 0.376 \tTrain Accuracy: 82.00\n",
            "Epoch: 1/10 \tIteration: 2000 \t\tTrain Loss: 0.191 \tTrain Accuracy: 94.00\n",
            "Epoch: 1/10 \tIteration: 2100 \t\tTrain Loss: 0.227 \tTrain Accuracy: 88.00\n",
            "Epoch: 1/10 \tIteration: 2200 \t\tTrain Loss: 0.260 \tTrain Accuracy: 86.00\n",
            "Epoch: 1/10 \tIteration: 2300 \t\tTrain Loss: 0.201 \tTrain Accuracy: 94.00\n",
            "Epoch: 1/10 \tIteration: 2400 \t\tTrain Loss: 0.157 \tTrain Accuracy: 96.00\n",
            "Epoch: 1/10 \tIteration: 2500 \t\tTrain Loss: 0.327 \tTrain Accuracy: 86.00\n",
            "Epoch: 1/10 \tIteration: 2600 \t\tTrain Loss: 0.213 \tTrain Accuracy: 90.00\n",
            "Epoch: 1/10 \tIteration: 2700 \t\tTrain Loss: 0.366 \tTrain Accuracy: 82.00\n",
            "Epoch: 1/10 \tIteration: 2800 \t\tTrain Loss: 0.268 \tTrain Accuracy: 90.00\n",
            "Epoch: 1/10 \tIteration: 2900 \t\tTrain Loss: 0.372 \tTrain Accuracy: 86.00\n",
            "Epoch: 1/10 \tIteration: 3000 \t\tTrain Loss: 0.164 \tTrain Accuracy: 96.00\n",
            "Epoch: 1/10 \tIteration: 3100 \t\tTrain Loss: 0.122 \tTrain Accuracy: 98.00\n",
            "Epoch: 1/10 \tIteration: 3200 \t\tTrain Loss: 0.291 \tTrain Accuracy: 82.00\n",
            "Epoch: 1/10 \tIteration: 3300 \t\tTrain Loss: 0.212 \tTrain Accuracy: 90.00\n",
            "\tTrain Loss: 0.247 \tTrain Acc: 89.764\n",
            "\t\tVal Loss: 0.257 \t\tVal Acc: 88.935\n",
            "Time to train epoch: 62.58182883262634 s\n",
            "Epoch: 2/10 \tIteration: 3400 \t\tTrain Loss: 0.155 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 3500 \t\tTrain Loss: 0.237 \tTrain Accuracy: 88.00\n",
            "Epoch: 2/10 \tIteration: 3600 \t\tTrain Loss: 0.138 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 3700 \t\tTrain Loss: 0.113 \tTrain Accuracy: 98.00\n",
            "Epoch: 2/10 \tIteration: 3800 \t\tTrain Loss: 0.164 \tTrain Accuracy: 96.00\n",
            "Epoch: 2/10 \tIteration: 3900 \t\tTrain Loss: 0.167 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 4000 \t\tTrain Loss: 0.178 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 4100 \t\tTrain Loss: 0.243 \tTrain Accuracy: 88.00\n",
            "Epoch: 2/10 \tIteration: 4200 \t\tTrain Loss: 0.300 \tTrain Accuracy: 86.00\n",
            "Epoch: 2/10 \tIteration: 4300 \t\tTrain Loss: 0.151 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 4400 \t\tTrain Loss: 0.157 \tTrain Accuracy: 90.00\n",
            "Epoch: 2/10 \tIteration: 4500 \t\tTrain Loss: 0.214 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 4600 \t\tTrain Loss: 0.147 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 4700 \t\tTrain Loss: 0.200 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 4800 \t\tTrain Loss: 0.300 \tTrain Accuracy: 86.00\n",
            "Epoch: 2/10 \tIteration: 4900 \t\tTrain Loss: 0.281 \tTrain Accuracy: 88.00\n",
            "\tTrain Loss: 0.212 \tTrain Acc: 91.399\n",
            "\t\tVal Loss: 0.264 \t\tVal Acc: 89.019\n",
            "Time to train epoch: 62.67874717712402 s\n",
            "Epoch: 3/10 \tIteration: 5000 \t\tTrain Loss: 0.151 \tTrain Accuracy: 98.00\n",
            "Epoch: 3/10 \tIteration: 5100 \t\tTrain Loss: 0.357 \tTrain Accuracy: 84.00\n",
            "Epoch: 3/10 \tIteration: 5200 \t\tTrain Loss: 0.097 \tTrain Accuracy: 92.00\n",
            "Epoch: 3/10 \tIteration: 5300 \t\tTrain Loss: 0.201 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 5400 \t\tTrain Loss: 0.275 \tTrain Accuracy: 90.00\n",
            "Epoch: 3/10 \tIteration: 5500 \t\tTrain Loss: 0.122 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 5600 \t\tTrain Loss: 0.249 \tTrain Accuracy: 88.00\n",
            "Epoch: 3/10 \tIteration: 5700 \t\tTrain Loss: 0.142 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 5800 \t\tTrain Loss: 0.154 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 5900 \t\tTrain Loss: 0.109 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 6000 \t\tTrain Loss: 0.190 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 6100 \t\tTrain Loss: 0.212 \tTrain Accuracy: 92.00\n",
            "Epoch: 3/10 \tIteration: 6200 \t\tTrain Loss: 0.157 \tTrain Accuracy: 92.00\n",
            "Epoch: 3/10 \tIteration: 6300 \t\tTrain Loss: 0.108 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 6400 \t\tTrain Loss: 0.210 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 6500 \t\tTrain Loss: 0.130 \tTrain Accuracy: 92.00\n",
            "Epoch: 3/10 \tIteration: 6600 \t\tTrain Loss: 0.122 \tTrain Accuracy: 94.00\n",
            "\tTrain Loss: 0.176 \tTrain Acc: 93.167\n",
            "\t\tVal Loss: 0.303 \t\tVal Acc: 88.815\n",
            "Time to train epoch: 62.62905740737915 s\n",
            "Epoch: 4/10 \tIteration: 6700 \t\tTrain Loss: 0.062 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 6800 \t\tTrain Loss: 0.116 \tTrain Accuracy: 94.00\n",
            "Epoch: 4/10 \tIteration: 6900 \t\tTrain Loss: 0.034 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 7000 \t\tTrain Loss: 0.119 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 7100 \t\tTrain Loss: 0.156 \tTrain Accuracy: 92.00\n",
            "Epoch: 4/10 \tIteration: 7200 \t\tTrain Loss: 0.058 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 7300 \t\tTrain Loss: 0.237 \tTrain Accuracy: 88.00\n",
            "Epoch: 4/10 \tIteration: 7400 \t\tTrain Loss: 0.048 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7500 \t\tTrain Loss: 0.222 \tTrain Accuracy: 92.00\n",
            "Epoch: 4/10 \tIteration: 7600 \t\tTrain Loss: 0.061 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 7700 \t\tTrain Loss: 0.184 \tTrain Accuracy: 90.00\n",
            "Epoch: 4/10 \tIteration: 7800 \t\tTrain Loss: 0.273 \tTrain Accuracy: 92.00\n",
            "Epoch: 4/10 \tIteration: 7900 \t\tTrain Loss: 0.065 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 8000 \t\tTrain Loss: 0.062 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 8100 \t\tTrain Loss: 0.096 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 8200 \t\tTrain Loss: 0.189 \tTrain Accuracy: 92.00\n",
            "\tTrain Loss: 0.138 \tTrain Acc: 94.873\n",
            "\t\tVal Loss: 0.391 \t\tVal Acc: 88.278\n",
            "Time to train epoch: 62.746774673461914 s\n",
            "Epoch: 5/10 \tIteration: 8300 \t\tTrain Loss: 0.060 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 8400 \t\tTrain Loss: 0.214 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 8500 \t\tTrain Loss: 0.047 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 8600 \t\tTrain Loss: 0.185 \tTrain Accuracy: 94.00\n",
            "Epoch: 5/10 \tIteration: 8700 \t\tTrain Loss: 0.031 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 8800 \t\tTrain Loss: 0.024 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 8900 \t\tTrain Loss: 0.090 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9000 \t\tTrain Loss: 0.074 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9100 \t\tTrain Loss: 0.074 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9200 \t\tTrain Loss: 0.010 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9300 \t\tTrain Loss: 0.103 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 9400 \t\tTrain Loss: 0.039 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9500 \t\tTrain Loss: 0.114 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 9600 \t\tTrain Loss: 0.115 \tTrain Accuracy: 92.00\n",
            "Epoch: 5/10 \tIteration: 9700 \t\tTrain Loss: 0.096 \tTrain Accuracy: 94.00\n",
            "Epoch: 5/10 \tIteration: 9800 \t\tTrain Loss: 0.257 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 9900 \t\tTrain Loss: 0.067 \tTrain Accuracy: 98.00\n",
            "\tTrain Loss: 0.104 \tTrain Acc: 96.314\n",
            "\t\tVal Loss: 0.529 \t\tVal Acc: 88.037\n",
            "Time to train epoch: 62.79249429702759 s\n",
            "Epoch: 6/10 \tIteration: 10000 \t\tTrain Loss: 0.044 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10100 \t\tTrain Loss: 0.134 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10200 \t\tTrain Loss: 0.158 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10300 \t\tTrain Loss: 0.056 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10400 \t\tTrain Loss: 0.031 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 10500 \t\tTrain Loss: 0.031 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 10600 \t\tTrain Loss: 0.077 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 10700 \t\tTrain Loss: 0.038 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10800 \t\tTrain Loss: 0.057 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 10900 \t\tTrain Loss: 0.057 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 11000 \t\tTrain Loss: 0.118 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 11100 \t\tTrain Loss: 0.089 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 11200 \t\tTrain Loss: 0.230 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 11300 \t\tTrain Loss: 0.045 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 11400 \t\tTrain Loss: 0.028 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 11500 \t\tTrain Loss: 0.072 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 11600 \t\tTrain Loss: 0.042 \tTrain Accuracy: 100.00\n",
            "\tTrain Loss: 0.080 \tTrain Acc: 97.166\n",
            "\t\tVal Loss: 0.798 \t\tVal Acc: 87.898\n",
            "Time to train epoch: 62.93988513946533 s\n",
            "Epoch: 7/10 \tIteration: 11700 \t\tTrain Loss: 0.005 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 11800 \t\tTrain Loss: 0.040 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 11900 \t\tTrain Loss: 0.018 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12000 \t\tTrain Loss: 0.021 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12100 \t\tTrain Loss: 0.171 \tTrain Accuracy: 90.00\n",
            "Epoch: 7/10 \tIteration: 12200 \t\tTrain Loss: 0.086 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12300 \t\tTrain Loss: 0.075 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12400 \t\tTrain Loss: 0.149 \tTrain Accuracy: 94.00\n",
            "Epoch: 7/10 \tIteration: 12500 \t\tTrain Loss: 0.151 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12600 \t\tTrain Loss: 0.011 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12700 \t\tTrain Loss: 0.082 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 12800 \t\tTrain Loss: 0.073 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12900 \t\tTrain Loss: 0.027 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 13000 \t\tTrain Loss: 0.047 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 13100 \t\tTrain Loss: 0.112 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 13200 \t\tTrain Loss: 0.031 \tTrain Accuracy: 100.00\n",
            "\tTrain Loss: 0.067 \tTrain Acc: 97.610\n",
            "\t\tVal Loss: 0.931 \t\tVal Acc: 87.741\n",
            "Time to train epoch: 62.83555293083191 s\n",
            "Epoch: 8/10 \tIteration: 13300 \t\tTrain Loss: 0.106 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 13400 \t\tTrain Loss: 0.019 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13500 \t\tTrain Loss: 0.015 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13600 \t\tTrain Loss: 0.031 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13700 \t\tTrain Loss: 0.128 \tTrain Accuracy: 96.00\n",
            "Epoch: 8/10 \tIteration: 13800 \t\tTrain Loss: 0.083 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 13900 \t\tTrain Loss: 0.110 \tTrain Accuracy: 96.00\n",
            "Epoch: 8/10 \tIteration: 14000 \t\tTrain Loss: 0.040 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14100 \t\tTrain Loss: 0.037 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14200 \t\tTrain Loss: 0.043 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14300 \t\tTrain Loss: 0.098 \tTrain Accuracy: 96.00\n",
            "Epoch: 8/10 \tIteration: 14400 \t\tTrain Loss: 0.018 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14500 \t\tTrain Loss: 0.033 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14600 \t\tTrain Loss: 0.024 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14700 \t\tTrain Loss: 0.064 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14800 \t\tTrain Loss: 0.044 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14900 \t\tTrain Loss: 0.010 \tTrain Accuracy: 100.00\n",
            "\tTrain Loss: 0.054 \tTrain Acc: 98.047\n",
            "\t\tVal Loss: 1.238 \t\tVal Acc: 87.889\n",
            "Time to train epoch: 62.91811490058899 s\n",
            "Epoch: 9/10 \tIteration: 15000 \t\tTrain Loss: 0.073 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15100 \t\tTrain Loss: 0.042 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15200 \t\tTrain Loss: 0.032 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15300 \t\tTrain Loss: 0.017 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15400 \t\tTrain Loss: 0.022 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15500 \t\tTrain Loss: 0.010 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15600 \t\tTrain Loss: 0.055 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15700 \t\tTrain Loss: 0.035 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15800 \t\tTrain Loss: 0.121 \tTrain Accuracy: 94.00\n",
            "Epoch: 9/10 \tIteration: 15900 \t\tTrain Loss: 0.011 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16000 \t\tTrain Loss: 0.023 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16100 \t\tTrain Loss: 0.122 \tTrain Accuracy: 96.00\n",
            "Epoch: 9/10 \tIteration: 16200 \t\tTrain Loss: 0.024 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16300 \t\tTrain Loss: 0.005 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16400 \t\tTrain Loss: 0.014 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16500 \t\tTrain Loss: 0.045 \tTrain Accuracy: 96.00\n",
            "\tTrain Loss: 0.043 \tTrain Acc: 98.518\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2022-05-15 05:10:00,270]\u001b[0m Trial 3 finished with value: 87.889 and parameters: {'optimizer': 'RMSprop', 'lr': 0.007813443444372781, 'dropout': 0.400174578315842, 'hidden_size': 94, 'embed_size': 70, 'filter_size': 105}. Best is trial 2 with value: 88.139.\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\t\tVal Loss: 1.469 \t\tVal Acc: 87.889\n",
            "Time to train epoch: 62.794313192367554 s\n",
            "Epoch: 0/10 \tIteration: 0 \t\tTrain Loss: 0.735 \tTrain Accuracy: 20.00\n",
            "Epoch: 0/10 \tIteration: 100 \t\tTrain Loss: 0.526 \tTrain Accuracy: 78.00\n",
            "Epoch: 0/10 \tIteration: 200 \t\tTrain Loss: 0.308 \tTrain Accuracy: 88.00\n",
            "Epoch: 0/10 \tIteration: 300 \t\tTrain Loss: 0.377 \tTrain Accuracy: 82.00\n",
            "Epoch: 0/10 \tIteration: 400 \t\tTrain Loss: 0.231 \tTrain Accuracy: 90.00\n",
            "Epoch: 0/10 \tIteration: 500 \t\tTrain Loss: 0.226 \tTrain Accuracy: 90.00\n",
            "Epoch: 0/10 \tIteration: 600 \t\tTrain Loss: 0.186 \tTrain Accuracy: 92.00\n",
            "Epoch: 0/10 \tIteration: 700 \t\tTrain Loss: 0.333 \tTrain Accuracy: 82.00\n",
            "Epoch: 0/10 \tIteration: 800 \t\tTrain Loss: 0.296 \tTrain Accuracy: 88.00\n",
            "Epoch: 0/10 \tIteration: 900 \t\tTrain Loss: 0.098 \tTrain Accuracy: 100.00\n",
            "Epoch: 0/10 \tIteration: 1000 \t\tTrain Loss: 0.300 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 1100 \t\tTrain Loss: 0.311 \tTrain Accuracy: 86.00\n",
            "Epoch: 0/10 \tIteration: 1200 \t\tTrain Loss: 0.226 \tTrain Accuracy: 88.00\n",
            "Epoch: 0/10 \tIteration: 1300 \t\tTrain Loss: 0.142 \tTrain Accuracy: 96.00\n",
            "Epoch: 0/10 \tIteration: 1400 \t\tTrain Loss: 0.250 \tTrain Accuracy: 88.00\n",
            "Epoch: 0/10 \tIteration: 1500 \t\tTrain Loss: 0.255 \tTrain Accuracy: 94.00\n",
            "Epoch: 0/10 \tIteration: 1600 \t\tTrain Loss: 0.190 \tTrain Accuracy: 90.00\n",
            "\tTrain Loss: 0.304 \tTrain Acc: 86.953\n",
            "\t\tVal Loss: 0.265 \t\tVal Acc: 88.583\n",
            "Time to train epoch: 76.04414677619934 s\n",
            "Epoch: 1/10 \tIteration: 1700 \t\tTrain Loss: 0.222 \tTrain Accuracy: 92.00\n",
            "Epoch: 1/10 \tIteration: 1800 \t\tTrain Loss: 0.298 \tTrain Accuracy: 86.00\n",
            "Epoch: 1/10 \tIteration: 1900 \t\tTrain Loss: 0.329 \tTrain Accuracy: 82.00\n",
            "Epoch: 1/10 \tIteration: 2000 \t\tTrain Loss: 0.159 \tTrain Accuracy: 96.00\n",
            "Epoch: 1/10 \tIteration: 2100 \t\tTrain Loss: 0.215 \tTrain Accuracy: 90.00\n",
            "Epoch: 1/10 \tIteration: 2200 \t\tTrain Loss: 0.278 \tTrain Accuracy: 88.00\n",
            "Epoch: 1/10 \tIteration: 2300 \t\tTrain Loss: 0.166 \tTrain Accuracy: 96.00\n",
            "Epoch: 1/10 \tIteration: 2400 \t\tTrain Loss: 0.156 \tTrain Accuracy: 94.00\n",
            "Epoch: 1/10 \tIteration: 2500 \t\tTrain Loss: 0.356 \tTrain Accuracy: 90.00\n",
            "Epoch: 1/10 \tIteration: 2600 \t\tTrain Loss: 0.233 \tTrain Accuracy: 90.00\n",
            "Epoch: 1/10 \tIteration: 2700 \t\tTrain Loss: 0.327 \tTrain Accuracy: 88.00\n",
            "Epoch: 1/10 \tIteration: 2800 \t\tTrain Loss: 0.202 \tTrain Accuracy: 94.00\n",
            "Epoch: 1/10 \tIteration: 2900 \t\tTrain Loss: 0.319 \tTrain Accuracy: 88.00\n",
            "Epoch: 1/10 \tIteration: 3000 \t\tTrain Loss: 0.180 \tTrain Accuracy: 94.00\n",
            "Epoch: 1/10 \tIteration: 3100 \t\tTrain Loss: 0.169 \tTrain Accuracy: 94.00\n",
            "Epoch: 1/10 \tIteration: 3200 \t\tTrain Loss: 0.300 \tTrain Accuracy: 82.00\n",
            "Epoch: 1/10 \tIteration: 3300 \t\tTrain Loss: 0.189 \tTrain Accuracy: 90.00\n",
            "\tTrain Loss: 0.235 \tTrain Acc: 90.167\n",
            "\t\tVal Loss: 0.269 \t\tVal Acc: 88.917\n",
            "Time to train epoch: 76.20155930519104 s\n",
            "Epoch: 2/10 \tIteration: 3400 \t\tTrain Loss: 0.162 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 3500 \t\tTrain Loss: 0.211 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 3600 \t\tTrain Loss: 0.085 \tTrain Accuracy: 96.00\n",
            "Epoch: 2/10 \tIteration: 3700 \t\tTrain Loss: 0.104 \tTrain Accuracy: 98.00\n",
            "Epoch: 2/10 \tIteration: 3800 \t\tTrain Loss: 0.144 \tTrain Accuracy: 92.00\n",
            "Epoch: 2/10 \tIteration: 3900 \t\tTrain Loss: 0.170 \tTrain Accuracy: 96.00\n",
            "Epoch: 2/10 \tIteration: 4000 \t\tTrain Loss: 0.146 \tTrain Accuracy: 96.00\n",
            "Epoch: 2/10 \tIteration: 4100 \t\tTrain Loss: 0.197 \tTrain Accuracy: 88.00\n",
            "Epoch: 2/10 \tIteration: 4200 \t\tTrain Loss: 0.265 \tTrain Accuracy: 90.00\n",
            "Epoch: 2/10 \tIteration: 4300 \t\tTrain Loss: 0.098 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 4400 \t\tTrain Loss: 0.130 \tTrain Accuracy: 94.00\n",
            "Epoch: 2/10 \tIteration: 4500 \t\tTrain Loss: 0.112 \tTrain Accuracy: 96.00\n",
            "Epoch: 2/10 \tIteration: 4600 \t\tTrain Loss: 0.142 \tTrain Accuracy: 96.00\n",
            "Epoch: 2/10 \tIteration: 4700 \t\tTrain Loss: 0.233 \tTrain Accuracy: 88.00\n",
            "Epoch: 2/10 \tIteration: 4800 \t\tTrain Loss: 0.341 \tTrain Accuracy: 90.00\n",
            "Epoch: 2/10 \tIteration: 4900 \t\tTrain Loss: 0.307 \tTrain Accuracy: 86.00\n",
            "\tTrain Loss: 0.188 \tTrain Acc: 92.525\n",
            "\t\tVal Loss: 0.312 \t\tVal Acc: 88.685\n",
            "Time to train epoch: 76.36857271194458 s\n",
            "Epoch: 3/10 \tIteration: 5000 \t\tTrain Loss: 0.168 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 5100 \t\tTrain Loss: 0.229 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 5200 \t\tTrain Loss: 0.066 \tTrain Accuracy: 100.00\n",
            "Epoch: 3/10 \tIteration: 5300 \t\tTrain Loss: 0.133 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 5400 \t\tTrain Loss: 0.226 \tTrain Accuracy: 86.00\n",
            "Epoch: 3/10 \tIteration: 5500 \t\tTrain Loss: 0.122 \tTrain Accuracy: 92.00\n",
            "Epoch: 3/10 \tIteration: 5600 \t\tTrain Loss: 0.143 \tTrain Accuracy: 92.00\n",
            "Epoch: 3/10 \tIteration: 5700 \t\tTrain Loss: 0.214 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 5800 \t\tTrain Loss: 0.150 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 5900 \t\tTrain Loss: 0.090 \tTrain Accuracy: 98.00\n",
            "Epoch: 3/10 \tIteration: 6000 \t\tTrain Loss: 0.193 \tTrain Accuracy: 94.00\n",
            "Epoch: 3/10 \tIteration: 6100 \t\tTrain Loss: 0.152 \tTrain Accuracy: 90.00\n",
            "Epoch: 3/10 \tIteration: 6200 \t\tTrain Loss: 0.080 \tTrain Accuracy: 98.00\n",
            "Epoch: 3/10 \tIteration: 6300 \t\tTrain Loss: 0.127 \tTrain Accuracy: 98.00\n",
            "Epoch: 3/10 \tIteration: 6400 \t\tTrain Loss: 0.206 \tTrain Accuracy: 96.00\n",
            "Epoch: 3/10 \tIteration: 6500 \t\tTrain Loss: 0.087 \tTrain Accuracy: 98.00\n",
            "Epoch: 3/10 \tIteration: 6600 \t\tTrain Loss: 0.087 \tTrain Accuracy: 98.00\n",
            "\tTrain Loss: 0.136 \tTrain Acc: 94.970\n",
            "\t\tVal Loss: 0.434 \t\tVal Acc: 88.231\n",
            "Time to train epoch: 76.70600867271423 s\n",
            "Epoch: 4/10 \tIteration: 6700 \t\tTrain Loss: 0.063 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 6800 \t\tTrain Loss: 0.125 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 6900 \t\tTrain Loss: 0.039 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7000 \t\tTrain Loss: 0.061 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 7100 \t\tTrain Loss: 0.132 \tTrain Accuracy: 94.00\n",
            "Epoch: 4/10 \tIteration: 7200 \t\tTrain Loss: 0.021 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7300 \t\tTrain Loss: 0.170 \tTrain Accuracy: 88.00\n",
            "Epoch: 4/10 \tIteration: 7400 \t\tTrain Loss: 0.039 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7500 \t\tTrain Loss: 0.200 \tTrain Accuracy: 94.00\n",
            "Epoch: 4/10 \tIteration: 7600 \t\tTrain Loss: 0.041 \tTrain Accuracy: 100.00\n",
            "Epoch: 4/10 \tIteration: 7700 \t\tTrain Loss: 0.073 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 7800 \t\tTrain Loss: 0.151 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 7900 \t\tTrain Loss: 0.063 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 8000 \t\tTrain Loss: 0.033 \tTrain Accuracy: 98.00\n",
            "Epoch: 4/10 \tIteration: 8100 \t\tTrain Loss: 0.060 \tTrain Accuracy: 96.00\n",
            "Epoch: 4/10 \tIteration: 8200 \t\tTrain Loss: 0.094 \tTrain Accuracy: 98.00\n",
            "\tTrain Loss: 0.090 \tTrain Acc: 96.913\n",
            "\t\tVal Loss: 0.672 \t\tVal Acc: 87.778\n",
            "Time to train epoch: 76.53943085670471 s\n",
            "Epoch: 5/10 \tIteration: 8300 \t\tTrain Loss: 0.015 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 8400 \t\tTrain Loss: 0.222 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 8500 \t\tTrain Loss: 0.025 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 8600 \t\tTrain Loss: 0.119 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 8700 \t\tTrain Loss: 0.028 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 8800 \t\tTrain Loss: 0.027 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 8900 \t\tTrain Loss: 0.064 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 9000 \t\tTrain Loss: 0.077 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9100 \t\tTrain Loss: 0.011 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9200 \t\tTrain Loss: 0.028 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9300 \t\tTrain Loss: 0.082 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 9400 \t\tTrain Loss: 0.023 \tTrain Accuracy: 100.00\n",
            "Epoch: 5/10 \tIteration: 9500 \t\tTrain Loss: 0.053 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 9600 \t\tTrain Loss: 0.082 \tTrain Accuracy: 96.00\n",
            "Epoch: 5/10 \tIteration: 9700 \t\tTrain Loss: 0.028 \tTrain Accuracy: 98.00\n",
            "Epoch: 5/10 \tIteration: 9800 \t\tTrain Loss: 0.168 \tTrain Accuracy: 94.00\n",
            "Epoch: 5/10 \tIteration: 9900 \t\tTrain Loss: 0.038 \tTrain Accuracy: 96.00\n",
            "\tTrain Loss: 0.069 \tTrain Acc: 97.595\n",
            "\t\tVal Loss: 1.041 \t\tVal Acc: 87.824\n",
            "Time to train epoch: 76.42547726631165 s\n",
            "Epoch: 6/10 \tIteration: 10000 \t\tTrain Loss: 0.045 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 10100 \t\tTrain Loss: 0.032 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10200 \t\tTrain Loss: 0.037 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 10300 \t\tTrain Loss: 0.026 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10400 \t\tTrain Loss: 0.024 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 10500 \t\tTrain Loss: 0.031 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10600 \t\tTrain Loss: 0.040 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10700 \t\tTrain Loss: 0.031 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 10800 \t\tTrain Loss: 0.115 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 10900 \t\tTrain Loss: 0.049 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 11000 \t\tTrain Loss: 0.160 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 11100 \t\tTrain Loss: 0.050 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 11200 \t\tTrain Loss: 0.066 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 11300 \t\tTrain Loss: 0.019 \tTrain Accuracy: 100.00\n",
            "Epoch: 6/10 \tIteration: 11400 \t\tTrain Loss: 0.034 \tTrain Accuracy: 98.00\n",
            "Epoch: 6/10 \tIteration: 11500 \t\tTrain Loss: 0.115 \tTrain Accuracy: 96.00\n",
            "Epoch: 6/10 \tIteration: 11600 \t\tTrain Loss: 0.018 \tTrain Accuracy: 100.00\n",
            "\tTrain Loss: 0.056 \tTrain Acc: 98.011\n",
            "\t\tVal Loss: 1.155 \t\tVal Acc: 87.972\n",
            "Time to train epoch: 76.52487826347351 s\n",
            "Epoch: 7/10 \tIteration: 11700 \t\tTrain Loss: 0.059 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 11800 \t\tTrain Loss: 0.031 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 11900 \t\tTrain Loss: 0.003 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12000 \t\tTrain Loss: 0.061 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12100 \t\tTrain Loss: 0.008 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12200 \t\tTrain Loss: 0.123 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12300 \t\tTrain Loss: 0.032 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12400 \t\tTrain Loss: 0.085 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 12500 \t\tTrain Loss: 0.092 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 12600 \t\tTrain Loss: 0.025 \tTrain Accuracy: 100.00\n",
            "Epoch: 7/10 \tIteration: 12700 \t\tTrain Loss: 0.023 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 12800 \t\tTrain Loss: 0.036 \tTrain Accuracy: 96.00\n",
            "Epoch: 7/10 \tIteration: 12900 \t\tTrain Loss: 0.040 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 13000 \t\tTrain Loss: 0.025 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 13100 \t\tTrain Loss: 0.029 \tTrain Accuracy: 98.00\n",
            "Epoch: 7/10 \tIteration: 13200 \t\tTrain Loss: 0.048 \tTrain Accuracy: 98.00\n",
            "\tTrain Loss: 0.041 \tTrain Acc: 98.577\n",
            "\t\tVal Loss: 1.515 \t\tVal Acc: 87.861\n",
            "Time to train epoch: 76.54927253723145 s\n",
            "Epoch: 8/10 \tIteration: 13300 \t\tTrain Loss: 0.098 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 13400 \t\tTrain Loss: 0.030 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13500 \t\tTrain Loss: 0.025 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 13600 \t\tTrain Loss: 0.017 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13700 \t\tTrain Loss: 0.062 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 13800 \t\tTrain Loss: 0.003 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 13900 \t\tTrain Loss: 0.020 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14000 \t\tTrain Loss: 0.052 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14100 \t\tTrain Loss: 0.007 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14200 \t\tTrain Loss: 0.037 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14300 \t\tTrain Loss: 0.021 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14400 \t\tTrain Loss: 0.041 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14500 \t\tTrain Loss: 0.070 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14600 \t\tTrain Loss: 0.028 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14700 \t\tTrain Loss: 0.077 \tTrain Accuracy: 98.00\n",
            "Epoch: 8/10 \tIteration: 14800 \t\tTrain Loss: 0.017 \tTrain Accuracy: 100.00\n",
            "Epoch: 8/10 \tIteration: 14900 \t\tTrain Loss: 0.026 \tTrain Accuracy: 98.00\n",
            "\tTrain Loss: 0.034 \tTrain Acc: 98.863\n",
            "\t\tVal Loss: 1.741 \t\tVal Acc: 87.981\n",
            "Time to train epoch: 76.55458474159241 s\n",
            "Epoch: 9/10 \tIteration: 15000 \t\tTrain Loss: 0.018 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15100 \t\tTrain Loss: 0.054 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15200 \t\tTrain Loss: 0.018 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15300 \t\tTrain Loss: 0.020 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15400 \t\tTrain Loss: 0.032 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15500 \t\tTrain Loss: 0.026 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 15600 \t\tTrain Loss: 0.062 \tTrain Accuracy: 96.00\n",
            "Epoch: 9/10 \tIteration: 15700 \t\tTrain Loss: 0.013 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15800 \t\tTrain Loss: 0.007 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 15900 \t\tTrain Loss: 0.018 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 16000 \t\tTrain Loss: 0.025 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 16100 \t\tTrain Loss: 0.033 \tTrain Accuracy: 98.00\n",
            "Epoch: 9/10 \tIteration: 16200 \t\tTrain Loss: 0.011 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16300 \t\tTrain Loss: 0.006 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16400 \t\tTrain Loss: 0.008 \tTrain Accuracy: 100.00\n",
            "Epoch: 9/10 \tIteration: 16500 \t\tTrain Loss: 0.075 \tTrain Accuracy: 98.00\n",
            "\tTrain Loss: 0.032 \tTrain Acc: 98.894\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[32m[I 2022-05-15 05:22:44,735]\u001b[0m Trial 4 finished with value: 87.741 and parameters: {'optimizer': 'SGD', 'lr': 0.4709919111090412, 'dropout': 0.30983404481599963, 'hidden_size': 108, 'embed_size': 125, 'filter_size': 147}. Best is trial 2 with value: 88.139.\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\t\tVal Loss: 1.776 \t\tVal Acc: 87.741\n",
            "Time to train epoch: 76.5160927772522 s\n",
            "Study statistics: \n",
            "  Number of finished trials:  5\n",
            "  Number of complete trials:  5\n",
            "Best trial:\n",
            "  Value:  88.139\n",
            "  Params: \n",
            "    optimizer: Adam\n",
            "    lr: 0.3541538761638133\n",
            "    dropout: 0.14804085015911742\n",
            "    hidden_size: 188\n",
            "    embed_size: 40\n",
            "    filter_size: 92\n"
          ]
        }
      ],
      "source": [
        "import optuna\n",
        "from optuna.trial import TrialState\n",
        "study = optuna.create_study(direction=\"maximize\",pruner=optuna.pruners.HyperbandPruner(\n",
        "        min_resource=1, max_resource=epochs, reduction_factor=3\n",
        "    ),)\n",
        "study.optimize(objective, n_trials=5, callbacks=[callback])\n",
        "complete_trials = study.get_trials(deepcopy=False, states=[TrialState.COMPLETE])\n",
        "print(\"Study statistics: \")\n",
        "print(\"  Number of finished trials: \", len(study.trials))\n",
        "print(\"  Number of complete trials: \", len(complete_trials))\n",
        "print(\"Best trial:\")\n",
        "trial = study.best_trial\n",
        "best_model=study.user_attrs[\"best_model\"]\n",
        "print(\"  Value: \", trial.value)\n",
        "print(\"  Params: \")\n",
        "for key, value in trial.params.items():\n",
        "    print(\"    {}: {}\".format(key, value))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "z4Tjh75SWd8c"
      },
      "outputs": [],
      "source": [
        "torch.save(best_model.state_dict(),\"./lstm-cnn-amazon-hyperband.pth\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "X-7GSEt-tGtF"
      },
      "outputs": [],
      "source": [
        "criterion = torch.nn.BCELoss().to(device)\n",
        "def test(model, test_dl, epochs):\n",
        "    with torch.no_grad():\n",
        "        model.eval()\n",
        "        test_acc = []\n",
        "        test_loss = []\n",
        "        for inputs, labels in test_dl:\n",
        "            input_test, labels_test = inputs.cuda(), labels.cuda()\n",
        "            logits_test,accuracy_test = model(input_test,labels_test)\n",
        "            loss_test = criterion(logits_test,labels_test.float())\n",
        "            test_acc.append(accuracy_test.item()*100/batch_size)\n",
        "            test_loss.append(loss_test.item())\n",
        "        print(\"Test Loss: {:.3f}\".format(np.mean(test_loss)), \"\\tTest Acc: {:.3f}\".format(np.mean(test_acc)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "c5S_4Ju7zg2z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "39788c60-938b-45b3-9317-a7caaa86c63e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/rnn.py:762: UserWarning: RNN module weights are not part of single contiguous chunk of memory. This means they need to be compacted at every call, possibly greatly increasing memory usage. To compact weights again call flatten_parameters(). (Triggered internally at  ../aten/src/ATen/native/cudnn/RNN.cpp:926.)\n",
            "  self.dropout, self.training, self.bidirectional, self.batch_first)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test Loss: 1.594 \tTest Acc: 87.491\n"
          ]
        }
      ],
      "source": [
        "test(best_model, test_dl, epochs)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "lstm-cnn-amazon-hyperband.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.6"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}