220921_chatbot_miniproject.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyPa8J47aAWk1gyYiaXtGs36",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Junseokee/Deep_Learning/blob/main/220921_chatbot_miniproject.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 라이브러리 호출"
      ],
      "metadata": {
        "id": "0WXPwuYrao58"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers\n",
        "!pip install ratsnlp\n",
        "!pip install pytorch_lightning"
      ],
      "metadata": {
        "id": "Uzu0D1VTao59"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 69,
      "metadata": {
        "id": "guz7qMihao5-"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "from pytorch_lightning import Trainer\n",
        "from pytorch_lightning.callbacks import ModelCheckpoint\n",
        "from pytorch_lightning.core.lightning import LightningModule\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from transformers.optimization import AdamW, get_cosine_schedule_with_warmup\n",
        "from transformers import PreTrainedTokenizerFast, GPT2LMHeadModel\n",
        "import re"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YU8dO2tJU7K5",
        "outputId": "9470108a-42a6-4cf1-c84b-b050aa17457a"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### KoGPT2 모델 사용"
      ],
      "metadata": {
        "id": "0j-_ZKila2ia"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 모델 호출"
      ],
      "metadata": {
        "id": "-xAc_L-8bFwn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = GPT2LMHeadModel.from_pretrained('skt/kogpt2-base-v2')"
      ],
      "metadata": {
        "id": "PEVMbV0ebHMp"
      },
      "execution_count": 71,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 토큰 정의\n"
      ],
      "metadata": {
        "id": "n-yksXKregFn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "Q_TKN = \"<usr>\"\n",
        "A_TKN = \"<sys>\"\n",
        "BOS = '</s>'\n",
        "EOS = '</s>'\n",
        "MASK = '<unused0>'\n",
        "SENT = '<unused1>'\n",
        "PAD = '<pad>'"
      ],
      "metadata": {
        "id": "39wrPALvef4n"
      },
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 허깅페이스 사전 학습모델 호출"
      ],
      "metadata": {
        "id": "h703vrV1ekhO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "koGPT2_TOKENIZER = PreTrainedTokenizerFast.from_pretrained(\"skt/kogpt2-base-v2\",\n",
        "            bos_token=BOS, eos_token=EOS, unk_token='<unk>',\n",
        "            pad_token=PAD, mask_token=MASK,\n",
        "            downstream_model_dir = '/content/drive/MyDrive/nlpbook/checkpoint-generation') \n",
        "model = GPT2LMHeadModel.from_pretrained('skt/kogpt2-base-v2')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Whx9BX0qenRb",
        "outputId": "271b9069-4db7-49cc-957b-16aa38628c68"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. \n",
            "The tokenizer class you load from this checkpoint is 'GPT2Tokenizer'. \n",
            "The class this function is called from is 'PreTrainedTokenizerFast'.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 데이터 다운로드"
      ],
      "metadata": {
        "id": "qlL2p1T6ao5_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import urllib.request\n",
        "\n",
        "urllib.request.urlretrieve(\n",
        "    \"https://raw.githubusercontent.com/songys/Chatbot_data/master/ChatbotData.csv\",\n",
        "    filename=\"ChatBotData.csv\",\n",
        ")\n",
        "Chatbot_Data = pd.read_csv(\"ChatBotData.csv\")\n",
        "# Test 용으로 300개 데이터만 처리한다.\n",
        "Chatbot_Data = Chatbot_Data[:5000]\n",
        "Chatbot_Data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "1fc782d3-0e6a-4c68-ae92-97c18c439c80",
        "id": "cQSBH2tvao5_"
      },
      "execution_count": 83,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                 Q            A  label\n",
              "0           12시 땡!   하루가 또 가네요.      0\n",
              "1      1지망 학교 떨어졌어    위로해 드립니다.      0\n",
              "2     3박4일 놀러가고 싶다  여행은 언제나 좋죠.      0\n",
              "3  3박4일 정도 놀러가고 싶다  여행은 언제나 좋죠.      0\n",
              "4          PPL 심하네   눈살이 찌푸려지죠.      0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f70ad09a-7982-4836-a697-609f787b7c21\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Q</th>\n",
              "      <th>A</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>12시 땡!</td>\n",
              "      <td>하루가 또 가네요.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1지망 학교 떨어졌어</td>\n",
              "      <td>위로해 드립니다.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3박4일 놀러가고 싶다</td>\n",
              "      <td>여행은 언제나 좋죠.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3박4일 정도 놀러가고 싶다</td>\n",
              "      <td>여행은 언제나 좋죠.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>PPL 심하네</td>\n",
              "      <td>눈살이 찌푸려지죠.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f70ad09a-7982-4836-a697-609f787b7c21')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f70ad09a-7982-4836-a697-609f787b7c21 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f70ad09a-7982-4836-a697-609f787b7c21');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "Chatbot_Data.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mu4CTVlNagZT",
        "outputId": "b685d55e-5061-4101-b766-e1944482b069"
      },
      "execution_count": 85,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 5000 entries, 0 to 4999\n",
            "Data columns (total 3 columns):\n",
            " #   Column  Non-Null Count  Dtype \n",
            "---  ------  --------------  ----- \n",
            " 0   Q       5000 non-null   object\n",
            " 1   A       5000 non-null   object\n",
            " 2   label   5000 non-null   int64 \n",
            "dtypes: int64(1), object(2)\n",
            "memory usage: 117.3+ KB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 챗봇 데이터를 처리하는 클래스를 만든다.\n",
        "class ChatbotDataset(Dataset):\n",
        "    def __init__(self, chats, max_len=40):  # 데이터셋의 전처리를 해주는 부분\n",
        "        self._data = chats\n",
        "        self.max_len = max_len\n",
        "        self.q_token = Q_TKN\n",
        "        self.a_token = A_TKN\n",
        "        self.sent_token = SENT\n",
        "        self.eos = EOS\n",
        "        self.mask = MASK\n",
        "        self.tokenizer = koGPT2_TOKENIZER\n",
        "\n",
        "    def __len__(self):  # chatbotdata 의 길이를 리턴한다.\n",
        "        return len(self._data)\n",
        "\n",
        "    def __getitem__(self, idx):  # 로드한 챗봇 데이터를 차례차례 DataLoader로 넘겨주는 메서드\n",
        "        turn = self._data.iloc[idx]\n",
        "        q = turn[\"Q\"]  # 질문을 가져온다.\n",
        "        q = re.sub(r\"([?.!,])\", r\" \", q)  # 구둣점들을 제거한다.\n",
        "\n",
        "        a = turn[\"A\"]  # 답변을 가져온다.\n",
        "        a = re.sub(r\"([?.!,])\", r\" \", a)  # 구둣점들을 제거한다.\n",
        "\n",
        "        q_toked = self.tokenizer.tokenize(self.q_token + q + self.sent_token)\n",
        "        q_len = len(q_toked)\n",
        "\n",
        "        a_toked = self.tokenizer.tokenize(self.a_token + a + self.eos)\n",
        "        a_len = len(a_toked)\n",
        "\n",
        "        #질문의 길이가 최대길이보다 크면\n",
        "        if q_len > self.max_len:\n",
        "            a_len = self.max_len - q_len        #답변의 길이를 최대길이 - 질문길이\n",
        "            if a_len <= 0:       #질문의 길이가 너무 길어 질문만으로 최대 길이를 초과 한다면\n",
        "                q_toked = q_toked[-(int(self.max_len / 2)) :]   #질문길이를 최대길이의 반으로 \n",
        "                q_len = len(q_toked)\n",
        "                a_len = self.max_len - q_len              #답변의 길이를 최대길이 - 질문길이\n",
        "            a_toked = a_toked[:a_len]\n",
        "            a_len = len(a_toked)\n",
        "\n",
        "        #질문의 길이 + 답변의 길이가 최대길이보다 크면\n",
        "        if q_len + a_len > self.max_len:\n",
        "            a_len = self.max_len - q_len        #답변의 길이를 최대길이 - 질문길이\n",
        "            if a_len <= 0:       #질문의 길이가 너무 길어 질문만으로 최대 길이를 초과 한다면\n",
        "                q_toked = q_toked[-(int(self.max_len / 2)) :]   #질문길이를 최대길이의 반으로 \n",
        "                q_len = len(q_toked)\n",
        "                a_len = self.max_len - q_len              #답변의 길이를 최대길이 - 질문길이\n",
        "            a_toked = a_toked[:a_len]\n",
        "            a_len = len(a_toked)\n",
        "\n",
        "        # 답변 labels = [mask, mask, ...., mask, ..., <bos>,..답변.. <eos>, <pad>....]\n",
        "        labels = [self.mask,] * q_len + a_toked[1:]\n",
        "\n",
        "        # mask = 질문길이 0 + 답변길이 1 + 나머지 0\n",
        "        mask = [0] * q_len + [1] * a_len + [0] * (self.max_len - q_len - a_len)\n",
        "        # 답변 labels을 index 로 만든다.\n",
        "        labels_ids = self.tokenizer.convert_tokens_to_ids(labels)\n",
        "        # 최대길이만큼 PADDING\n",
        "        while len(labels_ids) < self.max_len:\n",
        "            labels_ids += [self.tokenizer.pad_token_id]\n",
        "\n",
        "        # 질문 + 답변을 index 로 만든다.    \n",
        "        token_ids = self.tokenizer.convert_tokens_to_ids(q_toked + a_toked)\n",
        "        # 최대길이만큼 PADDING\n",
        "        while len(token_ids) < self.max_len:\n",
        "            token_ids += [self.tokenizer.pad_token_id]\n",
        "\n",
        "        #질문+답변, 마스크, 답변\n",
        "        return (token_ids, np.array(mask), labels_ids)"
      ],
      "metadata": {
        "id": "MkgF1xz9ZFzG"
      },
      "execution_count": 86,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 배치 데이터를 만들기 위해 collate_batch 함수 정의"
      ],
      "metadata": {
        "id": "1XZR2v4gYSiY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def collate_batch(batch):\n",
        "    data = [item[0] for item in batch]\n",
        "    mask = [item[1] for item in batch]\n",
        "    label = [item[2] for item in batch]\n",
        "    return torch.LongTensor(data), torch.LongTensor(mask), torch.LongTensor(label)"
      ],
      "metadata": {
        "id": "9P76tFZMWCIN"
      },
      "execution_count": 87,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Dataset 선언"
      ],
      "metadata": {
        "id": "Y4QVFXJ2e34Y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "train_set = ChatbotDataset(\n",
        "    Chatbot_Data, \n",
        "    max_len=40)\n",
        "train_dataloader = DataLoader(train_set, \n",
        "                              batch_size=32, \n",
        "                              num_workers=0, \n",
        "                              shuffle=True, \n",
        "                              collate_fn=collate_batch,)"
      ],
      "metadata": {
        "id": "JUaE5HCSezC_"
      },
      "execution_count": 88,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 모델 추가 학습을 위한 .train()"
      ],
      "metadata": {
        "id": "VieeHgcefdNY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = model.to(device)\n",
        "model.train()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z6vsoAvMfaRT",
        "outputId": "f841203d-8a56-4cbd-9753-729d8219b6ab"
      },
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GPT2LMHeadModel(\n",
              "  (transformer): GPT2Model(\n",
              "    (wte): Embedding(51200, 768)\n",
              "    (wpe): Embedding(1024, 768)\n",
              "    (drop): Dropout(p=0.1, inplace=False)\n",
              "    (h): ModuleList(\n",
              "      (0): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (1): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (2): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (3): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (4): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (5): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (6): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (7): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (8): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (9): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (10): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (11): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "  )\n",
              "  (lm_head): Linear(in_features=768, out_features=51200, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 하이퍼 파라메터 선언"
      ],
      "metadata": {
        "id": "qOFT_pyBfhSn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "learning_rate = 5e-5\n",
        "criterion = torch.nn.CrossEntropyLoss(reduction=\"none\")\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
        "\n",
        "epoch = 2\n",
        "Sneg = -1e18"
      ],
      "metadata": {
        "id": "B6AAUeXXffi-"
      },
      "execution_count": 90,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 학습"
      ],
      "metadata": {
        "id": "XTPPZhCRgFlo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "train_step = 0\n",
        "train_start_index = train_step + 1 if train_step != 0 else 0\n",
        "total_train_step = len(train_dataloader)\n",
        "\n",
        "print (\"start\")\n",
        "model = model.to(device)\n",
        "with tqdm(total=total_train_step, desc=f\"Train({epoch})\") as pbar:\n",
        "  pbar.update(train_step)\n",
        "  for i in range(epoch):\n",
        "    for batch_idx, samples in enumerate(train_dataloader):\n",
        "        optimizer.zero_grad()\n",
        "        token_ids, mask, label = samples\n",
        "        token_ids = token_ids.to(device)\n",
        "        out = model(token_ids)\n",
        "        out = out.logits      #Returns a new tensor with the logit of the elements of input\n",
        "        mask_3d = mask.unsqueeze(dim=2).repeat_interleave(repeats=out.shape[2], dim=2)\n",
        "        mask_3d = mask_3d.to(device)\n",
        "        mask_out = torch.where(mask_3d == 1, out, Sneg * torch.ones_like(out))\n",
        "        mask_out = mask_out.to('cpu')\n",
        "        loss = criterion(mask_out.transpose(2, 1), label)\n",
        "        loss = loss.to(device)\n",
        "        # 평균 loss 만들기 avg_loss[0] / avg_loss[1] <- loss 정규화\n",
        "        avg_loss = loss.sum() / mask.sum()\n",
        "        avg_loss.backward()\n",
        "        # 학습 끝\n",
        "        optimizer.step()\n",
        "print('end')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ODuGsh0OvaES",
        "outputId": "a3428fbf-4bb8-4ab7-ff16-4dea8d583a85"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "start\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rTrain(2):   0%|          | 0/157 [00:00<?, ?it/s]"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# print (\"start\")\n",
        "# for i in range(epoch):\n",
        "#     for batch_idx, samples in enumerate(train_dataloader):\n",
        "#         optimizer.zero_grad()\n",
        "#         token_ids, mask, label = samples\n",
        "#         token_ids = token_ids.to(device)\n",
        "#         out = model(token_ids)\n",
        "#         out = out.logits      #Returns a new tensor with the logit of the elements of input\n",
        "#         mask_3d = mask.unsqueeze(dim=2).repeat_interleave(repeats=out.shape[2], dim=2)\n",
        "#         mask_3d = mask_3d.to(device)\n",
        "#         mask_out = torch.where(mask_3d == 1, out, Sneg * torch.ones_like(out))\n",
        "#         mask_out = mask_out.to('cpu')\n",
        "#         loss = criterion(mask_out.transpose(2, 1), label)\n",
        "#         loss = loss.to(device)\n",
        "#         # 평균 loss 만들기 avg_loss[0] / avg_loss[1] <- loss 정규화\n",
        "#         avg_loss = loss.sum() / mask.sum()\n",
        "#         avg_loss.backward()\n",
        "#         # 학습 끝\n",
        "#         optimizer.step()\n",
        "# print('end')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JgwvikWzfjmc",
        "outputId": "08cf7d22-fa77-4220-fd2c-d10ce58c3981"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "start\n",
            "end\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 모델 테스트"
      ],
      "metadata": {
        "id": "Nzv_7qy16Z04"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gTTs\n",
        "!pip install librosa"
      ],
      "metadata": {
        "id": "WA5DZT5SgLW9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from gtts import gTTS\n",
        "from IPython.display import Audio\n",
        "import time\n",
        "import librosa\n",
        "\n",
        "text = '안녕하세요 반갑습니다. 감사합니다. 안녕히계세요 @'\n",
        "\n"
      ],
      "metadata": {
        "id": "kno6NmVCkCLY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import warnings\n",
        "\n",
        "# 경고메세지 끄기\n",
        "warnings.filterwarnings(action='ignore')\n",
        "\n",
        "with torch.no_grad():\n",
        "    while 1:\n",
        "        q = input(\"user > \").strip()\n",
        "        if q == \"종료\":\n",
        "            break\n",
        "        a = \"\"\n",
        "        while 1:\n",
        "            input_ids = torch.LongTensor(koGPT2_TOKENIZER.encode(Q_TKN + q + SENT + A_TKN + a)).unsqueeze(dim=0)\n",
        "            model = model.cpu()\n",
        "            pred = model(input_ids)\n",
        "            pred = pred.logits\n",
        "            gen = koGPT2_TOKENIZER.convert_ids_to_tokens(torch.argmax(pred, dim=-1).squeeze().numpy().tolist())[-1]\n",
        "            if gen == EOS:\n",
        "                break\n",
        "            a += gen.replace(\"▁\", \" \")\n",
        "        text = a.strip()\n",
        "        tts = gTTS(text=text, lang='ko')\n",
        "        tts.save('hello_ko.wav')\n",
        "        y, sr = librosa.load('hello_ko.wav')\n",
        "        display(Audio('hello_ko.wav', autoplay=True))\n",
        "        time.sleep((len(y)/sr)+1)\n",
        "        # print(\"Chatbot > {}\".format(a.strip()))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185
        },
        "id": "bmqUESafc5Tm",
        "outputId": "ba13b4c1-4252-47fc-d1a1-ebf1f7a554c7"
      },
      "execution_count": 45,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "user > 반가워\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.lib.display.Audio object>"
            ],
            "text/html": [
              "\n",
              "                <audio  controls=\"controls\" autoplay=\"autoplay\">\n",
              "                    <source src=\"data:audio/x-wav;base64,//NExAAQixIAAGBEuTUrFAdxIDsrEkZjuJCdpbG5ezQyNr/7jZuepSRAj/r///yf/k/2/////+3IQhEP/kIQhPt9CHOc4c4cWcQQhCABBCo7N7BSEMBIYHsHa5J4TLpq//NExBAR4E4wAVsQAOyy/S5V+X74SJS5Bhd9b1y6VQ9QfRNC3AgYvOFxwArNOKbS7JwaGL+vKU5xLrzjKCo/OaloQ3l6Q8M2dzG5n31ZeZ//9v///7f2+6GuYK7Ev/Mi//NExBsY6yqkAYdQACxuYYQEhOF2IhQtCLEYVBZIf/0HlGUfk44e4sERKNC4ixAj8VxgBITiZ//555ISM5Ixcxj0bGIXAWwK4iAXguxQF+CsGgEMQzUl+cpf//p///////NExAoTIt7EAcEQAf/6vTSzHcQ5KnUqFiBRlQErGYEZyGcKhQQghjKRwaHdUDpg6POpiGcSprOrqcwo5FMxjwFIYyqwZihQ0I5VLI5gCZV1u1buHqzvtDOwpkXAwwwm//NExBAVkl60ABCGuc+rEfdFUNsqnVuy4oxkWpOX50Kr0gwE4kMtjVZhiM18spWBG0mVJlWcKOqlGZ1WGeTmomMLnQoo+YrCq+bIblsmJVFZf/6VrREEvA34PhiBJvKN//NExAwTwUqkAMMGlM/HF0TsTEuMgB+8XISWZExkewaotrE917/ub0zM2tdOnraUp5bJAbMhX1OUgbChxlgAMEiw9AOPG0ml0+k7oSe0Tw3CjDBA1bIwy0gYqAFmbshB//NExBAVsWqkAMySlKczVugYYmOgGz5XCWAoJVD6BQxrAmoxSiMErOQdaJhW041b1v3Ox/W0zGz/vbC7VvjSCOMJFDOY3pyORlNHysK3f//kP0ZBlseGKBoaOhY9x5WB//NExAwU+V6sAMzSlI9+lDDmVpFUFbTAZYFhLCSBZ4/AU0CwyGALKKx0dzolB0DXac0Hq3fz+s9LfjUy/gYaUAnVFfTKQwmoCh+bedblEvbd////0OygQyKQrCBCHsgy//NExAsSUVqsAMxQlYl4gSJ5pGlM2xsKEGkUQ5Yix0AtDJJAvioOsZg1IGVZf0ak6tWly3MaXkMxjSJ4kImQ5rpRCywbNYi1BqpVaqeMGCmYw46Ql6hYjEXlYQbvx3hb//NExBQSQVKcAMvMlZbKoRpRkSOGIYIsLKyq5lAUQFU5VDWDE3rHk3i28f5r5l5xq9Vuy38naxLa/dHmooisciyKThC7onYIQgLKGoBRA4ZGaMyEF6NCtrLbDtU8trX7//NExB4Q2VJoAVkwATzUqv1YzpqSOESr8y1evP///ef9+7yM5oLBHDq17x5Y9HxnOGzVdiSOvBpYIySQZOXGVKJT356xJ4j+AQUBpdgDf4RBBRYSHh4C3/yzxrGCw0SD//NExC0dkyKcAZhAAQT//oXZ/eLow7//5hP3M4SmX//+DIk+hc9zBcGo4SFsVf///94oKIQnUvTjRIJjx0odA43/////e97r3iss/ymMcweQQDkniGQVwiNY7UViv4Zy//NExAkRgUasAdlQAJtmpCcJkKuXw1C9biFWvLozWfT4ZSHd4EQNASxgNyZjk1fM21mo1zJjHT97mJvn1IGQs0EP///o/9n//iXccHR5qSDrzBmybpPEYeGHFX6T8UYx//NExBYSGU6oANNKlEARDRIFZKQzgA+DgEHBsBxICXBTOiSv77a9H2y0AZs9SMh3s0RF6gjCQpEbmf//+9f9CrNMhNO8EU4fsIgJ5KgAmEOLw+wpTOjqI52KwzxJB2QI//NExCARMUakANYEcPREncICt9vW/yH/01TN9HaF1V7HmyozChBgbiIMBL///6P6iCa5ESEMZuS/TcDD4AwggNEFk1xZ/ZUKgsCmCfRmyGg1kAkZND5AQNGshcmtT/Vq//NExC4RGVqcANTOlOvO0bI1FPL0EZqc2aFXik8xCe7///01tSgdATPUZ135MVIyI4ckEC5yQExS0SBNtgSLgiNkaOUC38YZgHcNlsQuRXV+vV1a9s7V6jhkQr8lmi+o//NExDwRsU6cAN0OlG6Hkv//+2Q+uhxlnbagBwhOvxSsENq/oPAhcbYU02JAAPOUwKNB/kQaFA1MIuUCDFpygXqilq7f9evfGtVMaxwilzY1lOHlOCuwwehv///bd9CU//NExEgSCVqYANzKlCp+nTEUY/4pQYEAxUoYsIFSZgQICBM+WHWbbgUCqDHMSAETcFpDW10zsupscqtnmW///1+W//9fkvVq9WzOVjBWpuXUTQUFHS0KBk4jVzARGaiY//NExFIR6U6MANYElTCHDVGJCGAImFCnOWiQ1wWIwQ5srQ0BxKtsnoxP2ECiEBPkiXLVuU9INBICjgZOzwKnEO/6f////6ExoOMWBUyD1S0FGghBQoPlAaAgIAB6qi1u//NExF0Q8G5cANaeKNerlVmrNWpWu1bwaalkmmQTYhi5UY4GnvQq6+nolPZz3RXbX03ubspZt0f8+tVXswZJaeBh0Re36a6qjEbW81QSdhrAXxGC+JgIYBTEydWC0D0K//NExGwRkGZEAVsAAASYToOMvEvtkQgAt58vjlLR2nhhf1EpEYEwJQkhzHSWHu38eay+PBabEM3YoJOa/8vugm90FEoklMVGSaH/5gaJmZuXEKjSZoKc6Y0jJE0ZD//z//NExHgiAxqIAZloAXTbQzMvuaMno1o1JHTZKHwMhyXcTf8pJ5iOp4tnAICIpZS1EhiZCvKr1+rvv+fYjP9fSbNG92gDWFRZ9I0BnROJAyBAKeGjHS5znFpVtdOLICRz//NExEMRcRasAcgwAOOdoWIgJwJxMDzoVJ2pRB2I40JzwGbEEg4FAb56JcUUvzEx3ZC2zehQ6kh0ggpgZHT9evVMTQTAoaBtSxY+kzbe5SHdVZ0a6lWG3/IntAjKx7cM//NExFARUVqwAGoGlFfkN5PpS149OzcJjcDyyKtllUMS0MEq7RBgrbFDcussA+oj1G3TxlMPUSd////9tH2peRWHl2Z23ev1JaoeK5GxCkCUrH1Ki/DUUK3Za2lYrGWo//NExF0RMKawAVhIAUH1fhIQTQ7wDIXoQcgoYZC1OUo8x6iVieFCGQdpYyFE7RLBLChj/YF5fhCZqN+q9WbWqSPMzOjrXLcqGeNFQudszS9swNU3ExnEePiJHpR/PqDj//NExGsh+j6wAZh4AP1j4li3xj/2k3//////b/////49dxD3Lk/9D9nSuQ8Nbh12EY9rEvtwm3xcKlcmXuip6VNKt1Yaqy23KUHkJ9KeMiladB9EJxbqudrWfd67NOtt//NExDYSERq0AdhgAX61+sXNbXKuhr+/jZLTUlulduqVQaGgXbQ7B0d/gqPUvQTNIXKijIH+EJ1GrOgibEE8xM4VWWl29ZKY6JI6l4Rkc7MD2gKGdDTGqMUIGAUDO1Ov//NExEARoWqYANMElZnU3lhnybpw0OBstIDjwGjZr4krsLEYYtvWRFzZkOJvKHfqpZTDtOiCVO8F/dFDPlt3FuVnsRQ01qbFHMVqI2+IeQkJu2dUV/////7f7a6PEgGC//NExEwR8PqEANvScHnEQCMA5vQ4qEUDRp/YWY4MvOwmNkSFV2Ku6lBqOkS55Lo/RKMr763nYMWytzNpnJH2ySfsUTdptRErd9C3f///jf7BVVVgBfQ5yoHiwsOCzmAG//NExFcSSPKEANvWcJgRma2BUEPWW04+NBjRWocevCvKt2x0gGf+39d87a5W4EwUHwA2WCJrzP/9ss56P/6YMXasKtMKEcAzZhYZAiKhFMTFkETUIVqP3GnFna0gbM1o//NExGARGOqIAVowAI4+TJFyTKgfCMcQ1SdhkyMJANsDBwCpBqIDrquZEsXymT7CURjj5FCuv7s7Jums0cun//9NBzcoGaH/TN006uZpE6XCcWtSH/vTpp+ptbr771////NExG4fYyqQAZqQAP/oM10/UpBai4mxmeUoyTq9bGICyYkquAjtyYWGAQbS7RMV5Qu3I6kQnK+dPnK3/+N6MzBu+QwA1aY4Wo20/vr/uze6/+fZbfuvX+lZMu/D7CAH//NExEMSqPasAdhgAAfB8Lq4dK9VyqkJBbdM2FpVLVDjO9NCFSWN8OCNzMpdsQeD5PPbeLXsgZ5Q06pNQo2Nri3g7/njqTRa6hZFCDbvHXq+Y0WMw2aUpErnIsT2o6Ii//NExEsQ6Va4AMPOlQ2MkCWEGg3LNZgDclNqY+vCGtQw0IwH+GFdElZvrvXh67V/e3zbzeR9yWbASPsO5bzPj3qBJMFmwRPZ3///6+YEIxsVdbD63HrR/mmwgAkvjhYH//NExFoRgWq4AMPQlCDaj85EEJkjh8hPKYyBg17Oor6iq6HqTbNlylYFMUCW7BAFVN5AZQ4FWMkXH8tkPSJ2Awpp1lI8OE1U19B4X8RsLpU8NteZ+qJXq8DQIIdC1FWJ//NExGcQOUa0AMYEcUXj8twwFH+jG6l/ew92M1QmM2NsQFY28g0FgIMAzMR7naMAGSZdCTAlIS1QyReXQ6B2jgrIVE4gOGgjlgEowLBQ/ECR8RqZFwvGhIOgfp+pdf9J//NExHkRgUasAMYGcWGeforGc2dkaZlGFg6gNx6Aoj////9S22AwMkmlgDUBfgYIBWzOQMcfAKaY8q41tEV+SVKKSigMXBSKjq0lmK17ne8po9vdVu69UVbuqWZBBUTM//NExIYSaUqgANUGlHxZQTBQzVXcrO65QIrNSbmYxPc4Z6qWYDysnUDXrDjC2vNNMokgx1TNAECQuPEaG3Ysb3r/3v+c3v/1y5a7uRDmM1p7ila46FvF3GNF8iqaVlGA//NExI8QiUKgAM4EcUFgpUtFQrNf8SLSCPi3i1dKwjGPYXYvYm4GUJKAKjeCpJcIWrkMswRaU3n+1K4+84xsSzXp2OYw95yY9cvtn/T5a4S5+CZKvyEpK4AmWMKw0jaw//NExJ8RyWKkAMYKleqBhDRkKlzpkuV5LmAxZE0aSEC3E+JWJYT4oybrlRwX/pilqZp9Y3jYeJarOm0DY7/xMY3fvzdYedQNl3uonYPRRcTXfZocbUKpBIMJByLbiZwL//NExKoSoVaoAMPMlZw1YgLCDNQZajj+uy8kyxC+Bw4SLXz1Hqdva9r1pWlfM4M4P988y/nj+b1wwVQhFSqUEiihAwwWU+i1mMuMjoxIRKC9MDl8L8nD92pWCGrFc8RV//NExLISOUqsAMPMldIzJNKGmFINw9EjzGxzKHVvK0ZiQ1Lalw+5dRpj91X19TTTe985gkWEJBOqm4CTmxKxKYK2ltybo0WGX5ke5fdd2Jy2UzlFY5qxOxmPtJFIBQZA//NExLwRiUKsAMYWcKUIli0GqvNa1FIipF2YAuPc8LrXWsuIQy9K/oVo5UqPVEDzURY6S00zxaUOhAgFQmLw6U8dClUq9pRSMdIMbbayISzGE7HQmYbExhuqWZrG9Lg5//NExMgR+PKkAMPYcQRDkIULRkefo9gIWhIv///qRzKlJMuPSmLGmEgYkXFkmjEGZiZbq1gcI0tiyRC2mOsDbk/j45tMpzpDjJ0HAVo/4yEsO4dqx4sX6t41oucQNvZ4//NExNMSIOaYAMYScO4QGN775vjUOtc53nWva0sHcKFU8ck9stYN///+//+SNAE7ahTg6DwEeD0hxEUzMkM8ABpMwVDjMFtAPAEAAOYz0gMqJHApAYGMwgxBIHQwEJxr//NExN0R6PqcAMPMcRSS7pSo7QRo+nl5exrb+xt/6Tk/DlHjT54Us7Xsxm8/b+Q9JZiNxurejcbsRmGe55U0TuS7lv5yPyuMVLHa9SYlnz1cTvyhlB+rb/+3//6KC04L//NExOgYgVKUAVl4AETESPUsLuHY8MoGFOYhwhWBQBtkmbUZJp7CGKObJYlcAUC5ZsBAa00VTJCFBAEKaEIO1YlayVOyaQXgcA8kstw9XmrVLK7lNRS+3nhc1qOxCGYA//NExNkgoXaYAZnIAJHdppZQWY3XztUkFVKaXQzGYrKvluUMS6WV7N2ht81apMs8v/f9+n/P////Czn/jfoqMN0yYSTU1BkKQH0PFAVMw2C/g1SaRIjYAsgakaCYeucD//NExKkh0cqcAZnIAAYwCYxSRUCUSBmhwRY4SAJ8S1lzXlpZxKp9bFV8Xcp88c8rWp2tymrzlvOxT7s0tyHoBy3ejsYfvdSvuIUtqkfaUym98Wrw5M01fs7TP5MV7WWV//NExHQiKcKYAZnQAN7r///q1P7///3j/+n/6tXz0n8WXWoxgwSGBxImCbELI+GBmBQGlsjJQkQBoOLQNRg4jMYMwg3MpCjAiYgJg4LMuDDHCoebwEumWCQkNmmDGDJJ//NExD4hIY6EAdt4AAwpFGw3izTRoDLaTe40WDC8ObUXcLOfGtLqsSie04F+Tzkw5Vs60rY6HRV05tbA+jrNYO8X1SNWZo5v///FUPAijgq1D96VvLoAqQTwsIEJp4o8//NExAwVoW6UAM4GlBQs7woeCuI0BBS1DVQodsDQw0Q8BGYHHQVBRQc2Vw8xaXU1HzX87+Xf7n/f/y1yLz+cbtMkYoC+JYKgCgtd0gmlhQOiNP///8VYlaCCjW8bCkcb//NExAgSQIKgAMYwSEr9JbKOsNUNDrp0igE/WopfApsFmQjvqKlHk1VkOLqzC6WY2EjrSBe0WvYt7lWkAOKILiImTKBs0I3Gi6v////rpX3AkFmchCg6ZiREepWikROM//NExBIRiWKcANTElSPTuEkCApLiMhTxmgjMaonALpC4D4zxxJaS2r09t69N9+6/Z1chlLVJg5RQcEwgKi7t2Z0q23Y3cUKT0Ly0TcWbRhFRgzAEUX4FwBc0NYMXCNhR//NExB4RQWacAMQElcBlEhFbBcCQUghOnS87orZ19e+9d/Xf9d4WrIY6hXkMglAjoq1TqpymO8WABxhOpwwgaFKDyZhD01V3IcPgQIcgLEMEL8A5LChLixrp/Gja1//j//NExCwQyWqUAMvElT9118/4s6tf/+uWmZ2DHKzlJYUt9VzK1X6EP54xDgAzDIktFjpVkQUfEgaRl1g7zgJgwAhAEYg4jMhPoylT0O+L51X1ta9de3x8wMLUbWN3h568//NExDsSmVJ4AMvGlcHgiz1WBnDOFVWhroSkUipKQxqzB6yYSHGsljilACpujKqulSth4obe9wMk0XAuw6oTUrUSrDSq1Z9s0rWtfvXznW/FUxmLnst9W28rmo8hiVEw//NExEMScV5gANvElKZJoCta7q4y2gZuC2D1iCkY9aYYiZxeVXyVKVo5g6iYj9PYhS4Lkxoa8Q6O9itsz7eskx7GpMarRIYUpIqVJHfbf/uU2UUUK/6XGhXTRFxVjMiV//NExEwQ4Mo4AMvGcWMbyLMBxu81gYvogV1C0oKhygofLGjQTDIqgGWCqBwqKlUcSPYK7jSha96v1irhUUurutd+8186mNIdl6IFaJotScdtLQJiQ4QNlyDS4SA1oYgo//NExFsRMJX8AHmGTGKng4kwaNFxgeFF2iqHuQx+uxXTehLtdq1sbe9KHmkz7FqaxcmQvMoe8uUsWprBVVF5lCZlE0VhdzLVwADJoU+bYcWYGBvNAgRDAIDT+t7jgPiC//NExGkScGHsAMJGKFz4ICcQCAafYSE44Inw+QLlDgYaQE7j+IxxxvsQ6Xpf+Q0Fz/KDmddG4XoAKYMkW9LZY2HDALKihyBAcONs86gcSYJg2YC4aOrCAUmz5EFjDEaV//NExHISYGXwAHmGKKjxoAJA8meMWdK9HRY31UbN7d3empBRlIHblHUVABgFgE6GCRkg9lgupPFJGyQqB4iE4k72B0lPMTcjRDDDbEGInsQsmYMmT8UTWYov35P0rp5t//NExHsRGFX0AMJMJDlvjpqqlOjuEe7qDIohKV8owzWUlRh3a/PMH2dqPoE/2obOjTHtWmYrYX0b0n9vXXU86H4EUw0691DM3fgx5GyPRGa0NbmoeBAiQik0WDBEaHxW//NExIkZoSXkAHpMcUpkjbGFA+QEJ2DKNuLLDk7UghioaKOt8DFLnJr1NNaZq5RtICaVOohGpIKsMfsafZUQ5dtl6hqCGWyCM4+/95b1v23+02YnUZfflN/EZm7j3TMz//NExHUhEw3gAMJMuJd76ermWiPOyqO7vOdnP1U1NEaTsg1nCiC6aPjptbXe1iA4pANqDofiryVthogE6CRMgTl1rWTkdJZIWontQqE6BU4jzQsXUXW+3iNlSPb+ouC2//NExEMgAwnwAMJMubqANebsP3e8tFlzTPGx81i1xr4lnnz8XmQvvUzu/0b3K79qhttpbtvfxqBO57xbW7+43c761Y+en3HNohwSYHyJIKIcO2QhQ5T/PF8UOBSk0frz//NExBYXCQXUAHmGcYdcsRhL/ndjXwokMK2BjMxrw/WHVoCBLemaSxYXMcht7f0VFNhfjCbN/ps0JM3klFeCon4ojnG26bk+yJ+LxXXYuCv/hUUqTEFNRTMuMTAwqqqq//NExAwAAANIAAAAAKqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq\" type=\"audio/x-wav\" />\n",
              "                    Your browser does not support the audio element.\n",
              "                </audio>\n",
              "              "
            ]
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "user > 끌래\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.lib.display.Audio object>"
            ],
            "text/html": [
              "\n",
              "                <audio  controls=\"controls\" autoplay=\"autoplay\">\n",
              "                    <source src=\"data:audio/x-wav;base64,//NExAARCxYAAEhEuQAEIiDgG0ABxEOiQVqLpOkxM9xsxy8AP83/r/f/P/dG/b//0af/////tf///+QhCEIQhDhwMDAwMDAwMAAAAAAAAAEMKjvXUBDRVETIxlI5rsYi//NExA4SkEYwAVsYAPLZbTb5axrVyoMzLYWGQTraUTKNPl8owgU1n8vqEo8eQfIVF2HC9yfLxPaoWaA+oIcauTAhs2xaCpA0mmoBPPIDwFmgcwux/MveRCZ//4OAlP4A//NExBYXaxpYAZloARgL6E7/xKQV8tEP/4NskBLyEOT/82HoUBwDwT//ygmmX1Gh7//9RoUE0DQ6bjz////Qc3ZzdCgm6f////+cNCUOm6ablNDNJZf2IuW7poFyUQ/A//NExAsS+Na0AZhgAGwwu3Q/hizekhO8glkB4lk9m09+zcAgHCwImP0prrvOWPHF7b48q0yhoILen01ZMuh7Ufp5chFytbvuyf/Zcz8wy03Ige/s4tpSMZwRwTCYk+gs//NExBIVYQq4Adh4AK2luNKcanXPS1JIvyeoDuVrwgh1ltIGGmSVuWEdWG1XiRIuIEuo8+rN9q6eSxIkeFEqXByaU1IADYnOAgXeKN////6aq4N3OZIXZQFwdfbA0DTZ//NExA8UiWa0AMYQlUBSk+IxV8ZWrKNCv3E34rkjI7NiACIUiuFtLeKhlNzTm3ty7/mLTSZyLxIwIRwhguOxasbdkErv8nNBt428hTYvbd3VvX4oT6QiNhYVi9tMhLma//NExA8WGXKsAM4SlHdJxLNMICEw7rcXivZIIsqiqzzVJUpljGF7NAyUOXfR6ePLF9cvrZ3ZDtlY3xTNUDLaxCJPEk2Qiz366WUttSa9PhFab2////0KuysqOF/1XELn//NExAkTMRakAMPecCdTR3un27EHFJfJIL4Y2z9Li8bzRcX24NWE5XinE1aHMYQsUeR9fXzvFv7brW3tCzhinrWtrRrB0hqPLdDQ5/////5KUCBQoVUMKBQVrBWWvSOu//NExA8RcRZsAMsMcFg4lQUAoHfWviDYrR80Sj6pyuZigouCh+HJc1Ven9ekv3/aqb61a/NNROqROlg1hI9z3/7v/lYz8LDn0wYQOlXULHVQxlDjMBbs06al1+yKGNum//NExBwSeWJIANmGlJqLU/Kg4CByMBKsnGaq/Y1ukLUoLKGwoKdU8/5GLn/0ZLtuj9X/d29Kdd3/X9UFmILTQv+B1HQCg8LgiLFJnNWtQSfUqtXDI420sYjyvcyLe5F9//NExCURePY8AVQYAKAhJbhptZ0xXWyRUWQh0drU+19XVeZ/dfY9nF1OvarFUhiShORgCst/7OrwAACf/tEFyKPkoYlMYcDWFq8lzhoaCADeSIwP1u6p0hgrA6DD/st3//NExDIdcyqEAY9oAMcImZUOYlF//4cxFy4eUJZ/7N+IwXRaHGGDN1En///+MOS4XQTge5AcuEwYQsPf////6ZfGAMByDyGEPAhA2m5fHYPCztZl9wcuJQwjwgAZvqC4//NExA8QYPK0AdgYAH9b7vedvuesMMNiCJxZ8Q8ARMAEQ4hW5b54iMVNin0rcz/2+SXpUs6rTWVLVft/xgTBatYxIySFrS1uYHmXRjS0CkJqZcNNc6TbN7OQYJYzQNJH//NExCARQPqwAMJYcIdntioQRiYA0H6iw4eow3FMLSxhQqichtasVbtIGxwuVv///461yKHy0mtAYMDSwaBACKTWdo1FhIVRpEjG5ugoBUSSutaJKAZCkAE0Mj5dn1ra//NExC4RoO6gAMmYcMuvXawLrA0+WR8qPd///6Vu9uR/+upR7qoKC4wGAEvOWPTeBswIFBxSZYTiwgXhMYCzBwAHMJial10DVnRo62VZSURUvNkRBZtACU4rXjUUMfFZ//NExDoSGPJgANtScDkrNIHSx7/7v9f/89Z9dTX0hiAhBzfdkFKoJCS5LAoUvowEITiiudnvMt6xx/L/zP+GdVQoUE5UFiR6p4lFWNkEZH+eu/ZVf72a1f//2J/rMkaQ//NExEQQQKZIAVsYAEGgCiQ89OkLjTBNpMy0DJ71287h0vnct2B/BXDF1zHoDIKZcGOp1I1orrGGCcAA0ChAz3Zbtq7AUcaguASAT0Ln1su9WzAX4gwOwDMGOMsSv/1N//NExFYgkyJQAZtoAfckDw8C+fNjQ3M1/X+v9bZsUC4eNx7kkPAYAzNyX/t/1//5gaFBOaLuggaFlRDzzUEFDdMi7XavppmiOPnoIwKAgGirKYQwNKEQVorYXsixcYHo//NExCYbmtqwAY9AAUYLhIHBcDiimIo95mEukLMi4xFaWWU9li+6br6/0/+LX+rnqb+v3r63/SToiVHFLZdVUd9y0RSU73/y9f++01dlXWT+9NvaP6rL3iNSnZzlbA2n//NExAoREO60AdhAANPEmcTMvlcbl9CyhYk5bjkOVc9EIYDlCGcDSiNxgeGEGQ76VCZloYW5B7RxE07nBU/VS0e53//X/sqqx3MAljqW8FMO7ulA6amawTadsHWjaMZD//NExBgSOU7AAMPMlPW12hVarj0YzujZHIZMWZkrryf0JzmJc9IjIsejmT+z47ft+2XkLxhoMA8b///9/9PmqEZvFfeBhuFZxFRbm0ko3KBx1hwWu3nNvL9G6wVaNiJ8//NExCIR8VLAAMPMlNOaGo1TWA113mNXMrXCZ4OSqyq/z8/O2/5u3U6eceDMz///0v+pz1swSe+UDAGPy99TSCegGY5Ya/GBNKAVMwmBlnSgm1oAU+4HyI5ETl00e+kL//NExC0QGPq4AMJYcbkR9RspH3Lz3sTcnxf/tYAlSuhKhyViihqZrDNyeNXyE0yRAYSxwBR5HuDivWuPXsusdLact5shMpDQGg+gEwmFUyaNKkMTpCZTETWJExr1nU4C//NExD8RyO6UAMsScDolGFTv///H1bc4xYjDKFPAospZTLlXu44CyCZyNPoSrt/UL+9r3cC5HY0GGSMicvKCD43y4gOCcofkEiOxr3JFF22nOX/12/u/9vsSrlHvX1My//NExEoRcPqYANCScKY5fJBCKytXQZGh7HafHeQMlM4hRJfOvvWoe7uUSmiH8SJopiAo+siHzVXSEVwpHDt6r+GNzz39PJr4Tk2Ks99vIgXHuJj1ztckOWnKnqMrhdVX//NExFcRoVqsAMtelHRvHga1FPKmyBjWuoovRG6VhPiEwAjhcS7Qs1g2lUZM6MKusUy8yTfjX+y5TKUAQmlA1It///5ShdY1jOitVSEYmetywDhExlHVpqg7FOspaZ6o//NExGMSIV6wAMtSlJ7x9KdRbnBO3MwWkplgN0kMBkc0Szki3qAbuSpfkk3uv/0m8K1wvO3f//lKVf5BZtqyG4ooPTXZYaHq3cIz1rg2atHHwVzrk3UQyvuL8JxHPGET//NExG0RkVqwAMtWlGd6X0JpRrcb2i3wy20xskfYih1XEUNFzoq7///qzQjYGCOuCdKqON4L5ze/5xm2htfMrKQpSgQEKVtf+k0v///+/ft//5951dTmIQn/J9CNU5Gf//NExHkQwP6wAMpecH1Od/5xc7uQQc5zoQXc8/OdGw49aSIUKDvomKH//////Px8fM8L3URrU/FvrSsqUtO8DnHaDw9LQLHywiTBYkEMfbVQdvIg4waUKCCfQcwKjdha//NExIkScxK8AHiEuYxRgyBUoa6HGB2YgohcKP5n/////78cf/P/O/HTJ0g0e+8TCpEc+7T11BZI8xlLRokaQdJByVosGDHosXRjFF2FxcNUUiOOHjLEASkDRo+6BPDt//NExJIS+x64AGgQvMrlIqdMbomAuhm+Z7XTAc7L/yMh2///WjszW26SldDGVnMhUhjbZi9BToa6srIWGFFY2jspSGCtNKhnIBFYMKuxQTOrIGFCnNUTCsVqjkaQ7ms6//NExJkQoyLAADgQvH3SQXbqzrW2nyOLuQOBQWJFFtEzZ0jTcjBdIiHRBTprMRlKr3Pt7T+Xl3/Qeu7xKjbzH97/2/eMZ9+OnpZQxE/TwiWBa5VdVmnc0Nz29Itb/6bC//NExKkT0xK8ABBEudsoA5i1L3URK6+g0AhTwAANdmhABYWVIzlB2Cuiy93IasOnYg9eb/Pe5bnv6/kYo5a/lDyUY85YkwxGikkdJDKsAc60jwMcatPMzYpPnlZEImIE//NExKwV8ZKkAMJMlBI8uKWcLhtzf/0SNijtYeE4qHjSzCqko2AEeE3ivOUhlsrYH4XGAnCmsbg6c6O9GrtaQ1+n1wzSHAAGAECEPaMOSbe+Zjqm46Wxo2C2Pm2Jhh9S//NExKcacYKoANYMlEyFBOkqFB9H///+jVXL3wFvW3jX7SMgnKRYAgrIwCxOznY9NZzOVkJmeOsOCnQlfAMLng3NcYPY2Oo3VYaI4qLs1Xp2k0cwxEDEnqAoKCwca/////NExJASWRrAAHvQcP/6Vc3D6CaU5LVSfyigKMlrZGhQ3syhhShRKGMh2gQ4CUqO5S3/1/9flb/2/6dP9XozIV73ahOjXU6v1JU+SRzyIcDPf/O84GBmZEIQjIRT1Azn//NExJkSER68AMPQcD0YgQkcPSBEwBMUVKsV//////3f/3/Lq/l+FbK731UPmz6sv/Fhnwr8ti+wjOScUEMZRLGCkT1pRSVo4tBeZJhPMUE6ARlwWLq9ZZQ4BDBcgIZm//NExKMVcwK4AHiEucUOIiUsquuqHD////////07ZNSKrEMhhN0lcQOyMZCKKIHI8aRQ9FhkVOIDROQoormFY9oDHNOJGsHxpDGFhUgsIAdCAQVWBwSE1S/8////9VP///NExKAVCxqwACgSvf//////16ftqSa7H6msw1zHMhimd1RVYlSs7VIZDuaJCToUaw0rR+hSnGII3QVcpwKVhMrsxyymHsI7gZi9Z2GdrOzlrM/88RmE2YV79WXk7/+n//NExJ4Qixa8AAgKuUqilLpe3/ns6y4Z2USZ1KZ2Qy9djGexZQokxZdjPlNoapWN0dQFnQy6KAgAW7KljTFQMusnO2ygCxF1srUHBIdFXQcGU7y+gvaJpi84TsuCYbRo//NExK4QaxbAAAhKud6NHOOyUQR+MIGOAYnW8McH4qHwOFlk////+j//9FXKwIeKbNjHmPS1JFFHQMqrFFoFqlxGfvHilMkq45Zt52GDH1mMvkoOwnGoeR5CQt2Hc7PX//NExL8RwvKwABBEucrIqjW/AS/KUfOTvy30375nPv11/2dfeZhBkKnwuZ//+1TaQAgq0BTOHQbqICDHUEjJrEZaIbpxeX4Xupr7DVLTHe3NpUE8EYUE4sXOlBZARLkx//NExMsQyPKoAMIScBskhtYTtG/u3Dw9b89+86Drxx1MSAhGqsXoRbn0KOs/exNydwIuO5Dt2arY2OT1uxhXxuSos08eOUmkSykqByJCsEk2VPHSEPDvUHpqRMJRPebN//NExNoVeWKsAMMYlK/6id3/zUK516azPX2sTTGmOmkyUDOWBt5oUhhR79ltSc0yHQVMXDWgiNAYmqqVSS6KFNRllSPspYa0Bpbzg07oOw11pL7RZNEusWjbsqNaCmhM//NExNcSEWK4AHsSlUWK5rNIKT9Wq/0FrlYcqSTTkNw+8kJrSiYsxplMJ+1GbtR/IHhiKWsaKdqyDCxlGoJlVuHflmOH///9Jlh+///q0tvn//9v7//9KhZw+ShDUewR//NExOESaWq4AVhYAJL4CHDFDqEAxh1gIdAMSZhqfQa+us9VgKEgcqBQfksm1BUVUFWBpGNdHkQlPQsCV2o8CVIptyUHehuTgtuteJv5EJa16Gvqv6190JDS2pY8ctqS//NExOojmcacAZnAAPr9ghr0W5KaLcN1IPfyvLaSch2G95W+RqPzUajWN+wQTLgTho65/T///R//1TOLNV4lwBbBnpjT46qCZRIQKKgEgYXELQQCZQ6sZ3HlkQIMZAne//NExK4h+X6cAZnAAKaPXLkei4RnIIEg0RQFYqyxAGAugyLNXpvv9GY7EJXN2pZLKR/qbVamj7wRmWxChlcfo4cwuUsMzuMqi1LDD7u7KrNLqpQWcNdyuvrTRqXbrXr3//NExHkfWYacAZnAAFD0M4KqMYhAqoLQDMBQEjFpRgQ5iQxhQZnhw8TMSQCRAKfmWFF8DUlACQMcQwjwuMyAxkDSERYZaYI4DeFgWvQ0a7KV8D3QGE60hsxedmaCVy/s//NExE4iEcaYAZrIAK6ftmzZmauchgeLQzLqGH5XZh+G6e7DsqoI1Lo27EqjdJ97HOtKJyfxlztZ2Px7Y5////zn5///92lqFElBAsIBPL7GISISVK8Eam0C4TWBBCYi//NExBgXsXKwAZh4ABehdK+kcBIg7BCBugLYMcJ8BOBbR4uAlDbPsn63HpnWa2p9X9Kbvnwr2b3m6eZvf61bFKVpHfZpXVojm81S+cv7zqN8u7gSBGwl4WkB+DMAE5yj//NExAwVqW64AY94AJiTgmiUDiAThGgbI5gVBfY5oAJw7gbYJwdAfYMMxrYwWK+smJHp//jGt43r7+NZzXEKXOM+DG+d+BNvEbeLee7ym9Z3mPNdLerxBZp0gc5WFQEa//NExAgSUU6oAdhoARP0jK665V8Q4FCpdq6Yiy5HJi6n21YsRDDACiFO4eDdLhgaukivU9B1/anvfez33djY3MFpXQOJscOKBgmOLc/fFco6IrXbbGDqIoo+mzGRCjio//NExBESSWagAMyKlXo2UgYQB1F8NtJ4X4F2EFwsmAuBdFRMS4SLHUbetrfa29dt67YMNcXVGEBc4TFCC6nDCFHhvXS5qsZUYJJ5UhCfqj6sW2KpRwCIj9IsCsUw4oNr//NExBoRAVqkAMQElR/CwkN/HNAMx6IELZJp1oup969W9Wo1/T+hylPMxmcwGFDDBQCMTWkwCpXEDBgWY+4XEn0/2gTQoIqKokZAxGhmSmIQBe4ZwBaBtQ6wEwbRDy6x//NExCkSmWagAMxKlIPU9q9f85btL//MjD2dmFkYWcWFGFkHAzhgyYPJd////VW5DYrux1u4ZTGEJr8swVdL2btq6A7TRC/JcWMJo0SFADItJ01YIdY9aZr/vGMb//+G//NExDEQ8YaYAMvKlEPN//9lM12jlDsQOo0igqG1xyFuAqSaOCcZK6RAoyCQ1OXqiCmrkww7y2SU6kJIcMcGyBZc6J1mYWpXXrrePb5m9vX513s9paZaezUlLqVjCjBO//NExEARkUqIAMvElUCwuPdNjtUwMFC66dkIjoOZ2WTyPzlISWdMCkQknKGofB4Gp6UAKpnz2ERYQaqc+fM+Vnt4Ym7/9/nQosn/UVCjoj3blx5pMCTTmKk0hOOJTDCj//NExEwP8M5sANsEcXDCBDNElE5ltBKi3I5OxFydKRFxMwfy5cYMVTRI2cb/8ar7Hs1VUAnATOkg76xDhrLEgKQPLkm9X///LJ/qFJAwi4mMSaSF8LumkDWxg4oFFHmD//NExF8SgN44ANvGcIsx9uVoUVzSoLQ1LeWVPKcU7kR0jGDjCDrV0q0L05tNC1sFcCmpKo1vaLauVp9KBrxwLb5FuAUMoD55wRHZxwMTCJg+QabIExQgBRYSNAp4JCR4//NExGgQ6RX8AHmGcBRETJI9P/pGDGscZT///RFR4wWDxoDBJp80TE1BtwsLVQ4ADwnjOg47AnHEEBKTWwQGIthYu4bU0MAwoCLQ+8a1KAVaKknGxR9OKUOi+jZU+h0X//NExHcRCEn4AHpGJBegw5QrSb0m6hSgxX16TEUrbNvUKBEJl6q7KmvSMFFFypkkBrDiRBEHA6GQ+55FziTHCzXiiCkRtn2HD7j8o9EyamjLHlWuKsQWpWpNetWr37kW//NExIURuHXwAHmMSCZ5evzkUpFKALx1xDzRcFOlQFmrAl1P7IXSa9PZULkzsDQUiKUxQsTcHOuqxytcM2zNqK1DEqzCxxIqKsNFVWBY7gWOlSTWZVgEbUBLoUSxsKCn//NExJERYEn0AMJEJLUT0gJG2NS1Um41Xar8Y/Y7qTMpRjOqXVL2P431f1L2a7CqqhUAmCkGFMwUBKFVBJTadWJAImBAmFio5cXqWN56wpBWG2goSFw2sGDSBMnSYEZs//NExJ4eSxnAAMoGvUAbKChVHNG2o3VzqKN6BoKGCMn2509RAwmTTbDEHTibaDAjLu9jCCDtBZNkCgh9jL2Gj+Pv8Q+/34/aMu/3vYQe/8vYe7bPfjDyaQPa6H9P6HtN//NExHcfioHgAMJMuTIeH5pEY8eqM00wVZnrRCBSVjDQFJrrXI3rkSGH4nT3EokRLO6QoNPYkm+9ZbtHS5w8wgggUSQE+gvOCFRgo5/tZGQRS8mEDTvTUYbFG6xgiSj8//NExEsgmwXwAMJMuVu9Xp5kXVudJ2ZCFQeYW1rLpBr5P+nzGZsp0v0H3fexWNOHeC/ZSF7TP514/9Q092uv/3v9NOGVRIXoYJcVlPRrIlRJ5mleq0jBJFiRLQUAhKjS//NExBsWken0AHmEmCRyq8zWvjzMzVf1Vea/oGM+pSlLmZDGlKVspSl0/UpTaP/9QpnUKUYDSga2ZIqDX4KnSowGomPeVBUFQWBp5VX+IJEkJFAYELIDxxEGEgQsQeYU//NExBMSUJDsAAmMTVnGlFmHwr/+ClaaaaqqoldNNIlLTTTCJVVVU00wxH/+gYlVVXTTSBrU2+iqqq//////////2lVMQU1FMy4xMDBVVVVVVVVVVVVVVVVVVVVVVVVV\" type=\"audio/x-wav\" />\n",
              "                    Your browser does not support the audio element.\n",
              "                </audio>\n",
              "              "
            ]
          },
          "metadata": {}
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "user > quit\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import warnings\n",
        "\n",
        "# 경고메세지 끄기\n",
        "warnings.filterwarnings(action='ignore')\n",
        "\n",
        "with torch.no_grad():\n",
        "    while 1:\n",
        "        q = input(\"user > \").strip()\n",
        "        if q == \"종료\":\n",
        "            break\n",
        "        a = \"\"\n",
        "        while 1:\n",
        "            input_ids = torch.LongTensor(koGPT2_TOKENIZER.encode(Q_TKN + q + SENT + A_TKN + a)).unsqueeze(dim=0)\n",
        "            model = model.cpu()\n",
        "            pred = model(input_ids)\n",
        "            pred = pred.logits\n",
        "            gen = koGPT2_TOKENIZER.convert_ids_to_tokens(torch.argmax(pred, dim=-1).squeeze().numpy().tolist())[-1]\n",
        "            if gen == EOS:\n",
        "                break\n",
        "            a += gen.replace(\"▁\", \" \")\n",
        "        # text = a.strip()\n",
        "        # tts = gTTS(text=text, lang='ko')\n",
        "        # tts.save('hello_ko.wav')\n",
        "        # y, sr = librosa.load('hello_ko.wav')\n",
        "        # display(Audio('hello_ko.wav', autoplay=True))\n",
        "        # time.sleep((len(y)/sr)+1)ㅇㅇ\n",
        "        print(\"Chatbot > {}\".format(a.strip()))"
      ],
      "metadata": {
        "id": "T12auNnKdRNH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 모델 저장"
      ],
      "metadata": {
        "id": "U0RQo1Ck5O-p"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_dir = '/content/drive/MyDrive/nlpbook/chat_bot'"
      ],
      "metadata": {
        "id": "8k5hnF6P5VSo"
      },
      "execution_count": 64,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(model.state_dict(),model_dir)"
      ],
      "metadata": {
        "id": "YYB4cPfYxJb5"
      },
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 모델 불러오기"
      ],
      "metadata": {
        "id": "o9oY73W25SiF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.load_state_dict(torch.load(model_dir))\n",
        "model.eval()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KTF6yTDd2wof",
        "outputId": "18ee2935-3906-4bbb-8992-540d6e7b49a9"
      },
      "execution_count": 97,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GPT2LMHeadModel(\n",
              "  (transformer): GPT2Model(\n",
              "    (wte): Embedding(51200, 768)\n",
              "    (wpe): Embedding(1024, 768)\n",
              "    (drop): Dropout(p=0.1, inplace=False)\n",
              "    (h): ModuleList(\n",
              "      (0): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (1): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (2): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (3): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (4): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (5): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (6): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (7): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (8): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (9): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (10): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (11): GPT2Block(\n",
              "        (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPT2Attention(\n",
              "          (c_attn): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "        (mlp): GPT2MLP(\n",
              "          (c_fc): Conv1D()\n",
              "          (c_proj): Conv1D()\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "  )\n",
              "  (lm_head): Linear(in_features=768, out_features=51200, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 97
        }
      ]
    }
  ]
}