chapter10/10-2d-llm-jp-eval-jcommonsenseqa-flexeval.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# 第10章 性能評価"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## 10.2 評価指標を用いた自動評価"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### 10.2.4 多肢選択式質問応答タスクによる自動評価"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### ツールを使用した評価"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dkam4BHiBMaE",
        "outputId": "e259e9eb-207e-41af-cc57-0c7dca3bef2f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting flexeval\n",
            "  Downloading flexeval-0.6.3-py3-none-any.whl.metadata (4.1 kB)\n",
            "Collecting datasets<3.0.0,>=2.14.6 (from flexeval)\n",
            "  Downloading datasets-2.21.0-py3-none-any.whl.metadata (21 kB)\n",
            "Collecting evaluate<0.5.0,>=0.4.1 (from flexeval)\n",
            "  Downloading evaluate-0.4.2-py3-none-any.whl.metadata (9.3 kB)\n",
            "Collecting fuzzywuzzy<0.19.0,>=0.18.0 (from flexeval)\n",
            "  Downloading fuzzywuzzy-0.18.0-py2.py3-none-any.whl.metadata (4.9 kB)\n",
            "Requirement already satisfied: google-api-python-client<3.0.0,>=2.131.0 in /usr/local/lib/python3.10/dist-packages (from flexeval) (2.137.0)\n",
            "Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /usr/local/lib/python3.10/dist-packages (from flexeval) (3.1.4)\n",
            "Collecting jiwer<4.0.0,>=3.0.4 (from flexeval)\n",
            "  Downloading jiwer-3.0.4-py3-none-any.whl.metadata (2.6 kB)\n",
            "Collecting jsonargparse<5.0.0,>=4.26.1 (from jsonargparse[jsonnet]<5.0.0,>=4.26.1->flexeval)\n",
            "  Downloading jsonargparse-4.32.1-py3-none-any.whl.metadata (12 kB)\n",
            "Collecting loguru<0.8.0,>=0.7.2 (from flexeval)\n",
            "  Downloading loguru-0.7.2-py3-none-any.whl.metadata (23 kB)\n",
            "Collecting openai<2.0.0,>=1.16.1 (from flexeval)\n",
            "  Downloading openai-1.43.0-py3-none-any.whl.metadata (22 kB)\n",
            "Collecting peft<0.11.0,>=0.10.0 (from flexeval)\n",
            "  Downloading peft-0.10.0-py3-none-any.whl.metadata (13 kB)\n",
            "Collecting python-levenshtein<0.24.0,>=0.23.0 (from flexeval)\n",
            "  Downloading python_Levenshtein-0.23.0-py3-none-any.whl.metadata (3.8 kB)\n",
            "Collecting rouge<2.0.0,>=1.0.1 (from flexeval)\n",
            "  Downloading rouge-1.0.1-py3-none-any.whl.metadata (4.1 kB)\n",
            "Collecting sacrebleu<3.0.0,>=2.4.1 (from sacrebleu[ja]<3.0.0,>=2.4.1->flexeval)\n",
            "  Downloading sacrebleu-2.4.3-py3-none-any.whl.metadata (51 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.8/51.8 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: transformers<5.0.0,>=4.34.1 in /usr/local/lib/python3.10/dist-packages (from transformers[ja,sentencepiece,torch]<5.0.0,>=4.34.1->flexeval) (4.42.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets<3.0.0,>=2.14.6->flexeval) (3.15.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets<3.0.0,>=2.14.6->flexeval) (1.26.4)\n",
            "Collecting pyarrow>=15.0.0 (from datasets<3.0.0,>=2.14.6->flexeval)\n",
            "  Downloading pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (3.3 kB)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets<3.0.0,>=2.14.6->flexeval)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets<3.0.0,>=2.14.6->flexeval) (2.1.4)\n",
            "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.10/dist-packages (from datasets<3.0.0,>=2.14.6->flexeval) (2.32.3)\n",
            "Requirement already satisfied: tqdm>=4.66.3 in /usr/local/lib/python3.10/dist-packages (from datasets<3.0.0,>=2.14.6->flexeval) (4.66.5)\n",
            "Collecting xxhash (from datasets<3.0.0,>=2.14.6->flexeval)\n",
            "  Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
            "Collecting multiprocess (from datasets<3.0.0,>=2.14.6->flexeval)\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n",
            "Requirement already satisfied: fsspec<=2024.6.1,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.6.1,>=2023.1.0->datasets<3.0.0,>=2.14.6->flexeval) (2024.6.1)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets<3.0.0,>=2.14.6->flexeval) (3.10.5)\n",
            "Requirement already satisfied: huggingface-hub>=0.21.2 in /usr/local/lib/python3.10/dist-packages (from datasets<3.0.0,>=2.14.6->flexeval) (0.23.5)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets<3.0.0,>=2.14.6->flexeval) (24.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets<3.0.0,>=2.14.6->flexeval) (6.0.2)\n",
            "Requirement already satisfied: httplib2<1.dev0,>=0.19.0 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client<3.0.0,>=2.131.0->flexeval) (0.22.0)\n",
            "Requirement already satisfied: google-auth!=2.24.0,!=2.25.0,<3.0.0.dev0,>=1.32.0 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client<3.0.0,>=2.131.0->flexeval) (2.27.0)\n",
            "Requirement already satisfied: google-auth-httplib2<1.0.0,>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client<3.0.0,>=2.131.0->flexeval) (0.2.0)\n",
            "Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0.dev0,>=1.31.5 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client<3.0.0,>=2.131.0->flexeval) (2.19.1)\n",
            "Requirement already satisfied: uritemplate<5,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client<3.0.0,>=2.131.0->flexeval) (4.1.1)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2<4.0.0,>=3.1.2->flexeval) (2.1.5)\n",
            "Requirement already satisfied: click<9.0.0,>=8.1.3 in /usr/local/lib/python3.10/dist-packages (from jiwer<4.0.0,>=3.0.4->flexeval) (8.1.7)\n",
            "Collecting rapidfuzz<4,>=3 (from jiwer<4.0.0,>=3.0.4->flexeval)\n",
            "  Downloading rapidfuzz-3.9.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
            "Collecting jsonnet>=0.13.0 (from jsonargparse[jsonnet]<5.0.0,>=4.26.1->flexeval)\n",
            "  Downloading jsonnet-0.20.0.tar.gz (594 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m594.2/594.2 kB\u001b[0m \u001b[31m34.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.16.1->flexeval) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai<2.0.0,>=1.16.1->flexeval) (1.7.0)\n",
            "Collecting httpx<1,>=0.23.0 (from openai<2.0.0,>=1.16.1->flexeval)\n",
            "  Downloading httpx-0.27.2-py3-none-any.whl.metadata (7.1 kB)\n",
            "Collecting jiter<1,>=0.4.0 (from openai<2.0.0,>=1.16.1->flexeval)\n",
            "  Downloading jiter-0.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.6 kB)\n",
            "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.16.1->flexeval) (2.8.2)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.16.1->flexeval) (1.3.1)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.11 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.16.1->flexeval) (4.12.2)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from peft<0.11.0,>=0.10.0->flexeval) (5.9.5)\n",
            "Requirement already satisfied: torch>=1.13.0 in /usr/local/lib/python3.10/dist-packages (from peft<0.11.0,>=0.10.0->flexeval) (2.4.0+cu121)\n",
            "Requirement already satisfied: accelerate>=0.21.0 in /usr/local/lib/python3.10/dist-packages (from peft<0.11.0,>=0.10.0->flexeval) (0.32.1)\n",
            "Requirement already satisfied: safetensors in /usr/local/lib/python3.10/dist-packages (from peft<0.11.0,>=0.10.0->flexeval) (0.4.4)\n",
            "Collecting Levenshtein==0.23.0 (from python-levenshtein<0.24.0,>=0.23.0->flexeval)\n",
            "  Downloading Levenshtein-0.23.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.4 kB)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from rouge<2.0.0,>=1.0.1->flexeval) (1.16.0)\n",
            "Collecting portalocker (from sacrebleu<3.0.0,>=2.4.1->sacrebleu[ja]<3.0.0,>=2.4.1->flexeval)\n",
            "  Downloading portalocker-2.10.1-py3-none-any.whl.metadata (8.5 kB)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from sacrebleu<3.0.0,>=2.4.1->sacrebleu[ja]<3.0.0,>=2.4.1->flexeval) (2024.5.15)\n",
            "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.10/dist-packages (from sacrebleu<3.0.0,>=2.4.1->sacrebleu[ja]<3.0.0,>=2.4.1->flexeval) (0.9.0)\n",
            "Collecting colorama (from sacrebleu<3.0.0,>=2.4.1->sacrebleu[ja]<3.0.0,>=2.4.1->flexeval)\n",
            "  Downloading colorama-0.4.6-py2.py3-none-any.whl.metadata (17 kB)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from sacrebleu<3.0.0,>=2.4.1->sacrebleu[ja]<3.0.0,>=2.4.1->flexeval) (4.9.4)\n",
            "Collecting mecab-python3<2.0.0,>=1.0.9 (from sacrebleu[ja]<3.0.0,>=2.4.1->flexeval)\n",
            "  Downloading mecab_python3-1.0.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.1 kB)\n",
            "Collecting ipadic<2.0,>=1.0 (from sacrebleu[ja]<3.0.0,>=2.4.1->flexeval)\n",
            "  Downloading ipadic-1.0.0.tar.gz (13.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.4/13.4 MB\u001b[0m \u001b[31m82.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.34.1->transformers[ja,sentencepiece,torch]<5.0.0,>=4.34.1->flexeval) (0.19.1)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from transformers[ja,sentencepiece,torch]<5.0.0,>=4.34.1->flexeval) (3.20.3)\n",
            "Requirement already satisfied: sentencepiece!=0.1.92,>=0.1.91 in /usr/local/lib/python3.10/dist-packages (from transformers[ja,sentencepiece,torch]<5.0.0,>=4.34.1->flexeval) (0.1.99)\n",
            "Collecting fugashi>=1.0 (from transformers[ja,sentencepiece,torch]<5.0.0,>=4.34.1->flexeval)\n",
            "  Downloading fugashi-1.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.0 kB)\n",
            "Collecting rhoknp<1.3.1,>=1.1.0 (from transformers[ja,sentencepiece,torch]<5.0.0,>=4.34.1->flexeval)\n",
            "  Downloading rhoknp-1.3.0-py3-none-any.whl.metadata (7.8 kB)\n",
            "Collecting sudachidict-core>=20220729 (from transformers[ja,sentencepiece,torch]<5.0.0,>=4.34.1->flexeval)\n",
            "  Downloading SudachiDict_core-20240716-py3-none-any.whl.metadata (2.5 kB)\n",
            "Collecting sudachipy>=0.6.6 (from transformers[ja,sentencepiece,torch]<5.0.0,>=4.34.1->flexeval)\n",
            "  Downloading SudachiPy-0.6.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
            "Collecting unidic>=1.0.2 (from transformers[ja,sentencepiece,torch]<5.0.0,>=4.34.1->flexeval)\n",
            "  Downloading unidic-1.1.0.tar.gz (7.7 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting unidic-lite>=1.0.7 (from transformers[ja,sentencepiece,torch]<5.0.0,>=4.34.1->flexeval)\n",
            "  Downloading unidic-lite-1.0.8.tar.gz (47.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.4/47.4 MB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.16.1->flexeval) (3.8)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.16.1->flexeval) (1.2.2)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets<3.0.0,>=2.14.6->flexeval) (2.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets<3.0.0,>=2.14.6->flexeval) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets<3.0.0,>=2.14.6->flexeval) (24.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets<3.0.0,>=2.14.6->flexeval) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets<3.0.0,>=2.14.6->flexeval) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets<3.0.0,>=2.14.6->flexeval) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets<3.0.0,>=2.14.6->flexeval) (4.0.3)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0.dev0,>=1.31.5->google-api-python-client<3.0.0,>=2.131.0->flexeval) (1.64.0)\n",
            "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /usr/local/lib/python3.10/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0.dev0,>=1.31.5->google-api-python-client<3.0.0,>=2.131.0->flexeval) (1.24.0)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth!=2.24.0,!=2.25.0,<3.0.0.dev0,>=1.32.0->google-api-python-client<3.0.0,>=2.131.0->flexeval) (5.5.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth!=2.24.0,!=2.25.0,<3.0.0.dev0,>=1.32.0->google-api-python-client<3.0.0,>=2.131.0->flexeval) (0.4.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth!=2.24.0,!=2.25.0,<3.0.0.dev0,>=1.32.0->google-api-python-client<3.0.0,>=2.131.0->flexeval) (4.9)\n",
            "Requirement already satisfied: pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 in /usr/local/lib/python3.10/dist-packages (from httplib2<1.dev0,>=0.19.0->google-api-python-client<3.0.0,>=2.131.0->flexeval) (3.1.4)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.16.1->flexeval) (2024.7.4)\n",
            "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai<2.0.0,>=1.16.1->flexeval)\n",
            "  Downloading httpcore-1.0.5-py3-none-any.whl.metadata (20 kB)\n",
            "Collecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai<2.0.0,>=1.16.1->flexeval)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl.metadata (8.2 kB)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai<2.0.0,>=1.16.1->flexeval) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai<2.0.0,>=1.16.1->flexeval) (2.20.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets<3.0.0,>=2.14.6->flexeval) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets<3.0.0,>=2.14.6->flexeval) (2.0.7)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft<0.11.0,>=0.10.0->flexeval) (1.13.2)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.0->peft<0.11.0,>=0.10.0->flexeval) (3.3)\n",
            "Collecting wasabi<1.0.0,>=0.6.0 (from unidic>=1.0.2->transformers[ja,sentencepiece,torch]<5.0.0,>=4.34.1->flexeval)\n",
            "  Downloading wasabi-0.10.1-py3-none-any.whl.metadata (28 kB)\n",
            "Collecting plac<2.0.0,>=1.1.3 (from unidic>=1.0.2->transformers[ja,sentencepiece,torch]<5.0.0,>=4.34.1->flexeval)\n",
            "  Downloading plac-1.4.3-py2.py3-none-any.whl.metadata (5.9 kB)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets<3.0.0,>=2.14.6->flexeval) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets<3.0.0,>=2.14.6->flexeval) (2024.1)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets<3.0.0,>=2.14.6->flexeval) (2024.1)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth!=2.24.0,!=2.25.0,<3.0.0.dev0,>=1.32.0->google-api-python-client<3.0.0,>=2.131.0->flexeval) (0.6.0)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.13.0->peft<0.11.0,>=0.10.0->flexeval) (1.3.0)\n",
            "Downloading flexeval-0.6.3-py3-none-any.whl (247 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m247.6/247.6 kB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading datasets-2.21.0-py3-none-any.whl (527 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m527.3/527.3 kB\u001b[0m \u001b[31m36.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading evaluate-0.4.2-py3-none-any.whl (84 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fuzzywuzzy-0.18.0-py2.py3-none-any.whl (18 kB)\n",
            "Downloading jiwer-3.0.4-py3-none-any.whl (21 kB)\n",
            "Downloading jsonargparse-4.32.1-py3-none-any.whl (207 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.3/207.3 kB\u001b[0m \u001b[31m18.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading loguru-0.7.2-py3-none-any.whl (62 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.5/62.5 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading openai-1.43.0-py3-none-any.whl (365 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m365.7/365.7 kB\u001b[0m \u001b[31m29.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading peft-0.10.0-py3-none-any.whl (199 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.1/199.1 kB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading python_Levenshtein-0.23.0-py3-none-any.whl (9.4 kB)\n",
            "Downloading Levenshtein-0.23.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (169 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m169.4/169.4 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading rouge-1.0.1-py3-none-any.whl (13 kB)\n",
            "Downloading sacrebleu-2.4.3-py3-none-any.whl (103 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m104.0/104.0 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m12.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading fugashi-1.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (600 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m600.9/600.9 kB\u001b[0m \u001b[31m30.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading httpx-0.27.2-py3-none-any.whl (76 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.4/76.4 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading jiter-0.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (318 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m318.9/318.9 kB\u001b[0m \u001b[31m26.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading mecab_python3-1.0.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (581 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m581.7/581.7 kB\u001b[0m \u001b[31m41.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl (39.9 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m39.9/39.9 MB\u001b[0m \u001b[31m20.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading rapidfuzz-3.9.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m103.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading rhoknp-1.3.0-py3-none-any.whl (86 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading SudachiDict_core-20240716-py3-none-any.whl (72.0 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.0/72.0 MB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading SudachiPy-0.6.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.6 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.6/2.6 MB\u001b[0m \u001b[31m79.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
            "Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading portalocker-2.10.1-py3-none-any.whl (18 kB)\n",
            "Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m17.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading plac-1.4.3-py2.py3-none-any.whl (22 kB)\n",
            "Downloading wasabi-0.10.1-py3-none-any.whl (26 kB)\n",
            "Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: ipadic, jsonnet, unidic, unidic-lite\n",
            "  Building wheel for ipadic (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ipadic: filename=ipadic-1.0.0-py3-none-any.whl size=13556704 sha256=72b9d1ba03e21bf7466ac5cd9be846e9001f5056beb3f126c494614fb954328d\n",
            "  Stored in directory: /root/.cache/pip/wheels/5b/ea/e3/2f6e0860a327daba3b030853fce4483ed37468bbf1101c59c3\n",
            "  Building wheel for jsonnet (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for jsonnet: filename=jsonnet-0.20.0-cp310-cp310-linux_x86_64.whl size=6406880 sha256=ac0b5e4a351332d7aaac30cf064cc084d6e7550ee4425684a7af7936c6af6a04\n",
            "  Stored in directory: /root/.cache/pip/wheels/63/0d/6b/5467dd1db9332ba4bd5cf4153e2870c5f89bb4db473d989cc2\n",
            "  Building wheel for unidic (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for unidic: filename=unidic-1.1.0-py3-none-any.whl size=7402 sha256=3c69ff99c3979c2674c262c9fc80f9f9e31d2be99fc782491144305cc1919881\n",
            "  Stored in directory: /root/.cache/pip/wheels/7a/72/72/1f3d654c345ea69d5d51b531c90daf7ba14cc555eaf2c64ab0\n",
            "  Building wheel for unidic-lite (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for unidic-lite: filename=unidic_lite-1.0.8-py3-none-any.whl size=47658818 sha256=b50f1b77ed4fa17899ec45b216f3841351823fdea7f715ddf899040ca345f08e\n",
            "  Stored in directory: /root/.cache/pip/wheels/89/e8/68/f9ac36b8cc6c8b3c96888cd57434abed96595d444f42243853\n",
            "Successfully built ipadic jsonnet unidic unidic-lite\n",
            "Installing collected packages: wasabi, unidic-lite, sudachipy, plac, mecab-python3, jsonnet, ipadic, fuzzywuzzy, xxhash, sudachidict-core, rouge, rhoknp, rapidfuzz, pyarrow, portalocker, loguru, jsonargparse, jiter, h11, fugashi, dill, colorama, unidic, sacrebleu, multiprocess, Levenshtein, jiwer, httpcore, python-levenshtein, httpx, openai, datasets, peft, evaluate, flexeval\n",
            "  Attempting uninstall: wasabi\n",
            "    Found existing installation: wasabi 1.1.3\n",
            "    Uninstalling wasabi-1.1.3:\n",
            "      Successfully uninstalled wasabi-1.1.3\n",
            "  Attempting uninstall: pyarrow\n",
            "    Found existing installation: pyarrow 14.0.2\n",
            "    Uninstalling pyarrow-14.0.2:\n",
            "      Successfully uninstalled pyarrow-14.0.2\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "cudf-cu12 24.4.1 requires pyarrow<15.0.0a0,>=14.0.1, but you have pyarrow 17.0.0 which is incompatible.\n",
            "ibis-framework 8.0.0 requires pyarrow<16,>=2, but you have pyarrow 17.0.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed Levenshtein-0.23.0 colorama-0.4.6 datasets-2.21.0 dill-0.3.8 evaluate-0.4.2 flexeval-0.6.3 fugashi-1.3.2 fuzzywuzzy-0.18.0 h11-0.14.0 httpcore-1.0.5 httpx-0.27.2 ipadic-1.0.0 jiter-0.5.0 jiwer-3.0.4 jsonargparse-4.32.1 jsonnet-0.20.0 loguru-0.7.2 mecab-python3-1.0.9 multiprocess-0.70.16 openai-1.43.0 peft-0.10.0 plac-1.4.3 portalocker-2.10.1 pyarrow-17.0.0 python-levenshtein-0.23.0 rapidfuzz-3.9.6 rhoknp-1.3.0 rouge-1.0.1 sacrebleu-2.4.3 sudachidict-core-20240716 sudachipy-0.6.8 unidic-1.1.0 unidic-lite-1.0.8 wasabi-0.10.1 xxhash-3.5.0\n"
          ]
        }
      ],
      "source": [
        "!pip install flexeval"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f-ZqvW74BQcs",
        "outputId": "cb0f03e9-2462-491d-f601-b40a0b242dbb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "\n",
        "# Googleドライブを\"drive\"ディレクトリ以下にマウントする\n",
        "drive.mount(\"drive\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Glc5-00vBWBR",
        "outputId": "d239c54f-7c9d-4cda-986a-180f5b14707b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2024-08-12 14:40:18.498039: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-08-12 14:40:18.798777: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-08-12 14:40:18.883767: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-08-12 14:40:19.334942: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-08-12 14:40:21.634020: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "/*\n",
            "JCommonsenseQA is a Japanese version of CommonsenseQA, which is a multiple-choice question answering dataset that requires commonsense reasoning ability.\n",
            "The dataset is built using crowdsourcing with seeds extracted from the knowledge base ConceptNet.\n",
            "This is a setup for generating answers based on the choices provided.\n",
            "\n",
            "References:\n",
            "\n",
            "* [Hugging Face Dataset](https://huggingface.co/datasets/llm-book/JGLUE)\n",
            "* [Original Repository](https://github.com/yahoojapan/JGLUE)\n",
            "* [JGLUE: Japanese General Language Understanding Evaluation](https://aclanthology.org/2022.lrec-1.317)\n",
            "* [JGLUE: 日本語言語理解ベンチマーク](https://www.anlp.jp/proceedings/annual_meeting/2022/pdf_dir/E8-4.pdf)\n",
            "*/\n",
            "local dataset_base_args = {\n",
            "  class_path: 'HFGenerationDataset',\n",
            "  init_args: {\n",
            "    path: 'llm-book/JGLUE',\n",
            "    subset: 'JCommonsenseQA',\n",
            "    reference_template: '{% set choices = [choice0, choice1, choice2, choice3, choice4] %}{{ choices[label] }}',\n",
            "    dataset_kwargs: { trust_remote_code: true },\n",
            "  },\n",
            "};\n",
            "\n",
            "{\n",
            "  class_path: 'Generation',\n",
            "  init_args: {\n",
            "    eval_dataset: dataset_base_args { init_args+: { split: 'validation' } },\n",
            "    few_shot_generator: {\n",
            "      class_path: 'RandomFewShotGenerator',\n",
            "      init_args: {\n",
            "        dataset: dataset_base_args { init_args+: { split: 'train' } },\n",
            "        num_shots: 2,\n",
            "      },\n",
            "    },\n",
            "    prompt_template: {\n",
            "      class_path: 'Jinja2PromptTemplate',\n",
            "      init_args: {\n",
            "        template: |||\n",
            "          正しい答えは何でしょう？\n",
            "          {% for item in few_shot_data %}\n",
            "          0.「{{ item.choice0 }}」\n",
            "          1.「{{ item.choice1 }}」\n",
            "          2.「{{ item.choice2 }}」\n",
            "          3.「{{ item.choice3 }}」\n",
            "          4.「{{ item.choice4 }}」\n",
            "          問題：{{ item.question }}\n",
            "          回答：「{{ item.references[0] }}」\n",
            "          {% endfor %}\n",
            "          0.「{{ choice0 }}」\n",
            "          1.「{{ choice1 }}」\n",
            "          2.「{{ choice2 }}」\n",
            "          3.「{{ choice3 }}」\n",
            "          4.「{{ choice4 }}」\n",
            "          問題：{{question}}\n",
            "        ||| + '回答：「',\n",
            "      },\n",
            "    },\n",
            "    metrics: [\n",
            "      { class_path: 'ExactMatch' },\n",
            "    ],\n",
            "    gen_kwargs: { max_new_tokens: 40, stop_sequences: ['」'] },\n",
            "  },\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "!flexeval_presets jcommonsenseqa"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dbXigXbqBXa3",
        "outputId": "e630e2f2-2ab5-49c7-dcea-ebba8412fcd6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2024-08-12 14:40:50.224967: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-08-12 14:40:50.244444: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-08-12 14:40:50.250473: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-08-12 14:40:50.264695: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-08-12 14:40:51.531449: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[32m2024-08-12 14:40:53.586\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.scripts.flexeval_lm\u001b[0m:\u001b[36mmain\u001b[0m:\u001b[36m143\u001b[0m - \u001b[1mNamespace(eval_setup=Namespace(class_path='flexeval.Generation', init_args=Namespace(eval_dataset=Namespace(class_path='flexeval.HFGenerationDataset', init_args=Namespace(path='llm-book/JGLUE', split='validation', reference_template='{% set choices = [choice0, choice1, choice2, choice3, choice4] %}{{ choices[label] }}', reference_list_template=None, input_templates=None, subset='JCommonsenseQA', template_filters=None, dataset_kwargs={'trust_remote_code': True})), prompt_template=Namespace(class_path='flexeval.Jinja2PromptTemplate', init_args=Namespace(template='正しい答えは何でしょう？\\n{% for item in few_shot_data %}\\n0.「{{ item.choice0 }}」\\n1.「{{ item.choice1 }}」\\n2.「{{ item.choice2 }}」\\n3.「{{ item.choice3 }}」\\n4.「{{ item.choice4 }}」\\n問題：{{ item.question }}\\n回答：「{{ item.references[0] }}」\\n{% endfor %}\\n0.「{{ choice0 }}」\\n1.「{{ choice1 }}」\\n2.「{{ choice2 }}」\\n3.「{{ choice3 }}」\\n4.「{{ choice4 }}」\\n問題：{{question}}\\n回答：「')), gen_kwargs={'max_new_tokens': 40, 'stop_sequences': ['」']}, few_shot_generator=Namespace(class_path='flexeval.RandomFewShotGenerator', init_args=Namespace(dataset=Namespace(class_path='flexeval.HFGenerationDataset', init_args=Namespace(path='llm-book/JGLUE', split='train', reference_template='{% set choices = [choice0, choice1, choice2, choice3, choice4] %}{{ choices[label] }}', reference_list_template=None, input_templates=None, subset='JCommonsenseQA', template_filters=None, dataset_kwargs={'trust_remote_code': True})), num_shots=2, seed=42, num_trials_to_avoid_leak=3)), metrics=[Namespace(class_path='flexeval.ExactMatch', init_args=Namespace(processor=None, reference_processor=None))], batch_size=4, max_instances=None)), eval_setups=None, save_dir='drive/MyDrive/llm_book/eval/jcommonsenseqa_flexeval', force=False, result_recorder=None, config=None, metadata={}, language_model=Namespace(class_path='flexeval.HuggingFaceLM', init_args=Namespace(model='tokyotech-llm/Swallow-7b-hf', model_kwargs=None, tokenizer=None, tokenizer_kwargs=None, add_special_tokens=False, amp_dtype=None, random_seed=42, load_peft=False, custom_chat_template=None)))\u001b[0m\n",
            "\u001b[32m2024-08-12 14:40:53.587\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.scripts.flexeval_lm\u001b[0m:\u001b[36mmain\u001b[0m:\u001b[36m144\u001b[0m - \u001b[1mflexeval version: 0.6.1\u001b[0m\n",
            "tokenizer_config.json: 100% 773/773 [00:00<00:00, 5.33MB/s]\n",
            "tokenizer.model: 100% 914k/914k [00:00<00:00, 9.17MB/s]\n",
            "special_tokens_map.json: 100% 457/457 [00:00<00:00, 3.13MB/s]\n",
            "config.json: 100% 685/685 [00:00<00:00, 4.80MB/s]\n",
            "pytorch_model.bin.index.json: 100% 26.8k/26.8k [00:00<00:00, 79.8MB/s]\n",
            "Downloading shards:   0% 0/2 [00:00<?, ?it/s]\n",
            "pytorch_model-00001-of-00002.bin:   0% 0.00/9.98G [00:00<?, ?B/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   0% 10.5M/9.98G [00:00<08:57, 18.6MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   0% 21.0M/9.98G [00:00<05:57, 27.8MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   0% 31.5M/9.98G [00:00<04:27, 37.2MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   0% 41.9M/9.98G [00:01<03:24, 48.5MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   1% 62.9M/9.98G [00:01<02:23, 69.3MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   1% 83.9M/9.98G [00:01<01:47, 91.7MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   1% 105M/9.98G [00:01<01:26, 114MB/s]  \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   1% 126M/9.98G [00:01<01:13, 134MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   1% 147M/9.98G [00:01<01:06, 147MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   2% 168M/9.98G [00:01<01:10, 139MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   2% 189M/9.98G [00:02<01:03, 154MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   2% 210M/9.98G [00:02<01:00, 163MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   2% 231M/9.98G [00:02<01:00, 161MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   3% 252M/9.98G [00:02<00:58, 166MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   3% 273M/9.98G [00:02<00:58, 165MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   3% 294M/9.98G [00:02<00:58, 164MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   3% 315M/9.98G [00:02<00:56, 172MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   3% 336M/9.98G [00:02<00:53, 179MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   4% 367M/9.98G [00:03<00:50, 191MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   4% 398M/9.98G [00:03<00:47, 200MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   4% 419M/9.98G [00:03<00:48, 196MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   4% 440M/9.98G [00:03<00:48, 195MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   5% 461M/9.98G [00:03<00:48, 198MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   5% 482M/9.98G [00:03<00:47, 199MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   5% 503M/9.98G [00:03<00:47, 200MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   5% 524M/9.98G [00:03<00:47, 198MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   5% 545M/9.98G [00:07<08:52, 17.7MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   6% 577M/9.98G [00:07<05:42, 27.5MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   6% 608M/9.98G [00:07<03:54, 40.0MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   6% 640M/9.98G [00:07<02:48, 55.4MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   7% 661M/9.98G [00:08<02:20, 66.4MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   7% 682M/9.98G [00:08<01:57, 79.1MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   7% 703M/9.98G [00:08<01:38, 94.4MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   7% 724M/9.98G [00:08<01:24, 110MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   7% 744M/9.98G [00:08<01:13, 126MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   8% 765M/9.98G [00:08<01:05, 140MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   8% 786M/9.98G [00:08<00:59, 155MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   8% 818M/9.98G [00:08<00:52, 173MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   9% 849M/9.98G [00:09<00:47, 192MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   9% 881M/9.98G [00:09<00:46, 197MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   9% 912M/9.98G [00:09<00:46, 196MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:   9% 933M/9.98G [00:09<00:46, 194MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  10% 954M/9.98G [00:09<00:46, 193MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  10% 975M/9.98G [00:09<00:46, 195MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  10% 996M/9.98G [00:09<00:46, 193MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  10% 1.02G/9.98G [00:09<00:46, 193MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  10% 1.04G/9.98G [00:11<02:50, 52.5MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  11% 1.06G/9.98G [00:11<02:14, 66.3MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  11% 1.08G/9.98G [00:11<01:56, 76.1MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  11% 1.10G/9.98G [00:11<01:45, 84.0MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  11% 1.12G/9.98G [00:11<01:42, 86.3MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  11% 1.14G/9.98G [00:11<01:36, 92.0MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  12% 1.16G/9.98G [00:12<01:43, 84.8MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  12% 1.18G/9.98G [00:12<01:25, 103MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  12% 1.21G/9.98G [00:12<01:36, 91.3MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  12% 1.23G/9.98G [00:12<01:25, 102MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  13% 1.25G/9.98G [00:12<01:22, 105MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  13% 1.27G/9.98G [00:13<01:20, 109MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  13% 1.29G/9.98G [00:13<01:12, 119MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  13% 1.31G/9.98G [00:13<01:07, 128MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  13% 1.33G/9.98G [00:13<01:03, 136MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  14% 1.35G/9.98G [00:13<00:56, 152MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  14% 1.37G/9.98G [00:13<00:52, 164MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  14% 1.41G/9.98G [00:13<00:48, 178MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  14% 1.43G/9.98G [00:13<00:48, 176MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  15% 1.45G/9.98G [00:14<00:49, 171MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  15% 1.47G/9.98G [00:14<00:51, 165MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  15% 1.49G/9.98G [00:14<00:51, 166MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  15% 1.51G/9.98G [00:14<00:51, 164MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  15% 1.53G/9.98G [00:14<00:51, 163MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  16% 1.55G/9.98G [00:15<02:06, 66.4MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  16% 1.57G/9.98G [00:17<06:22, 22.0MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  16% 1.60G/9.98G [00:17<04:06, 33.9MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  16% 1.64G/9.98G [00:18<02:51, 48.7MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  17% 1.67G/9.98G [00:18<02:06, 65.9MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  17% 1.69G/9.98G [00:18<01:46, 78.2MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  17% 1.71G/9.98G [00:18<01:31, 90.4MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  17% 1.73G/9.98G [00:18<01:18, 104MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  18% 1.75G/9.98G [00:18<01:12, 114MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  18% 1.77G/9.98G [00:18<01:05, 125MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  18% 1.80G/9.98G [00:19<00:54, 149MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  18% 1.84G/9.98G [00:19<00:47, 170MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  19% 1.87G/9.98G [00:19<00:44, 183MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  19% 1.89G/9.98G [00:19<00:43, 185MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  19% 1.91G/9.98G [00:19<00:42, 190MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  19% 1.93G/9.98G [00:19<00:44, 181MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  20% 1.95G/9.98G [00:19<00:42, 187MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  20% 1.97G/9.98G [00:19<00:42, 190MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  20% 1.99G/9.98G [00:19<00:41, 190MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  20% 2.01G/9.98G [00:20<00:41, 192MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  20% 2.03G/9.98G [00:21<03:07, 42.3MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  21% 2.06G/9.98G [00:21<02:23, 55.2MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  21% 2.08G/9.98G [00:21<01:53, 69.5MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  21% 2.10G/9.98G [00:21<01:31, 85.9MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  21% 2.12G/9.98G [00:21<01:16, 102MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  22% 2.15G/9.98G [00:22<01:00, 129MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  22% 2.17G/9.98G [00:22<01:14, 105MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  22% 2.19G/9.98G [00:22<01:34, 82.6MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  22% 2.22G/9.98G [00:22<01:12, 107MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  22% 2.24G/9.98G [00:23<01:03, 121MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  23% 2.26G/9.98G [00:23<00:57, 134MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  23% 2.29G/9.98G [00:23<00:52, 146MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  23% 2.31G/9.98G [00:23<00:49, 156MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  23% 2.33G/9.98G [00:23<00:58, 132MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  24% 2.35G/9.98G [00:23<01:03, 119MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  24% 2.37G/9.98G [00:24<01:02, 121MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  24% 2.39G/9.98G [00:24<01:03, 119MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  24% 2.41G/9.98G [00:24<01:24, 89.8MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  24% 2.43G/9.98G [00:24<01:17, 96.9MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  25% 2.45G/9.98G [00:25<01:23, 90.1MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  25% 2.46G/9.98G [00:25<01:30, 83.0MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  25% 2.47G/9.98G [00:25<01:40, 74.4MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  25% 2.50G/9.98G [00:25<01:41, 73.4MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  25% 2.52G/9.98G [00:25<01:28, 84.0MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  25% 2.53G/9.98G [00:25<01:29, 83.1MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  25% 2.54G/9.98G [00:26<01:32, 80.5MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  26% 2.56G/9.98G [00:26<01:17, 95.8MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  26% 2.58G/9.98G [00:26<01:18, 93.9MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  26% 2.60G/9.98G [00:26<01:07, 109MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  26% 2.62G/9.98G [00:26<01:00, 121MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  26% 2.64G/9.98G [00:26<00:56, 131MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  27% 2.66G/9.98G [00:27<00:52, 141MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  27% 2.68G/9.98G [00:27<00:48, 151MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  27% 2.71G/9.98G [00:27<00:48, 150MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  27% 2.73G/9.98G [00:27<00:44, 162MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  28% 2.75G/9.98G [00:27<00:44, 163MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  28% 2.77G/9.98G [00:27<00:52, 137MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  28% 2.79G/9.98G [00:27<00:49, 144MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  28% 2.81G/9.98G [00:27<00:45, 157MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  28% 2.83G/9.98G [00:28<00:47, 149MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  29% 2.85G/9.98G [00:28<00:44, 160MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  29% 2.87G/9.98G [00:28<00:42, 167MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  29% 2.89G/9.98G [00:28<00:43, 164MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  29% 2.92G/9.98G [00:28<00:44, 160MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  29% 2.94G/9.98G [00:28<00:43, 162MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  30% 2.96G/9.98G [00:28<00:43, 161MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  30% 2.99G/9.98G [00:29<00:39, 176MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  30% 3.01G/9.98G [00:29<00:38, 182MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  30% 3.03G/9.98G [00:29<00:37, 183MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  31% 3.06G/9.98G [00:29<00:35, 196MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  31% 3.08G/9.98G [00:29<00:53, 129MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  31% 3.10G/9.98G [00:29<00:49, 139MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  31% 3.12G/9.98G [00:29<00:47, 146MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  32% 3.15G/9.98G [00:30<00:44, 154MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  32% 3.17G/9.98G [00:30<00:41, 166MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  32% 3.19G/9.98G [00:30<00:39, 173MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  32% 3.21G/9.98G [00:30<00:38, 178MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  32% 3.23G/9.98G [00:30<00:36, 183MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  33% 3.25G/9.98G [00:30<00:37, 182MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  33% 3.27G/9.98G [00:30<00:35, 187MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  33% 3.29G/9.98G [00:30<00:36, 185MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  33% 3.31G/9.98G [00:30<00:35, 186MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  33% 3.33G/9.98G [00:31<00:35, 188MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  34% 3.36G/9.98G [00:31<00:34, 192MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  34% 3.39G/9.98G [00:31<00:33, 196MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  34% 3.41G/9.98G [00:31<00:33, 197MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  34% 3.44G/9.98G [00:31<00:34, 187MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  35% 3.46G/9.98G [00:31<00:33, 192MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  35% 3.49G/9.98G [00:31<00:32, 202MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  35% 3.51G/9.98G [00:31<00:32, 199MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  35% 3.53G/9.98G [00:32<00:32, 198MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  36% 3.57G/9.98G [00:32<00:31, 204MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  36% 3.60G/9.98G [00:32<00:30, 209MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  36% 3.63G/9.98G [00:32<00:29, 212MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  37% 3.66G/9.98G [00:32<00:29, 215MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  37% 3.69G/9.98G [00:32<00:31, 200MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  37% 3.71G/9.98G [00:32<00:32, 194MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  37% 3.73G/9.98G [00:33<00:32, 195MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  38% 3.76G/9.98G [00:33<00:31, 199MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  38% 3.79G/9.98G [00:33<00:30, 200MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  38% 3.82G/9.98G [00:33<00:29, 208MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  39% 3.85G/9.98G [00:33<00:28, 212MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  39% 3.88G/9.98G [00:33<00:28, 212MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  39% 3.91G/9.98G [00:33<00:28, 213MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  40% 3.94G/9.98G [00:34<00:27, 217MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  40% 3.97G/9.98G [00:34<00:28, 209MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  40% 4.01G/9.98G [00:34<00:28, 210MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  40% 4.04G/9.98G [00:34<00:27, 213MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  41% 4.07G/9.98G [00:34<00:28, 207MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  41% 4.09G/9.98G [00:35<01:47, 54.9MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  41% 4.11G/9.98G [00:36<01:31, 64.4MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  41% 4.13G/9.98G [00:36<01:14, 78.3MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  42% 4.16G/9.98G [00:36<00:57, 101MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  42% 4.18G/9.98G [00:36<00:54, 106MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  42% 4.20G/9.98G [00:37<02:24, 39.9MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  42% 4.23G/9.98G [00:38<02:00, 47.5MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  43% 4.25G/9.98G [00:38<01:38, 58.2MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  43% 4.27G/9.98G [00:38<01:38, 58.2MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  43% 4.28G/9.98G [00:38<01:42, 55.8MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  43% 4.29G/9.98G [00:39<01:41, 56.3MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  43% 4.30G/9.98G [00:39<01:35, 59.7MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  43% 4.31G/9.98G [00:39<01:37, 58.4MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  43% 4.32G/9.98G [00:39<01:28, 63.7MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  43% 4.33G/9.98G [00:39<01:33, 60.2MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  44% 4.34G/9.98G [00:39<01:35, 58.9MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  44% 4.35G/9.98G [00:40<01:25, 66.0MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  44% 4.36G/9.98G [00:40<01:30, 62.3MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  44% 4.37G/9.98G [00:40<01:35, 58.6MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  44% 4.39G/9.98G [00:40<01:04, 86.2MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  44% 4.42G/9.98G [00:40<00:47, 116MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  45% 4.45G/9.98G [00:40<00:43, 126MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  45% 4.47G/9.98G [00:41<00:41, 134MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  45% 4.49G/9.98G [00:41<00:37, 147MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  45% 4.51G/9.98G [00:41<00:35, 156MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  45% 4.53G/9.98G [00:41<00:50, 109MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  46% 4.55G/9.98G [00:41<00:42, 127MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  46% 4.57G/9.98G [00:41<00:38, 140MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  46% 4.59G/9.98G [00:41<00:35, 153MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  46% 4.61G/9.98G [00:41<00:32, 167MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  46% 4.63G/9.98G [00:42<00:33, 158MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  47% 4.66G/9.98G [00:42<00:33, 160MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  47% 4.68G/9.98G [00:42<00:32, 162MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  47% 4.70G/9.98G [00:42<00:32, 164MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  47% 4.72G/9.98G [00:42<00:31, 165MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  48% 4.74G/9.98G [00:44<02:27, 35.6MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  48% 4.76G/9.98G [00:44<01:50, 47.4MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  48% 4.79G/9.98G [00:44<01:16, 67.7MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  48% 4.81G/9.98G [00:44<01:02, 82.2MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  48% 4.83G/9.98G [00:44<00:52, 98.9MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  49% 4.85G/9.98G [00:44<00:44, 114MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  49% 4.88G/9.98G [00:44<00:39, 130MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  49% 4.90G/9.98G [00:45<00:34, 146MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  49% 4.92G/9.98G [00:45<00:32, 158MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  49% 4.94G/9.98G [00:45<00:30, 166MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  50% 4.97G/9.98G [00:45<00:27, 179MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  50% 4.99G/9.98G [00:45<00:27, 185MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  50% 5.02G/9.98G [00:45<00:26, 190MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  51% 5.05G/9.98G [00:45<00:25, 197MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  51% 5.08G/9.98G [00:45<00:25, 195MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  51% 5.10G/9.98G [00:46<00:24, 198MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  51% 5.13G/9.98G [00:46<00:24, 201MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  52% 5.15G/9.98G [00:46<00:24, 199MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  52% 5.17G/9.98G [00:46<00:24, 199MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  52% 5.20G/9.98G [00:46<00:22, 215MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  52% 5.23G/9.98G [00:46<00:21, 217MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  53% 5.26G/9.98G [00:46<00:21, 215MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  53% 5.30G/9.98G [00:47<00:21, 214MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  53% 5.33G/9.98G [00:47<00:23, 202MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  54% 5.35G/9.98G [00:47<00:23, 201MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  54% 5.37G/9.98G [00:47<00:22, 202MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  54% 5.39G/9.98G [00:47<00:22, 202MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  54% 5.42G/9.98G [00:47<00:22, 204MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  55% 5.45G/9.98G [00:47<00:21, 207MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  55% 5.48G/9.98G [00:47<00:21, 210MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  55% 5.52G/9.98G [00:48<00:21, 211MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  56% 5.55G/9.98G [00:48<00:20, 214MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  56% 5.58G/9.98G [00:48<00:21, 204MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  56% 5.60G/9.98G [00:48<00:21, 200MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  56% 5.62G/9.98G [00:48<00:21, 201MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  57% 5.64G/9.98G [00:48<00:21, 202MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  57% 5.66G/9.98G [00:49<00:50, 85.8MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  57% 5.68G/9.98G [00:49<00:42, 102MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  57% 5.70G/9.98G [00:49<00:36, 117MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  57% 5.73G/9.98G [00:49<00:32, 130MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  58% 5.75G/9.98G [00:49<00:29, 145MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  58% 5.77G/9.98G [00:49<00:26, 157MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  58% 5.79G/9.98G [00:50<00:24, 168MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  58% 5.82G/9.98G [00:50<00:22, 185MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  59% 5.84G/9.98G [00:50<00:21, 189MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  59% 5.86G/9.98G [00:50<00:21, 191MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  59% 5.88G/9.98G [00:50<00:21, 192MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  59% 5.91G/9.98G [00:50<00:21, 193MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  59% 5.93G/9.98G [00:50<00:27, 147MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  60% 5.96G/9.98G [00:51<00:31, 128MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  60% 5.98G/9.98G [00:51<00:35, 112MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  60% 6.00G/9.98G [00:51<00:39, 100MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  60% 6.02G/9.98G [00:52<00:59, 66.7MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  61% 6.04G/9.98G [00:52<00:47, 82.3MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  61% 6.06G/9.98G [00:52<00:39, 99.0MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  61% 6.08G/9.98G [00:52<00:35, 111MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  61% 6.11G/9.98G [00:52<00:28, 138MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  61% 6.13G/9.98G [00:52<00:28, 136MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  62% 6.16G/9.98G [00:53<00:30, 125MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  62% 6.18G/9.98G [00:53<00:27, 138MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  62% 6.20G/9.98G [00:53<00:24, 152MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  62% 6.22G/9.98G [00:53<00:23, 160MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  63% 6.25G/9.98G [00:53<00:20, 179MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  63% 6.28G/9.98G [00:53<00:19, 191MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  63% 6.30G/9.98G [00:53<00:21, 170MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  63% 6.32G/9.98G [00:53<00:21, 169MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  64% 6.34G/9.98G [00:54<00:21, 167MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  64% 6.36G/9.98G [00:54<00:25, 144MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  64% 6.39G/9.98G [00:54<00:24, 148MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  64% 6.41G/9.98G [00:54<00:23, 150MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  64% 6.43G/9.98G [00:54<00:22, 159MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  65% 6.45G/9.98G [00:54<00:21, 165MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  65% 6.47G/9.98G [00:54<00:23, 146MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  65% 6.49G/9.98G [00:55<00:30, 113MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  65% 6.51G/9.98G [00:55<00:38, 90.3MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  65% 6.53G/9.98G [00:55<00:44, 76.6MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  66% 6.54G/9.98G [00:56<00:43, 78.2MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  66% 6.56G/9.98G [00:56<00:37, 91.2MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  66% 6.59G/9.98G [00:56<00:33, 101MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  66% 6.61G/9.98G [00:56<00:30, 110MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  66% 6.63G/9.98G [00:56<00:27, 123MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  67% 6.65G/9.98G [00:56<00:24, 139MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  67% 6.67G/9.98G [00:56<00:22, 150MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  67% 6.69G/9.98G [00:56<00:20, 157MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  67% 6.71G/9.98G [00:57<00:19, 168MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  68% 6.74G/9.98G [00:57<00:17, 187MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  68% 6.77G/9.98G [00:57<00:16, 196MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  68% 6.81G/9.98G [00:57<00:15, 203MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  68% 6.83G/9.98G [00:57<00:16, 193MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  69% 6.85G/9.98G [00:57<00:16, 194MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  69% 6.87G/9.98G [00:57<00:15, 197MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  69% 6.89G/9.98G [00:57<00:15, 201MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  69% 6.92G/9.98G [00:58<00:14, 204MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  70% 6.94G/9.98G [00:58<00:15, 201MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  70% 6.96G/9.98G [00:58<00:14, 203MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  70% 6.98G/9.98G [00:58<00:15, 199MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  70% 7.00G/9.98G [00:58<00:14, 200MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  70% 7.03G/9.98G [00:58<00:14, 202MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  71% 7.05G/9.98G [00:58<00:14, 200MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  71% 7.07G/9.98G [00:58<00:14, 203MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  71% 7.10G/9.98G [00:58<00:13, 210MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  71% 7.13G/9.98G [00:59<00:13, 213MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  72% 7.16G/9.98G [00:59<00:13, 216MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  72% 7.19G/9.98G [00:59<00:12, 216MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  72% 7.22G/9.98G [00:59<00:12, 221MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  73% 7.26G/9.98G [00:59<00:13, 204MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  73% 7.28G/9.98G [00:59<00:13, 200MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  73% 7.31G/9.98G [00:59<00:13, 203MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  73% 7.33G/9.98G [01:00<00:13, 203MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  74% 7.35G/9.98G [01:00<00:13, 199MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  74% 7.37G/9.98G [01:00<00:13, 198MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  74% 7.40G/9.98G [01:00<00:12, 206MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  75% 7.43G/9.98G [01:00<00:12, 210MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  75% 7.46G/9.98G [01:00<00:12, 209MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  75% 7.48G/9.98G [01:00<00:12, 207MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  75% 7.51G/9.98G [01:00<00:11, 211MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  76% 7.54G/9.98G [01:01<00:11, 207MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  76% 7.56G/9.98G [01:01<00:11, 206MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  76% 7.58G/9.98G [01:01<00:11, 206MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  76% 7.60G/9.98G [01:01<00:11, 207MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  76% 7.62G/9.98G [01:01<00:11, 205MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  77% 7.65G/9.98G [01:04<01:42, 22.6MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  77% 7.69G/9.98G [01:05<01:09, 33.0MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  77% 7.72G/9.98G [01:05<00:49, 45.9MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  78% 7.75G/9.98G [01:05<00:36, 61.7MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  78% 7.77G/9.98G [01:05<00:30, 73.0MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  78% 7.79G/9.98G [01:05<00:25, 86.5MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  78% 7.82G/9.98G [01:05<00:19, 108MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  79% 7.84G/9.98G [01:05<00:17, 121MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  79% 7.86G/9.98G [01:06<00:15, 134MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  79% 7.89G/9.98G [01:06<00:14, 147MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  79% 7.92G/9.98G [01:06<00:12, 167MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  80% 7.94G/9.98G [01:06<00:13, 157MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  80% 7.96G/9.98G [01:06<00:12, 163MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  80% 7.98G/9.98G [01:06<00:11, 167MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  80% 8.00G/9.98G [01:06<00:11, 174MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  80% 8.02G/9.98G [01:06<00:10, 179MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  81% 8.04G/9.98G [01:06<00:10, 186MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  81% 8.06G/9.98G [01:11<01:59, 16.1MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  81% 8.10G/9.98G [01:11<01:14, 25.1MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  81% 8.13G/9.98G [01:11<00:50, 36.7MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  82% 8.16G/9.98G [01:11<00:35, 50.9MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  82% 8.18G/9.98G [01:11<00:28, 62.4MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  82% 8.20G/9.98G [01:11<00:23, 74.5MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  82% 8.22G/9.98G [01:11<00:19, 89.2MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  83% 8.24G/9.98G [01:11<00:16, 105MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  83% 8.26G/9.98G [01:12<00:14, 121MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  83% 8.28G/9.98G [01:12<00:12, 137MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  83% 8.32G/9.98G [01:12<00:10, 159MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  84% 8.35G/9.98G [01:12<00:09, 176MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  84% 8.38G/9.98G [01:12<00:08, 190MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  84% 8.41G/9.98G [01:12<00:07, 198MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  85% 8.44G/9.98G [01:12<00:07, 205MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  85% 8.47G/9.98G [01:13<00:07, 208MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  85% 8.50G/9.98G [01:13<00:07, 208MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  86% 8.54G/9.98G [01:13<00:06, 209MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  86% 8.57G/9.98G [01:13<00:07, 198MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  86% 8.59G/9.98G [01:13<00:07, 196MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  86% 8.61G/9.98G [01:13<00:06, 198MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  86% 8.63G/9.98G [01:13<00:06, 200MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  87% 8.65G/9.98G [01:14<00:17, 77.7MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  87% 8.67G/9.98G [01:14<00:13, 93.5MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  87% 8.69G/9.98G [01:14<00:11, 110MB/s] \u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  87% 8.71G/9.98G [01:14<00:10, 125MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  88% 8.73G/9.98G [01:15<00:09, 137MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  88% 8.76G/9.98G [01:15<00:08, 151MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  88% 8.79G/9.98G [01:15<00:07, 169MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  88% 8.81G/9.98G [01:15<00:06, 174MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  88% 8.83G/9.98G [01:15<00:06, 181MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  89% 8.86G/9.98G [01:15<00:05, 195MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  89% 8.88G/9.98G [01:15<00:05, 193MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  89% 8.90G/9.98G [01:15<00:05, 193MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  89% 8.92G/9.98G [01:15<00:05, 189MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  90% 8.95G/9.98G [01:16<00:05, 195MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  90% 8.98G/9.98G [01:16<00:05, 196MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  90% 9.00G/9.98G [01:16<00:05, 192MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  90% 9.02G/9.98G [01:16<00:05, 192MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  91% 9.04G/9.98G [01:16<00:04, 194MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  91% 9.07G/9.98G [01:16<00:04, 199MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  91% 9.09G/9.98G [01:16<00:04, 198MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  91% 9.11G/9.98G [01:16<00:04, 197MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  92% 9.13G/9.98G [01:17<00:04, 196MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  92% 9.16G/9.98G [01:17<00:03, 207MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  92% 9.20G/9.98G [01:17<00:03, 214MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  92% 9.23G/9.98G [01:17<00:03, 219MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  93% 9.26G/9.98G [01:17<00:03, 208MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  93% 9.28G/9.98G [01:17<00:03, 203MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  93% 9.30G/9.98G [01:17<00:03, 198MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  93% 9.32G/9.98G [01:17<00:03, 199MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  94% 9.34G/9.98G [01:18<00:03, 199MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  94% 9.36G/9.98G [01:18<00:03, 199MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  94% 9.38G/9.98G [01:18<00:03, 198MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  94% 9.41G/9.98G [01:18<00:02, 196MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  95% 9.44G/9.98G [01:18<00:02, 205MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  95% 9.47G/9.98G [01:18<00:02, 210MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  95% 9.50G/9.98G [01:18<00:02, 214MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  96% 9.53G/9.98G [01:18<00:02, 215MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  96% 9.56G/9.98G [01:19<00:01, 216MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  96% 9.59G/9.98G [01:19<00:01, 202MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  96% 9.62G/9.98G [01:19<00:01, 201MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  97% 9.64G/9.98G [01:19<00:01, 196MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  97% 9.66G/9.98G [01:19<00:01, 184MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  97% 9.68G/9.98G [01:19<00:01, 185MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  97% 9.70G/9.98G [01:19<00:01, 181MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  97% 9.72G/9.98G [01:19<00:01, 176MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  98% 9.74G/9.98G [01:20<00:01, 172MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  98% 9.76G/9.98G [01:20<00:01, 169MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  98% 9.78G/9.98G [01:20<00:01, 168MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  98% 9.80G/9.98G [01:20<00:01, 171MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  98% 9.83G/9.98G [01:20<00:00, 171MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  99% 9.85G/9.98G [01:20<00:00, 171MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  99% 9.87G/9.98G [01:20<00:00, 162MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  99% 9.89G/9.98G [01:20<00:00, 167MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin:  99% 9.91G/9.98G [01:21<00:00, 165MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin: 100% 9.93G/9.98G [01:21<00:00, 166MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin: 100% 9.95G/9.98G [01:21<00:00, 143MB/s]\u001b[A\n",
            "pytorch_model-00001-of-00002.bin: 100% 9.98G/9.98G [01:21<00:00, 122MB/s]\n",
            "Downloading shards:  50% 1/2 [01:21<01:21, 81.80s/it]\n",
            "pytorch_model-00002-of-00002.bin:   0% 0.00/3.68G [00:00<?, ?B/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   0% 10.5M/3.68G [00:00<01:28, 41.7MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   1% 21.0M/3.68G [00:00<01:01, 59.7MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   1% 41.9M/3.68G [00:00<00:42, 85.5MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   2% 62.9M/3.68G [00:00<00:33, 109MB/s] \u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   2% 83.9M/3.68G [00:00<00:28, 125MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   3% 105M/3.68G [00:00<00:26, 137MB/s] \u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   3% 126M/3.68G [00:01<00:23, 150MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   4% 147M/3.68G [00:01<00:22, 154MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   5% 168M/3.68G [00:01<00:33, 105MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   5% 199M/3.68G [00:01<00:25, 134MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   6% 231M/3.68G [00:01<00:22, 157MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   7% 262M/3.68G [00:01<00:19, 175MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   8% 283M/3.68G [00:02<00:18, 179MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   8% 304M/3.68G [00:02<00:18, 179MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   9% 325M/3.68G [00:02<00:18, 185MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:   9% 346M/3.68G [00:02<00:17, 189MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  10% 367M/3.68G [00:02<00:17, 194MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  11% 388M/3.68G [00:02<00:17, 192MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  11% 409M/3.68G [00:03<00:40, 80.7MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  12% 430M/3.68G [00:03<00:33, 96.7MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  12% 451M/3.68G [00:03<00:28, 114MB/s] \u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  13% 472M/3.68G [00:03<00:24, 130MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  13% 493M/3.68G [00:03<00:21, 146MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  14% 514M/3.68G [00:03<00:19, 159MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  15% 535M/3.68G [00:03<00:18, 168MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  15% 556M/3.68G [00:03<00:18, 173MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  16% 577M/3.68G [00:04<00:17, 178MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  17% 608M/3.68G [00:04<00:15, 198MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  17% 629M/3.68G [00:04<00:15, 194MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  18% 661M/3.68G [00:04<00:15, 198MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  19% 692M/3.68G [00:04<00:14, 206MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  19% 713M/3.68G [00:04<00:15, 198MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  20% 744M/3.68G [00:04<00:14, 204MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  21% 765M/3.68G [00:04<00:14, 202MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  21% 786M/3.68G [00:05<00:14, 199MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  22% 807M/3.68G [00:05<00:14, 199MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  22% 828M/3.68G [00:05<00:14, 192MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  23% 860M/3.68G [00:05<00:14, 199MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  24% 881M/3.68G [00:05<00:14, 198MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  24% 902M/3.68G [00:05<00:13, 200MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  25% 923M/3.68G [00:05<00:13, 203MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  26% 954M/3.68G [00:05<00:12, 211MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  27% 986M/3.68G [00:06<00:12, 214MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  28% 1.02G/3.68G [00:06<00:12, 216MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  28% 1.05G/3.68G [00:06<00:13, 202MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  29% 1.07G/3.68G [00:06<00:13, 199MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  30% 1.09G/3.68G [00:06<00:12, 200MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  30% 1.11G/3.68G [00:06<00:12, 199MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  31% 1.13G/3.68G [00:06<00:13, 192MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  32% 1.16G/3.68G [00:06<00:13, 194MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  32% 1.20G/3.68G [00:07<00:12, 204MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  33% 1.23G/3.68G [00:07<00:11, 209MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  34% 1.26G/3.68G [00:07<00:11, 209MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  35% 1.29G/3.68G [00:07<00:11, 211MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  36% 1.32G/3.68G [00:07<00:11, 210MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  37% 1.35G/3.68G [00:07<00:11, 203MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  37% 1.37G/3.68G [00:07<00:11, 202MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  38% 1.39G/3.68G [00:08<00:11, 201MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  38% 1.42G/3.68G [00:08<00:11, 200MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  39% 1.44G/3.68G [00:08<00:11, 198MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  40% 1.46G/3.68G [00:08<00:11, 201MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  40% 1.48G/3.68G [00:08<00:10, 203MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  41% 1.51G/3.68G [00:08<00:10, 205MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  42% 1.53G/3.68G [00:08<00:10, 205MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  42% 1.55G/3.68G [00:08<00:10, 203MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  43% 1.57G/3.68G [00:08<00:10, 198MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  43% 1.59G/3.68G [00:09<00:11, 190MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  44% 1.61G/3.68G [00:09<00:11, 186MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  44% 1.64G/3.68G [00:09<00:10, 189MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  45% 1.67G/3.68G [00:09<00:10, 198MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  46% 1.70G/3.68G [00:09<00:09, 207MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  47% 1.73G/3.68G [00:09<00:09, 208MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  48% 1.75G/3.68G [00:09<00:09, 205MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  48% 1.78G/3.68G [00:10<00:09, 211MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  49% 1.80G/3.68G [00:10<00:08, 210MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  50% 1.82G/3.68G [00:10<00:10, 177MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  50% 1.85G/3.68G [00:10<00:12, 143MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  51% 1.87G/3.68G [00:10<00:16, 112MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  51% 1.89G/3.68G [00:10<00:14, 120MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  52% 1.91G/3.68G [00:11<00:16, 104MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  52% 1.93G/3.68G [00:11<00:17, 98.7MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  53% 1.95G/3.68G [00:11<00:16, 103MB/s] \u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  54% 1.97G/3.68G [00:12<00:20, 82.3MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  54% 1.98G/3.68G [00:12<00:21, 78.5MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  54% 1.99G/3.68G [00:12<00:21, 79.0MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  54% 2.00G/3.68G [00:12<00:20, 80.4MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  55% 2.01G/3.68G [00:12<00:21, 77.9MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  55% 2.03G/3.68G [00:12<00:19, 86.3MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  56% 2.04G/3.68G [00:13<00:27, 59.2MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  56% 2.07G/3.68G [00:13<00:20, 79.2MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  57% 2.09G/3.68G [00:13<00:16, 97.3MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  57% 2.11G/3.68G [00:13<00:15, 104MB/s] \u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  58% 2.13G/3.68G [00:13<00:12, 123MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  58% 2.15G/3.68G [00:13<00:13, 109MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  59% 2.17G/3.68G [00:14<00:11, 126MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  60% 2.20G/3.68G [00:14<00:12, 121MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  60% 2.22G/3.68G [00:14<00:10, 137MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  61% 2.24G/3.68G [00:14<00:12, 115MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  62% 2.26G/3.68G [00:14<00:13, 106MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  62% 2.29G/3.68G [00:15<00:15, 92.6MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  63% 2.31G/3.68G [00:15<00:18, 76.1MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  63% 2.32G/3.68G [00:15<00:18, 74.0MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  63% 2.33G/3.68G [00:15<00:19, 70.6MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  64% 2.34G/3.68G [00:16<00:19, 70.5MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  64% 2.36G/3.68G [00:16<00:14, 89.2MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  65% 2.38G/3.68G [00:16<00:11, 111MB/s] \u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  65% 2.40G/3.68G [00:16<00:10, 126MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  66% 2.42G/3.68G [00:16<00:08, 141MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  66% 2.44G/3.68G [00:16<00:07, 156MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  67% 2.46G/3.68G [00:16<00:07, 169MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  67% 2.49G/3.68G [00:16<00:06, 180MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  68% 2.51G/3.68G [00:20<01:03, 18.4MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  69% 2.53G/3.68G [00:20<00:45, 25.4MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  69% 2.55G/3.68G [00:20<00:33, 34.3MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  70% 2.58G/3.68G [00:20<00:21, 51.4MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  71% 2.61G/3.68G [00:20<00:15, 70.0MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  71% 2.63G/3.68G [00:20<00:12, 83.8MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  72% 2.65G/3.68G [00:21<00:10, 99.8MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  73% 2.67G/3.68G [00:21<00:08, 114MB/s] \u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  73% 2.69G/3.68G [00:21<00:07, 129MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  74% 2.72G/3.68G [00:21<00:06, 143MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  74% 2.74G/3.68G [00:21<00:06, 156MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  75% 2.76G/3.68G [00:21<00:05, 166MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  76% 2.79G/3.68G [00:21<00:04, 187MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  77% 2.82G/3.68G [00:21<00:04, 198MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  77% 2.85G/3.68G [00:22<00:04, 204MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  78% 2.88G/3.68G [00:22<00:03, 213MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  79% 2.92G/3.68G [00:22<00:03, 204MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  80% 2.95G/3.68G [00:22<00:03, 205MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  81% 2.97G/3.68G [00:22<00:03, 203MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  81% 2.99G/3.68G [00:22<00:03, 203MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  82% 3.01G/3.68G [00:22<00:03, 201MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  82% 3.03G/3.68G [00:23<00:06, 108MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  83% 3.05G/3.68G [00:23<00:05, 124MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  83% 3.07G/3.68G [00:23<00:04, 137MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  84% 3.09G/3.68G [00:23<00:03, 149MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  85% 3.11G/3.68G [00:23<00:03, 158MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  85% 3.14G/3.68G [00:23<00:03, 166MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  86% 3.16G/3.68G [00:23<00:03, 173MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  86% 3.18G/3.68G [00:23<00:02, 181MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  87% 3.20G/3.68G [00:24<00:02, 181MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  88% 3.23G/3.68G [00:24<00:02, 190MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  88% 3.25G/3.68G [00:24<00:02, 194MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  89% 3.27G/3.68G [00:24<00:02, 193MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  89% 3.29G/3.68G [00:24<00:02, 194MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  90% 3.31G/3.68G [00:24<00:01, 190MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  91% 3.33G/3.68G [00:24<00:01, 191MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  91% 3.37G/3.68G [00:24<00:01, 197MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  92% 3.39G/3.68G [00:25<00:01, 197MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  93% 3.41G/3.68G [00:25<00:01, 198MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  93% 3.43G/3.68G [00:25<00:01, 179MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  94% 3.45G/3.68G [00:25<00:01, 172MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  94% 3.47G/3.68G [00:25<00:01, 179MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  95% 3.49G/3.68G [00:25<00:01, 186MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  95% 3.51G/3.68G [00:25<00:00, 188MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  96% 3.53G/3.68G [00:25<00:00, 191MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  97% 3.55G/3.68G [00:25<00:00, 180MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  97% 3.58G/3.68G [00:26<00:00, 163MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  98% 3.60G/3.68G [00:26<00:00, 136MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  98% 3.62G/3.68G [00:26<00:00, 113MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  99% 3.64G/3.68G [00:26<00:00, 124MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin:  99% 3.66G/3.68G [00:27<00:00, 94.3MB/s]\u001b[A\n",
            "pytorch_model-00002-of-00002.bin: 100% 3.68G/3.68G [00:27<00:00, 134MB/s] \n",
            "Downloading shards: 100% 2/2 [01:49<00:00, 54.71s/it]\n",
            "Loading checkpoint shards: 100% 2/2 [00:58<00:00, 29.43s/it]\n",
            "generation_config.json: 100% 203/203 [00:00<00:00, 1.33MB/s]\n",
            "\u001b[32m2024-08-12 14:43:45.129\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.core.language_model.hf_lm\u001b[0m:\u001b[36m__init__\u001b[0m:\u001b[36m213\u001b[0m - \u001b[1mmodel device: cuda:0\u001b[0m\n",
            "\u001b[32m2024-08-12 14:43:45.129\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.core.language_model.hf_lm\u001b[0m:\u001b[36m__init__\u001b[0m:\u001b[36m214\u001b[0m - \u001b[1mmodel dtype: torch.bfloat16\u001b[0m\n",
            "\u001b[32m2024-08-12 14:43:45.129\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.core.language_model.hf_lm\u001b[0m:\u001b[36m__init__\u001b[0m:\u001b[36m215\u001b[0m - \u001b[1mamp_dtype: None\u001b[0m\n",
            "\u001b[32m2024-08-12 14:43:45.129\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.core.language_model.hf_lm\u001b[0m:\u001b[36m__init__\u001b[0m:\u001b[36m216\u001b[0m - \u001b[1mrandom seed: 42\u001b[0m\n",
            "Downloading builder script: 100% 14.0k/14.0k [00:00<00:00, 40.3MB/s]\n",
            "Downloading readme: 100% 3.08k/3.08k [00:00<00:00, 13.9MB/s]\n",
            "Downloading extra modules: 100% 9.03k/9.03k [00:00<00:00, 28.6MB/s]\n",
            "Downloading data: 1.88MB [00:00, 54.4MB/s]      \n",
            "Downloading data: 236kB [00:00, 18.7MB/s]        \n",
            "Generating train split: 8939 examples [00:01, 7852.96 examples/s]\n",
            "Generating validation split: 1119 examples [00:00, 7543.83 examples/s]\n",
            "\u001b[32m2024-08-12 14:43:50.201\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.scripts.flexeval_lm\u001b[0m:\u001b[36mmain\u001b[0m:\u001b[36m171\u001b[0m - \u001b[1mEvaluating with the setup: {'class_path': 'flexeval.Generation', 'init_args': {'eval_dataset': {'class_path': 'flexeval.HFGenerationDataset', 'init_args': {'path': 'llm-book/JGLUE', 'split': 'validation', 'reference_template': '{% set choices = [choice0, choice1, choice2, choice3, choice4] %}{{ choices[label] }}', 'reference_list_template': None, 'input_templates': None, 'subset': 'JCommonsenseQA', 'template_filters': None, 'dataset_kwargs': {'trust_remote_code': True}}}, 'prompt_template': {'class_path': 'flexeval.Jinja2PromptTemplate', 'init_args': {'template': '正しい答えは何でしょう？\\n{% for item in few_shot_data %}\\n0.「{{ item.choice0 }}」\\n1.「{{ item.choice1 }}」\\n2.「{{ item.choice2 }}」\\n3.「{{ item.choice3 }}」\\n4.「{{ item.choice4 }}」\\n問題：{{ item.question }}\\n回答：「{{ item.references[0] }}」\\n{% endfor %}\\n0.「{{ choice0 }}」\\n1.「{{ choice1 }}」\\n2.「{{ choice2 }}」\\n3.「{{ choice3 }}」\\n4.「{{ choice4 }}」\\n問題：{{question}}\\n回答：「'}}, 'gen_kwargs': {'max_new_tokens': 40, 'stop_sequences': ['」']}, 'few_shot_generator': {'class_path': 'flexeval.RandomFewShotGenerator', 'init_args': {'dataset': {'class_path': 'flexeval.HFGenerationDataset', 'init_args': {'path': 'llm-book/JGLUE', 'split': 'train', 'reference_template': '{% set choices = [choice0, choice1, choice2, choice3, choice4] %}{{ choices[label] }}', 'reference_list_template': None, 'input_templates': None, 'subset': 'JCommonsenseQA', 'template_filters': None, 'dataset_kwargs': {'trust_remote_code': True}}}, 'num_shots': 2, 'seed': 42, 'num_trials_to_avoid_leak': 3}}, 'metrics': [{'class_path': 'flexeval.ExactMatch', 'init_args': {'processor': None, 'reference_processor': None}}], 'batch_size': 4, 'max_instances': None}}\u001b[0m\n",
            "fatal: not a git repository (or any of the parent directories): .git\n",
            "\u001b[32m2024-08-12 14:43:51.229\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.core.result_recorder.local_recorder\u001b[0m:\u001b[36mrecord_config\u001b[0m:\u001b[36m70\u001b[0m - \u001b[1mSaved the config to drive/MyDrive/llm_book/eval/jcommonsenseqa_flexeval/config.json\u001b[0m\n",
            "\u001b[32m2024-08-12 14:43:51.229\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.core.evaluate_generation\u001b[0m:\u001b[36mevaluate_generation\u001b[0m:\u001b[36m26\u001b[0m - \u001b[1mEvaluate the model with gen_kwargs: {'max_new_tokens': 40, 'stop_sequences': ['」']}\u001b[0m\n",
            "\u001b[32m2024-08-12 14:43:51.229\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.core.evaluate_generation\u001b[0m:\u001b[36mevaluate_generation\u001b[0m:\u001b[36m27\u001b[0m - \u001b[1mPrompt template: Jinja2PromptTemplate(template='正しい答えは何でしょう？\\n{% for item in few_shot_data %}\\n0.「{{ item.choice0 }}」\\n1.「{{ item.choice1 }}」\\n2.「{{ item.choice2 }}」\\n3.「{{ item.choice3 }}」\\n4.「{{ item.choice4 }}」\\n問題：{{ item.question }}\\n回答：「{{ item.references[0] }}」\\n{% endfor %}\\n0.「{{ choice0 }}」\\n1.「{{ choice1 }}」\\n2.「{{ choice2 }}」\\n3.「{{ choice3 }}」\\n4.「{{ choice4 }}」\\n問題：{{question}}\\n回答：「')\u001b[0m\n",
            "  0% 0/1119 [00:00<?, ?it/s]\u001b[32m2024-08-12 14:43:57.858\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.core.evaluate_generation\u001b[0m:\u001b[36mevaluate_generation\u001b[0m:\u001b[36m60\u001b[0m - \u001b[1mExample of the model inputs and outputs:\u001b[0m\n",
            "\u001b[32m2024-08-12 14:43:57.858\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.core.evaluate_generation\u001b[0m:\u001b[36mevaluate_generation\u001b[0m:\u001b[36m61\u001b[0m - \u001b[1mlm_prompts: 正しい答えは何でしょう？\n",
            "\n",
            "0.「川蜘蛛」\n",
            "1.「足長蜘蛛」\n",
            "2.「アヘアヘ」\n",
            "3.「子守蜘蛛」\n",
            "4.「アメマ」\n",
            "問題：アメンボの別名は？\n",
            "回答：「川蜘蛛」\n",
            "\n",
            "0.「ファスナー」\n",
            "1.「ファンデ」\n",
            "2.「後ろ指」\n",
            "3.「後ろ足」\n",
            "4.「後ろ身頃」\n",
            "問題：衣服の背中を覆う部分のことは？\n",
            "回答：「後ろ身頃」\n",
            "\n",
            "0.「掲示板」\n",
            "1.「パソコン」\n",
            "2.「マザーボード」\n",
            "3.「ハードディスク」\n",
            "4.「まな板」\n",
            "問題：電子機器で使用される最も主要な電子回路基板の事をなんと言う？\n",
            "回答：「\u001b[0m\n",
            "\u001b[32m2024-08-12 14:43:57.858\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.core.evaluate_generation\u001b[0m:\u001b[36mevaluate_generation\u001b[0m:\u001b[36m62\u001b[0m - \u001b[1mlm_outputs: マザーボード\u001b[0m\n",
            "100% 1119/1119 [24:53<00:00,  1.33s/it]\n",
            "\u001b[32m2024-08-12 15:08:44.915\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.core.evaluate_generation\u001b[0m:\u001b[36mevaluate_generation\u001b[0m:\u001b[36m85\u001b[0m - \u001b[1m{'exact_match': 0.7908847184986595}\u001b[0m\n",
            "\u001b[32m2024-08-12 15:08:45.137\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.scripts.flexeval_lm\u001b[0m:\u001b[36mmain\u001b[0m:\u001b[36m191\u001b[0m - \u001b[1mElapsed time: 1493.91 sec\u001b[0m\n",
            "\u001b[32m2024-08-12 15:08:45.148\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.core.result_recorder.local_recorder\u001b[0m:\u001b[36mrecord_metrics\u001b[0m:\u001b[36m81\u001b[0m - \u001b[1mSaved the metrics to drive/MyDrive/llm_book/eval/jcommonsenseqa_flexeval/metrics.json\u001b[0m\n",
            "\u001b[32m2024-08-12 15:08:45.179\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mflexeval.core.result_recorder.local_recorder\u001b[0m:\u001b[36mrecord_model_outputs\u001b[0m:\u001b[36m92\u001b[0m - \u001b[1mSaved the outputs to drive/MyDrive/llm_book/eval/jcommonsenseqa_flexeval/outputs.jsonl\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!flexeval_lm \\\n",
        "  --language_model HuggingFaceLM \\\n",
        "  --language_model.model \"tokyotech-llm/Swallow-7b-hf\" \\\n",
        "  --eval_setup \"jcommonsenseqa\" \\\n",
        "  --save_dir \"./drive/MyDrive/llm_book/eval/jcommonsenseqa_flexeval\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dgd04MwtCM9T",
        "outputId": "0b02e4d6-b304-43e2-cb91-a49fbac89443"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{\n",
            "    \"exact_match\": 0.7908847184986595,\n",
            "    \"elapsed_time\": 1493.9078678909998\n",
            "}"
          ]
        }
      ],
      "source": [
        "!cat drive/MyDrive/llm_book/eval/jcommonsenseqa_flexeval/metrics.json"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6d6QkCBqJhby",
        "outputId": "6e192994-b9a7-45e0-f87c-176100b193e3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{\"lm_prompt\": \"正しい答えは何でしょう？\\n\\n0.「川蜘蛛」\\n1.「足長蜘蛛」\\n2.「アヘアヘ」\\n3.「子守蜘蛛」\\n4.「アメマ」\\n問題：アメンボの別名は？\\n回答：「川蜘蛛」\\n\\n0.「ファスナー」\\n1.「ファンデ」\\n2.「後ろ指」\\n3.「後ろ足」\\n4.「後ろ身頃」\\n問題：衣服の背中を覆う部分のことは？\\n回答：「後ろ身頃」\\n\\n0.「掲示板」\\n1.「パソコン」\\n2.「マザーボード」\\n3.「ハードディスク」\\n4.「まな板」\\n問題：電子機器で使用される最も主要な電子回路基板の事をなんと言う？\\n回答：「\", \"lm_output\": \"マザーボード\", \"task_inputs\": {\"q_id\": 8939, \"question\": \"電子機器で使用される最も主要な電子回路基板の事をなんと言う？\", \"choice0\": \"掲示板\", \"choice1\": \"パソコン\", \"choice2\": \"マザーボード\", \"choice3\": \"ハードディスク\", \"choice4\": \"まな板\", \"label\": 2}, \"references\": [\"マザーボード\"], \"exact_match\": true}\n"
          ]
        }
      ],
      "source": [
        "# ファイルの先頭1 行目を表示\n",
        "!head -n 1 drive/MyDrive/llm_book/eval/jcommonsenseqa_flexeval/outputs.jsonl"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-UVW6O0DIwlh",
        "outputId": "9116824a-8b37-4eb6-cbd2-701084546470"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "===== タスク入力=====\n",
            "{'q_id': 8941, 'question': 'しゃがんだりする様を何という？', 'choice0': '腰を下す', 'choice1': '座る', 'choice2': '仮眠を取る', 'choice3': '寝る', 'choice4': '起きる', 'label': 0}\n",
            "===== 正解=====\n",
            "腰を下す\n",
            "===== モデルの予測=====\n",
            "座る\n",
            "\n",
            "===== タスク入力=====\n",
            "{'q_id': 8949, 'question': '伸び縮するものは？', 'choice0': 'ムーブメント', 'choice1': 'ナット', 'choice2': '動作', 'choice3': 'ボルト', 'choice4': 'バネ', 'label': 4}\n",
            "===== 正解=====\n",
            "バネ\n",
            "===== モデルの予測=====\n",
            "ムーブメント\n",
            "\n",
            "===== タスク入力=====\n",
            "{'q_id': 8951, 'question': '買い物に必ず必要なのは？', 'choice0': 'お金を持っていく', 'choice1': '本を注文する', 'choice2': '読書する', 'choice3': '風', 'choice4': '雨', 'label': 0}\n",
            "===== 正解=====\n",
            "お金を持っていく\n",
            "===== モデルの予測=====\n",
            "お金\n",
            "\n"
          ]
        }
      ],
      "source": [
        "import json\n",
        "from pathlib import Path\n",
        "\n",
        "save_dir = \"./drive/MyDrive/llm_book/eval/jcommonsenseqa_flexeval\"\n",
        "# 不正解の出力を収集\n",
        "wrong_outputs: list[dict] = []\n",
        "with open(Path(save_dir) / \"outputs.jsonl\", \"r\") as f:\n",
        "    for line in f:\n",
        "        output = json.loads(line)\n",
        "        if not output[\"exact_match\"]:\n",
        "            wrong_outputs.append(output)\n",
        "\n",
        "# 不正解の出力を表示\n",
        "for output in wrong_outputs[:3]:\n",
        "    print(\"===== タスク入力=====\")\n",
        "    print(output[\"task_inputs\"])\n",
        "    print(\"===== 正解=====\")\n",
        "    print(output[\"references\"][0])\n",
        "    print(\"===== モデルの予測=====\")\n",
        "    print(output[\"lm_output\"])\n",
        "    print()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}