regmean_demo.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "TiJlTqqf-QJM"
      },
      "outputs": [],
      "source": [
        "import os \n",
        "import re\n",
        "\n",
        "import numpy as np\n",
        "import torch\n",
        "import transformers\n",
        "from datasets import load_dataset, load_metric\n",
        "from torch import nn\n",
        "from tqdm import tqdm\n",
        "from transformers import AutoModelForSequenceClassification, AutoTokenizer, TrainingArguments, Trainer, DataCollatorWithPadding"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sTw_tLKv_VgN",
        "outputId": "23c76776-afd1-4e98-83ca-c5db3b634878"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "('4.20.1', '1.13.1+cu116')"
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "transformers.__version__, torch.__version__"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "umZVhXcc_fvQ"
      },
      "source": [
        "## Train individual models"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "nq80HYsu_aLc"
      },
      "outputs": [],
      "source": [
        "def get_metrics_func(task_name):\n",
        "    metric = load_metric(\"glue\", task_name)\n",
        "    def compute_metrics(p):\n",
        "        preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions\n",
        "        preds = np.argmax(preds, axis=1)\n",
        "        result = metric.compute(predictions=preds, references=p.label_ids)\n",
        "        if len(result) > 1:\n",
        "            result[\"combined_score\"] = np.mean(list(result.values())).item()\n",
        "        return result\n",
        "    return compute_metrics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "rorXK0zL_hc-"
      },
      "outputs": [],
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained('roberta-base')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "gR2kgbw3_pbr"
      },
      "outputs": [],
      "source": [
        "def train_glue_model(task_name, checkpoint_name):\n",
        "    model = AutoModelForSequenceClassification.from_pretrained('roberta-base').cuda()\n",
        "    ds = load_dataset('glue', task_name)\n",
        "    metric = get_metrics_func(task_name)\n",
        "    enc_ds = ds.map(lambda examples: tokenizer(examples[\"sentence\"], max_length=128, truncation=True), batched=True)\n",
        "    training_args = TrainingArguments(\n",
        "        output_dir=f'./results/{task_name}',  # output directory\n",
        "        num_train_epochs=3,                   # total # of training epochs\n",
        "        per_device_train_batch_size=16,       # batch size per device during training\n",
        "        per_device_eval_batch_size=16,        # batch size for evaluation\n",
        "        warmup_steps=500,                     # number of warmup steps for learning rate scheduler\n",
        "        weight_decay=0.01,                    # strength of weight decay\n",
        "    )\n",
        "    trainer = Trainer(\n",
        "        model=model,                        # the instantiated 🤗 Transformers model to be trained\n",
        "        args=training_args,                 # training arguments, defined above\n",
        "        train_dataset=enc_ds['train'],      # training dataset\n",
        "        eval_dataset=enc_ds['validation'],  # evaluation dataset\n",
        "        compute_metrics=metric,\n",
        "        tokenizer=tokenizer,\n",
        "    )\n",
        "\n",
        "    checkpoint_path = os.path.join(training_args.output_dir, checkpoint_name)\n",
        "    if os.path.exists(checkpoint_path):\n",
        "        model = AutoModelForSequenceClassification.from_pretrained(checkpoint_path).cuda()\n",
        "    else:\n",
        "        trainer.train()\n",
        "\n",
        "    return trainer, model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 960,
          "referenced_widgets": [
            "e19ad1a040a84e1a9e24a287aea7bf3a",
            "b312213f8aff41ea8d7fa0cb55faeed7",
            "fbee3a73421b4b41b5d4735e15cb6801",
            "efda9deadaad4955840db5c2bc4c2120",
            "10add3bae44346a19df233fb22b6a505",
            "f245ec0a619541bfbf70620df4f8d711",
            "0b036071e8634c3cbec1952d59e6c390",
            "ba8b216cc3764c03a444bda8b5286664",
            "0c808e4692e24c6393df72317eff5881",
            "3b17a615103747d4a599a1096ba3032f",
            "51e4141f59904cf6a2dc38955bd60025",
            "059e5c32ea9c48c4b0ca47f7334afdeb",
            "e9bd93f7e909421caafad292b7700569",
            "a74fda5a864642ceb2da5bf64cff589d",
            "faaaba09b3254b098e2d519dc187f86b",
            "0c88132a652042c8a31dac8227b06178",
            "44d0af08a7944c83b40247edf6a66169",
            "6dd3c0a270a14c2db56eeffd288a9e8f",
            "0b75db6ecd684d23a79841ef40c346fd",
            "33aa094ede9741cdbc13e4121beafad4",
            "d5653db415454c729210daf8a28a8c2a",
            "c49990927b1847df840b334d8b63f801",
            "cf9febd1ce864bf19584a1ae317659d8",
            "34f11f26440c409eb301ad306936ddc3",
            "9698453f3d1d48f188ea46e11e1d87d3",
            "779a1bfdaf7547b6ab76aa9fa6afa7de",
            "dd628d6fdb0d43948a327b7171f4f5a7",
            "cef9cc3424394642a8e66fed3f705db2",
            "18f7e4ffa3a2447d8f0c89ba0befbe04",
            "81eff330045a40bab5f3b04b0eb8545e",
            "809c25dc8d784b77850515c9d7781e52",
            "8585b4e9c42a4f1aacf92ad38b1a78f8",
            "681d2065b7e245519356397e756f2721",
            "6a1e1c367f7f4f19964d65b3924e3339",
            "f75e97ad7ee74f2d9fd1480de114913d",
            "d3f3e299cadc43e1aa3550cdda2483b9",
            "8b0f020360ca4f5caf5abf167a8ebd9b",
            "491251d0ea0d43be814066efe65d70a0",
            "e8c03e546fe649faaa755a57ed58f3f2",
            "858de7cf253742878c45a51064782a00",
            "fc6f87d531aa45c08bc55a495e2370f6",
            "a3ab257806cd4312b4103e9e4a277276",
            "23765e7147d54bee996d5a6a7eff99b8",
            "0e968f62e46b485d8132261eb12e440b"
          ]
        },
        "id": "pitwN9X4_2dT",
        "outputId": "0a53ea19-c806-4156-9a2c-5aa015faeb2e"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.weight', 'lm_head.dense.bias', 'lm_head.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "WARNING:datasets.builder:Reusing dataset glue (/root/.cache/huggingface/datasets/glue/cola/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e19ad1a040a84e1a9e24a287aea7bf3a",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:datasets.fingerprint:Parameter 'function'=<function train_glue_model.<locals>.<lambda> at 0x7f57f0a8cee0> of the transform datasets.arrow_dataset.Dataset._map_single couldn't be hashed properly, a random hash was used instead. Make sure your transforms and parameters are serializable with pickle or dill for the dataset fingerprinting and caching to work. If you reuse this transform, the caching mechanism will consider it to be different from the previous calls and recompute everything. This warning is only showed once. Subsequent hashing failures won't be showed.\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "059e5c32ea9c48c4b0ca47f7334afdeb",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/9 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cf9febd1ce864bf19584a1ae317659d8",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6a1e1c367f7f4f19964d65b3924e3339",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "The following columns in the training set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: sentence, idx. If sentence, idx are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n",
            "***** Running training *****\n",
            "  Num examples = 8551\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 1605\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1605' max='1605' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1605/1605 01:46, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.546800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.395400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.240300</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Saving model checkpoint to ./results/cola/checkpoint-500\n",
            "Configuration saved in ./results/cola/checkpoint-500/config.json\n",
            "Model weights saved in ./results/cola/checkpoint-500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/cola/checkpoint-500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/cola/checkpoint-500/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/cola/checkpoint-1000\n",
            "Configuration saved in ./results/cola/checkpoint-1000/config.json\n",
            "Model weights saved in ./results/cola/checkpoint-1000/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/cola/checkpoint-1000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/cola/checkpoint-1000/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/cola/checkpoint-1500\n",
            "Configuration saved in ./results/cola/checkpoint-1500/config.json\n",
            "Model weights saved in ./results/cola/checkpoint-1500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/cola/checkpoint-1500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/cola/checkpoint-1500/special_tokens_map.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "trainer1, model1 = train_glue_model('cola', 'checkpoint-1500')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "afe2793a24dc4e55b31a6fac8ccf8371",
            "7c6c341121174128969595f8c57e55ad",
            "5531d838c5b84481915caee4ecf7133b",
            "5c172c0f40f547078cf00872ae9ec304",
            "96553724a6a8454a85e7c850c47d84d0",
            "fe7d59ee8b2c48d98a41823c9cbf42ec",
            "256d6ccc340f45d6bf8ebfa4fd216316",
            "42bf4f90fc91457181570be40540dbed",
            "561d6c95a2444cc49467b3b9808a613a",
            "cfde00f311b94723ba907d8f1e77f9f9",
            "c707c30a030d4060877d6b55ec98c7fb"
          ]
        },
        "id": "LIwR_Pbg_4vc",
        "outputId": "d997fb5b-b651-4c41-cbc9-02a673248ec5"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loading configuration file https://huggingface.co/roberta-base/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/733bade19e5f0ce98e6531021dd5180994bb2f7b8bd7e80c7968805834ba351e.35205c6cfc956461d8515139f0f8dd5d207a2f336c0c3a83b4bc8dca3518e37b\n",
            "Model config RobertaConfig {\n",
            "  \"_name_or_path\": \"roberta-base\",\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.20.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file https://huggingface.co/roberta-base/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/51ba668f7ff34e7cdfa9561e8361747738113878850a7d717dbc69de8683aaad.c7efaa30a0d80b2958b876969faa180e485944a849deee4ad482332de65365a7\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.weight', 'lm_head.dense.bias', 'lm_head.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "WARNING:datasets.builder:Reusing dataset glue (/root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "afe2793a24dc4e55b31a6fac8ccf8371",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-1c80317fa3b1799d.arrow\n",
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-bdd640fb06671ad1.arrow\n",
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-3eb13b9046685257.arrow\n",
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "The following columns in the training set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: sentence, idx. If sentence, idx are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n",
            "***** Running training *****\n",
            "  Num examples = 67349\n",
            "  Num Epochs = 3\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 12630\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='12630' max='12630' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [12630/12630 14:29, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.450900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.366700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.318300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.304100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.290100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.260700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>0.274300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>0.256000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4500</td>\n",
              "      <td>0.219100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5000</td>\n",
              "      <td>0.215400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5500</td>\n",
              "      <td>0.206000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6000</td>\n",
              "      <td>0.174200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6500</td>\n",
              "      <td>0.185400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7000</td>\n",
              "      <td>0.183000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7500</td>\n",
              "      <td>0.185600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8000</td>\n",
              "      <td>0.175600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8500</td>\n",
              "      <td>0.165500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9000</td>\n",
              "      <td>0.127600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9500</td>\n",
              "      <td>0.115300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10000</td>\n",
              "      <td>0.120300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10500</td>\n",
              "      <td>0.116900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11000</td>\n",
              "      <td>0.117200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11500</td>\n",
              "      <td>0.124100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12000</td>\n",
              "      <td>0.116400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12500</td>\n",
              "      <td>0.122800</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Saving model checkpoint to ./results/sst2/checkpoint-500\n",
            "Configuration saved in ./results/sst2/checkpoint-500/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-500/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-1000\n",
            "Configuration saved in ./results/sst2/checkpoint-1000/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-1000/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-1000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-1000/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-1500\n",
            "Configuration saved in ./results/sst2/checkpoint-1500/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-1500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-1500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-1500/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-2000\n",
            "Configuration saved in ./results/sst2/checkpoint-2000/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-2000/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-2000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-2000/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-2500\n",
            "Configuration saved in ./results/sst2/checkpoint-2500/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-2500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-2500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-2500/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-3000\n",
            "Configuration saved in ./results/sst2/checkpoint-3000/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-3000/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-3000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-3000/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-3500\n",
            "Configuration saved in ./results/sst2/checkpoint-3500/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-3500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-3500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-3500/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-4000\n",
            "Configuration saved in ./results/sst2/checkpoint-4000/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-4000/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-4000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-4000/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-4500\n",
            "Configuration saved in ./results/sst2/checkpoint-4500/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-4500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-4500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-4500/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-5000\n",
            "Configuration saved in ./results/sst2/checkpoint-5000/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-5000/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-5000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-5000/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-5500\n",
            "Configuration saved in ./results/sst2/checkpoint-5500/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-5500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-5500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-5500/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-6000\n",
            "Configuration saved in ./results/sst2/checkpoint-6000/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-6000/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-6000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-6000/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-6500\n",
            "Configuration saved in ./results/sst2/checkpoint-6500/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-6500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-6500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-6500/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-7000\n",
            "Configuration saved in ./results/sst2/checkpoint-7000/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-7000/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-7000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-7000/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-7500\n",
            "Configuration saved in ./results/sst2/checkpoint-7500/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-7500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-7500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-7500/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-8000\n",
            "Configuration saved in ./results/sst2/checkpoint-8000/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-8000/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-8000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-8000/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-8500\n",
            "Configuration saved in ./results/sst2/checkpoint-8500/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-8500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-8500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-8500/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-9000\n",
            "Configuration saved in ./results/sst2/checkpoint-9000/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-9000/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-9000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-9000/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-9500\n",
            "Configuration saved in ./results/sst2/checkpoint-9500/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-9500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-9500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-9500/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-10000\n",
            "Configuration saved in ./results/sst2/checkpoint-10000/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-10000/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-10000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-10000/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-10500\n",
            "Configuration saved in ./results/sst2/checkpoint-10500/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-10500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-10500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-10500/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-11000\n",
            "Configuration saved in ./results/sst2/checkpoint-11000/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-11000/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-11000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-11000/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-11500\n",
            "Configuration saved in ./results/sst2/checkpoint-11500/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-11500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-11500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-11500/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-12000\n",
            "Configuration saved in ./results/sst2/checkpoint-12000/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-12000/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-12000/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-12000/special_tokens_map.json\n",
            "Saving model checkpoint to ./results/sst2/checkpoint-12500\n",
            "Configuration saved in ./results/sst2/checkpoint-12500/config.json\n",
            "Model weights saved in ./results/sst2/checkpoint-12500/pytorch_model.bin\n",
            "tokenizer config file saved in ./results/sst2/checkpoint-12500/tokenizer_config.json\n",
            "Special tokens file saved in ./results/sst2/checkpoint-12500/special_tokens_map.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "trainer2, model2 = train_glue_model('sst2', 'checkpoint-12500')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tcMrxGCfCkM-"
      },
      "source": [
        "## Compute inner-product matrices"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "1M3b6xqUBD98"
      },
      "outputs": [],
      "source": [
        "def filter_params_to_merge(param_names, exclude_param_regex):\n",
        "    params_to_merge = []\n",
        "    for name in param_names:\n",
        "        valid = not any([re.match(patt, name) for patt in exclude_param_regex])\n",
        "        if valid:\n",
        "            params_to_merge.append(name)\n",
        "    return params_to_merge\n",
        "\n",
        "\n",
        "def filter_modules_by_regex(base_module, include_patterns, include_type):\n",
        "    modules = {}\n",
        "    for name, module in base_module.named_modules():\n",
        "        valid_name = not include_patterns or any([re.match(patt, name) for patt in include_patterns])\n",
        "        valid_type = not include_type or any([isinstance(module, md_cls) for md_cls in include_type])\n",
        "        if valid_type and valid_name:\n",
        "            modules[name] = module\n",
        "    return modules"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "UKMJm-vDCvDW"
      },
      "outputs": [],
      "source": [
        "def compute_gram(model, trainer):\n",
        "    train_dataloader = trainer.get_train_dataloader()\n",
        "    grams = {} # gram matrices for each linear layer inputs\n",
        "    xn = {} # number of examples used for computing gram\n",
        "\n",
        "    def get_gram(name):\n",
        "        def hook(module, input, output):\n",
        "            x = input[0].detach() # $[b,t,h]\n",
        "            x = x.view(-1, x.size(-1))\n",
        "            xtx = torch.matmul(x.transpose(0,1), x) # [h,h]\n",
        "            if name not in grams:\n",
        "                grams[name] = xtx / x.size(0)\n",
        "                xn[name] = x.size(0)\n",
        "            else:\n",
        "                grams[name] = (grams[name] * xn[name] + xtx) / (x.size(0) + xn[name])\n",
        "                xn[name] += x.size(0)\n",
        "        return hook\n",
        "\n",
        "    linear_modules = filter_modules_by_regex(model, None, [nn.Linear])\n",
        "    handles = []\n",
        "    for name, module in linear_modules.items():\n",
        "        handle = module.register_forward_hook(get_gram(name))\n",
        "        handles.append(handle)\n",
        "\n",
        "    n_step = 1000\n",
        "    total = n_step if n_step > 0 else len(train_dataloader)\n",
        "    for step, inputs in tqdm(enumerate(train_dataloader), total=total, desc='Computing gram matrix'):\n",
        "        if n_step > 0 and step == n_step:\n",
        "            break\n",
        "\n",
        "        inputs = trainer._prepare_inputs(inputs)\n",
        "        outputs = model(**inputs)\n",
        "\n",
        "    for handle in handles:\n",
        "        handle.remove()\n",
        "\n",
        "    return grams"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WyWXHgFWCyaj",
        "outputId": "e8688ca2-ba58-4f85-f06a-760d02ab63c3"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "The following columns in the training set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: sentence, idx. If sentence, idx are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "Computing gram matrix:  54%|█████▎    | 535/1000 [00:13<00:11, 40.73it/s]\n"
          ]
        }
      ],
      "source": [
        "with torch.no_grad():\n",
        "    grams1 = compute_gram(model1, trainer1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Is-NisUjCzKK",
        "outputId": "a93e3e3e-9020-41c2-88e6-1ea319adcaf1"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "The following columns in the training set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: sentence, idx. If sentence, idx are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "Computing gram matrix: 100%|██████████| 1000/1000 [00:26<00:00, 38.00it/s]\n"
          ]
        }
      ],
      "source": [
        "with torch.no_grad():\n",
        "    grams2 = compute_gram(model2, trainer2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "SbDnvQUFC1_F"
      },
      "outputs": [],
      "source": [
        "def avg_merge(local_models, global_model, regmean_grams=None, **kwargs):\n",
        "    params = {}\n",
        "    for local_model in local_models:\n",
        "        n2p = {k: v for k,v in local_model.named_parameters()}\n",
        "        merge_param_names = filter_params_to_merge([n for n in n2p], ['.*classifier.*']) # for glue label spaces are different\n",
        "        for n in merge_param_names:\n",
        "            if n not in params:\n",
        "                params[n] = []\n",
        "            params[n].append(n2p[n])\n",
        "\n",
        "    if regmean_grams: # regmean average\n",
        "        avg_params = regmean_merge(params, regmean_grams)\n",
        "\n",
        "    else: # simple average\n",
        "        avg_params = {k: torch.stack(v,0).mean(0) for k, v in params.items()}\n",
        "\n",
        "    return avg_params\n",
        "\n",
        "def copy_params_to_model(avg_params, model):\n",
        "    for n, p in model.named_parameters():\n",
        "        if n in avg_params:\n",
        "            p.data.copy_(avg_params[n])\n",
        "\n",
        "def reduce_non_diag(cov_mat, a):\n",
        "    diag_weight = torch.diag(torch.ones(cov_mat.size(0)) - a).to(cov_mat.device)\n",
        "    non_diag_weight = torch.zeros_like(diag_weight).fill_(a)\n",
        "    weight = diag_weight + non_diag_weight\n",
        "    ret = cov_mat * weight\n",
        "    return ret\n",
        "\n",
        "def regmean_merge(all_params, all_grams):\n",
        "    avg_params = {}\n",
        "    n_model = len(all_grams)\n",
        "    for name in all_params:\n",
        "        h_avged = False\n",
        "        if name.endswith('.weight'):\n",
        "            print(f'Regmean: {name}')\n",
        "            module_name = name[:-len('.weight')]\n",
        "            if module_name in all_grams[0]:\n",
        "                gram_m_ws, grams = [], []\n",
        "\n",
        "                for model_id, model_grams in enumerate(all_grams):\n",
        "                    param_grams = model_grams[module_name]\n",
        "\n",
        "                    # for roberta we dont need this; but it is important for deberta and t5\n",
        "                    #param_grams = reduce_non_diag(param_grams, a=0.9)\n",
        "\n",
        "                    param = all_params[name][model_id]\n",
        "                    gram_m_ws.append(torch.matmul(param_grams, param.transpose(0,1)))\n",
        "                    grams.append(param_grams)\n",
        "                sum_gram = sum(grams)\n",
        "                sum_gram_m_ws = sum(gram_m_ws)\n",
        "                sum_gram_inv = torch.inverse(sum_gram)\n",
        "                wt = torch.matmul(sum_gram_inv, sum_gram_m_ws)\n",
        "                w = wt.transpose(0,1)\n",
        "                avg_params[name] = w\n",
        "                h_avged = True\n",
        "        if not h_avged: # if not averaged with regmean, then do simple avg\n",
        "            avg_params[name] = torch.stack(all_params[name],0).mean(0)\n",
        "           \n",
        "    return avg_params"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SWJLbbJoC5BK"
      },
      "source": [
        "## Performance before merging"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 211
        },
        "id": "5ma2-jDoC20w",
        "outputId": "5cfe6fbf-6202-4fef-fdd3-d6de48ba4eea"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: sentence, idx. If sentence, idx are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1043\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='66' max='66' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [66/66 00:00]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "{'eval_loss': 0.531173825263977,\n",
              " 'eval_matthews_correlation': 0.6011053198493792,\n",
              " 'eval_runtime': 1.0405,\n",
              " 'eval_samples_per_second': 1002.385,\n",
              " 'eval_steps_per_second': 63.43,\n",
              " 'epoch': 3.0}"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "trainer1.evaluate()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 211
        },
        "id": "gRnmUbHAC85h",
        "outputId": "4d58c263-ad50-4881-eb97-cca6311a45a9"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: sentence, idx. If sentence, idx are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 872\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='55' max='55' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [55/55 00:00]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "{'eval_loss': 0.290436327457428,\n",
              " 'eval_accuracy': 0.9277522935779816,\n",
              " 'eval_runtime': 0.9222,\n",
              " 'eval_samples_per_second': 945.572,\n",
              " 'eval_steps_per_second': 59.64,\n",
              " 'epoch': 3.0}"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "trainer2.evaluate()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uKKEe_NjC_1A"
      },
      "source": [
        "## Merging with RegMean"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JjOPsTyiDCWz",
        "outputId": "9c89ce3d-ca64-42f5-83e8-627b00fc8f5b"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loading configuration file https://huggingface.co/roberta-base/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/733bade19e5f0ce98e6531021dd5180994bb2f7b8bd7e80c7968805834ba351e.35205c6cfc956461d8515139f0f8dd5d207a2f336c0c3a83b4bc8dca3518e37b\n",
            "Model config RobertaConfig {\n",
            "  \"_name_or_path\": \"roberta-base\",\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.20.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file https://huggingface.co/roberta-base/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/51ba668f7ff34e7cdfa9561e8361747738113878850a7d717dbc69de8683aaad.c7efaa30a0d80b2958b876969faa180e485944a849deee4ad482332de65365a7\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.weight', 'lm_head.dense.bias', 'lm_head.bias', 'lm_head.decoder.weight', 'roberta.pooler.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "merged_model = AutoModelForSequenceClassification.from_pretrained('roberta-base').cuda()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hXfjDyCvDEdS",
        "outputId": "3df82b9b-4b08-4ab4-f3a7-ec815f49fe37"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Regmean: roberta.embeddings.word_embeddings.weight\n",
            "Regmean: roberta.embeddings.position_embeddings.weight\n",
            "Regmean: roberta.embeddings.token_type_embeddings.weight\n",
            "Regmean: roberta.embeddings.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.0.attention.self.query.weight\n",
            "Regmean: roberta.encoder.layer.0.attention.self.key.weight\n",
            "Regmean: roberta.encoder.layer.0.attention.self.value.weight\n",
            "Regmean: roberta.encoder.layer.0.attention.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.0.attention.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.0.intermediate.dense.weight\n",
            "Regmean: roberta.encoder.layer.0.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.0.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.1.attention.self.query.weight\n",
            "Regmean: roberta.encoder.layer.1.attention.self.key.weight\n",
            "Regmean: roberta.encoder.layer.1.attention.self.value.weight\n",
            "Regmean: roberta.encoder.layer.1.attention.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.1.attention.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.1.intermediate.dense.weight\n",
            "Regmean: roberta.encoder.layer.1.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.1.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.2.attention.self.query.weight\n",
            "Regmean: roberta.encoder.layer.2.attention.self.key.weight\n",
            "Regmean: roberta.encoder.layer.2.attention.self.value.weight\n",
            "Regmean: roberta.encoder.layer.2.attention.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.2.attention.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.2.intermediate.dense.weight\n",
            "Regmean: roberta.encoder.layer.2.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.2.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.3.attention.self.query.weight\n",
            "Regmean: roberta.encoder.layer.3.attention.self.key.weight\n",
            "Regmean: roberta.encoder.layer.3.attention.self.value.weight\n",
            "Regmean: roberta.encoder.layer.3.attention.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.3.attention.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.3.intermediate.dense.weight\n",
            "Regmean: roberta.encoder.layer.3.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.3.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.4.attention.self.query.weight\n",
            "Regmean: roberta.encoder.layer.4.attention.self.key.weight\n",
            "Regmean: roberta.encoder.layer.4.attention.self.value.weight\n",
            "Regmean: roberta.encoder.layer.4.attention.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.4.attention.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.4.intermediate.dense.weight\n",
            "Regmean: roberta.encoder.layer.4.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.4.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.5.attention.self.query.weight\n",
            "Regmean: roberta.encoder.layer.5.attention.self.key.weight\n",
            "Regmean: roberta.encoder.layer.5.attention.self.value.weight\n",
            "Regmean: roberta.encoder.layer.5.attention.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.5.attention.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.5.intermediate.dense.weight\n",
            "Regmean: roberta.encoder.layer.5.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.5.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.6.attention.self.query.weight\n",
            "Regmean: roberta.encoder.layer.6.attention.self.key.weight\n",
            "Regmean: roberta.encoder.layer.6.attention.self.value.weight\n",
            "Regmean: roberta.encoder.layer.6.attention.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.6.attention.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.6.intermediate.dense.weight\n",
            "Regmean: roberta.encoder.layer.6.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.6.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.7.attention.self.query.weight\n",
            "Regmean: roberta.encoder.layer.7.attention.self.key.weight\n",
            "Regmean: roberta.encoder.layer.7.attention.self.value.weight\n",
            "Regmean: roberta.encoder.layer.7.attention.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.7.attention.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.7.intermediate.dense.weight\n",
            "Regmean: roberta.encoder.layer.7.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.7.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.8.attention.self.query.weight\n",
            "Regmean: roberta.encoder.layer.8.attention.self.key.weight\n",
            "Regmean: roberta.encoder.layer.8.attention.self.value.weight\n",
            "Regmean: roberta.encoder.layer.8.attention.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.8.attention.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.8.intermediate.dense.weight\n",
            "Regmean: roberta.encoder.layer.8.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.8.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.9.attention.self.query.weight\n",
            "Regmean: roberta.encoder.layer.9.attention.self.key.weight\n",
            "Regmean: roberta.encoder.layer.9.attention.self.value.weight\n",
            "Regmean: roberta.encoder.layer.9.attention.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.9.attention.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.9.intermediate.dense.weight\n",
            "Regmean: roberta.encoder.layer.9.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.9.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.10.attention.self.query.weight\n",
            "Regmean: roberta.encoder.layer.10.attention.self.key.weight\n",
            "Regmean: roberta.encoder.layer.10.attention.self.value.weight\n",
            "Regmean: roberta.encoder.layer.10.attention.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.10.attention.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.10.intermediate.dense.weight\n",
            "Regmean: roberta.encoder.layer.10.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.10.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.11.attention.self.query.weight\n",
            "Regmean: roberta.encoder.layer.11.attention.self.key.weight\n",
            "Regmean: roberta.encoder.layer.11.attention.self.value.weight\n",
            "Regmean: roberta.encoder.layer.11.attention.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.11.attention.output.LayerNorm.weight\n",
            "Regmean: roberta.encoder.layer.11.intermediate.dense.weight\n",
            "Regmean: roberta.encoder.layer.11.output.dense.weight\n",
            "Regmean: roberta.encoder.layer.11.output.LayerNorm.weight\n"
          ]
        }
      ],
      "source": [
        "regmean_avg_params = avg_merge([model1, model2], merged_model, regmean_grams=[grams1, grams2])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "bxriZ67QDGnk"
      },
      "outputs": [],
      "source": [
        "copy_params_to_model(regmean_avg_params, merged_model)\n",
        "merged_model.classifier = model1.classifier  # we didn't merge classification heads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "OI9sqvJSDIww"
      },
      "outputs": [],
      "source": [
        "evaluator_cola = Trainer(\n",
        "    model=merged_model,                    # the instantiated 🤗 Transformers model to be trained\n",
        "    args=trainer1.args,                    # training arguments, defined above\n",
        "    train_dataset=trainer1.train_dataset,  # training dataset\n",
        "    eval_dataset=trainer1.eval_dataset,    # evaluation dataset\n",
        "    compute_metrics=get_metrics_func('cola'),\n",
        "    tokenizer=tokenizer,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        },
        "id": "iWNCKH-eDKqO",
        "outputId": "38e2f8bd-a3bf-48a2-a305-3389949633f3"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: sentence, idx. If sentence, idx are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1043\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='66' max='66' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [66/66 00:01]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "{'eval_loss': 0.66486656665802,\n",
              " 'eval_matthews_correlation': 0.08987761934408492,\n",
              " 'eval_runtime': 1.0497,\n",
              " 'eval_samples_per_second': 993.636,\n",
              " 'eval_steps_per_second': 62.876}"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "evaluator_cola.evaluate()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "NU347AJADMMQ"
      },
      "outputs": [],
      "source": [
        "copy_params_to_model(regmean_avg_params, merged_model)\n",
        "merged_model.classifier = model2.classifier  # we didn't merge classification heads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "oTBTtE-ODOI1"
      },
      "outputs": [],
      "source": [
        "evaluator_sst = Trainer(\n",
        "    model=merged_model,                    # the instantiated 🤗 Transformers model to be trained\n",
        "    args=trainer2.args,                    # training arguments, defined above\n",
        "    train_dataset=trainer2.train_dataset,  # training dataset\n",
        "    eval_dataset=trainer2.eval_dataset,    # evaluation dataset\n",
        "    compute_metrics=get_metrics_func('sst2'),\n",
        "    tokenizer=tokenizer,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        },
        "id": "x7h6FrrJDPfZ",
        "outputId": "6ed41f08-b692-47aa-f58a-6f4cff5b1242"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: sentence, idx. If sentence, idx are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 872\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='55' max='55' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [55/55 00:00]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "{'eval_loss': 0.23991066217422485,\n",
              " 'eval_accuracy': 0.9357798165137615,\n",
              " 'eval_runtime': 0.9036,\n",
              " 'eval_samples_per_second': 964.997,\n",
              " 'eval_steps_per_second': 60.866}"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "evaluator_sst.evaluate()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QW9X4EPtDROg"
      },
      "source": [
        "## Merging with Simple Avg"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        },
        "id": "CQCqT8mbDTop",
        "outputId": "768795aa-a18e-490d-ad3e-409d4e9e0d1e"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: sentence, idx. If sentence, idx are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1043\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='132' max='66' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [66/66 00:15]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "{'eval_loss': 0.6459254026412964,\n",
              " 'eval_matthews_correlation': 0.020369871666239114,\n",
              " 'eval_runtime': 1.1313,\n",
              " 'eval_samples_per_second': 921.966,\n",
              " 'eval_steps_per_second': 58.341}"
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "simple_avg_params = avg_merge([model1, model2], merged_model)\n",
        "copy_params_to_model(simple_avg_params, merged_model)\n",
        "merged_model.classifier = model1.classifier  # we didn't merge classification heads\n",
        "evaluator_cola.evaluate()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        },
        "id": "q1WPrrGBDVK4",
        "outputId": "f069b0ca-bd63-4e97-96ad-61ac3defd02f"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "The following columns in the evaluation set don't have a corresponding argument in `RobertaForSequenceClassification.forward` and have been ignored: sentence, idx. If sentence, idx are not expected by `RobertaForSequenceClassification.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 872\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='110' max='55' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [55/55 00:05]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "{'eval_loss': 0.1879292130470276,\n",
              " 'eval_accuracy': 0.9392201834862385,\n",
              " 'eval_runtime': 1.1327,\n",
              " 'eval_samples_per_second': 769.839,\n",
              " 'eval_steps_per_second': 48.556}"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "simple_avg_params = avg_merge([model1, model2], merged_model)\n",
        "copy_params_to_model(simple_avg_params, merged_model)\n",
        "merged_model.classifier = model2.classifier  # we didn't merge classification heads\n",
        "evaluator_sst.evaluate()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3_ti8l-mGOSh"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "authorship_tag": "ABX9TyM0ZDj+W7sCBPdRNQVJyAQ8",
      "include_colab_link": true,
      "machine_shape": "hm",
      "provenance": []
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "059e5c32ea9c48c4b0ca47f7334afdeb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e9bd93f7e909421caafad292b7700569",
              "IPY_MODEL_a74fda5a864642ceb2da5bf64cff589d",
              "IPY_MODEL_faaaba09b3254b098e2d519dc187f86b"
            ],
            "layout": "IPY_MODEL_0c88132a652042c8a31dac8227b06178"
          }
        },
        "0b036071e8634c3cbec1952d59e6c390": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0b75db6ecd684d23a79841ef40c346fd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0c808e4692e24c6393df72317eff5881": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0c88132a652042c8a31dac8227b06178": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0e968f62e46b485d8132261eb12e440b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "10add3bae44346a19df233fb22b6a505": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "18f7e4ffa3a2447d8f0c89ba0befbe04": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "23765e7147d54bee996d5a6a7eff99b8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "256d6ccc340f45d6bf8ebfa4fd216316": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "33aa094ede9741cdbc13e4121beafad4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "34f11f26440c409eb301ad306936ddc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cef9cc3424394642a8e66fed3f705db2",
            "placeholder": "​",
            "style": "IPY_MODEL_18f7e4ffa3a2447d8f0c89ba0befbe04",
            "value": "100%"
          }
        },
        "3b17a615103747d4a599a1096ba3032f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "42bf4f90fc91457181570be40540dbed": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "44d0af08a7944c83b40247edf6a66169": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "491251d0ea0d43be814066efe65d70a0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "51e4141f59904cf6a2dc38955bd60025": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5531d838c5b84481915caee4ecf7133b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_42bf4f90fc91457181570be40540dbed",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_561d6c95a2444cc49467b3b9808a613a",
            "value": 3
          }
        },
        "561d6c95a2444cc49467b3b9808a613a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5c172c0f40f547078cf00872ae9ec304": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cfde00f311b94723ba907d8f1e77f9f9",
            "placeholder": "​",
            "style": "IPY_MODEL_c707c30a030d4060877d6b55ec98c7fb",
            "value": " 3/3 [00:00&lt;00:00, 176.65it/s]"
          }
        },
        "681d2065b7e245519356397e756f2721": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6a1e1c367f7f4f19964d65b3924e3339": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f75e97ad7ee74f2d9fd1480de114913d",
              "IPY_MODEL_d3f3e299cadc43e1aa3550cdda2483b9",
              "IPY_MODEL_8b0f020360ca4f5caf5abf167a8ebd9b"
            ],
            "layout": "IPY_MODEL_491251d0ea0d43be814066efe65d70a0"
          }
        },
        "6dd3c0a270a14c2db56eeffd288a9e8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "779a1bfdaf7547b6ab76aa9fa6afa7de": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8585b4e9c42a4f1aacf92ad38b1a78f8",
            "placeholder": "​",
            "style": "IPY_MODEL_681d2065b7e245519356397e756f2721",
            "value": " 2/2 [00:00&lt;00:00, 56.41ba/s]"
          }
        },
        "7c6c341121174128969595f8c57e55ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fe7d59ee8b2c48d98a41823c9cbf42ec",
            "placeholder": "​",
            "style": "IPY_MODEL_256d6ccc340f45d6bf8ebfa4fd216316",
            "value": "100%"
          }
        },
        "809c25dc8d784b77850515c9d7781e52": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "81eff330045a40bab5f3b04b0eb8545e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8585b4e9c42a4f1aacf92ad38b1a78f8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "858de7cf253742878c45a51064782a00": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8b0f020360ca4f5caf5abf167a8ebd9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_23765e7147d54bee996d5a6a7eff99b8",
            "placeholder": "​",
            "style": "IPY_MODEL_0e968f62e46b485d8132261eb12e440b",
            "value": " 2/2 [00:00&lt;00:00, 55.18ba/s]"
          }
        },
        "96553724a6a8454a85e7c850c47d84d0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9698453f3d1d48f188ea46e11e1d87d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_81eff330045a40bab5f3b04b0eb8545e",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_809c25dc8d784b77850515c9d7781e52",
            "value": 2
          }
        },
        "a3ab257806cd4312b4103e9e4a277276": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a74fda5a864642ceb2da5bf64cff589d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b75db6ecd684d23a79841ef40c346fd",
            "max": 9,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_33aa094ede9741cdbc13e4121beafad4",
            "value": 9
          }
        },
        "afe2793a24dc4e55b31a6fac8ccf8371": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7c6c341121174128969595f8c57e55ad",
              "IPY_MODEL_5531d838c5b84481915caee4ecf7133b",
              "IPY_MODEL_5c172c0f40f547078cf00872ae9ec304"
            ],
            "layout": "IPY_MODEL_96553724a6a8454a85e7c850c47d84d0"
          }
        },
        "b312213f8aff41ea8d7fa0cb55faeed7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f245ec0a619541bfbf70620df4f8d711",
            "placeholder": "​",
            "style": "IPY_MODEL_0b036071e8634c3cbec1952d59e6c390",
            "value": "100%"
          }
        },
        "ba8b216cc3764c03a444bda8b5286664": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c49990927b1847df840b334d8b63f801": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c707c30a030d4060877d6b55ec98c7fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cef9cc3424394642a8e66fed3f705db2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cf9febd1ce864bf19584a1ae317659d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_34f11f26440c409eb301ad306936ddc3",
              "IPY_MODEL_9698453f3d1d48f188ea46e11e1d87d3",
              "IPY_MODEL_779a1bfdaf7547b6ab76aa9fa6afa7de"
            ],
            "layout": "IPY_MODEL_dd628d6fdb0d43948a327b7171f4f5a7"
          }
        },
        "cfde00f311b94723ba907d8f1e77f9f9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d3f3e299cadc43e1aa3550cdda2483b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fc6f87d531aa45c08bc55a495e2370f6",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a3ab257806cd4312b4103e9e4a277276",
            "value": 2
          }
        },
        "d5653db415454c729210daf8a28a8c2a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dd628d6fdb0d43948a327b7171f4f5a7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e19ad1a040a84e1a9e24a287aea7bf3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b312213f8aff41ea8d7fa0cb55faeed7",
              "IPY_MODEL_fbee3a73421b4b41b5d4735e15cb6801",
              "IPY_MODEL_efda9deadaad4955840db5c2bc4c2120"
            ],
            "layout": "IPY_MODEL_10add3bae44346a19df233fb22b6a505"
          }
        },
        "e8c03e546fe649faaa755a57ed58f3f2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e9bd93f7e909421caafad292b7700569": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_44d0af08a7944c83b40247edf6a66169",
            "placeholder": "​",
            "style": "IPY_MODEL_6dd3c0a270a14c2db56eeffd288a9e8f",
            "value": "100%"
          }
        },
        "efda9deadaad4955840db5c2bc4c2120": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3b17a615103747d4a599a1096ba3032f",
            "placeholder": "​",
            "style": "IPY_MODEL_51e4141f59904cf6a2dc38955bd60025",
            "value": " 3/3 [00:00&lt;00:00, 179.97it/s]"
          }
        },
        "f245ec0a619541bfbf70620df4f8d711": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f75e97ad7ee74f2d9fd1480de114913d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e8c03e546fe649faaa755a57ed58f3f2",
            "placeholder": "​",
            "style": "IPY_MODEL_858de7cf253742878c45a51064782a00",
            "value": "100%"
          }
        },
        "faaaba09b3254b098e2d519dc187f86b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d5653db415454c729210daf8a28a8c2a",
            "placeholder": "​",
            "style": "IPY_MODEL_c49990927b1847df840b334d8b63f801",
            "value": " 9/9 [00:00&lt;00:00, 17.24ba/s]"
          }
        },
        "fbee3a73421b4b41b5d4735e15cb6801": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ba8b216cc3764c03a444bda8b5286664",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0c808e4692e24c6393df72317eff5881",
            "value": 3
          }
        },
        "fc6f87d531aa45c08bc55a495e2370f6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fe7d59ee8b2c48d98a41823c9cbf42ec": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}