build.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024-06-24 11:25:43,622] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tyfei/anaconda3/envs/pytorch/compiler_compat/ld: cannot find -laio: No such file or directory\n",
      "collect2: error: ld returned 1 exit status\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[93m [WARNING] \u001b[0m async_io requires the dev libaio .so object and headers but these were not found.\n",
      "\u001b[93m [WARNING] \u001b[0m async_io: please install the libaio-dev package with apt\n",
      "\u001b[93m [WARNING] \u001b[0m If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found.\n",
      "\u001b[93m [WARNING] \u001b[0m Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH\n",
      "\u001b[93m [WARNING] \u001b[0m sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.2\n",
      "\u001b[93m [WARNING] \u001b[0m please install triton==1.0.0 if you want to use sparse attention\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import esm \n",
    "import models \n",
    "import importlib \n",
    "import pandas as pd \n",
    "import os \n",
    "import numpy as np\n",
    "import utils \n",
    "import torch.nn as nn\n",
    "from tqdm import tqdm\n",
    "from random import sample\n",
    "importlib.reload(utils)\n",
    "importlib.reload(models) \n",
    "from torch import optim\n",
    "import re\n",
    "import datautils\n",
    "importlib.reload(datautils)\n",
    "import pytorch_lightning as L\n",
    "import torch.nn.functional as F\n",
    "# from pytorch_lightning.strategies import DeepSpeedStrategy\n",
    "importlib.reload(L)\n",
    "strategy = L.strategies.DeepSpeedStrategy()\n",
    "import torch.utils.tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'read': {'fix_layers': [1, 2, 3, 4]},\n",
       " 'train': {'epoch': 15, 'lr': 0.0005, 'step': 1.5, 'step_lambda': True}}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import yaml\n",
    "a = {\"train\":{\"epoch\":15, \"lr\":5e-4, \"step_lambda\":True, \"step\":1.5}, \"read\":{\"fix_layers\":[1,2,3,4]}}\n",
    "with open(\"./file.yaml\", mode=\"wt\", encoding=\"utf-8\") as file:\n",
    "    yaml.dump(a, file)\n",
    "with open(\"./file.yaml\") as f:\n",
    "    b = yaml.safe_load(f)\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import models \n",
    "import importlib \n",
    "importlib.reload(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 3, 4]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[i for i in range(5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys \n",
    "sys.path.append(\"/home/tyfei/fun/utils\")\n",
    "import VirusDataset \n",
    "import esm \n",
    "import random\n",
    "model, alphabet = esm.pretrained.esm2_t6_8M_UR50D()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "import torch\n",
    "\n",
    "torch.set_float32_matmul_precision(\"medium\")  # make lightning happy\n",
    "# sys.path.append(\"/home/tyfei/fun/utils\")\n",
    "\n",
    "import pytorch_lightning as L\n",
    "from pytorch_lightning.callbacks import ModelCheckpoint\n",
    "from pytorch_lightning.loggers import TensorBoardLogger\n",
    "from pytorch_lightning.profilers import PyTorchProfiler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    }
   ],
   "source": [
    "k = 2\n",
    "pf = 2\n",
    "checkpoint_callback = ModelCheckpoint(\n",
    "    monitor=\"validate_acc\",  # Replace with your validation metric\n",
    "    mode=\"max\",  # 'min' if the metric should be minimized (e.g., loss), 'max' for maximization (e.g., accuracy)\n",
    "    save_top_k=k,  # Save top k checkpoints based on the monitored metric\n",
    "    save_last=True,  # Save the last checkpoint at the end of training\n",
    "    dirpath=\"/home/tyfei/ionChannel/ckpts/701pf2\",  # Directory where the checkpoints will be saved\n",
    "    filename=\"{pf}-{epoch}-{validate_acc:.2f}\",  # Checkpoint file naming pattern\n",
    ")\n",
    "logger = TensorBoardLogger(\"tb_logs\", name=\"ion_test\")\n",
    "trainer = L.Trainer(\n",
    "    logger=logger,\n",
    "    # strategy=\"DDP\",\n",
    "    accelerator=\"gpu\",\n",
    "    devices=[0],\n",
    "    max_epochs=15,\n",
    "    callbacks=[checkpoint_callback],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd5089f7ce2c4e1197827721468e57a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\">      Validate metric      </span>┃<span style=\"font-weight: bold\">       DataLoader 0        </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080\">            acc            </span>│<span style=\"color: #800080; text-decoration-color: #800080\">    0.5677664279937744     </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080\">      adversial loss       </span>│<span style=\"color: #800080; text-decoration-color: #800080\">    1.3970738649368286     </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080\">       predict loss        </span>│<span style=\"color: #800080; text-decoration-color: #800080\">     0.687456488609314     </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080\">       validate_acc        </span>│<span style=\"color: #800080; text-decoration-color: #800080\">    0.5677664279937744     </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080\">       validate_loss       </span>│<span style=\"color: #800080; text-decoration-color: #800080\">    0.8272002935409546     </span>│\n",
       "└───────────────────────────┴───────────────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1m     Validate metric     \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      DataLoader 0       \u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[36m \u001b[0m\u001b[36m           acc           \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.5677664279937744    \u001b[0m\u001b[35m \u001b[0m│\n",
       "│\u001b[36m \u001b[0m\u001b[36m     adversial loss      \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   1.3970738649368286    \u001b[0m\u001b[35m \u001b[0m│\n",
       "│\u001b[36m \u001b[0m\u001b[36m      predict loss       \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m    0.687456488609314    \u001b[0m\u001b[35m \u001b[0m│\n",
       "│\u001b[36m \u001b[0m\u001b[36m      validate_acc       \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.5677664279937744    \u001b[0m\u001b[35m \u001b[0m│\n",
       "│\u001b[36m \u001b[0m\u001b[36m      validate_loss      \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.8272002935409546    \u001b[0m\u001b[35m \u001b[0m│\n",
       "└───────────────────────────┴───────────────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[{'predict loss': 0.687456488609314,\n",
       "  'adversial loss': 1.3970738649368286,\n",
       "  'acc': 0.5677664279937744,\n",
       "  'validate_loss': 0.8272002935409546,\n",
       "  'validate_acc': 0.5677664279937744}]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import esm\n",
    "\n",
    "import VirusDataset\n",
    "\n",
    "model, alphabet = esm.pretrained.esm2_t12_35M_UR50D()\n",
    "batch_converter = alphabet.get_batch_converter()\n",
    "import random\n",
    "\n",
    "random.seed(1509)\n",
    "torch.manual_seed(1509)\n",
    "\n",
    "ds = VirusDataset.SeqdataModule()\n",
    "\n",
    "import models\n",
    "\n",
    "# model, alphabet = esm.pretrained.esm2_t12_35M_UR50D()\n",
    "# batch_converter = alphabet.get_batch_converter()\n",
    "\n",
    "clsmodel = models.ionclf(model, max_lambda=6, unfix=[\"11\"])\n",
    "trainer.validate(clsmodel, ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">A0A010SQC8|IPR025692(491...809)|Potassium transporter KefA\n",
      "MPTLRSFFAIALLGLSLSVGSLQAAEPPSAETVQKSLDKIAERKLPEADQKALQLVLQQTLTQLSNRDDYEKRLNDLKQQ\n",
      "LSTAPKQNIENQRELERLKASKAVPVEQRYANLSVPQLEQMLTERSTQQSDLQKALADANSLIITAQTRPERAQAEISTS\n",
      "QTRILQINGILKLGRDNGKALTNEQRDTLNAELAALNALIPLRRQELAGNSQLQDLGNSQHDLLLEKTARMEREIQDLQT\n",
      "LINQKRLAQSQQTVTQQSIEAQKAGGSNLLATESAANLKLSDYLLKSTDRLNELTQQNLQTKQQLDSVTQSDAALDEQIS\n",
      "VLKGSLLLSKILYKQKQALPRLTLDRNLANDIADIRLYQFEVNQQRELISTPGAYVDNLLASQPPDQVTPQLRKNLLDLA\n",
      "VTRADLLERLSRELSALLNESITLQLNQKQLLTTAQNLRATLDEQMFWIPSNKPLDFEWMRGVPDRLEKQVVTLPWASSL\n",
      "SELADGLTQRPLLFLPLVLLIGFLVWRRKYLYARLNKVHQDIGHFKRDSQWHTPQAILINVLLAMPVSLALALCGFALQI\n",
      "DARGQNANLGAALLQIAQAWLVFYTAYRILAPGGVAQLHFRWERPQVEFLRGWIRRLGLVVLALVAVVAVAELQPAALTD\n",
      "DVIGIAVVLTCYALMAWLLSRLLINSPTHNSASLFRKAVGVLFTVLPIALFIAVCFGYFYTALKLSDRLINTLYLLMFWL\n"
     ]
    }
   ],
   "source": [
    "!head /home/tyfei/datasets/ion_channel/Interprot/ion_channel/0.99/IPR025692.fasta.0.99.fas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['A', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'V', 'W', 'Y'])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mapping = {'A': 0,'C': 1,'D': 2,'E': 3,'F': 4,'G': 5,'H': 6,'I': 7,'K': 8,\n",
    " 'L': 9,'M': 10,'N': 11,'P': 12,'Q': 13,'R': 14,'S': 15,'T': 16,'V': 17,'W': 18,'Y': 19}\n",
    "mapping.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model, alphabet = esm.pretrained.esm2_t12_35M_UR50D()\n",
    "batch_converter = alphabet.get_batch_converter()\n",
    "model.eval()\n",
    "model.token_dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ion = models.ionclf(model, addadversial=False)\n",
    "ion.fixParameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IPR025692.fasta.0.99.fas True\n",
      "IPR003873.fasta.0.99.fas True\n",
      "IPR005403.fasta.0.99.fas True\n",
      "IPR003929.fasta.0.99.fas True\n",
      "cd16972.fasta.0.99.fas True\n",
      "IPR003972.fasta.0.99.fas True\n",
      "IPR006859.fasta.0.99.fas True\n",
      "IPR028746.fasta.0.99.fas True\n",
      "PF17175.fasta.0.99.fas True\n",
      "PF12795.fasta.0.99.fas True\n",
      "IPR024393.fasta.0.99.fas True\n",
      "cd01034.fasta.0.99.fas True\n",
      "IPR015449.fasta.0.99.fas True\n",
      "IPR028744.fasta.0.99.fas True\n",
      "IPR049913.fasta.0.99.fas True\n",
      "IPR028747.fasta.0.99.fas True\n",
      "IPR003947.fasta.0.99.fas True\n",
      "IPR003974.fasta.0.99.fas True\n",
      "IPR000482.fasta.0.99.fas True\n",
      "IPR044379.fasta.0.99.fas True\n",
      "IPR003950.fasta.0.99.fas True\n",
      "IPR011555.fasta.0.99.fas True\n",
      "IPR016449.fasta.0.99.fas True\n",
      "IPR029601.fasta.0.99.fas True\n",
      "cd18415.fasta.0.99.fas True\n",
      "IPR003272.fasta.0.99.fas True\n",
      "TIGR00862.fasta.0.99.fas True\n",
      "IPR008132.fasta.0.99.fas True\n",
      "cd15241.fasta.0.99.fas True\n",
      "IPR001185.fasta.0.99.fas True\n",
      "IPR024587.fasta.0.99.fas True\n",
      "IPR044377.fasta.0.99.fas True\n",
      "IPR010291.fasta.0.99.fas True\n",
      "IPR005404.fasta.0.99.fas True\n",
      "IPR005425.fasta.0.99.fas True\n",
      "IPR003270.fasta.0.99.fas True\n",
      "IPR003967.fasta.0.99.fas True\n",
      "cd10301.fasta.0.99.fas True\n",
      "IPR001520.fasta.0.99.fas True\n",
      "PF11504.fasta.0.99.fas True\n",
      "IPR004054.fasta.0.99.fas True\n",
      "IPR005405.fasta.0.99.fas True\n",
      "IPR023271.fasta.0.99.fas True\n",
      "cd15028.fasta.0.99.fas True\n",
      "PF16940.fasta.0.99.fas True\n",
      "IPR004048.fasta.0.99.fas True\n",
      "IPR003938.fasta.0.99.fas True\n",
      "IPR008133.fasta.0.99.fas True\n",
      "IPR019823.fasta.0.99.fas True\n",
      "IPR003970.fasta.0.99.fas True\n",
      "IPR006686.fasta.0.99.fas True\n",
      "IPR000431.fasta.0.99.fas True\n",
      "IPR005424.fasta.0.99.fas True\n",
      "PF02723.fasta.0.99.fas True\n",
      "IPR048786.fasta.0.99.fas True\n",
      "IPR048788.fasta.0.99.fas True\n",
      "PR00168.fasta.0.99.fas True\n",
      "IPR003973.fasta.0.99.fas True\n",
      "TIGR01271.fasta.0.99.fas True\n",
      "IPR004726.fasta.0.99.fas True\n",
      "SM01420.fasta.0.99.fas True\n",
      "cd18316.fasta.0.99.fas True\n",
      "IPR031892.fasta.0.99.fas True\n",
      "IPR000450.fasta.0.99.fas True\n",
      "IPR046988.fasta.0.99.fas True\n",
      "IPR005821.fasta.0.99.fas True\n",
      "IPR005399.fasta.0.99.fas True\n",
      "IPR010308.fasta.0.99.fas True\n",
      "IPR013821.fasta.0.99.fas True\n",
      "IPR050216.fasta.0.99.fas True\n",
      "cd15240.fasta.0.99.fas True\n",
      "IPR000377.fasta.0.99.fas True\n",
      "PR01333.fasta.0.99.fas True\n",
      "IPR005408.fasta.0.99.fas True\n",
      "IPR024701.fasta.0.99.fas True\n",
      "IPR003582.fasta.0.99.fas True\n",
      "IPR005401.fasta.0.99.fas True\n",
      "IPR044496.fasta.0.99.fas True\n",
      "IPR023408.fasta.0.99.fas True\n",
      "cd07912.fasta.0.99.fas True\n",
      "IPR001069.fasta.0.99.fas True\n",
      "IPR008074.fasta.0.99.fas True\n",
      "cd03684.fasta.0.99.fas True\n",
      "cd20323.fasta.0.99.fas True\n",
      "IPR008134.fasta.0.99.fas True\n",
      "IPR035940.fasta.0.99.fas True\n",
      "cd15238.fasta.0.99.fas True\n",
      "IPR004056.fasta.0.99.fas True\n",
      "IPR000455.fasta.0.99.fas True\n",
      "IPR003969.fasta.0.99.fas True\n",
      "cd15244.fasta.0.99.fas True\n",
      "IPR008059.fasta.0.99.fas True\n",
      "IPR005400.fasta.0.99.fas True\n",
      "IPR003278.fasta.0.99.fas True\n",
      "IPR003930.fasta.0.99.fas True\n",
      "IPR002394.fasta.0.99.fas True\n",
      "IPR003279.fasta.0.99.fas True\n",
      "IPR003269.fasta.0.99.fas True\n",
      "IPR003948.fasta.0.99.fas True\n",
      "PF15968.fasta.0.99.fas True\n",
      "cd15239.fasta.0.99.fas True\n",
      "IPR020383.fasta.0.99.fas True\n",
      "IPR002147.fasta.0.99.fas True\n",
      "IPR003268.fasta.0.99.fas True\n",
      "IPR004055.fasta.0.99.fas True\n",
      "cd11861.fasta.0.99.fas True\n",
      "IPR012897.fasta.0.99.fas True\n",
      "IPR004053.fasta.0.99.fas True\n",
      "IPR013555.fasta.0.99.fas True\n",
      "IPR003275.fasta.0.99.fas True\n",
      "IPR000369.fasta.0.99.fas True\n",
      "IPR003273.fasta.0.99.fas True\n",
      "TIGR00860.fasta.0.99.fas True\n",
      "IPR008187.fasta.0.99.fas True\n",
      "IPR038727.fasta.0.99.fas True\n",
      "IPR028751.fasta.0.99.fas True\n",
      "IPR001779.fasta.0.99.fas True\n",
      "IPR002231.fasta.0.99.fas True\n",
      "PF17523.fasta.0.99.fas True\n",
      "cd20328.fasta.0.99.fas True\n",
      "IPR003280.fasta.0.99.fas True\n",
      "PF12794.fasta.0.99.fas True\n",
      "IPR005826.fasta.0.99.fas True\n",
      "cd16965.fasta.0.99.fas True\n",
      "IPR002232.fasta.0.99.fas True\n",
      "IPR013673.fasta.0.99.fas True\n",
      "PTHR23294.fasta.0.99.fas True\n",
      "cd18372.fasta.0.99.fas True\n",
      "IPR005409.fasta.0.99.fas True\n",
      "IPR001696.fasta.0.99.fas True\n",
      "cd15242.fasta.0.99.fas True\n",
      "IPR000610.fasta.0.99.fas True\n",
      "PS51926.fasta.0.99.fas True\n",
      "IPR003976.fasta.0.99.fas True\n",
      "IPR038951.fasta.0.99.fas True\n",
      "PF17055.fasta.0.99.fas True\n",
      "cd15243.fasta.0.99.fas True\n",
      "IPR003274.fasta.0.99.fas True\n",
      "IPR030259.fasta.0.99.fas True\n",
      "IPR008058.fasta.0.99.fas True\n",
      "IPR048789.fasta.0.99.fas True\n",
      "IPR003271.fasta.0.99.fas True\n",
      "TIGR00870.fasta.0.99.fas True\n",
      "IPR021105.fasta.0.99.fas True\n",
      "IPR004350.fasta.0.99.fas True\n",
      "IPR002490.fasta.0.99.fas True\n",
      "PF05978.fasta.0.99.fas True\n",
      "IPR012496.fasta.0.99.fas True\n",
      "cd16971.fasta.0.99.fas True\n",
      "IPR045863.fasta.0.99.fas True\n",
      "IPR005827.fasta.0.99.fas True\n",
      "IPR043506.fasta.0.99.fas True\n",
      "IPR004049.fasta.0.99.fas True\n",
      "IPR029597.fasta.0.99.fas True\n",
      "PTHR10582.fasta.0.99.fas True\n",
      "IPR003691.fasta.0.99.fas True\n",
      "cd19010.fasta.0.99.fas True\n",
      "IPR014258.fasta.0.99.fas True\n",
      "IPR037673.fasta.0.99.fas True\n",
      "cd17073.fasta.0.99.fas True\n",
      "IPR005402.fasta.0.99.fas True\n",
      "IPR005410.fasta.0.99.fas True\n",
      "IPR045042.fasta.0.99.fas True\n",
      "IPR008348.fasta.0.99.fas True\n",
      "cd18594.fasta.0.99.fas True\n",
      "IPR013518.fasta.0.99.fas True\n",
      "PF10505.fasta.0.99.fas True\n",
      "IPR013122.fasta.0.99.fas True\n",
      "IPR047871.fasta.0.99.fas True\n",
      "IPR002089.fasta.0.99.fas True\n",
      "IPR000505.fasta.0.99.fas True\n",
      "IPR028823.fasta.0.99.fas True\n",
      "PF08344.fasta.0.99.fas True\n",
      "IPR000245.fasta.0.99.fas True\n",
      "IPR004052.fasta.0.99.fas True\n",
      "IPR008061.fasta.0.99.fas True\n",
      "IPR038900.fasta.0.99.fas True\n",
      "IPR001397.fasta.0.99.fas True\n",
      "cd21088.fasta.0.99.fas True\n",
      "cd11722.fasta.0.99.fas True\n",
      "TIGR00861.fasta.0.99.fas True\n",
      "IPR003949.fasta.0.99.fas True\n",
      "PTHR31618.fasta.0.99.fas True\n",
      "PS51670.fasta.0.99.fas True\n",
      "IPR008057.fasta.0.99.fas True\n",
      "cd09323.fasta.0.99.fas True\n",
      "IPR020467.fasta.0.99.fas True\n",
      "cd21873.fasta.0.99.fas True\n",
      "IPR003276.fasta.0.99.fas True\n",
      "cd19056.fasta.0.99.fas True\n",
      "PF00654.fasta.0.99.fas True\n",
      "IPR008688.fasta.0.99.fas True\n",
      "IPR003975.fasta.0.99.fas True\n",
      "IPR029569.fasta.0.99.fas True\n",
      "cd18371.fasta.0.99.fas True\n",
      "PF07810.fasta.0.99.fas True\n",
      "IPR003971.fasta.0.99.fas True\n",
      "cd14965.fasta.0.99.fas True\n",
      "IPR005983.fasta.0.99.fas True\n",
      "IPR036019.fasta.0.99.fas True\n",
      "cd21087.fasta.0.99.fas True\n",
      "IPR004050.fasta.0.99.fas True\n",
      "IPR043507.fasta.0.99.fas True\n",
      "IPR008054.fasta.0.99.fas True\n",
      "PF16799.fasta.0.99.fas True\n",
      "IPR005426.fasta.0.99.fas True\n",
      "cd18364.fasta.0.99.fas True\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_excel(\"/home/tyfei/datasets/ion_channel/Interprot/Interprot-ion-channel-judgement.xls\")\n",
    "seqs = {}\n",
    "for i in os.listdir(\"/home/tyfei/datasets/ion_channel/Interprot/ion_channel/0.99\"):\n",
    "    try:\n",
    "        if i.endswith(\".fas\"):\n",
    "            print(i, i[:i.find(\".\")] in df[\"Accession\"].values)\n",
    "            seqs[i[:i.find(\".\")]] = utils.readDataset(\"/home/tyfei/datasets/ion_channel/Interprot/ion_channel/0.99/\"+i, 300, trunc=1198)\n",
    "    except:\n",
    "        print(i, \"failed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "seqs[\"negative\"] = utils.readDataset(\"/home/tyfei/datasets/ion_channel/Interprot/Negative_sample/decoy_1m_new.fasta\", 40000, 1200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered = []\n",
    "labels = [] \n",
    "\n",
    "for i, tt in seqs.items():\n",
    "    if i == \"negative\":\n",
    "        labels.extend([0]*len(tt))\n",
    "    else:\n",
    "        labels.extend([1]*len(tt))\n",
    "    for name, seq in tt:\n",
    "        if \"*\" in seq:\n",
    "            filtered.append((name, seq[:-1].upper()))\n",
    "        else:\n",
    "            filtered.append((name, seq.upper()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(93404, 53404)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(labels), sum(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_labels, _, batch_tokens = batch_converter(filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "678139"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "allvirus = []\n",
    "for i in os.listdir(\"/home/tyfei/datasets/NCBI_virus/genbank_csv/\"):\n",
    "    df = pd.read_csv(\"/home/tyfei/datasets/NCBI_virus/genbank_csv/\"+i)\n",
    "    for j in df[\"translation\"]:\n",
    "        seq = re.sub(\"[^a-z,A-Z]\", \"\", str(j)) \n",
    "        if len(seq) < 10:\n",
    "            continue\n",
    "        if \"J\" in seq:\n",
    "            continue\n",
    "        if len(seq) > 1200:\n",
    "            seq = seq[:1200]\n",
    "        allvirus.append((\"1\", seq))\n",
    "len(allvirus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "allvirus = allvirus[:30000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_labels2, _, batch_tokens2 = batch_converter(allvirus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([30000, 1202])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_tokens2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tyfei/ionChannel/datautils.py:60: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  self.seq = torch.tensor(seq).long()\n",
      "/home/tyfei/ionChannel/datautils.py:62: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  self.seqtest = torch.tensor(seqtest).long()\n"
     ]
    }
   ],
   "source": [
    "# i = sample(range(batch_tokens2.shape[0]), len(batch_tokens))\n",
    "trainset = datautils.SeqDataset2(batch_tokens, labels, batch_tokens2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tyfei/ionChannel/datautils.py:74: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  self.seq = torch.tensor(seq).long()\n"
     ]
    }
   ],
   "source": [
    "testset  = datautils.TestDataset(batch_tokens2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = datautils.SeqdataModule(trainset, testset, batch_size=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "You are using a CUDA device ('NVIDIA GeForce RTX 4090') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "You are using a CUDA device ('NVIDIA GeForce RTX 4090') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5]\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5]\n",
      "\n",
      "  | Name      | Type             | Params | Mode \n",
      "-------------------------------------------------------\n",
      "0 | esm_model | ESM2             | 33.5 M | eval \n",
      "1 | cls       | Sequential       | 145 K  | train\n",
      "2 | dis       | Sequential       | 145 K  | train\n",
      "3 | reverse   | GradientReversal | 0      | train\n",
      "4 | acc       | BinaryAccuracy   | 0      | train\n",
      "-------------------------------------------------------\n",
      "8.6 M     Trainable params\n",
      "25.2 M    Non-trainable params\n",
      "33.8 M    Total params\n",
      "135.167   Total estimated model params size (MB)\n",
      "\n",
      "  | Name      | Type             | Params | Mode \n",
      "-------------------------------------------------------\n",
      "0 | esm_model | ESM2             | 33.5 M | eval \n",
      "1 | cls       | Sequential       | 145 K  | train\n",
      "2 | dis       | Sequential       | 145 K  | train\n",
      "3 | reverse   | GradientReversal | 0      | train\n",
      "4 | acc       | BinaryAccuracy   | 0      | train\n",
      "-------------------------------------------------------\n",
      "8.6 M     Trainable params\n",
      "25.2 M    Non-trainable params\n",
      "33.8 M    Total params\n",
      "135.167   Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c809f31f3674ad0a5963a94232c8daf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tyfei/anaconda3/envs/pytorch/lib/python3.12/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:475: Your `val_dataloader`'s sampler has shuffling enabled, it is strongly recommended that you turn shuffling off for val/test dataloaders.\n",
      "/home/tyfei/anaconda3/envs/pytorch/lib/python3.12/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:424: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=127` in the `DataLoader` to improve performance.\n",
      "/home/tyfei/anaconda3/envs/pytorch/lib/python3.12/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:424: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=127` in the `DataLoader` to improve performance.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c64ae56f27304737a04981d0dd24f1d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a054234751fd43e09cd1681fb3697d86",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2dd573b9828347b0acbdc5645f25cf40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "139d157634564d0f9b3e0644c43a3297",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "149a1415a082482fb991ef674a7d8807",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97179b6a9e0b44dbaf86e6e77437ac39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: |          | 0/? [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=5` reached.\n",
      "`Trainer.fit` stopped: `max_epochs=5` reached.\n"
     ]
    }
   ],
   "source": [
    "trainer = L.Trainer(\n",
    "        # strategy=\"ddp\",\n",
    "        devices=[1],\n",
    "        min_epochs=1,\n",
    "        max_epochs=5)\n",
    "trainer.fit(ion, ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tyfei/ionChannel/utils.py:27: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  self.seq = torch.tensor(seq).long()\n"
     ]
    }
   ],
   "source": [
    "ds = utils.SeqDataset(batch_tokens, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainloader, valloader = utils.splitDataset(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17538681344"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
      "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
      "\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "torch.cuda.memory_allocated(\"cuda:2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "esm_model.layers.9.self_attn.k_proj.weight Parameter containing:\n",
      "tensor([[-0.0335,  0.0458, -0.0815,  ..., -0.0226, -0.0247, -0.0179],\n",
      "        [ 0.0969, -0.1143,  0.1702,  ...,  0.0959, -0.0573,  0.1472],\n",
      "        [ 0.0635, -0.0958, -0.1116,  ..., -0.0831,  0.0536,  0.0624],\n",
      "        ...,\n",
      "        [-0.1055,  0.0552,  0.1266,  ..., -0.0116, -0.0106, -0.0596],\n",
      "        [ 0.0890, -0.0312, -0.1110,  ..., -0.1038, -0.0019,  0.0107],\n",
      "        [-0.1689, -0.0557, -0.0856,  ..., -0.0821,  0.0670,  0.1293]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.9.self_attn.k_proj.bias Parameter containing:\n",
      "tensor([-1.0323e-02, -4.9219e-01,  1.7517e-01, -1.0437e-01, -1.6904e+00,\n",
      "         2.5996e+00,  2.7695e+00, -6.0195e+00,  4.7266e-01, -1.9385e+00,\n",
      "        -1.1455e+00,  7.6709e-01,  8.9990e-01,  1.8384e-01,  5.8057e-01,\n",
      "         1.0752e+00, -2.4304e-01, -5.8252e-01,  2.0293e+00,  5.3076e-01,\n",
      "         2.6392e-01, -3.8354e-01,  1.2659e-01,  5.0244e-01, -2.3453e-02,\n",
      "        -1.8494e-02,  1.1517e-01, -3.2318e-02,  7.4425e-03, -4.9292e-01,\n",
      "        -1.1631e+00, -1.2280e-01, -5.6152e-02,  1.8823e-01,  6.8164e-01,\n",
      "         1.9348e-01,  1.0022e-01,  1.1269e-02, -4.1412e-02, -2.8442e-02,\n",
      "        -3.3081e-01,  7.8796e-02,  1.2688e-02,  2.0098e+00,  3.0347e-01,\n",
      "        -3.1030e-01,  1.3391e-01, -4.1260e-01, -1.1973e+00,  2.2510e-01,\n",
      "         2.4988e-01, -6.3770e-01, -1.0420e+00,  2.6001e-02,  3.9629e+00,\n",
      "         2.1069e-01,  4.3433e-01, -1.0264e+00,  1.4844e+00, -1.4648e+00,\n",
      "         5.5762e-01, -3.2397e-01,  3.5571e-01,  2.3608e-01,  1.9482e-01,\n",
      "        -2.1758e+00, -4.8584e-01,  5.6836e+00, -6.0693e-01,  2.6113e+00,\n",
      "        -4.0845e-01, -4.4586e-02,  2.4719e-02, -4.2084e-02, -7.9468e-02,\n",
      "         1.5601e-01,  8.8806e-02,  1.9421e-01,  1.7090e-01, -2.3779e-01,\n",
      "        -8.0139e-02,  2.8833e-01, -6.3843e-02,  3.7964e-01, -4.1290e-02,\n",
      "        -1.5306e-03,  9.3140e-02, -1.4893e-01,  3.2446e-01,  5.2002e-01,\n",
      "        -1.3594e+00,  2.7676e+00,  4.7485e-02, -2.7710e-01,  2.6587e-01,\n",
      "        -1.5051e-01,  1.8433e-01, -1.1591e-01, -2.6904e-01, -4.1077e-02,\n",
      "         4.8071e-01,  6.0852e-02,  2.1423e-01,  4.6758e+00,  2.5162e-02,\n",
      "        -3.0859e-01, -9.6094e-01, -3.9453e-01,  4.4586e-02,  2.7542e-02,\n",
      "         1.2451e-01, -3.7036e-01,  3.8306e-01, -1.4551e+00, -2.7812e+00,\n",
      "         1.2286e-01, -1.3416e-01,  8.6523e-01,  2.5806e-01, -3.4821e-02,\n",
      "        -1.4075e-01, -1.5332e-01,  2.2510e-01, -2.8198e-01,  5.4736e-01,\n",
      "        -1.2109e+00,  3.4302e-01,  1.6724e-01,  1.6357e-01,  5.9131e-01,\n",
      "         3.9990e-01, -1.4328e-02, -1.4441e-01, -8.8928e-02, -2.1753e-01,\n",
      "        -1.4282e-01, -4.7778e-01,  3.7598e-01,  2.3281e+00,  4.2422e+00,\n",
      "         5.6104e-01,  2.3853e-01, -2.8345e-01, -5.3284e-02, -2.2415e-02,\n",
      "         2.8580e-02,  1.9913e-02,  2.0935e-02,  7.0312e-02, -1.7847e-01,\n",
      "         5.4248e-01,  6.6467e-02,  1.3940e-01,  2.0300e-01, -1.4294e-01,\n",
      "        -1.5271e-01,  2.4048e-02,  1.2779e-02,  3.2253e-03, -1.6312e-02,\n",
      "        -4.8523e-02, -5.0323e-02,  3.6697e-03,  1.0459e+00,  5.6976e-02,\n",
      "         1.9989e-02,  3.2410e-02, -6.5765e-03, -3.6163e-02,  3.3813e-02,\n",
      "         6.4209e-02, -3.5919e-02, -1.3025e-01,  4.1357e-01,  1.2666e+00,\n",
      "        -1.3750e+00,  7.2998e-02, -1.8298e-01,  2.2827e-01,  1.3440e-01,\n",
      "         3.9215e-02, -2.9266e-02, -7.5645e-03, -1.7847e-01,  1.3586e-01,\n",
      "        -2.4561e-01, -7.9407e-02,  7.8076e-01, -1.8814e-02,  2.4292e-01,\n",
      "        -1.6870e-01, -1.4404e-01,  1.4137e-02, -6.1829e-02, -1.4758e-01,\n",
      "        -1.2805e-01,  2.5562e-01,  3.6841e-01, -9.5398e-02,  1.8379e+00,\n",
      "         1.5955e-01,  1.0638e-01,  2.8540e-01, -3.4253e-01, -8.7463e-02,\n",
      "        -1.2146e-01,  1.5454e-01,  8.2245e-03, -2.7686e-01, -7.4512e-01,\n",
      "        -2.4180e+00, -1.6416e+00,  8.4351e-02, -4.8523e-02,  5.4993e-02,\n",
      "        -3.9581e-02,  1.7538e-03,  2.0630e-02, -3.7781e-02, -2.4841e-02,\n",
      "         1.7102e-01, -1.6296e-02, -1.0522e-01,  2.2424e-01, -7.1045e-02,\n",
      "         3.2202e-01,  2.0215e-01, -2.2351e-01,  3.5858e-03, -4.4403e-03,\n",
      "        -1.1429e-02,  3.4058e-02, -5.1384e-03, -8.8730e-03,  3.1738e-01,\n",
      "         2.4475e-01, -7.6416e-02, -4.2065e-01, -4.4189e-02,  3.5596e-01,\n",
      "        -1.2268e-02,  3.0022e-03, -1.2779e-02,  1.3901e-02, -7.5378e-02,\n",
      "        -1.0468e-01, -6.7932e-02, -3.2166e-02,  1.0651e-01,  1.9153e-01,\n",
      "        -1.1145e-01,  1.9373e-01,  2.5955e-02, -4.2755e-02,  3.9642e-02,\n",
      "        -1.4587e-02,  4.4922e-02,  5.2681e-03, -2.4243e-01, -7.5928e-01,\n",
      "        -2.1271e-02,  8.2336e-02, -4.3610e-02, -1.1597e-01,  3.4326e-01,\n",
      "        -1.3098e-01,  3.6108e-01, -7.1826e-01,  8.0420e-01, -1.8469e-01,\n",
      "         1.2383e+00,  1.5186e-01,  6.5576e-01,  5.0000e-01, -7.3364e-02,\n",
      "         1.0615e+00,  3.7036e-01, -1.8896e-01, -3.8306e-01, -2.4719e-01,\n",
      "         6.8506e-01,  2.3027e+00, -2.1680e+00, -5.8984e+00, -2.2766e-01,\n",
      "        -5.5371e-01,  5.2344e-01, -5.7220e-02,  5.1666e-02,  5.2673e-02,\n",
      "        -4.6173e-02, -5.3444e-03, -6.3324e-03,  4.9408e-02,  1.0176e+00,\n",
      "         3.3145e+00, -2.5244e-01,  1.4580e-02,  5.5273e-01, -3.3105e-01,\n",
      "         5.5267e-02,  1.6251e-02, -2.3572e-01, -6.9946e-02,  3.0469e-01,\n",
      "        -6.8408e-01,  5.9961e-01, -7.8979e-02,  1.3748e-02,  1.4905e-01,\n",
      "         2.8275e-02,  2.1411e-01,  1.8555e-02,  1.0269e-02, -2.1408e-02,\n",
      "        -5.1498e-03, -3.9032e-02,  1.0849e-02,  1.9153e-01,  7.9895e-02,\n",
      "        -9.4971e-02, -1.0907e-01,  6.4453e-02, -1.5662e-01,  2.7664e-02,\n",
      "        -3.6774e-02,  4.0588e-02, -1.2756e-01, -1.1612e-02,  1.2457e-01,\n",
      "         1.3208e-01, -7.5928e-01, -1.5979e-01,  1.6235e-01, -4.4019e-01,\n",
      "        -8.3191e-02, -4.5471e-02, -7.0496e-02,  1.9989e-02, -9.1370e-02,\n",
      "         1.4734e-01,  1.9849e-01, -5.8075e-02, -7.3013e-03, -3.2471e-02,\n",
      "         1.7175e-01, -5.5695e-02,  1.2317e-01, -7.2327e-02, -1.4267e-03,\n",
      "         8.8135e-02, -1.1658e-01,  1.6528e-01, -4.1260e-01, -1.6807e+00,\n",
      "        -2.6562e+00, -7.0984e-02,  4.6973e-01,  1.8909e-01, -2.1228e-01,\n",
      "        -1.2337e-02,  3.9368e-02, -1.8875e-02, -2.4750e-02,  8.3740e-02,\n",
      "         1.0352e-01, -3.5522e-02,  4.6310e-03, -3.4033e-01,  1.7615e-01,\n",
      "        -1.2274e-01, -2.4927e-01, -2.8885e-02, -3.8376e-03, -9.0179e-03,\n",
      "         4.3678e-03,  2.9205e-02,  1.2585e-01, -1.3879e-01,  1.3838e+00,\n",
      "        -5.1880e-03,  2.3438e-01, -2.2363e-01, -1.6284e-01,  1.4453e-01,\n",
      "         6.6452e-03, -4.4327e-03,  6.8604e-02, -5.5518e-01, -1.3105e+00,\n",
      "        -2.0337e-01, -3.9688e+00,  8.5876e-02,  4.7656e-01,  4.1382e-01,\n",
      "        -6.0156e-01, -4.0405e-01,  1.3904e-01,  5.4626e-02, -2.5903e-01,\n",
      "        -1.6492e-01, -1.7310e-01,  2.6992e+00,  1.9531e-01, -7.9712e-02,\n",
      "        -2.5195e-01,  1.5833e-01,  3.8208e-02,  2.7969e-02, -1.7914e-02,\n",
      "         5.3680e-02, -3.1143e-02,  9.5215e-02, -1.2253e-02,  1.3196e-01,\n",
      "         6.9434e-01, -8.4152e-03,  2.6505e-02, -8.2458e-02,  1.7883e-01,\n",
      "        -1.1917e-02, -3.8834e-03, -1.4206e-02,  7.4951e-02,  4.9316e-02,\n",
      "        -8.4229e-02,  3.4619e-01,  4.3652e-01, -3.3154e-01, -2.3328e-01,\n",
      "        -4.5074e-02, -8.6487e-02,  1.3037e-03, -3.3142e-02,  3.2471e-02,\n",
      "         5.1605e-02,  6.5796e-02,  7.1167e-02, -3.9038e-01, -2.4281e-03,\n",
      "        -1.0620e-01, -5.0879e-01, -2.7954e-01,  2.0798e-02,  1.7853e-03,\n",
      "         7.0915e-03, -1.0933e-02,  5.1147e-02,  4.2664e-02,  1.1414e-01,\n",
      "        -1.0175e-01,  1.2158e+00,  3.6963e-01, -7.0190e-02,  3.1769e-02,\n",
      "         6.0577e-02, -1.6373e-02, -7.3719e-04,  2.0154e-01, -1.3171e-01,\n",
      "        -3.9697e-01,  3.0762e-01,  4.4189e-01,  1.2939e+00, -1.3269e-01,\n",
      "         5.9863e-01,  3.8110e-01, -2.1423e-01, -7.2693e-02,  9.5032e-02,\n",
      "        -3.2867e-02, -1.6785e-01, -3.5596e-01,  1.0742e+00, -2.3535e+00,\n",
      "        -2.7852e+00, -2.7856e-01, -4.2163e-01,  7.7734e-01, -1.6357e-01],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.9.self_attn.v_proj.weight Parameter containing:\n",
      "tensor([[ 0.1119,  0.0597,  0.0826,  ...,  0.0280,  0.0332, -0.0181],\n",
      "        [ 0.0610, -0.0928, -0.0203,  ..., -0.0709, -0.0416,  0.0767],\n",
      "        [ 0.1244,  0.1273, -0.0111,  ...,  0.0921,  0.1567, -0.0716],\n",
      "        ...,\n",
      "        [-0.0421,  0.1654, -0.0770,  ..., -0.0450, -0.1622,  0.0303],\n",
      "        [-0.0424,  0.1130, -0.0215,  ...,  0.0213, -0.0255, -0.1409],\n",
      "        [-0.0458, -0.0550,  0.0297,  ...,  0.0419,  0.0720,  0.0118]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.9.self_attn.v_proj.bias Parameter containing:\n",
      "tensor([-1.6083e-02,  4.7699e-02,  1.4999e-02, -3.1677e-02, -2.9785e-02,\n",
      "        -2.8748e-02,  1.0216e-02,  1.3840e-02, -9.9640e-03,  5.4596e-02,\n",
      "        -3.5980e-02,  1.0895e-02,  2.9633e-02,  3.2410e-02, -1.1398e-02,\n",
      "        -6.7787e-03,  3.7964e-02,  2.3788e-02, -2.1439e-02, -5.1392e-02,\n",
      "        -1.1414e-02,  4.3823e-02,  1.1551e-02, -2.3331e-02, -6.9389e-03,\n",
      "         9.1171e-03,  2.4002e-02,  7.2136e-03,  3.0289e-03, -3.7823e-03,\n",
      "         3.8567e-03,  5.2071e-03, -1.1017e-02, -1.3130e-02,  3.4821e-02,\n",
      "        -1.3359e-02, -1.3176e-02,  6.2656e-04, -4.9782e-03, -1.0471e-03,\n",
      "        -4.5967e-03,  3.9612e-02,  5.6877e-03,  2.1057e-02,  1.4099e-02,\n",
      "        -8.5068e-04, -3.2196e-02,  4.2648e-03, -2.4724e-04, -1.7349e-02,\n",
      "        -1.8635e-03,  1.8097e-02,  4.9255e-02,  2.8515e-04,  4.0863e-02,\n",
      "        -2.3193e-03, -3.5278e-02,  1.4481e-02,  3.2406e-03, -1.2436e-02,\n",
      "        -5.9624e-03, -1.5343e-02,  9.5062e-03, -2.7237e-02,  2.6733e-02,\n",
      "        -1.2070e-02,  3.5278e-02,  1.6449e-02, -1.9974e-02, -2.9404e-02,\n",
      "        -3.0701e-02, -3.2562e-02, -4.2458e-03, -4.9782e-03,  1.1551e-02,\n",
      "         3.3966e-02,  3.0270e-03,  5.2376e-03,  2.5772e-02,  5.3406e-03,\n",
      "         4.1580e-03,  1.6174e-02, -4.4403e-03,  9.0942e-03, -3.1395e-03,\n",
      "         2.0111e-02,  2.8061e-02, -3.0632e-03, -4.0913e-04,  1.5297e-02,\n",
      "        -1.0033e-02, -2.4643e-02,  1.9241e-02,  3.1281e-02,  1.5320e-02,\n",
      "         1.3351e-02, -4.0985e-02,  1.7365e-02,  5.7037e-02,  4.4975e-03,\n",
      "         4.6654e-03,  3.3478e-02,  2.5997e-03,  2.6413e-02,  6.7482e-03,\n",
      "         7.4244e-04,  5.8250e-03,  1.8721e-03, -3.3173e-02, -3.4576e-02,\n",
      "         1.0815e-03,  7.8201e-03,  1.3471e-05,  2.2583e-02,  2.4780e-02,\n",
      "         1.3687e-02, -1.5671e-02,  1.0696e-02,  4.0770e-04, -1.8921e-02,\n",
      "         9.3231e-03, -1.1612e-02, -4.5441e-02, -2.8248e-03, -4.0161e-02,\n",
      "        -2.3285e-02,  1.1017e-02, -2.8824e-02,  1.3905e-03, -2.6657e-02,\n",
      "         2.1866e-02,  1.3336e-02,  1.5884e-02, -3.3722e-03,  4.0741e-02,\n",
      "         3.1921e-02, -5.5275e-03, -7.7820e-03,  1.3649e-02, -1.3977e-02,\n",
      "        -6.9046e-03, -1.8921e-02, -2.6779e-02,  4.8248e-02,  1.4763e-03,\n",
      "         2.5162e-02, -2.6596e-02, -5.7907e-03, -1.0170e-02,  6.3896e-03,\n",
      "        -1.0559e-02,  4.7455e-03, -1.3069e-02,  2.0294e-02, -8.9340e-03,\n",
      "        -1.1681e-02, -4.4830e-02, -2.8778e-02, -3.5339e-02,  1.9333e-02,\n",
      "        -2.3537e-03,  2.4834e-03,  1.1063e-02,  2.6245e-02,  2.8976e-02,\n",
      "         2.7573e-02, -1.4671e-02,  1.1620e-02,  9.0256e-03,  3.8223e-03,\n",
      "        -2.9709e-02,  7.6389e-04, -1.0941e-02,  4.4830e-02,  1.3336e-02,\n",
      "         8.5907e-03,  3.6297e-03,  2.1347e-02, -3.2997e-04, -2.0737e-02,\n",
      "         2.0691e-02, -1.9989e-02,  9.1629e-03, -2.2293e-02,  8.9645e-03,\n",
      "        -2.8305e-02,  2.0264e-02,  1.5915e-02,  1.5762e-02, -2.4757e-03,\n",
      "        -1.0239e-02, -6.6757e-03,  1.2177e-02,  1.9257e-02, -9.6359e-03,\n",
      "        -3.7575e-03, -1.9388e-03, -2.2030e-03, -5.3406e-03,  7.8201e-03,\n",
      "         2.7588e-02,  7.1869e-03, -1.7405e-05,  1.9470e-02,  3.8208e-02,\n",
      "         2.6062e-02, -3.8738e-03,  1.7014e-03, -7.2098e-03, -3.2539e-03,\n",
      "         1.9775e-02, -8.5602e-03, -1.0567e-03, -1.8295e-02, -9.1858e-03,\n",
      "        -1.1879e-02,  1.2100e-02,  2.2781e-02, -4.9019e-03, -8.8196e-03,\n",
      "        -1.9257e-02,  3.4294e-03, -8.6899e-03, -6.9666e-04,  3.8815e-03,\n",
      "         1.6365e-03, -8.7433e-03,  4.9858e-03,  4.4785e-03, -1.3847e-02,\n",
      "         3.6049e-03,  1.0284e-02,  7.3662e-03, -4.4403e-03, -9.7656e-04,\n",
      "        -1.1873e-03, -6.4621e-03,  2.7924e-03, -6.8970e-03, -8.8730e-03,\n",
      "        -1.4629e-03, -1.9485e-02, -3.7861e-04, -3.5133e-03, -1.0216e-02,\n",
      "         6.1188e-03,  1.6251e-02,  5.7640e-03, -1.5778e-02, -1.5160e-02,\n",
      "         6.1226e-03, -8.8959e-03,  1.0445e-02, -1.2085e-02,  2.6550e-03,\n",
      "        -5.0850e-03, -2.6733e-02,  1.4648e-02, -2.5391e-02,  1.9169e-03,\n",
      "         7.5340e-03,  9.5673e-03, -4.5837e-02,  8.8120e-03, -7.0534e-03,\n",
      "         2.9449e-02,  2.8095e-03, -4.5509e-03, -4.1847e-03, -1.1154e-02,\n",
      "         3.9749e-03, -1.1192e-02, -1.7090e-02, -1.9180e-02,  5.0583e-03,\n",
      "         1.6632e-02,  1.7944e-02,  1.7593e-02, -9.2697e-03, -1.2657e-02,\n",
      "         3.8891e-03, -2.2030e-03, -2.8198e-02,  7.0374e-02, -3.3081e-02,\n",
      "        -2.4048e-02,  3.5191e-03,  4.5532e-02, -1.6479e-02, -1.2642e-02,\n",
      "        -1.3390e-02,  1.4603e-02, -2.6047e-02, -1.0857e-02, -1.1459e-02,\n",
      "        -5.1689e-03,  2.7420e-02,  2.0477e-02, -3.0155e-03, -8.0414e-03,\n",
      "        -5.9586e-03,  5.1918e-03,  1.4420e-02,  1.3496e-02, -2.7065e-03,\n",
      "        -1.8677e-02, -2.1652e-02,  1.2741e-02, -8.0261e-03,  9.3002e-03,\n",
      "         1.0780e-02,  8.4839e-03,  3.5133e-03,  6.9466e-03,  4.3335e-03,\n",
      "        -2.3056e-02,  1.6525e-02, -7.9727e-03, -1.6357e-02, -4.6417e-02,\n",
      "         6.5575e-03,  2.2614e-02, -1.8272e-03, -2.8591e-03, -1.0271e-03,\n",
      "         1.4238e-03, -9.8114e-03, -2.0523e-03,  1.9012e-02,  1.3514e-03,\n",
      "        -2.3346e-02,  6.4545e-03, -2.2644e-02, -1.1833e-02, -3.1586e-03,\n",
      "        -1.5114e-02,  4.2648e-03, -7.8964e-03, -1.0185e-02, -9.0332e-03,\n",
      "        -9.3460e-03, -9.0408e-03, -3.5095e-03,  1.0475e-02, -2.5234e-03,\n",
      "        -5.8479e-03,  2.3209e-02,  6.8359e-03, -2.3132e-02, -6.4735e-03,\n",
      "        -1.3931e-02, -1.0506e-02,  7.1144e-03, -2.0187e-02,  2.2926e-03,\n",
      "        -1.0628e-02,  1.1772e-02, -7.0534e-03, -1.7166e-02,  4.2748e-04,\n",
      "        -1.1002e-02,  2.5635e-02, -8.3008e-03,  1.7443e-03,  6.0606e-04,\n",
      "        -2.1286e-02, -3.4752e-03, -1.7288e-02,  7.2365e-03,  1.0023e-03,\n",
      "         2.6367e-02, -2.7527e-02, -7.8201e-03, -1.3405e-02, -1.3443e-02,\n",
      "         1.2980e-03,  1.5060e-02, -5.9738e-03, -9.7961e-03,  3.8528e-03,\n",
      "        -5.8460e-04, -7.4692e-03,  6.5498e-03,  6.5804e-03,  5.5466e-03,\n",
      "         1.9287e-02,  3.5858e-02,  2.2324e-02, -3.1424e-04,  3.6102e-02,\n",
      "        -2.5620e-02,  2.3300e-02, -4.3793e-03,  1.5686e-02, -2.5711e-02,\n",
      "         6.7759e-04, -2.7512e-02,  1.0376e-02, -3.3356e-02,  2.0752e-03,\n",
      "         2.2034e-02,  3.6591e-02, -2.0695e-04,  1.0826e-02,  2.8244e-02,\n",
      "        -2.1301e-02, -6.0768e-03, -5.8479e-03, -5.9433e-03, -2.8725e-03,\n",
      "        -1.0429e-02, -1.3756e-02, -1.2344e-02,  1.8341e-02, -4.6234e-03,\n",
      "        -2.5158e-03, -1.9089e-02,  1.6266e-02, -1.1032e-02, -1.7822e-02,\n",
      "         6.1417e-03, -1.0025e-02,  5.4817e-03,  5.4092e-03,  9.4986e-03,\n",
      "         5.9319e-03, -1.2794e-02,  3.0670e-03,  1.3115e-02, -1.3565e-02,\n",
      "         2.2171e-02, -1.4099e-02, -2.1713e-02,  5.0774e-03,  1.3657e-02,\n",
      "        -3.3798e-03,  2.4384e-02, -3.7933e-02, -2.0409e-03,  3.3508e-02,\n",
      "        -1.3962e-02,  2.3163e-02,  2.2263e-02,  1.0216e-02,  1.3138e-02,\n",
      "        -3.1128e-03,  2.4948e-02,  1.2817e-02, -2.0325e-02,  2.4277e-02,\n",
      "         2.1820e-02,  1.2688e-02, -8.3008e-03, -1.3123e-02, -1.5282e-02,\n",
      "        -1.4397e-02,  1.5045e-02,  4.3182e-02, -6.6757e-03, -2.4063e-02,\n",
      "         1.0452e-02, -2.5833e-02, -1.2093e-02,  1.5541e-02, -3.3112e-02,\n",
      "        -3.2410e-02, -3.6602e-03, -1.0704e-02,  2.2812e-02, -4.7278e-04,\n",
      "         1.2642e-02,  4.3274e-02, -1.4641e-02,  3.9001e-02,  2.0706e-02,\n",
      "         2.2339e-02, -2.3632e-03,  1.5556e-02,  1.7715e-02,  3.7781e-02],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.9.self_attn.q_proj.weight Parameter containing:\n",
      "tensor([[-0.1218,  0.0409,  0.0202,  ...,  0.0365, -0.0443,  0.0035],\n",
      "        [-0.0601,  0.0576, -0.0375,  ..., -0.0513, -0.0050,  0.0098],\n",
      "        [-0.0804, -0.1359, -0.0231,  ..., -0.1005, -0.1031, -0.0858],\n",
      "        ...,\n",
      "        [-0.0674,  0.0615, -0.0456,  ..., -0.0389,  0.0511, -0.1294],\n",
      "        [ 0.1305,  0.1076, -0.0776,  ..., -0.0760, -0.0115, -0.0345],\n",
      "        [ 0.1063, -0.0371,  0.2515,  ..., -0.0269, -0.1229, -0.0515]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.9.self_attn.q_proj.bias Parameter containing:\n",
      "tensor([-3.1921e-02, -1.4795e-01, -1.8652e-01, -1.5417e-01, -3.4131e-01,\n",
      "         3.8208e-01,  2.5659e-01, -3.8257e-01, -6.0141e-05, -5.6427e-02,\n",
      "        -1.0052e-01,  1.7920e-01, -5.9521e-01, -3.5449e-01,  3.3691e-01,\n",
      "         3.6719e-01, -8.8684e-02, -3.7537e-02,  2.0020e-01,  8.8730e-03,\n",
      "        -2.0645e-02, -2.7771e-03,  1.4626e-02,  2.2375e-01, -1.6586e-02,\n",
      "         3.5477e-03,  2.1164e-02,  8.5327e-02, -1.0712e-01, -2.5146e-01,\n",
      "        -2.8784e-01, -7.7057e-03, -8.7128e-03,  5.6396e-02, -2.9602e-02,\n",
      "         4.3091e-02,  3.9795e-02, -3.9917e-02, -9.2834e-02,  1.4050e-01,\n",
      "        -2.5659e-01,  1.2018e-01, -5.0171e-02,  3.4863e-01,  2.1423e-02,\n",
      "        -1.5671e-02,  1.4473e-02,  2.7466e-02,  9.5825e-02, -2.4072e-01,\n",
      "         2.6978e-01, -3.0029e-01, -3.2422e-01, -1.1070e-02,  3.4619e-01,\n",
      "         4.0527e-02,  8.2703e-03, -3.6591e-02,  1.4648e-01, -2.2913e-01,\n",
      "        -5.5908e-01, -1.4636e-01,  1.3403e-01,  2.2778e-01,  1.1298e-01,\n",
      "        -3.4424e-01, -2.5589e-02,  3.5059e-01, -6.9771e-03,  1.3940e-01,\n",
      "        -7.5684e-02,  2.3376e-02, -5.3978e-03,  2.8107e-02,  4.8737e-02,\n",
      "         2.3285e-02, -5.0110e-02,  9.3384e-02,  7.8491e-02, -5.5267e-02,\n",
      "        -3.0899e-03,  3.6526e-04,  3.0014e-02,  1.0956e-02, -2.5635e-02,\n",
      "         1.4992e-02, -3.2471e-02,  2.0154e-01,  1.9543e-01,  3.0737e-01,\n",
      "        -3.4375e-01,  3.5205e-01, -1.1429e-02, -2.9648e-02,  2.1072e-02,\n",
      "        -3.9886e-02,  8.9783e-02,  1.0297e-01, -7.6111e-02,  1.8005e-01,\n",
      "         1.7004e-01,  9.1370e-02,  9.6130e-03,  3.7646e-01, -5.1460e-03,\n",
      "         1.6527e-03, -6.9237e-03, -2.1410e-04, -6.5369e-02,  1.1151e-01,\n",
      "        -2.4207e-01, -2.2742e-01,  2.4939e-01, -3.2373e-01, -3.4497e-01,\n",
      "         7.8735e-03,  2.4281e-03,  1.4755e-02, -2.0584e-02,  1.6769e-02,\n",
      "         2.9709e-02,  1.4648e-01, -1.7908e-01, -2.2644e-01,  1.4368e-01,\n",
      "        -2.9468e-01,  5.4749e-02,  1.6769e-02, -7.5378e-03,  2.3621e-02,\n",
      "         3.6102e-02, -3.5919e-02,  6.6711e-02, -1.9623e-02, -9.5825e-02,\n",
      "         4.7638e-02, -2.4976e-01,  1.4014e-01,  3.0835e-01,  3.4814e-01,\n",
      "        -1.5823e-02,  3.4088e-02,  3.5187e-02,  2.1759e-02,  6.3934e-03,\n",
      "         1.9272e-02,  3.4851e-02, -5.8563e-02, -1.0431e-01, -2.5049e-01,\n",
      "         3.0322e-01,  1.6754e-02,  1.5465e-02, -3.9825e-02,  2.5082e-03,\n",
      "        -3.7750e-02, -2.9633e-02,  2.9236e-02,  4.3945e-02,  1.0643e-02,\n",
      "         7.0312e-02, -3.6987e-02,  9.6817e-03,  4.2651e-01,  2.4307e-02,\n",
      "         2.2376e-04, -4.9774e-02,  5.2521e-02, -5.8327e-03, -6.3232e-02,\n",
      "        -1.1371e-01, -8.2153e-02,  1.4294e-01,  2.4756e-01,  3.2080e-01,\n",
      "        -3.1030e-01, -3.3417e-02,  1.7433e-03, -3.3203e-02, -9.4223e-03,\n",
      "        -4.4861e-03,  2.8137e-02,  5.2528e-03,  1.5259e-01,  1.7029e-01,\n",
      "        -1.8506e-01, -7.0679e-02,  2.0642e-01, -3.1494e-02,  2.4704e-02,\n",
      "        -1.0948e-02,  4.8828e-03,  3.1372e-02, -1.0425e-01,  1.2500e-01,\n",
      "         2.1802e-01,  1.6919e-01,  1.1066e-01, -4.5685e-02,  2.3035e-01,\n",
      "        -1.3603e-02,  2.9236e-02, -1.8829e-02, -7.5867e-02,  4.5319e-02,\n",
      "         1.4931e-02, -1.8066e-01,  8.8196e-02, -2.3840e-01, -2.5781e-01,\n",
      "        -3.3423e-01, -1.9080e-01,  4.6654e-03, -1.2444e-02,  1.2604e-02,\n",
      "        -1.1528e-02, -3.7231e-02,  4.4586e-02, -1.6968e-02, -3.4332e-02,\n",
      "         3.0487e-02, -1.1993e-02, -7.0740e-02,  1.0925e-01, -4.7241e-02,\n",
      "         2.7808e-01, -6.0638e-02,  9.6313e-02,  2.6536e-04, -1.5526e-02,\n",
      "        -7.3471e-03,  2.2568e-02,  8.3160e-03,  1.2598e-01,  1.2927e-01,\n",
      "         1.3232e-01,  9.3746e-04, -3.3545e-01, -4.0009e-02,  2.2974e-01,\n",
      "        -3.6407e-02,  3.4424e-02,  2.7939e-02, -6.8848e-02, -4.5593e-02,\n",
      "        -2.8662e-01, -9.3872e-02, -8.9111e-03, -3.9406e-03, -1.7517e-02,\n",
      "         1.5063e-01, -9.2041e-02, -4.4098e-03, -1.7319e-02,  6.6589e-02,\n",
      "         7.3914e-02,  1.1981e-01,  4.5471e-02, -2.8491e-01, -3.5425e-01,\n",
      "         1.0452e-02, -2.8824e-02,  5.9052e-02, -1.3782e-01, -2.4231e-01,\n",
      "         7.7515e-02,  3.1763e-01, -2.7539e-01,  2.1960e-01, -5.3467e-02,\n",
      "         1.3538e-01,  1.2909e-02,  2.8381e-02, -7.4921e-03,  6.0425e-03,\n",
      "         1.2152e-01, -1.9263e-01, -1.9019e-01, -9.3506e-02, -1.5173e-01,\n",
      "         2.1155e-01,  3.0493e-01, -2.3621e-01, -3.7476e-01,  2.0905e-02,\n",
      "        -5.1041e-03,  8.2855e-03, -1.5236e-02,  7.5745e-02,  8.8623e-02,\n",
      "        -2.6321e-02, -1.0724e-01,  1.9177e-01, -3.4637e-02,  2.9932e-01,\n",
      "         3.6060e-01, -2.6718e-02, -1.6602e-02,  4.0955e-02, -5.4283e-03,\n",
      "         5.2032e-02, -5.2551e-02, -1.9873e-01, -2.1716e-01,  2.5269e-01,\n",
      "        -3.2642e-01,  8.4045e-02, -1.5083e-02,  2.4414e-02,  5.9265e-02,\n",
      "         1.4740e-02, -1.1749e-02,  2.0538e-02,  4.8767e-02, -3.1872e-03,\n",
      "        -9.2041e-02, -1.2927e-01,  1.1786e-01,  2.0300e-01,  2.4750e-02,\n",
      "        -4.9927e-02, -1.6577e-01,  1.0553e-01,  1.2280e-01,  1.7090e-02,\n",
      "         9.1171e-03, -1.0681e-02, -3.3783e-02,  1.7099e-03,  1.5125e-01,\n",
      "         1.3220e-01, -3.0078e-01, -6.8909e-02,  3.7567e-02,  2.1399e-01,\n",
      "        -1.3647e-01,  3.7018e-02,  7.0190e-03,  9.8511e-02,  5.5359e-02,\n",
      "        -1.9312e-01,  1.9934e-01,  1.6739e-02, -2.5635e-02,  4.2511e-02,\n",
      "        -1.1055e-02,  2.3605e-02,  5.0659e-02, -5.9235e-02,  6.5460e-03,\n",
      "         2.2095e-02,  6.3843e-02,  1.5515e-01, -2.5391e-01, -3.3325e-01,\n",
      "        -3.3716e-01,  1.6586e-02,  1.5396e-02,  2.2018e-02, -7.3433e-03,\n",
      "         6.9275e-03,  6.0120e-03, -1.1864e-02,  3.7415e-02,  7.0190e-02,\n",
      "        -7.9163e-02,  3.2745e-02, -1.5457e-02, -9.2896e-02, -8.0750e-02,\n",
      "         1.4648e-03, -1.0583e-01, -4.0253e-02, -1.6846e-02, -3.4332e-02,\n",
      "         6.7854e-04,  1.0272e-01,  2.8076e-01,  1.2260e-02,  4.7412e-01,\n",
      "         4.3945e-02,  2.9892e-02,  5.0110e-02,  5.7434e-02,  1.4221e-01,\n",
      "         1.3220e-01, -9.5459e-02,  4.7516e-02, -3.1323e-01, -3.0322e-01,\n",
      "         2.7985e-02, -3.1348e-01, -4.1687e-02,  8.7500e-04,  2.0798e-02,\n",
      "        -4.9957e-02, -2.5610e-01,  1.1566e-01, -1.8509e-02, -2.8931e-01,\n",
      "        -1.1475e-01, -3.0518e-02,  3.1177e-01,  4.9438e-02,  4.6082e-02,\n",
      "         5.1270e-03, -3.6163e-02,  8.7097e-02,  3.3844e-02, -5.5542e-03,\n",
      "         6.2012e-02, -2.0950e-02, -2.3096e-01, -1.3745e-01,  1.5112e-01,\n",
      "         2.6831e-01, -6.2805e-02, -6.3721e-02,  2.3254e-02,  1.2585e-01,\n",
      "        -2.2491e-02, -1.9943e-02, -5.3162e-02,  1.1774e-01, -9.1431e-02,\n",
      "        -1.3269e-01,  2.6050e-01,  2.1008e-01, -1.1299e-02, -5.5359e-02,\n",
      "         1.9836e-02, -7.7637e-02,  2.6520e-02, -1.2035e-03,  6.1768e-02,\n",
      "        -4.1901e-02,  1.2305e-01,  2.0190e-01, -2.7075e-01,  6.3538e-02,\n",
      "        -8.0872e-02, -1.1932e-01, -2.2034e-02, -1.3840e-02, -8.6060e-03,\n",
      "         3.1708e-02,  1.8005e-02,  5.4901e-02, -8.0872e-02,  1.2659e-01,\n",
      "        -4.3457e-02,  3.8452e-01,  1.1420e-01,  3.8391e-02,  2.9236e-02,\n",
      "         9.7656e-03, -1.3538e-01,  6.0394e-02, -1.1310e-01, -1.0376e-01,\n",
      "        -2.6343e-01,  1.0431e-01,  7.2876e-02,  1.5564e-01, -2.4002e-02,\n",
      "         1.4740e-02,  6.6772e-02, -2.9278e-04, -8.4351e-02,  1.4429e-01,\n",
      "         2.0300e-01, -2.6196e-01, -2.3926e-01,  3.2617e-01, -3.4009e-01,\n",
      "        -2.7832e-01, -1.1749e-03,  8.6823e-03,  2.9190e-02, -1.1803e-02],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.9.self_attn.out_proj.weight Parameter containing:\n",
      "tensor([[ 0.1051, -0.0206, -0.1499,  ...,  0.0152, -0.0081, -0.1283],\n",
      "        [-0.1046,  0.0968,  0.0263,  ...,  0.0098, -0.0118,  0.0192],\n",
      "        [-0.1324, -0.1132, -0.0111,  ..., -0.0325, -0.0476, -0.0933],\n",
      "        ...,\n",
      "        [ 0.1225,  0.1914, -0.0230,  ..., -0.0167,  0.0267, -0.0193],\n",
      "        [ 0.0172, -0.0601,  0.0109,  ...,  0.0512, -0.1696,  0.0025],\n",
      "        [-0.0826, -0.0512,  0.1091,  ...,  0.0250,  0.1077,  0.0557]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.9.self_attn.out_proj.bias Parameter containing:\n",
      "tensor([ 4.2267e-02,  3.5458e-03, -5.3406e-02,  3.9482e-03,  2.5513e-02,\n",
      "        -4.5410e-02,  1.7065e-01, -8.7128e-03, -1.2512e-01,  6.5956e-03,\n",
      "        -1.1963e-01,  5.8411e-02,  6.7329e-04,  7.3662e-03,  6.0310e-03,\n",
      "         5.1331e-02,  2.1973e-03, -8.6121e-02, -1.5918e-01, -1.7029e-01,\n",
      "         5.4535e-02, -1.4990e-01, -7.8552e-02, -6.5918e-02, -1.6870e-01,\n",
      "         9.5459e-02, -2.0447e-01,  1.0303e-01, -1.1017e-02, -1.3953e-01,\n",
      "         1.0052e-01, -3.0346e-03,  1.3464e-01,  2.7191e-02, -1.3208e-01,\n",
      "        -1.3025e-01,  1.2622e-01, -1.8250e-02, -1.0126e-01, -7.7759e-02,\n",
      "        -2.4277e-02, -4.4647e-02,  3.1311e-02,  3.5767e-02, -1.2781e-01,\n",
      "        -9.2407e-02,  1.0645e+00, -3.0182e-02, -6.0822e-02, -1.3013e-01,\n",
      "         6.4575e-02, -6.1951e-02,  1.8567e-01,  9.5825e-03,  7.3730e-02,\n",
      "        -6.5552e-02, -1.0919e-01, -1.4084e-02, -1.6113e-01,  1.3542e-02,\n",
      "        -1.0657e-01, -3.7415e-02,  7.5745e-02,  1.3069e-02,  1.1951e-01,\n",
      "        -1.4209e-01, -1.7627e-01,  4.3091e-02,  7.4219e-02,  1.4551e-01,\n",
      "        -1.1780e-01,  6.9641e-02,  1.5100e-01, -1.0785e-01,  6.9336e-02,\n",
      "        -1.3176e-02,  1.1737e-01,  4.2755e-02,  4.2847e-02,  1.0156e-01,\n",
      "         7.3792e-02,  3.2776e-02, -4.1595e-02, -6.3660e-02,  1.8494e-02,\n",
      "        -7.7698e-02, -5.6976e-02,  3.5614e-02,  8.3374e-02, -2.8870e-02,\n",
      "        -1.0535e-01,  8.1787e-02, -8.1909e-02, -7.3471e-03, -8.5632e-02,\n",
      "         1.2756e-01,  8.8623e-02, -9.9976e-02,  7.7026e-02, -1.3135e-01,\n",
      "        -4.4464e-02,  1.2097e-01, -7.6782e-02,  1.3745e-01,  2.0813e-02,\n",
      "        -9.6817e-03,  1.6342e-02,  9.3506e-02,  3.9520e-02,  5.6671e-02,\n",
      "        -1.1926e-01,  1.1452e-02, -4.8096e-02,  3.7518e-03, -4.5532e-02,\n",
      "         1.4696e-03,  5.3009e-02,  1.6870e-01, -1.1432e-01, -1.4966e-01,\n",
      "        -2.5986e-02,  1.3733e-01, -2.2571e-01,  4.5441e-02, -1.2184e-02,\n",
      "         1.9119e-02,  2.6932e-02, -6.4026e-02,  4.7302e-02, -4.4922e-02,\n",
      "        -1.0278e-01, -1.1157e-01,  6.6109e-03, -9.7351e-02,  1.4062e-01,\n",
      "         3.9001e-02, -3.5431e-02,  1.5784e-01,  1.8811e-01,  9.5673e-03,\n",
      "         1.1560e-01, -1.1670e-01,  1.8341e-02, -1.2527e-02, -6.5193e-03,\n",
      "         7.0435e-02,  1.2524e-01,  1.4015e-02,  1.2335e-01,  8.8257e-02,\n",
      "         3.9482e-03, -5.2826e-02, -1.4290e-02, -2.2858e-02,  1.1261e-01,\n",
      "        -2.8442e-02, -8.7585e-02, -4.5349e-02,  2.2995e-02,  1.6614e-01,\n",
      "         3.9459e-02, -1.4267e-02,  7.1533e-02,  3.3905e-02, -7.5317e-02,\n",
      "        -1.4014e-01,  1.7166e-02, -1.0590e-01, -4.3060e-02,  4.2999e-02,\n",
      "         8.6243e-02,  3.1921e-02,  7.7515e-02, -5.5847e-03, -9.1187e-02,\n",
      "         8.9111e-02, -9.8724e-03, -7.1106e-02, -2.1191e-03,  1.1890e-01,\n",
      "        -4.0253e-02, -6.6040e-02, -6.0654e-03, -1.0016e-01,  1.8616e-02,\n",
      "         3.6713e-02, -1.0132e-02,  1.3293e-01, -7.2754e-02, -2.5070e-02,\n",
      "         9.0790e-03,  1.0278e-01,  8.2642e-02,  5.0964e-02, -1.2854e-01,\n",
      "         2.3102e-02, -9.1431e-02, -8.3496e-02, -5.6000e-02,  1.9531e-01,\n",
      "         5.2307e-02,  8.8318e-02,  2.0889e-02, -1.1711e-02, -2.9846e-02,\n",
      "         2.7206e-02, -3.1836e-01,  1.4026e-01, -7.3853e-02,  5.9937e-02,\n",
      "         3.5370e-02, -1.2256e-01,  1.1157e-01, -9.0942e-03, -3.5065e-02,\n",
      "         4.6539e-03,  7.8918e-02, -9.7046e-02, -3.0457e-02,  4.9347e-02,\n",
      "        -1.0187e-01,  7.6050e-02,  6.9641e-02, -2.8046e-02,  9.1492e-02,\n",
      "        -7.6294e-02,  1.7712e-01,  1.1298e-01,  2.0294e-02,  2.8854e-02,\n",
      "        -1.3672e-01, -1.6675e-01,  1.1432e-01,  1.4075e-01, -8.2581e-02,\n",
      "         4.1779e-02, -3.6743e-02,  6.0455e-02,  1.7981e-01, -1.4542e-02,\n",
      "         4.5288e-02, -5.6274e-02,  3.6224e-02, -2.6306e-02, -1.9440e-02,\n",
      "        -5.2124e-02,  8.8013e-02, -7.9956e-02, -2.0288e-01, -8.7280e-02,\n",
      "        -9.0271e-02, -1.1157e-01, -2.0398e-01,  1.8542e-01, -1.0693e-01,\n",
      "        -1.2500e-01, -2.0386e-01, -2.0337e-01, -5.6305e-02, -9.5825e-02,\n",
      "         1.0089e-01,  5.4413e-02, -2.7008e-03, -1.5129e-02,  7.6416e-02,\n",
      "        -6.3416e-02, -3.4210e-02,  2.8477e-03,  1.0645e-01,  6.7627e-02,\n",
      "        -1.3318e-01, -2.1622e-02,  6.9458e-02, -2.3755e-01, -7.4341e-02,\n",
      "         1.3831e-01, -3.6774e-03, -1.6125e-01, -2.4573e-01,  7.3608e-02,\n",
      "        -1.5186e-01,  9.6985e-02,  6.3629e-03, -1.3513e-01, -5.1514e-02,\n",
      "        -1.2280e-01,  9.9670e-02,  1.9482e-01, -2.2858e-02, -2.0215e-01,\n",
      "         3.4698e-02, -7.1777e-02,  1.5271e-01, -1.8665e-01,  9.1797e-02,\n",
      "        -2.3853e-01, -4.7112e-03,  1.1847e-01,  1.0175e-01,  1.3708e-01,\n",
      "         1.2286e-01, -1.0034e-01,  1.4673e-01,  5.2185e-02,  9.0820e-02,\n",
      "         5.0385e-02,  6.6101e-02, -1.8958e-01,  1.1603e-01, -5.8319e-02,\n",
      "        -1.9812e-01, -1.9097e-04, -1.4600e-01, -1.9608e-02, -7.2746e-03,\n",
      "        -1.5417e-01,  1.2250e-01,  8.8257e-02,  1.5793e-02, -7.1716e-02,\n",
      "        -1.0779e-01,  2.9327e-02,  1.4023e-02, -7.5012e-02, -1.9019e-01,\n",
      "        -1.4893e-01,  1.0516e-01,  2.0129e-01, -1.5087e-03,  6.2378e-02,\n",
      "         6.7978e-03,  1.3916e-01,  1.1340e-01, -1.1581e-02,  4.0344e-02,\n",
      "         7.6904e-02,  1.6309e-01,  4.7089e-02, -1.2561e-01,  7.1350e-02,\n",
      "         9.4116e-02,  9.4055e-02,  1.9592e-01,  7.9041e-02,  5.7831e-03,\n",
      "         4.9286e-02,  9.6191e-02, -2.5375e-02,  2.5195e-01,  4.4830e-02,\n",
      "         8.7036e-02,  1.6663e-01,  3.4607e-02, -1.3086e-01,  5.1605e-02,\n",
      "        -2.5146e-01,  1.6968e-01,  1.3928e-01,  3.6072e-02,  6.2683e-02,\n",
      "         1.4160e-02, -3.0884e-02,  3.2379e-02, -9.9609e-02,  1.5686e-01,\n",
      "        -1.1816e-01, -4.6265e-02,  1.2366e-01, -9.7656e-02,  1.2268e-01,\n",
      "        -7.9163e-02,  6.7024e-03, -1.0767e-01, -1.2451e-01, -7.4829e-02,\n",
      "        -7.4036e-02,  8.4839e-02,  4.0009e-02, -1.4572e-02,  8.7830e-02,\n",
      "         8.7433e-03,  3.7918e-03,  2.5803e-02, -1.8591e-01,  7.7454e-02,\n",
      "        -1.3879e-01,  7.8857e-02, -4.5410e-02,  1.9421e-01, -1.2732e-01,\n",
      "         3.4821e-02,  3.2005e-03,  1.1978e-02,  5.7617e-02, -4.3213e-02,\n",
      "        -9.5276e-02, -2.5024e-02,  1.3123e-01,  1.3013e-01, -3.3508e-02,\n",
      "         4.3457e-02,  4.8676e-03, -6.5613e-02,  1.0370e-01,  7.5623e-02,\n",
      "        -1.0803e-02,  6.5002e-02,  6.8359e-02, -9.8328e-02, -5.7465e-02,\n",
      "        -3.6926e-02, -1.0016e-01,  6.8604e-02, -1.1658e-01,  8.7524e-02,\n",
      "         1.0138e-01, -8.0505e-02, -6.0211e-02,  5.0385e-02, -7.1106e-02,\n",
      "        -3.3188e-03,  1.3208e-01, -7.2510e-02, -6.4514e-02, -6.7383e-02,\n",
      "         1.1444e-01,  1.3878e-02,  2.7145e-02,  6.4392e-03, -6.9885e-02,\n",
      "        -8.1177e-02,  1.7441e-02, -9.5276e-02, -6.0387e-03, -3.2654e-02,\n",
      "         4.5624e-02,  6.3843e-02,  1.0895e-01, -1.0034e-01,  5.4230e-02,\n",
      "        -6.5956e-03, -1.0706e-01,  9.1797e-02,  4.8065e-02, -1.0541e-01,\n",
      "         1.3191e-02, -7.5623e-02,  7.0984e-02, -1.2006e-01, -9.5947e-02,\n",
      "        -4.4189e-02, -7.6660e-02, -2.6367e-02, -7.4921e-03, -1.2207e-01,\n",
      "         3.9185e-02, -8.0919e-04, -1.1151e-01, -4.5593e-02, -3.0060e-02,\n",
      "        -3.0957e-01,  1.3684e-01, -5.2124e-02, -1.2042e-01,  6.6711e-02,\n",
      "        -1.7786e-01, -1.3892e-01, -2.2324e-02, -6.8665e-02, -1.7471e-03,\n",
      "        -8.1482e-02,  1.3989e-01,  3.8338e-03,  1.2720e-01,  5.5237e-02,\n",
      "        -1.1920e-01,  5.7983e-02,  7.6111e-02,  5.0751e-02,  9.5062e-03],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.9.self_attn_layer_norm.weight Parameter containing:\n",
      "tensor([0.5820, 0.5771, 0.5830, 0.5586, 0.6016, 0.5498, 0.5488, 0.6040, 0.5601,\n",
      "        0.5859, 0.5796, 0.5298, 0.5840, 0.5488, 0.5459, 0.5718, 0.5288, 0.5444,\n",
      "        0.5024, 0.5356, 0.5742, 0.6343, 0.5767, 0.5688, 0.5786, 0.5054, 0.5312,\n",
      "        0.5879, 0.5645, 0.5933, 0.5942, 0.5532, 0.5947, 0.5840, 0.5503, 0.6011,\n",
      "        0.5420, 0.5591, 0.6045, 0.5967, 0.5830, 0.5645, 0.6025, 0.5796, 0.5630,\n",
      "        0.5923, 0.2754, 0.5840, 0.5542, 0.5923, 0.5591, 0.5972, 0.5732, 0.5513,\n",
      "        0.5527, 0.5776, 0.5420, 0.5791, 0.5063, 0.5654, 0.5312, 0.5483, 0.5728,\n",
      "        0.5752, 0.5483, 0.5005, 0.2747, 0.5474, 0.5459, 0.5283, 0.5703, 0.5352,\n",
      "        0.5225, 0.5474, 0.6157, 0.5728, 0.5684, 0.5625, 0.5874, 0.5703, 0.5396,\n",
      "        0.5967, 0.5298, 0.6001, 0.5854, 0.5557, 0.5508, 0.5732, 0.5767, 0.5864,\n",
      "        0.5996, 0.5771, 0.6099, 0.5752, 0.5942, 0.5044, 0.5801, 0.5503, 0.5986,\n",
      "        0.6084, 0.5640, 0.5737, 0.5547, 0.6099, 0.5298, 0.6074, 0.5630, 0.5693,\n",
      "        0.5903, 0.5728, 0.5840, 0.6304, 0.6201, 0.5771, 0.5527, 0.5610, 0.5933,\n",
      "        0.5259, 0.5996, 0.5542, 0.5908, 0.5825, 0.5581, 0.6226, 0.5513, 0.6035,\n",
      "        0.5693, 0.5518, 0.5396, 0.5371, 0.5503, 0.6426, 0.5508, 0.5850, 0.5615,\n",
      "        0.5728, 0.6128, 0.4517, 0.5239, 0.5972, 0.5146, 0.5474, 0.5405, 0.6211,\n",
      "        0.5210, 0.5630, 0.5483, 0.6113, 0.4807, 0.5645, 0.5601, 0.5352, 0.5498,\n",
      "        0.5918, 0.5244, 0.5986, 0.5728, 0.5977, 0.6074, 0.5732, 0.5586, 0.5430,\n",
      "        0.5186, 0.5327, 0.5161, 0.5762, 0.5415, 0.5347, 0.5820, 0.5864, 0.5117,\n",
      "        0.6240, 0.5493, 0.5630, 0.4670, 0.5640, 0.6206, 0.6016, 0.5405, 0.5723,\n",
      "        0.5991, 0.5542, 0.5742, 0.5801, 0.5713, 0.6201, 0.5532, 0.5454, 0.5859,\n",
      "        0.5693, 0.5879, 0.5938, 0.5728, 0.5244, 0.6094, 0.5723, 0.5918, 0.5381,\n",
      "        0.5674, 0.4536, 0.5532, 0.4927, 0.5044, 0.5835, 0.5610, 0.5713, 0.4421,\n",
      "        0.5913, 0.5908, 0.5444, 0.5669, 0.5796, 0.5542, 0.5664, 0.5684, 0.5723,\n",
      "        0.5591, 0.5615, 0.5908, 0.5845, 0.5303, 0.5884, 0.5405, 0.5625, 0.5488,\n",
      "        0.6001, 0.5723, 0.5913, 0.6284, 0.5757, 0.5771, 0.6108, 0.5703, 0.5620,\n",
      "        0.5996, 0.5586, 0.5640, 0.6025, 0.5850, 0.5889, 0.5239, 0.6201, 0.6025,\n",
      "        0.5645, 0.5669, 0.4692, 0.5552, 0.5698, 0.5610, 0.4216, 0.5669, 0.5898,\n",
      "        0.5391, 0.5610, 0.5552, 0.5225, 0.5200, 0.5410, 0.5454, 0.5947, 0.5532,\n",
      "        0.5791, 0.5679, 0.5571, 0.5601, 0.5146, 0.5947, 0.5845, 0.5786, 0.5928,\n",
      "        0.5801, 0.5679, 0.5835, 0.5620, 0.5488, 0.5952, 0.5571, 0.5859, 0.5137,\n",
      "        0.5547, 0.5688, 0.5547, 0.5483, 0.5771, 0.6533, 0.5625, 0.5605, 0.4692,\n",
      "        0.5889, 0.5493, 0.5537, 0.6045, 0.5898, 0.4399, 0.5786, 0.4985, 0.5630,\n",
      "        0.5532, 0.5894, 0.3093, 0.5933, 0.6006, 0.6025, 0.5981, 0.5405, 0.5488,\n",
      "        0.5503, 0.5464, 0.6172, 0.5000, 0.5259, 0.5850, 0.5586, 0.5737, 0.5762,\n",
      "        0.4963, 0.6289, 0.5742, 0.6079, 0.6040, 0.5859, 0.5459, 0.6206, 0.5332,\n",
      "        0.5547, 0.5532, 0.5981, 0.6367, 0.6089, 0.5405, 0.5874, 0.5381, 0.6040,\n",
      "        0.5869, 0.5562, 0.5254, 0.5532, 0.5649, 0.5801, 0.5830, 0.5801, 0.5742,\n",
      "        0.4871, 0.6294, 0.5796, 0.5337, 0.5527, 0.5918, 0.4597, 0.5645, 0.5757,\n",
      "        0.5625, 0.5713, 0.5273, 0.5698, 0.5444, 0.5693, 0.5669, 0.5630, 0.5674,\n",
      "        0.5981, 0.5459, 0.5640, 0.5293, 0.5996, 0.5591, 0.5620, 0.6177, 0.5703,\n",
      "        0.5557, 0.5830, 0.5513, 0.5903, 0.5713, 0.5767, 0.5698, 0.6074, 0.5879,\n",
      "        0.5571, 0.5684, 0.5791, 0.5659, 0.4990, 0.5063, 0.5752, 0.6182, 0.5952,\n",
      "        0.5957, 0.5547, 0.6084, 0.5884, 0.5903, 0.5737, 0.5674, 0.5088, 0.5410,\n",
      "        0.6562, 0.5591, 0.5576, 0.5903, 0.5781, 0.6016, 0.5488, 0.5532, 0.4966,\n",
      "        0.5859, 0.6245, 0.5864, 0.5884, 0.5874, 0.5625, 0.6133, 0.5581, 0.5312,\n",
      "        0.5620, 0.5234, 0.5708, 0.5630, 0.5732, 0.6104, 0.5967, 0.5537, 0.6172,\n",
      "        0.6040, 0.6035, 0.5811, 0.5425, 0.5874, 0.5825, 0.5928, 0.6191, 0.5918,\n",
      "        0.5723, 0.5732, 0.5767, 0.5630, 0.6172, 0.5703, 0.5317, 0.5693, 0.5483,\n",
      "        0.5356, 0.5903, 0.5708, 0.6133, 0.5156, 0.5166, 0.5859, 0.5459, 0.5503,\n",
      "        0.5845, 0.5742, 0.5557, 0.5889, 0.5796, 0.5781, 0.6016, 0.5806, 0.5054,\n",
      "        0.5625, 0.2230, 0.5562, 0.5664, 0.6011, 0.5635, 0.5889, 0.5645, 0.5562,\n",
      "        0.5776, 0.5898, 0.5483, 0.5381, 0.5728, 0.5737, 0.5942, 0.5732, 0.5786,\n",
      "        0.5522, 0.5532, 0.5913], requires_grad=True)\n",
      "esm_model.layers.9.self_attn_layer_norm.bias Parameter containing:\n",
      "tensor([ 1.0620e-01, -1.8066e-02, -7.6370e-03, -4.4067e-02,  2.0618e-03,\n",
      "        -1.0178e-02,  7.6027e-03, -7.0312e-02,  5.9021e-02,  3.6560e-02,\n",
      "        -6.9542e-03, -2.4567e-02,  4.4464e-02, -2.3056e-02, -9.2346e-02,\n",
      "        -4.5837e-02, -4.1626e-02,  5.6305e-02, -5.3680e-02, -1.3626e-02,\n",
      "         7.0679e-02, -2.9480e-02, -2.0386e-02,  1.7624e-02, -6.0310e-03,\n",
      "        -4.4189e-02,  7.2327e-02,  8.0994e-02,  8.5068e-03, -3.3478e-02,\n",
      "        -3.9154e-02,  3.5492e-02,  2.2034e-02,  2.1896e-02, -1.9775e-02,\n",
      "        -3.0182e-02, -4.6356e-02, -1.5926e-03,  6.8604e-02, -7.1449e-03,\n",
      "        -9.8038e-04,  1.4534e-02, -9.6130e-03,  1.3916e-02,  2.3834e-02,\n",
      "        -1.7914e-02,  8.2520e-01,  7.9529e-02,  2.2522e-02,  3.5889e-02,\n",
      "        -1.5884e-02, -3.5057e-03,  5.6381e-03,  2.2217e-02, -2.5253e-02,\n",
      "        -8.3084e-03,  7.5500e-02, -2.3880e-03,  5.9906e-02,  5.7526e-03,\n",
      "         3.7598e-02,  6.2828e-03,  2.7649e-02,  1.8356e-02,  3.2187e-06,\n",
      "         1.7288e-02,  2.4353e-01,  1.7792e-02, -1.0742e-02, -1.0635e-02,\n",
      "         4.2175e-02, -2.2827e-02,  2.8641e-02, -5.0163e-03, -8.5983e-03,\n",
      "         4.4174e-03,  4.5135e-02,  6.6895e-02, -8.8043e-03, -3.3112e-02,\n",
      "         9.0271e-02, -4.8859e-02, -1.6235e-02,  4.0192e-02,  6.6467e-02,\n",
      "        -4.9858e-03,  4.2358e-02,  3.0014e-02,  3.6774e-02,  5.2338e-02,\n",
      "        -2.4948e-02, -3.4393e-02, -4.1618e-03, -1.1429e-02,  4.6631e-02,\n",
      "         6.8176e-02, -1.4477e-03,  1.0864e-02,  8.4900e-02,  7.3204e-03,\n",
      "         1.6388e-02,  3.9032e-02,  2.5879e-02, -1.1284e-02, -1.4359e-02,\n",
      "        -6.7078e-02, -3.5278e-02,  3.7598e-02,  4.3823e-02,  6.1768e-02,\n",
      "        -2.4567e-02,  6.9656e-03, -1.2100e-02,  3.6926e-02,  3.0396e-02,\n",
      "         5.8594e-02, -3.3539e-02, -3.1300e-03,  1.9547e-02,  9.9640e-03,\n",
      "         2.9327e-02,  1.5783e-04,  5.6488e-02,  5.6671e-02, -8.8196e-03,\n",
      "         1.7059e-02,  7.8392e-04,  3.1982e-02,  4.6539e-02,  2.5436e-02,\n",
      "        -2.0355e-02, -2.2797e-02, -1.4374e-02,  8.3435e-02,  1.8738e-02,\n",
      "         6.1188e-03, -8.3466e-03,  1.5762e-02, -2.2354e-02,  5.0171e-02,\n",
      "         2.9587e-02,  5.3009e-02,  5.2521e-02,  1.1856e-02, -9.3842e-03,\n",
      "         3.8055e-02, -2.0432e-02, -1.0643e-02, -8.8562e-02, -3.7140e-02,\n",
      "         5.9265e-02,  1.7014e-02,  2.4857e-02,  4.2480e-02, -9.4452e-03,\n",
      "         2.2259e-03,  7.0374e-02,  8.2703e-03,  5.1453e-02, -1.1230e-02,\n",
      "        -1.9363e-02,  1.8219e-02,  5.1056e-02, -3.3722e-02,  2.4384e-02,\n",
      "         2.1942e-02,  3.6835e-02,  4.5135e-02,  1.5427e-02,  1.1497e-02,\n",
      "        -6.2027e-03,  8.3542e-03,  3.7384e-02,  3.9825e-02,  4.7638e-02,\n",
      "         7.4829e-02,  7.4585e-02,  3.3051e-02,  2.1500e-02,  3.7048e-02,\n",
      "         2.4338e-02,  5.8929e-02,  6.5979e-02, -1.4038e-02,  1.6800e-02,\n",
      "        -7.9880e-03,  3.9093e-02, -9.4604e-03, -1.3992e-02,  6.5125e-02,\n",
      "        -1.6281e-02, -4.1008e-03,  5.8685e-02,  7.8201e-03,  8.8013e-02,\n",
      "         1.3641e-02,  4.1504e-02, -3.4119e-02,  3.7155e-03,  2.8503e-02,\n",
      "         3.8483e-02,  1.5857e-01, -4.4785e-03,  1.1650e-02, -2.9785e-02,\n",
      "        -2.8961e-02,  5.8807e-02,  2.7588e-02,  3.7903e-02,  4.7333e-02,\n",
      "         1.9485e-02,  2.0874e-02, -1.0971e-02, -1.3107e-02, -1.9897e-02,\n",
      "         2.9488e-03, -1.2085e-02, -5.5817e-02,  4.3152e-02,  1.5099e-02,\n",
      "        -5.3467e-02,  2.6398e-02,  9.2316e-03, -1.1091e-03,  3.1952e-02,\n",
      "        -9.7504e-03,  2.0996e-02,  3.7140e-02,  9.8328e-02,  4.2328e-02,\n",
      "        -1.0852e-01,  6.9504e-03, -4.8180e-03,  2.0676e-02,  6.2378e-02,\n",
      "         1.4587e-02, -7.2876e-02, -1.5945e-02, -1.9951e-03,  9.8705e-04,\n",
      "        -7.3204e-03,  1.3023e-02,  5.1239e-02,  1.2947e-02, -3.7476e-02,\n",
      "         1.4198e-02,  3.8727e-02,  8.7585e-02, -1.3100e-02,  1.2146e-02,\n",
      "         2.5730e-03,  1.6575e-03, -7.7553e-03, -6.7825e-03,  4.3762e-02,\n",
      "         1.4343e-02, -2.5208e-02, -1.8723e-02, -5.0751e-02,  1.0040e-02,\n",
      "        -2.7027e-03,  3.0670e-02,  7.8979e-02, -1.6968e-02,  4.3182e-02,\n",
      "         9.2392e-03,  7.1350e-02,  7.0618e-02, -2.2934e-02,  7.3318e-03,\n",
      "         4.8706e-02,  4.9530e-02,  6.8054e-02,  6.6284e-02,  4.4128e-02,\n",
      "         5.2490e-02, -6.9771e-03,  1.6693e-02,  4.4891e-02, -3.3905e-02,\n",
      "        -3.1281e-02,  9.9182e-03, -5.1941e-02,  2.1011e-02, -4.4098e-03,\n",
      "         4.5959e-02,  1.5762e-02, -2.4490e-02,  2.9449e-02,  5.9723e-02,\n",
      "        -1.0231e-02,  1.0040e-02,  3.7933e-02, -9.0027e-04, -2.9663e-02,\n",
      "        -1.5884e-02, -3.9764e-02,  6.4545e-03,  3.1982e-02,  1.2384e-01,\n",
      "        -9.2651e-02,  1.2100e-02, -1.2917e-02, -4.2816e-02,  2.2583e-02,\n",
      "        -3.9062e-02, -5.2795e-03, -5.4321e-02,  2.8564e-02,  3.5980e-02,\n",
      "         7.5684e-02,  4.2328e-02,  6.6895e-02, -1.9669e-02,  2.0752e-02,\n",
      "         3.2440e-02,  3.3783e-02, -1.2833e-02, -3.3173e-02, -4.4525e-02,\n",
      "         4.6692e-02, -3.1769e-02, -1.6602e-02, -2.2335e-03,  2.0920e-02,\n",
      "         4.0161e-02, -3.9154e-02,  4.3060e-02,  1.6174e-02,  1.5533e-02,\n",
      "         3.7933e-02, -5.2795e-03,  3.3966e-02, -4.2450e-02,  1.6418e-02,\n",
      "         3.6812e-04,  2.1500e-02,  6.8207e-03, -3.8300e-03,  1.0217e-01,\n",
      "         4.4220e-02,  3.9307e-02, -2.6855e-02, -7.0686e-03,  3.8147e-02,\n",
      "         1.0063e-02, -6.1646e-02,  9.1400e-03, -4.5288e-02,  3.1616e-02,\n",
      "        -5.8517e-03,  3.6072e-02, -3.9154e-02,  9.3842e-03, -6.3171e-02,\n",
      "         3.7262e-02,  7.1411e-02,  2.0264e-02,  4.9042e-02,  1.0836e-04,\n",
      "        -5.0537e-02,  7.8979e-02, -4.2267e-03,  5.0751e-02,  4.1412e-02,\n",
      "         8.5388e-02,  1.5015e-02, -2.2736e-02, -1.5297e-02, -8.0490e-03,\n",
      "         4.2145e-02,  5.1689e-04,  6.4209e-02,  6.5247e-02,  4.1771e-03,\n",
      "         6.0120e-02,  7.1594e-02, -1.9791e-02,  3.7567e-02, -4.5013e-03,\n",
      "        -1.2913e-03,  9.9365e-02, -2.5269e-02, -3.0304e-02, -3.9825e-02,\n",
      "        -1.8448e-02,  4.7821e-02,  2.5787e-02,  4.1748e-02,  1.1490e-02,\n",
      "         6.2622e-02,  1.0185e-02,  9.9411e-03,  3.3966e-02,  5.5359e-02,\n",
      "        -3.1525e-02,  1.8463e-02, -1.9775e-02,  1.5144e-02,  5.3741e-02,\n",
      "         5.8937e-04, -5.0812e-02,  2.9495e-02, -9.8877e-03,  6.2469e-02,\n",
      "         1.4992e-02,  1.2466e-02, -1.8158e-02,  1.9302e-02,  6.9275e-02,\n",
      "        -3.6377e-02,  1.0010e-02, -4.5074e-02,  4.9866e-02, -5.3930e-04,\n",
      "        -1.5459e-03,  2.3071e-02,  3.3813e-02,  1.2016e-02, -4.2480e-02,\n",
      "         3.5126e-02, -2.1820e-03, -8.7708e-02,  2.0752e-02,  1.7456e-02,\n",
      "        -4.6265e-02,  1.0262e-03,  3.9246e-02,  4.4403e-02,  1.9928e-02,\n",
      "         2.0798e-02,  8.8074e-02,  7.3853e-02, -1.5076e-02, -1.0155e-02,\n",
      "         3.0930e-02,  5.9937e-02,  6.4011e-03, -1.1925e-02,  5.0293e-02,\n",
      "         1.5213e-02,  2.2812e-02,  6.7932e-02,  3.6560e-02,  3.6285e-02,\n",
      "         8.5266e-02,  5.0293e-02, -6.6948e-03,  1.0094e-02,  9.7885e-03,\n",
      "         1.2627e-02, -2.2690e-02, -6.5063e-02, -1.2482e-02,  6.1829e-02,\n",
      "         9.3613e-03,  3.7842e-02,  5.7404e-02,  4.8370e-03, -9.0637e-03,\n",
      "        -2.8613e-01,  5.4016e-02,  4.5593e-02, -3.7170e-02,  2.9240e-03,\n",
      "         3.9978e-02,  2.4857e-02, -1.5823e-02, -3.7811e-02,  3.9124e-02,\n",
      "         4.1962e-02,  3.2562e-02,  8.5831e-03,  7.3738e-03,  1.0513e-02,\n",
      "        -5.0720e-02,  1.1955e-02,  4.7241e-02, -1.5762e-02, -5.1880e-02],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.9.fc1.weight Parameter containing:\n",
      "tensor([[ 0.0908, -0.1039, -0.0771,  ...,  0.0233, -0.1139, -0.0514],\n",
      "        [-0.0302,  0.1948,  0.0275,  ..., -0.0438,  0.0881,  0.1205],\n",
      "        [-0.0114,  0.0466,  0.0625,  ..., -0.0083,  0.0626,  0.0322],\n",
      "        ...,\n",
      "        [ 0.1870,  0.0575, -0.2078,  ..., -0.0255, -0.0190,  0.0900],\n",
      "        [-0.1484, -0.2006, -0.0100,  ..., -0.1077, -0.1603,  0.1020],\n",
      "        [ 0.0185,  0.0841,  0.1000,  ..., -0.1011, -0.0415, -0.1177]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.9.fc1.bias Parameter containing:\n",
      "tensor([-0.0781, -0.0703, -0.0186,  ..., -0.0314, -0.0714, -0.0568],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.9.fc2.weight Parameter containing:\n",
      "tensor([[ 0.0493, -0.0541,  0.1442,  ..., -0.0095, -0.0049,  0.2290],\n",
      "        [ 0.0336, -0.0134, -0.0133,  ..., -0.0829,  0.1893,  0.0873],\n",
      "        [ 0.0879,  0.1154,  0.0030,  ...,  0.0468,  0.0133, -0.0181],\n",
      "        ...,\n",
      "        [ 0.0845, -0.0207,  0.0558,  ..., -0.1099,  0.0932, -0.0047],\n",
      "        [-0.0956,  0.0331, -0.0888,  ..., -0.1161,  0.1696, -0.0851],\n",
      "        [-0.2103, -0.0563,  0.3066,  ..., -0.0284, -0.2131, -0.0504]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.9.fc2.bias Parameter containing:\n",
      "tensor([ 6.2805e-02,  3.2787e-03, -3.6804e-02,  5.4230e-02, -7.6355e-02,\n",
      "        -2.5497e-02,  9.1858e-02, -2.9877e-02, -6.2683e-02, -2.6184e-02,\n",
      "        -1.0455e-01,  8.1848e-02, -4.3671e-02, -6.8420e-02,  3.3836e-03,\n",
      "        -3.2654e-02,  4.6120e-03,  6.6284e-02, -3.5458e-03, -1.0040e-01,\n",
      "         1.9283e-03, -3.4760e-02, -9.6619e-02,  1.5579e-02, -3.8635e-02,\n",
      "        -7.4654e-03, -2.2842e-02,  9.0759e-02, -5.5298e-02, -1.0352e-01,\n",
      "         9.5276e-02, -4.8553e-02,  1.6342e-02,  2.3636e-02, -4.1809e-02,\n",
      "         2.6215e-02,  8.0444e-02,  2.6825e-02, -5.9387e-02, -5.0446e-02,\n",
      "         1.1055e-02,  4.9957e-02, -3.5400e-03, -3.1952e-02, -4.1771e-03,\n",
      "        -1.1658e-01,  6.2598e-01,  3.7231e-02, -3.3264e-03, -1.0681e-01,\n",
      "        -5.9723e-02, -2.3804e-02,  4.2572e-02,  3.9520e-02,  3.1677e-02,\n",
      "        -2.2156e-02,  1.3550e-02, -8.0383e-02,  1.7670e-02,  9.7046e-03,\n",
      "         2.1652e-02,  7.1983e-03,  1.1040e-02,  6.5552e-02,  1.6443e-01,\n",
      "        -6.1554e-02, -1.8127e-01, -4.4739e-02,  4.2328e-02,  9.5764e-02,\n",
      "        -6.2866e-02, -3.8544e-02,  1.0071e-01, -1.0938e-01,  6.8420e-02,\n",
      "        -4.8950e-02, -7.7362e-03,  1.9348e-02,  3.9764e-02, -1.0834e-02,\n",
      "         9.5276e-02,  6.6528e-02,  1.4282e-02, -6.3171e-02,  1.6467e-01,\n",
      "        -9.3994e-02, -1.4755e-02,  4.1626e-02, -2.6154e-02, -5.7587e-02,\n",
      "        -3.0563e-02,  6.6956e-02, -1.0474e-01,  2.6352e-02, -9.5947e-02,\n",
      "         4.0512e-03,  7.5806e-02, -7.4951e-02,  7.4921e-03, -2.5909e-02,\n",
      "         3.2444e-03,  2.6367e-02, -6.9214e-02,  5.6091e-02, -3.8879e-02,\n",
      "        -2.2552e-02,  6.7261e-02,  1.1224e-01,  4.4952e-02,  3.8261e-03,\n",
      "        -1.8933e-01, -2.0275e-03, -6.6711e-02,  3.0090e-02, -2.9785e-02,\n",
      "        -3.2471e-02,  6.5979e-02,  1.1090e-01, -1.5283e-01, -9.4116e-02,\n",
      "        -3.5248e-02,  8.5022e-02, -1.3916e-01,  3.3356e-02,  2.3972e-02,\n",
      "         4.7729e-02, -3.4275e-03,  4.4128e-02,  2.8778e-02,  2.6718e-02,\n",
      "        -5.4535e-02, -4.6570e-02,  1.8454e-03, -8.8745e-02,  4.8584e-02,\n",
      "        -4.9072e-02, -3.2715e-02,  8.1665e-02,  7.0190e-02,  9.7534e-02,\n",
      "         8.4961e-02, -9.3231e-03, -3.5583e-02,  1.4671e-02, -6.3658e-05,\n",
      "         5.7281e-02,  1.3313e-02,  6.9189e-04,  3.0975e-02,  6.0760e-02,\n",
      "        -5.1666e-02,  5.0415e-02,  2.6840e-02,  2.5387e-03,  5.9357e-02,\n",
      "         4.1695e-03, -4.6295e-02,  1.0117e-02, -1.3512e-02,  2.1255e-02,\n",
      "        -5.0774e-03,  3.9917e-02,  7.9269e-03,  3.7109e-02,  2.5940e-03,\n",
      "        -6.2622e-02,  6.0852e-02, -1.1330e-03, -6.3660e-02,  8.3130e-02,\n",
      "        -1.7029e-02,  1.9302e-02,  3.2806e-02,  1.8311e-02, -3.4210e-02,\n",
      "         5.3345e-02,  4.0359e-03, -3.0777e-02, -2.5848e-02,  5.1331e-02,\n",
      "        -6.2561e-02, -5.7922e-02,  3.1403e-02, -9.8755e-02, -6.1073e-03,\n",
      "        -6.4583e-03,  4.3030e-02,  4.7882e-02, -3.4363e-02, -1.1700e-01,\n",
      "         5.8685e-02,  3.9520e-02,  2.4536e-02, -6.2347e-02, -8.0078e-02,\n",
      "         1.8406e-03, -2.6245e-02,  1.9287e-02, -8.7097e-02,  7.2327e-02,\n",
      "        -1.7334e-02,  7.2517e-03, -3.5065e-02,  2.4986e-03, -2.1530e-02,\n",
      "         6.8176e-02, -1.4490e-01,  1.3977e-01, -7.0457e-03,  4.2328e-02,\n",
      "        -6.3744e-03, -1.3159e-01,  2.5463e-03,  2.3956e-02, -3.3417e-02,\n",
      "        -1.3641e-02,  2.3975e-01, -1.1650e-02, -6.7383e-02,  2.4948e-03,\n",
      "        -3.8605e-03, -4.9469e-02, -2.2186e-02,  5.6152e-03,  3.0594e-02,\n",
      "        -7.3929e-03,  9.1553e-02,  3.9581e-02,  6.7261e-02,  4.9927e-02,\n",
      "        -6.1646e-02, -1.2537e-01,  1.2433e-01,  4.6539e-02, -3.6896e-02,\n",
      "         6.6467e-02,  3.8471e-03, -1.7223e-03,  7.8186e-02,  2.0905e-02,\n",
      "         2.5665e-02, -2.6505e-02,  1.2794e-02, -9.5154e-02, -4.1748e-02,\n",
      "         3.4119e-02,  7.8674e-02, -5.8136e-02, -1.4819e-01,  8.1604e-02,\n",
      "         7.6065e-03, -6.5552e-02, -1.1560e-01,  1.1200e-01, -1.5671e-02,\n",
      "        -1.5945e-02, -5.2063e-02, -4.3396e-02,  7.1526e-03, -8.9844e-02,\n",
      "         3.8452e-02,  1.8524e-02, -3.5278e-02,  4.1321e-02,  5.5603e-02,\n",
      "        -1.0901e-01,  1.8143e-02,  4.6509e-02,  1.4453e-01, -1.7853e-02,\n",
      "         3.2082e-03, -2.2568e-02, -1.0963e-02, -1.0236e-01, -1.4124e-03,\n",
      "         4.1687e-02,  2.4857e-02, -3.4637e-02, -3.8849e-02, -1.4502e-01,\n",
      "        -3.2990e-02, -5.9166e-03,  3.7720e-02, -4.4769e-02, -9.4666e-02,\n",
      "        -3.7384e-02,  2.7008e-02,  7.3975e-02,  4.1138e-02, -1.4587e-01,\n",
      "        -1.8631e-02,  8.8623e-02,  3.2990e-02, -1.2610e-01, -1.7670e-02,\n",
      "        -2.2446e-02,  8.8577e-03,  1.4355e-01, -9.0599e-04,  5.6152e-02,\n",
      "         1.2708e-01, -6.7688e-02,  1.5222e-01,  2.2491e-02,  1.0292e-02,\n",
      "         3.3875e-02,  9.0866e-03, -9.1919e-02,  4.5868e-02,  3.1799e-02,\n",
      "        -9.9121e-02,  1.0254e-02, -3.7628e-02, -2.2614e-02,  6.6467e-02,\n",
      "        -6.2225e-02,  1.2189e-01,  1.3574e-01,  1.9684e-02, -1.0327e-01,\n",
      "        -1.1591e-01,  4.8004e-02,  1.1490e-02,  7.3120e-02, -1.1908e-01,\n",
      "        -8.3862e-02,  1.8188e-02,  6.6833e-02, -2.2827e-02, -3.4576e-02,\n",
      "        -2.8152e-03,  3.7201e-02,  7.6660e-02,  1.1368e-02, -3.7567e-02,\n",
      "         1.7029e-01,  1.9385e-01,  6.9809e-03, -1.0297e-01,  2.6245e-02,\n",
      "         8.4381e-03, -3.3226e-03,  1.2720e-01,  5.4443e-02,  2.6855e-02,\n",
      "        -4.2176e-04,  1.2830e-01, -4.7302e-02,  1.1835e-01,  9.8724e-03,\n",
      "         2.9831e-02,  6.0455e-02, -3.1403e-02, -1.6022e-02, -8.0505e-02,\n",
      "        -6.8359e-02,  5.9662e-02,  4.9225e-02,  8.8379e-02,  6.2180e-03,\n",
      "         2.0035e-02, -4.0863e-02,  7.4280e-02, -7.4768e-02,  1.0828e-01,\n",
      "        -1.0370e-01,  1.1917e-02,  7.0679e-02, -6.5491e-02,  5.7922e-02,\n",
      "        -1.0956e-01,  2.3056e-02, -4.2542e-02, -8.6548e-02, -6.9885e-02,\n",
      "        -1.3855e-01,  9.0515e-02,  4.9896e-02,  2.5635e-02, -1.1269e-02,\n",
      "        -5.4932e-03, -1.6155e-03, -1.8005e-02, -1.0083e-01, -1.8368e-03,\n",
      "        -3.6682e-02,  4.7455e-02, -4.3488e-02,  5.8899e-02, -4.3579e-02,\n",
      "        -4.3243e-02, -1.3123e-02, -9.0759e-02,  6.2866e-03, -5.3619e-02,\n",
      "        -5.2673e-02, -1.5305e-02,  7.3608e-02,  2.2842e-02,  4.1840e-02,\n",
      "         4.9286e-02, -1.2253e-02, -7.2815e-02,  6.4331e-02,  8.9264e-03,\n",
      "        -3.3356e-02,  7.2876e-02,  5.6305e-02, -1.5137e-02, -3.7476e-02,\n",
      "         2.4414e-02, -7.8583e-03,  5.7434e-02, -5.3101e-02, -6.6528e-02,\n",
      "         1.0248e-01, -2.9602e-03,  1.6937e-02,  1.2672e-02, -6.0333e-02,\n",
      "         1.0185e-02,  6.4697e-02, -5.2551e-02, -4.1046e-02, -6.7322e-02,\n",
      "         2.7832e-02, -2.5909e-02,  4.3671e-02, -2.7451e-02, -7.6904e-02,\n",
      "         3.8696e-02, -7.4036e-02,  4.0558e-02,  1.1246e-02, -5.2673e-02,\n",
      "        -2.7557e-02,  2.8702e-02,  1.8234e-02, -1.0565e-01, -3.8940e-02,\n",
      "        -2.7222e-02, -8.6670e-02,  6.5796e-02, -1.1116e-02, -7.2571e-02,\n",
      "        -5.0476e-02, -1.2170e-01,  5.1331e-02, -1.2219e-01, -8.0017e-02,\n",
      "        -1.8492e-03, -4.2572e-02, -6.0730e-02,  3.7842e-02, -1.1554e-01,\n",
      "         2.2888e-02, -6.1676e-02,  9.3201e-02,  5.2917e-02,  3.1311e-02,\n",
      "        -3.1982e-02,  4.9316e-02, -1.0999e-01, -9.3689e-03,  2.0370e-02,\n",
      "        -5.5176e-02, -7.0068e-02,  2.7962e-03, -5.7220e-02, -1.9104e-02,\n",
      "        -3.0685e-02, -1.4893e-02,  1.1871e-02,  3.6865e-02,  2.8290e-02,\n",
      "        -1.0339e-01,  8.2703e-03,  6.7078e-02,  8.4229e-02,  2.2995e-02],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.9.final_layer_norm.weight Parameter containing:\n",
      "tensor([0.6509, 0.6616, 0.6841, 0.6050, 0.6494, 0.6138, 0.6318, 0.6338, 0.6211,\n",
      "        0.6353, 0.6475, 0.6416, 0.6543, 0.6338, 0.6104, 0.6421, 0.6250, 0.6855,\n",
      "        0.6211, 0.5840, 0.6436, 0.6582, 0.6445, 0.6279, 0.6899, 0.6338, 0.6763,\n",
      "        0.6294, 0.5991, 0.6401, 0.6743, 0.6577, 0.6733, 0.6484, 0.6465, 0.6616,\n",
      "        0.6211, 0.6553, 0.6538, 0.6787, 0.6323, 0.6587, 0.6675, 0.6553, 0.6587,\n",
      "        0.6309, 0.3438, 0.6440, 0.6367, 0.6865, 0.6196, 0.6514, 0.6416, 0.6265,\n",
      "        0.6338, 0.6113, 0.6426, 0.6558, 0.6382, 0.6655, 0.6426, 0.6543, 0.6572,\n",
      "        0.6611, 0.6460, 0.6162, 0.3711, 0.6372, 0.6689, 0.6060, 0.6162, 0.6670,\n",
      "        0.5923, 0.6362, 0.6704, 0.6831, 0.6362, 0.6646, 0.6479, 0.6523, 0.6587,\n",
      "        0.6436, 0.6123, 0.6636, 0.6567, 0.6323, 0.6274, 0.6411, 0.6714, 0.6230,\n",
      "        0.6558, 0.6528, 0.6763, 0.6509, 0.6587, 0.6299, 0.5957, 0.6050, 0.6602,\n",
      "        0.6841, 0.6069, 0.6499, 0.6299, 0.6528, 0.6118, 0.6626, 0.6460, 0.6274,\n",
      "        0.6396, 0.6680, 0.6655, 0.6675, 0.6543, 0.6558, 0.6216, 0.6631, 0.6265,\n",
      "        0.6216, 0.6421, 0.6064, 0.6401, 0.6284, 0.6401, 0.6772, 0.6157, 0.6724,\n",
      "        0.6108, 0.6694, 0.6255, 0.5947, 0.6294, 0.6587, 0.6631, 0.6387, 0.6094,\n",
      "        0.6284, 0.6675, 0.6025, 0.6245, 0.6914, 0.6377, 0.6655, 0.6338, 0.6509,\n",
      "        0.5884, 0.6958, 0.6211, 0.6733, 0.5894, 0.6372, 0.6489, 0.6826, 0.6396,\n",
      "        0.6611, 0.6260, 0.6475, 0.6797, 0.6299, 0.6577, 0.6953, 0.6577, 0.6021,\n",
      "        0.6138, 0.6328, 0.6431, 0.6377, 0.6465, 0.6797, 0.6431, 0.6001, 0.6035,\n",
      "        0.6626, 0.6226, 0.6509, 0.5498, 0.6094, 0.6543, 0.6562, 0.6226, 0.6099,\n",
      "        0.6948, 0.6270, 0.6543, 0.6279, 0.6631, 0.6582, 0.6616, 0.6514, 0.6753,\n",
      "        0.6670, 0.6489, 0.6445, 0.6699, 0.6299, 0.6416, 0.6089, 0.6655, 0.6191,\n",
      "        0.6421, 0.5591, 0.6343, 0.5811, 0.5811, 0.6719, 0.5986, 0.6538, 0.5742,\n",
      "        0.6392, 0.6255, 0.6460, 0.6621, 0.6572, 0.6396, 0.6377, 0.6162, 0.6343,\n",
      "        0.6411, 0.6611, 0.6660, 0.6382, 0.6553, 0.6450, 0.6196, 0.6323, 0.6021,\n",
      "        0.6758, 0.6528, 0.6626, 0.6484, 0.6328, 0.6392, 0.6353, 0.6401, 0.6606,\n",
      "        0.6479, 0.6045, 0.6284, 0.7031, 0.6235, 0.6748, 0.6069, 0.6699, 0.6099,\n",
      "        0.6626, 0.6553, 0.6299, 0.6382, 0.6704, 0.6650, 0.7451, 0.6445, 0.6777,\n",
      "        0.6592, 0.6558, 0.6509, 0.6191, 0.6152, 0.6499, 0.6367, 0.6572, 0.6484,\n",
      "        0.6377, 0.6572, 0.6245, 0.6104, 0.6274, 0.6631, 0.6353, 0.6631, 0.6836,\n",
      "        0.6978, 0.6392, 0.6680, 0.6548, 0.6133, 0.6255, 0.6470, 0.6714, 0.6641,\n",
      "        0.6582, 0.6509, 0.6597, 0.6104, 0.6177, 0.7139, 0.6323, 0.6387, 0.6367,\n",
      "        0.6357, 0.5840, 0.6367, 0.6836, 0.6304, 0.4900, 0.6655, 0.6187, 0.6313,\n",
      "        0.6147, 0.6704, 0.4246, 0.6533, 0.6621, 0.6602, 0.6724, 0.6152, 0.6304,\n",
      "        0.5747, 0.6372, 0.6338, 0.6377, 0.6079, 0.6260, 0.6904, 0.6611, 0.6802,\n",
      "        0.5586, 0.6304, 0.6436, 0.6528, 0.6733, 0.6602, 0.6743, 0.6572, 0.5942,\n",
      "        0.6362, 0.6177, 0.6489, 0.6978, 0.6865, 0.6255, 0.6704, 0.6270, 0.6792,\n",
      "        0.6128, 0.6216, 0.6357, 0.6260, 0.6597, 0.6660, 0.6357, 0.6689, 0.6006,\n",
      "        0.6104, 0.6641, 0.6328, 0.6582, 0.6172, 0.6421, 0.5635, 0.6401, 0.6035,\n",
      "        0.6714, 0.6079, 0.6416, 0.6597, 0.6880, 0.6743, 0.6753, 0.6714, 0.6255,\n",
      "        0.5996, 0.6050, 0.6069, 0.6372, 0.6367, 0.6421, 0.6206, 0.6738, 0.6465,\n",
      "        0.6523, 0.6431, 0.6543, 0.6514, 0.6431, 0.6631, 0.6426, 0.6626, 0.6294,\n",
      "        0.6504, 0.6631, 0.6646, 0.6548, 0.6333, 0.5566, 0.6440, 0.6812, 0.6499,\n",
      "        0.6807, 0.6699, 0.6709, 0.6636, 0.6582, 0.6514, 0.6362, 0.5918, 0.5835,\n",
      "        0.6709, 0.6255, 0.6392, 0.6641, 0.6284, 0.6528, 0.6382, 0.6816, 0.6211,\n",
      "        0.6826, 0.6567, 0.6685, 0.6538, 0.6211, 0.6504, 0.6606, 0.6123, 0.6372,\n",
      "        0.6514, 0.5767, 0.6353, 0.6768, 0.6777, 0.6699, 0.6611, 0.6143, 0.6548,\n",
      "        0.6562, 0.6748, 0.6650, 0.6489, 0.6387, 0.6343, 0.6445, 0.6846, 0.6484,\n",
      "        0.6401, 0.6582, 0.6226, 0.6738, 0.6626, 0.6172, 0.6255, 0.6411, 0.6353,\n",
      "        0.6069, 0.7017, 0.6318, 0.6680, 0.6436, 0.6528, 0.6641, 0.6343, 0.6323,\n",
      "        0.6284, 0.6475, 0.6519, 0.6797, 0.6963, 0.6401, 0.6631, 0.6665, 0.5791,\n",
      "        0.6548, 0.2839, 0.6138, 0.6167, 0.6724, 0.6167, 0.6875, 0.5952, 0.6211,\n",
      "        0.6416, 0.6172, 0.6387, 0.6187, 0.6445, 0.6646, 0.6655, 0.6333, 0.6636,\n",
      "        0.6084, 0.6636, 0.6367], requires_grad=True)\n",
      "esm_model.layers.9.final_layer_norm.bias Parameter containing:\n",
      "tensor([ 3.4332e-02,  5.2856e-02, -2.0554e-02, -2.3636e-02,  3.2196e-02,\n",
      "        -1.9592e-02,  2.6505e-02, -2.9037e-02, -1.3323e-03,  1.9394e-02,\n",
      "        -2.8564e-02,  2.0401e-02,  5.5084e-02,  8.6823e-03, -1.0010e-02,\n",
      "         4.0619e-02,  4.3304e-02, -8.2947e-02, -1.2732e-01,  3.1433e-02,\n",
      "         7.0862e-02,  4.2816e-02, -1.4992e-02, -4.1046e-02, -9.7168e-02,\n",
      "         6.6223e-02, -3.4607e-02,  1.8951e-02,  2.4353e-02, -5.0110e-02,\n",
      "        -3.1677e-02,  2.5284e-02, -3.5461e-02, -1.6495e-02,  2.6169e-02,\n",
      "        -3.3051e-02, -1.0185e-02, -4.0894e-03, -2.4910e-03, -6.1096e-02,\n",
      "        -7.6416e-02, -5.1727e-02,  1.2230e-02, -2.6474e-03, -8.3801e-02,\n",
      "        -3.5431e-02, -1.1755e-01, -2.9602e-03, -5.4871e-02, -5.7495e-02,\n",
      "         4.3884e-02,  3.6983e-03,  5.4993e-02, -2.3071e-02,  4.4647e-02,\n",
      "         4.5776e-02, -3.7323e-02,  6.8237e-02, -7.2632e-02, -2.9465e-02,\n",
      "        -6.0699e-02, -2.7649e-02,  1.3171e-01, -3.8391e-02, -1.0065e-01,\n",
      "        -3.2837e-02,  1.5967e-01, -7.5417e-03,  1.2183e-01, -1.1734e-02,\n",
      "        -4.0955e-02,  1.0815e-01,  9.3994e-02, -5.0446e-02,  3.5919e-02,\n",
      "         1.0297e-01,  1.4722e-01,  5.5115e-02,  2.7418e-04, -3.4924e-03,\n",
      "         9.4849e-02,  1.7456e-02, -1.0321e-01,  1.4999e-02,  1.7517e-02,\n",
      "         6.8237e-02,  6.7177e-03, -2.8351e-02,  8.0811e-02, -6.2637e-03,\n",
      "         3.1525e-02,  2.3331e-02, -1.1345e-02,  2.0523e-02,  8.9355e-02,\n",
      "         7.1533e-02,  3.6743e-02,  2.5131e-02,  7.2754e-02, -7.9468e-02,\n",
      "         6.0616e-03,  2.8168e-02,  1.7654e-02,  5.4588e-03,  9.1980e-02,\n",
      "        -5.2216e-02, -6.0486e-02,  3.2104e-02,  1.6800e-02,  1.5427e-02,\n",
      "         6.8542e-02,  6.5369e-02,  6.7810e-02, -8.6670e-02, -7.9041e-03,\n",
      "         7.2449e-02, -4.1809e-02,  7.0312e-02,  3.5400e-02, -7.4997e-03,\n",
      "        -5.8960e-02,  7.5623e-02, -2.2690e-02, -2.4841e-02, -5.6061e-02,\n",
      "        -4.5929e-02, -6.8245e-03, -7.6782e-02,  9.0027e-02, -7.3891e-03,\n",
      "        -1.0071e-01, -8.5297e-03,  4.9011e-02,  4.6082e-02, -2.9221e-02,\n",
      "         5.8685e-02,  4.0863e-02,  7.3059e-02,  1.1157e-01, -1.8448e-02,\n",
      "         3.5004e-02, -5.4321e-02,  9.8389e-02, -6.7596e-03,  2.3712e-02,\n",
      "         1.1337e-02, -1.1536e-02,  6.8115e-02,  9.3018e-02,  4.9194e-02,\n",
      "         4.3762e-02, -7.8857e-02, -8.0261e-02, -8.6136e-03,  7.5806e-02,\n",
      "        -4.3915e-02, -4.4525e-02,  5.1971e-02,  1.0857e-02,  1.0754e-01,\n",
      "        -3.0613e-03, -5.2795e-02,  5.4504e-02, -7.2632e-02, -5.1208e-02,\n",
      "        -2.1667e-02,  4.9477e-03, -6.1005e-02, -3.9825e-03,  1.1330e-02,\n",
      "         1.3538e-01,  5.4312e-04,  4.5380e-02,  1.3908e-02, -9.2010e-03,\n",
      "         1.5915e-02, -6.3286e-03, -9.2773e-03,  2.1774e-02,  4.3274e-02,\n",
      "         4.9019e-03,  1.4198e-02, -7.2876e-02, -1.3359e-02,  5.2155e-02,\n",
      "         3.2349e-02, -3.5492e-02,  3.7750e-02, -8.7524e-02,  6.3904e-02,\n",
      "        -3.8177e-02,  6.6956e-02,  7.4585e-02,  8.8257e-02,  4.5135e-02,\n",
      "         4.9896e-02, -6.5369e-02, -8.6731e-02,  9.0149e-02,  1.1078e-01,\n",
      "         4.4281e-02,  1.1963e-01,  5.0110e-02, -1.0602e-01, -7.9834e-02,\n",
      "         3.9673e-02, -3.7689e-02, -1.9135e-02, -1.3710e-02,  2.3956e-02,\n",
      "         2.7390e-02, -1.8402e-02,  2.7191e-02, -1.2085e-02, -6.8545e-06,\n",
      "         3.7506e-02, -1.1603e-01, -6.8542e-02,  2.4918e-02, -8.7662e-03,\n",
      "        -1.4978e-01,  2.0416e-02,  6.3293e-02, -3.0930e-02,  2.5650e-02,\n",
      "        -4.6661e-02,  3.0411e-02,  3.4698e-02,  6.5002e-02,  3.8090e-03,\n",
      "        -2.4063e-02, -4.4327e-03, -7.0923e-02,  5.6366e-02, -4.0283e-02,\n",
      "        -2.4109e-02, -1.1499e-01,  8.6487e-02,  2.6443e-02, -2.5146e-02,\n",
      "        -1.9287e-02,  3.3356e-02,  4.1687e-02, -4.1161e-03, -4.3243e-02,\n",
      "        -1.5576e-01,  3.1677e-02,  4.2938e-02,  5.1300e-02, -1.1816e-01,\n",
      "        -5.4688e-02,  9.8038e-03, -6.8665e-02,  6.6162e-02,  4.6234e-03,\n",
      "        -7.1655e-02,  5.9662e-03, -2.9724e-02, -1.1951e-01, -3.0258e-02,\n",
      "         5.3619e-02, -1.6222e-03,  5.1331e-02, -2.8244e-02, -4.7272e-02,\n",
      "         8.1543e-02, -3.7659e-02,  4.7180e-02,  1.3962e-02,  7.2510e-02,\n",
      "        -7.9468e-02,  2.2736e-02,  1.1469e-01, -3.5645e-02,  1.5717e-03,\n",
      "         2.7832e-02, -3.8696e-02, -3.6865e-02, -1.4514e-01,  1.3708e-01,\n",
      "        -2.7725e-02,  1.0999e-01, -2.1225e-02, -3.4180e-03,  1.0674e-02,\n",
      "        -5.2681e-03,  9.4543e-02,  1.1542e-01, -7.9498e-03, -1.2817e-02,\n",
      "        -4.4495e-02, -9.9365e-02,  5.1727e-02,  4.0245e-03,  4.6143e-02,\n",
      "        -3.2654e-02, -4.2725e-02,  2.1118e-02,  1.2189e-01,  1.7590e-01,\n",
      "         9.7036e-04, -1.8250e-02,  5.1056e-02,  5.6976e-02, -8.9798e-03,\n",
      "        -1.3718e-02, -9.8648e-03, -8.9722e-02, -1.0796e-03, -1.5833e-01,\n",
      "         1.8097e-02,  1.6388e-02, -4.4250e-02, -4.1107e-02, -7.3425e-02,\n",
      "        -9.8877e-03,  5.8716e-02, -4.4067e-02,  1.8356e-02,  1.7929e-02,\n",
      "         1.5747e-02,  8.9539e-02,  1.3458e-02, -2.7725e-02,  3.5645e-02,\n",
      "        -3.7964e-02, -6.7863e-03,  9.8022e-02,  4.0436e-02,  1.4429e-01,\n",
      "         1.7242e-02,  1.5649e-01, -1.9684e-02,  4.9477e-03,  2.9251e-02,\n",
      "        -9.1492e-02,  1.1310e-01,  3.8910e-02,  7.4341e-02,  2.7008e-02,\n",
      "        -9.3918e-03,  8.0688e-02,  8.9783e-02,  1.7593e-02,  3.0273e-02,\n",
      "         8.0383e-02,  1.9102e-03,  1.6357e-02,  4.4830e-02, -2.2232e-02,\n",
      "         8.3923e-03,  5.5145e-02,  3.0231e-03, -6.9885e-02,  9.1187e-02,\n",
      "        -1.0907e-01,  9.7412e-02,  3.2440e-02, -7.9163e-02, -5.3215e-03,\n",
      "         1.6220e-02,  5.1270e-02, -4.7058e-02, -3.0960e-02,  6.8420e-02,\n",
      "         3.5828e-02, -5.3772e-02,  2.4200e-02,  1.1230e-02,  2.9877e-02,\n",
      "         4.1290e-02, -8.5983e-03,  1.4069e-02, -6.7810e-02, -4.6753e-02,\n",
      "         1.1017e-01,  1.9665e-03,  2.0493e-02, -4.4281e-02,  1.3466e-02,\n",
      "        -1.6327e-03,  3.2318e-02,  4.0894e-02,  9.6207e-03,  3.1143e-02,\n",
      "        -9.3307e-03,  4.9713e-02,  1.3382e-02,  4.2267e-02, -1.7128e-03,\n",
      "         9.1858e-03, -2.8824e-02,  1.5350e-02,  9.1171e-03,  1.1635e-02,\n",
      "        -1.7822e-02,  2.3575e-03,  7.5378e-02, -1.6327e-02, -4.3304e-02,\n",
      "         2.1881e-02, -1.1261e-01, -1.5717e-02, -2.2675e-02,  2.3666e-02,\n",
      "         4.3427e-02, -2.5711e-02,  1.5495e-02,  8.7738e-03,  2.8519e-02,\n",
      "        -3.2623e-02, -9.1171e-03,  3.6621e-02, -3.3813e-02,  1.8265e-02,\n",
      "         9.6130e-02,  4.9133e-02, -5.7770e-02, -7.1594e-02, -4.5044e-02,\n",
      "         3.9246e-02,  4.0436e-02, -7.8613e-02, -4.1779e-02,  7.6599e-02,\n",
      "         8.1970e-02,  7.0740e-02,  1.8784e-02,  1.1487e-01,  2.3834e-02,\n",
      "        -1.1848e-02,  6.6772e-02, -3.3081e-02,  3.5572e-03,  4.3182e-02,\n",
      "         1.0254e-01,  3.4058e-02,  5.2032e-02,  6.3972e-03,  2.4902e-02,\n",
      "         1.2779e-02, -3.8757e-02, -1.1604e-02,  7.4585e-02,  9.6863e-02,\n",
      "         6.8665e-02,  8.4106e-02,  1.3027e-03,  3.6224e-02, -6.6833e-02,\n",
      "         2.3254e-02, -1.3416e-01, -1.0117e-02,  5.7793e-03, -1.8890e-02,\n",
      "         3.9459e-02, -9.0759e-02, -4.6417e-02, -2.6413e-02, -9.6283e-03,\n",
      "         3.5645e-01,  8.7524e-02, -7.9651e-03, -6.7673e-03,  6.9580e-02,\n",
      "        -5.2887e-02,  3.3386e-02, -4.2114e-02,  3.5736e-02,  3.5431e-02,\n",
      "        -6.1401e-02,  8.9111e-02, -4.4891e-02,  6.0059e-02, -4.9988e-02,\n",
      "        -6.5155e-03, -2.1088e-02,  8.0994e-02,  7.6828e-03, -2.7756e-02],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.self_attn.k_proj.weight Parameter containing:\n",
      "tensor([[ 0.0235,  0.0966,  0.0686,  ...,  0.1081, -0.0353, -0.0275],\n",
      "        [-0.0338,  0.0033, -0.0396,  ...,  0.1133,  0.1161,  0.0605],\n",
      "        [ 0.2251,  0.0842, -0.0694,  ...,  0.0977, -0.0156,  0.0555],\n",
      "        ...,\n",
      "        [ 0.2230,  0.0675, -0.1879,  ..., -0.1481,  0.1117,  0.1310],\n",
      "        [ 0.1953,  0.0345,  0.1281,  ...,  0.1621,  0.0378,  0.0047],\n",
      "        [-0.0790,  0.0878, -0.0525,  ..., -0.0805, -0.1459,  0.0536]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.self_attn.k_proj.bias Parameter containing:\n",
      "tensor([ 1.7044e-02, -8.6914e-02, -8.2275e-02,  1.7627e-01, -1.3098e-01,\n",
      "        -4.5703e-01, -7.8955e-01,  4.0894e-02, -9.9658e-01, -5.0446e-02,\n",
      "         5.3467e-01,  2.0728e-01, -6.2523e-03,  1.9150e-02,  1.3135e-01,\n",
      "        -8.0383e-02, -2.1179e-01, -7.9651e-02, -1.1029e-01, -1.7471e+00,\n",
      "         3.1470e-01, -7.9834e-01, -5.5225e-01, -1.7297e-01, -4.5586e-03,\n",
      "         5.5145e-02,  1.1856e-02, -9.0027e-03, -8.7051e-03, -1.9043e-01,\n",
      "        -3.7305e-01,  4.3188e-01, -1.7371e-01,  1.8237e-01, -2.0715e-01,\n",
      "         6.2305e-01,  2.4700e-04, -2.5253e-02, -4.1687e-02, -9.1782e-03,\n",
      "         1.7627e-01,  1.8250e-02, -1.2451e-01, -5.5322e-01,  2.1887e-01,\n",
      "         2.6440e-01, -9.9414e-01, -4.1107e-02,  4.8866e-03,  3.6865e-02,\n",
      "        -4.5349e-02,  6.0974e-02, -5.3558e-02,  4.1443e-02, -4.7150e-02,\n",
      "         4.0869e-01, -7.2937e-02, -6.3477e-02, -2.6929e-01,  3.6890e-01,\n",
      "        -3.0270e-03, -3.1525e-02, -4.7211e-02, -3.1372e-02, -1.0840e-01,\n",
      "        -1.8127e-01, -2.1716e-01, -3.0078e-01, -9.9731e-02,  7.0740e-02,\n",
      "         3.6957e-02,  1.1719e-02, -1.7651e-01,  1.6675e-01,  1.5674e-01,\n",
      "         9.7998e-01, -1.4478e-01,  2.9434e+00, -2.5938e+00, -3.4336e+00,\n",
      "         6.2939e-01,  2.8735e-01, -1.8135e+00, -4.2578e-01,  5.2441e-01,\n",
      "         1.9800e-01, -6.1719e-01, -2.5586e-01, -2.1562e+00, -7.9150e-01,\n",
      "        -1.9814e+00, -3.5000e+00, -3.9331e-01, -1.0098e+00,  7.4854e-01,\n",
      "         2.7954e-01, -1.1904e+00,  4.2749e-01,  4.5508e-01, -8.3203e-01,\n",
      "         1.4365e+00,  2.7500e+00,  4.5195e+00, -5.5703e+00,  2.9922e-02,\n",
      "         1.3428e+00,  2.2229e-01, -1.4521e+00, -1.2061e-01,  1.9287e-02,\n",
      "         2.7368e-01, -2.9614e-01, -3.4546e-01,  4.6606e-01, -3.6896e-02,\n",
      "         4.8682e-01,  3.7646e-01,  9.5557e-01,  1.2871e+00,  8.6475e-01,\n",
      "         3.9722e-01,  4.9048e-01,  2.3523e-01, -3.5767e-01, -6.9043e-01,\n",
      "         1.5518e+00, -3.1953e+00,  1.6296e-01,  2.3230e-01, -3.2129e-01,\n",
      "        -2.7148e+00, -3.0811e-01, -1.2002e+00,  1.5100e-01,  3.7476e-01,\n",
      "        -4.1479e-01,  5.1611e-01, -2.5562e-01,  2.4490e-02, -4.5742e+00,\n",
      "         4.4189e-02,  1.0022e-01,  7.5830e-01, -1.4590e+00, -1.1053e-01,\n",
      "         4.9731e-01, -2.0508e-01, -6.1084e-01,  2.7539e-01,  3.8262e+00,\n",
      "        -6.3110e-02, -6.0234e+00,  9.3628e-02,  9.0723e-01,  1.9812e-01,\n",
      "         1.2520e+00, -8.7891e-01, -7.1655e-02, -5.4199e-01,  6.8701e-01,\n",
      "         1.7480e+00, -4.8004e-02,  5.1836e+00,  8.3789e-01, -5.4932e-01,\n",
      "        -2.2876e-01,  3.2383e+00, -5.4541e-01,  3.5156e-02,  1.2073e-01,\n",
      "         1.0297e-01,  2.9199e-01, -5.7812e-01, -7.6807e-01,  2.3965e+00,\n",
      "        -9.6069e-02, -7.2559e-01,  1.9836e-01, -5.4004e-01,  1.9226e-02,\n",
      "        -3.7689e-02, -5.1514e-02,  5.6183e-02,  1.8665e-01, -1.2789e-03,\n",
      "         6.6357e-01, -2.1924e-01, -4.3555e+00,  9.0271e-02, -3.5938e-01,\n",
      "        -2.5586e-01, -1.6833e-01, -8.7891e-02, -6.2988e-02,  5.4749e-02,\n",
      "         2.0483e-01,  4.4189e-01, -1.3977e-01, -1.0468e-01,  2.2003e-02,\n",
      "         1.1450e-01, -4.0894e-01,  1.0309e-01,  2.8839e-02,  3.7781e-02,\n",
      "         2.3514e-02, -1.4075e-01, -2.1545e-01, -2.8125e-01,  1.3115e+00,\n",
      "         2.4883e+00,  3.7754e+00, -1.9238e-01,  2.1655e-01,  1.1285e-01,\n",
      "         1.9421e-01, -3.8696e-02, -8.4106e-02, -1.2732e-01,  1.5967e-01,\n",
      "        -4.7791e-02,  1.1895e+00, -2.3516e+00, -1.4124e-01,  6.2866e-02,\n",
      "        -1.8274e-01,  8.7830e-02,  4.5258e-02,  4.1412e-02,  2.0645e-02,\n",
      "         1.4172e-01,  3.4912e-01, -6.2207e-01,  1.5649e-01, -2.3499e-01,\n",
      "        -3.8105e+00,  1.9348e-01, -6.1328e-01,  5.1807e-01, -3.7323e-02,\n",
      "        -1.4172e-01, -5.5054e-02, -1.5125e-01, -2.7124e-01,  1.2964e-01,\n",
      "         5.0720e-02,  1.8738e-01,  4.0039e+00, -1.8884e-01, -6.5002e-02,\n",
      "        -2.9272e-01,  6.4774e-03,  1.4053e-02, -6.0852e-02,  1.0217e-01,\n",
      "        -9.1553e-02, -5.4395e-01,  1.3564e+00,  2.5039e+00,  1.0840e-01,\n",
      "        -2.0447e-02,  2.7222e-01,  3.0469e-01, -5.7568e-01,  1.6589e-01,\n",
      "         2.0618e-01, -1.0193e-01, -4.6118e-01, -1.5137e+00,  2.5098e+00,\n",
      "        -7.4512e-01,  4.5820e+00,  5.3076e-01,  6.3672e-01,  1.2891e-01,\n",
      "         6.2695e-01, -4.3884e-02,  1.0565e-01,  5.2734e-01,  5.3223e-01,\n",
      "        -3.2471e-01,  3.4106e-01,  3.9453e+00,  1.9336e+00, -2.9565e-01,\n",
      "        -3.6816e-01, -1.3096e+00, -3.5614e-02, -1.0657e-01,  7.0801e-02,\n",
      "        -2.1683e-02,  7.8003e-02, -3.1372e-01, -4.3652e-01, -2.0273e+00,\n",
      "         2.2832e+00,  5.7297e-03, -1.3512e-02,  4.8926e-01, -4.2944e-01,\n",
      "         1.2302e-03,  5.4840e-02, -1.6052e-01, -8.7524e-02, -2.6929e-01,\n",
      "        -5.6396e-01, -6.6101e-02,  7.9785e-01, -2.3767e-01,  2.8296e-01,\n",
      "         1.5723e-01, -3.8849e-02,  2.1534e-03, -5.3436e-02,  2.8473e-02,\n",
      "        -2.1973e-02,  7.8735e-02, -2.1558e-01, -4.1138e-01, -1.4000e-02,\n",
      "         2.6230e-02,  3.3789e-01, -2.6062e-02,  4.7119e-02, -6.8398e-03,\n",
      "        -6.6833e-02,  1.3145e-02,  1.0162e-01,  8.0139e-02, -1.8408e-01,\n",
      "        -2.0504e-03,  9.0967e-01,  1.5625e-01,  1.6589e-01,  2.1423e-01,\n",
      "        -4.5483e-01, -1.4038e-01,  1.5869e-02,  3.2104e-01, -1.9263e-01,\n",
      "         7.3633e-01, -2.5527e+00,  5.3101e-02, -4.0820e-01,  5.6250e-01,\n",
      "        -2.4304e-01,  7.6367e-01,  8.1738e-01,  2.4060e-01,  2.5439e-01,\n",
      "        -1.0461e-01,  7.2559e-01,  7.4805e-01,  3.6914e-01,  4.7344e+00,\n",
      "        -5.8555e+00, -5.0635e-01,  9.5996e-01,  6.7969e-01,  2.8149e-01,\n",
      "        -8.1587e-04,  1.6479e-02, -3.3081e-02,  1.1223e-02, -1.0504e-01,\n",
      "         3.4375e-01, -1.9312e-01,  4.6167e-01,  2.8735e-01,  3.4131e-01,\n",
      "        -3.5986e-01, -3.7256e-01, -7.9575e-03, -6.8092e-03,  8.3466e-03,\n",
      "         1.6980e-01, -6.7688e-02, -1.3904e-01,  4.4043e-01,  6.6992e-01,\n",
      "         1.5466e-01,  7.4707e-02, -7.2708e-03,  1.8628e-01, -1.6724e-02,\n",
      "         4.3121e-02, -1.5015e-01, -1.2415e-01, -3.5352e-01,  6.7139e-01,\n",
      "         1.7256e+00, -8.7646e-02,  3.7451e-01,  3.6621e-02, -1.2779e-02,\n",
      "         2.6953e-01,  1.1795e-02,  1.0126e-01, -1.6406e-01,  1.4368e-01,\n",
      "        -5.1727e-03, -7.2327e-02, -1.9482e-01,  2.5703e+00,  6.3721e-01,\n",
      "         6.5308e-02, -2.3804e-01,  7.4387e-03, -3.4668e-02, -1.0364e-01,\n",
      "        -6.1371e-02,  1.0449e-01,  6.0181e-02, -6.5491e-02,  1.7200e-01,\n",
      "         3.5156e-01,  1.6068e-02,  2.4988e-01,  1.2421e-01,  5.0732e-01,\n",
      "         3.7842e-02, -1.5121e-02, -1.4122e-02,  8.1635e-03,  4.4189e-02,\n",
      "        -1.1487e-01, -1.7639e-01,  9.4543e-02, -1.2134e-01, -4.9530e-02,\n",
      "         8.7646e-02,  2.5586e-01, -7.1143e-01, -1.9360e-01, -2.0337e-01,\n",
      "         1.1896e-01,  1.0870e-01,  3.9429e-01,  3.3965e+00, -1.8188e-02,\n",
      "        -2.2119e-01,  1.1084e+00,  5.8057e-01,  6.6748e-01,  7.4768e-02,\n",
      "        -3.3844e-02, -1.6296e-01,  5.4492e-01,  1.4873e+00,  2.4062e+00,\n",
      "        -9.7070e-01, -6.6562e+00, -1.5137e-01,  1.1680e+00,  1.4404e+00,\n",
      "         3.5156e-01, -5.0934e-02, -8.7402e-02, -4.9316e-02,  3.0609e-02,\n",
      "         1.1066e-01,  3.0670e-02,  1.9318e-02, -1.1367e+00, -1.6296e-01,\n",
      "        -1.8408e-01,  3.2031e-01,  2.8662e-01, -4.2328e-02,  5.4077e-02,\n",
      "        -4.8065e-03,  3.9734e-02, -1.8018e-01, -3.3521e-01, -6.6406e-01,\n",
      "         6.1230e-01,  6.3135e-01, -2.3694e-01,  1.8396e-03,  2.1631e-01],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.self_attn.v_proj.weight Parameter containing:\n",
      "tensor([[-0.0801,  0.1498, -0.2395,  ..., -0.0128, -0.1044, -0.1015],\n",
      "        [-0.1556, -0.0836,  0.1040,  ...,  0.1588,  0.1416,  0.0708],\n",
      "        [-0.0814,  0.0323, -0.0892,  ...,  0.0922, -0.1565, -0.0083],\n",
      "        ...,\n",
      "        [-0.0253, -0.0773,  0.1637,  ...,  0.1519, -0.0618,  0.0821],\n",
      "        [-0.0073, -0.0905,  0.0311,  ...,  0.0166, -0.1691, -0.0861],\n",
      "        [ 0.2522, -0.2607, -0.0388,  ...,  0.0529, -0.0122,  0.0014]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.self_attn.v_proj.bias Parameter containing:\n",
      "tensor([-3.0785e-03, -1.0490e-02,  2.9419e-02, -6.3782e-03, -1.6373e-02,\n",
      "         5.1804e-03,  1.5778e-02,  1.3481e-02,  2.8076e-02,  9.6817e-03,\n",
      "        -1.1162e-02, -1.6464e-02,  1.2741e-02,  1.7838e-02,  8.8043e-03,\n",
      "         2.0691e-02,  1.4641e-02, -2.2949e-02, -1.5297e-02,  9.9945e-03,\n",
      "        -2.5650e-02, -1.1482e-02,  2.4139e-02, -8.5907e-03, -3.9978e-03,\n",
      "         1.9150e-02, -5.7068e-03, -3.7956e-03,  3.0766e-03,  2.3384e-03,\n",
      "         1.2512e-02,  2.0294e-02, -2.2430e-02, -6.2990e-04,  9.7733e-03,\n",
      "        -3.9482e-03,  2.7046e-03,  1.5358e-02, -2.2964e-02,  6.4125e-03,\n",
      "        -1.7929e-02, -8.0643e-03, -2.5055e-02,  1.3367e-02,  2.7485e-03,\n",
      "        -6.5994e-04,  1.1627e-02, -5.4054e-03, -1.8356e-02, -7.0953e-03,\n",
      "        -1.6689e-03, -1.4725e-02,  2.9793e-03,  2.0935e-02, -1.2543e-02,\n",
      "        -1.7303e-02,  1.8272e-03, -2.1553e-03,  2.4857e-02,  4.9782e-03,\n",
      "        -9.5596e-03,  1.8173e-02,  2.2919e-02, -1.8005e-02, -1.0710e-03,\n",
      "        -2.0813e-02,  1.9562e-02,  4.6883e-03, -2.2202e-02,  6.3057e-03,\n",
      "         1.2787e-02, -1.1230e-02, -4.7073e-03, -1.6571e-02, -4.8126e-02,\n",
      "        -1.0231e-02, -4.0955e-02,  1.1375e-02, -1.8143e-02,  1.3702e-02,\n",
      "         1.1780e-02, -1.5327e-02, -3.1853e-03, -2.2850e-03,  6.2943e-04,\n",
      "         3.9917e-02,  1.2863e-02,  1.9424e-02,  4.3983e-03,  3.6499e-02,\n",
      "         3.4981e-03,  2.3746e-03,  2.0950e-02, -4.3854e-02,  4.1107e-02,\n",
      "        -3.3752e-02,  2.5864e-02,  2.5296e-04, -2.0538e-02, -3.6049e-03,\n",
      "        -3.2776e-02,  3.6865e-02, -1.3718e-02, -3.1616e-02,  3.3245e-03,\n",
      "         1.6891e-02, -2.4384e-02,  5.2643e-02, -5.9853e-03,  1.1856e-02,\n",
      "        -2.2590e-05, -3.0470e-04,  1.3077e-02,  6.2943e-03, -1.7563e-02,\n",
      "        -1.3718e-02, -2.6733e-02,  1.2856e-02, -1.3489e-02,  3.2776e-02,\n",
      "        -4.8866e-03,  2.6321e-02, -3.0609e-02, -8.5115e-04, -4.2206e-02,\n",
      "        -3.3966e-02, -8.5297e-03, -4.3579e-02,  2.3773e-02,  1.3145e-02,\n",
      "         6.4812e-03, -1.0109e-02,  2.7069e-02, -1.3138e-02, -8.1635e-03,\n",
      "         7.6256e-03,  1.2535e-02,  1.7365e-02,  5.1231e-03, -9.2621e-03,\n",
      "         2.7634e-02,  3.9444e-03, -8.4229e-03,  1.6724e-02,  2.3590e-02,\n",
      "        -7.6141e-03, -2.3178e-02,  1.2482e-02,  3.0575e-03,  1.0841e-02,\n",
      "         1.7990e-02,  2.4292e-02,  8.0338e-03, -4.8584e-02, -1.1230e-02,\n",
      "        -2.0676e-02,  2.4841e-02,  3.4973e-02,  7.3051e-03, -1.5488e-02,\n",
      "         8.7402e-02,  1.8139e-03,  1.5841e-03, -1.3168e-02,  1.1959e-03,\n",
      "         8.1024e-03, -5.8777e-02,  1.1921e-03,  1.1780e-02, -2.2614e-02,\n",
      "         2.1530e-02, -4.4785e-03, -3.4313e-03,  1.7258e-02,  4.0474e-03,\n",
      "         2.5787e-02, -2.7130e-02, -1.5495e-02, -2.4094e-02, -1.1940e-02,\n",
      "         2.8648e-03, -1.8295e-02,  5.1346e-03, -2.4295e-04,  5.8327e-03,\n",
      "         2.4612e-02,  2.3163e-02,  8.2922e-04, -1.3153e-02,  9.3460e-03,\n",
      "        -7.5874e-03,  1.9852e-02, -2.2537e-02,  4.6814e-02,  3.4882e-02,\n",
      "         2.6871e-02,  2.6276e-02,  7.4524e-02, -1.8417e-02,  2.2812e-02,\n",
      "        -3.1616e-02,  2.4323e-02, -1.1436e-02,  4.1718e-02, -2.7634e-02,\n",
      "        -2.3239e-02,  3.9337e-02, -1.3710e-02,  2.2766e-02, -3.2593e-02,\n",
      "        -2.5528e-02,  1.0513e-02,  4.5197e-02,  3.8757e-02, -2.9316e-03,\n",
      "        -2.3956e-02,  2.1713e-02,  8.3618e-03,  2.0889e-02, -3.7567e-02,\n",
      "         2.6794e-02,  8.4209e-04, -1.6312e-02,  2.4979e-02,  1.4038e-02,\n",
      "        -2.6215e-02,  3.6926e-03,  1.7395e-02,  1.6129e-02, -4.3579e-02,\n",
      "        -1.7044e-02, -1.0166e-03, -6.9771e-03, -8.5754e-03,  3.6255e-02,\n",
      "        -6.9618e-03, -4.5319e-03, -2.7786e-02,  1.7578e-02,  8.4543e-04,\n",
      "        -7.7095e-03, -5.2528e-03, -1.2337e-02, -1.6968e-02, -1.2329e-02,\n",
      "         1.4542e-02,  1.9302e-02, -1.9501e-02,  1.4931e-02,  9.7504e-03,\n",
      "        -1.2016e-03,  2.0660e-02, -2.3972e-02, -2.8671e-02, -7.2174e-03,\n",
      "         1.5610e-02,  1.5450e-02,  1.5564e-02,  2.5574e-02, -4.4670e-03,\n",
      "        -1.3199e-02,  1.3496e-02,  1.8936e-02,  3.2558e-03,  1.3489e-02,\n",
      "         7.3967e-03, -3.6438e-02, -6.8787e-02,  5.6915e-02, -1.0399e-02,\n",
      "         2.1713e-02, -6.8848e-02,  6.6345e-02,  4.0092e-03,  5.7343e-02,\n",
      "         1.4931e-02,  3.1708e-02,  1.4557e-02, -9.5367e-04,  1.9821e-02,\n",
      "         1.3985e-02, -4.4365e-03,  1.7838e-02,  2.3544e-02,  3.9856e-02,\n",
      "         9.2173e-04,  2.1423e-02, -2.8343e-03,  1.1835e-03, -1.5884e-02,\n",
      "         3.4790e-02,  1.0254e-02, -8.1940e-03,  4.0321e-03, -2.3514e-02,\n",
      "         1.5388e-02,  1.2810e-02,  1.8784e-02, -1.6632e-02,  8.0261e-03,\n",
      "         6.1188e-03,  1.0193e-02,  1.4099e-02, -3.3356e-02,  1.1032e-02,\n",
      "         6.2828e-03, -2.2430e-02, -7.8964e-03,  1.5511e-02, -3.2368e-03,\n",
      "         8.0185e-03,  6.9084e-03,  5.6953e-03, -2.0477e-02,  8.1940e-03,\n",
      "         9.9106e-03, -2.1713e-02,  1.5434e-02, -2.3651e-03,  1.4648e-03,\n",
      "         2.4147e-03,  9.7504e-03,  4.1652e-04,  5.4359e-03,  5.3825e-03,\n",
      "         4.9095e-03,  4.8637e-03,  2.9335e-03, -1.3885e-02,  5.1727e-03,\n",
      "        -9.8114e-03,  8.6670e-03, -1.1429e-02, -6.3515e-03,  6.1646e-03,\n",
      "         1.0246e-02, -3.6560e-02, -3.5126e-02, -1.2306e-02,  1.3151e-03,\n",
      "        -1.9302e-02, -2.4681e-03,  8.0338e-03,  3.4122e-03,  5.9090e-03,\n",
      "        -8.2588e-04, -1.3397e-02, -1.1246e-02, -2.0485e-03, -1.8646e-02,\n",
      "        -4.0398e-03,  2.5375e-02,  2.8210e-03,  1.4954e-02, -1.4114e-03,\n",
      "        -3.0289e-02,  4.4189e-02, -8.2397e-03, -4.8340e-02,  2.6436e-03,\n",
      "        -7.2670e-03,  2.1267e-03,  2.0294e-02, -8.5602e-03, -6.8398e-03,\n",
      "         6.7139e-03, -3.7270e-03,  5.6763e-03, -3.3016e-03,  1.6041e-03,\n",
      "        -4.3945e-03,  2.7332e-03, -3.4485e-03, -1.8454e-03, -1.8402e-02,\n",
      "         9.3603e-04, -6.6109e-03,  7.6675e-03, -5.2147e-03, -9.5844e-04,\n",
      "        -2.9335e-03,  3.5071e-04, -9.0103e-03,  7.5302e-03,  4.4281e-02,\n",
      "        -4.3793e-03, -1.5244e-02, -8.9722e-03,  3.8818e-02, -4.0619e-02,\n",
      "         5.0354e-03,  4.8637e-03, -3.1464e-02,  4.1382e-02,  1.3092e-02,\n",
      "         1.2871e-02, -2.0493e-02, -3.6469e-02,  3.8574e-02,  2.4155e-02,\n",
      "         1.5091e-02, -5.1270e-02,  4.5593e-02, -1.6037e-02, -2.2217e-02,\n",
      "         3.3691e-02, -4.0894e-02, -1.8646e-02, -1.4664e-02, -2.2842e-02,\n",
      "         1.0681e-02, -2.6001e-02, -2.9297e-03, -1.1574e-02, -5.4703e-03,\n",
      "         1.9547e-02,  2.0874e-02, -2.7206e-02,  8.5373e-03,  7.4730e-03,\n",
      "         7.4844e-03,  1.1711e-02, -9.0790e-03, -2.0538e-02, -1.8005e-02,\n",
      "        -6.9504e-03, -1.2932e-02, -2.5711e-02, -1.6937e-02,  1.2039e-02,\n",
      "        -1.6785e-03, -3.1853e-03, -2.4857e-02,  9.4833e-03, -1.0347e-03,\n",
      "         6.9504e-03, -1.9318e-02, -3.3905e-02,  1.0139e-02,  4.6295e-02,\n",
      "         9.6054e-03,  2.3365e-03, -1.1909e-02,  2.0485e-03, -1.9913e-02,\n",
      "        -4.0283e-03,  4.2084e-02, -3.6041e-02,  1.5190e-02,  7.5417e-03,\n",
      "         1.4153e-03, -1.6527e-03, -3.2368e-03, -1.6846e-02,  2.7710e-02,\n",
      "        -1.8402e-02, -1.0078e-02,  4.8561e-03,  2.3209e-02,  9.9106e-03,\n",
      "        -4.3929e-05, -2.1484e-02,  9.8572e-03, -8.5754e-03,  1.3206e-02,\n",
      "         2.2163e-03,  1.8280e-02, -1.0201e-02, -4.2381e-03,  1.1093e-02,\n",
      "         2.4460e-02,  1.3123e-02,  3.0174e-03, -7.8812e-03, -7.5035e-03,\n",
      "        -4.9477e-03, -4.0007e-04, -6.7444e-03, -4.9744e-03,  1.5869e-02],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.self_attn.q_proj.weight Parameter containing:\n",
      "tensor([[ 0.0305, -0.0659,  0.1065,  ...,  0.0978,  0.1525,  0.0549],\n",
      "        [ 0.1394, -0.1971,  0.0316,  ...,  0.1998,  0.0665, -0.0623],\n",
      "        [-0.0798,  0.0301, -0.1467,  ...,  0.0620, -0.0076,  0.0220],\n",
      "        ...,\n",
      "        [ 0.1040,  0.2161,  0.0675,  ..., -0.0106, -0.0373, -0.0787],\n",
      "        [-0.3123,  0.0880,  0.2583,  ...,  0.1934,  0.0367, -0.0297],\n",
      "        [-0.0965,  0.0354, -0.0281,  ..., -0.0341,  0.2203,  0.0475]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.self_attn.q_proj.bias Parameter containing:\n",
      "tensor([ 3.0365e-02, -6.3110e-02, -1.1237e-01,  1.8384e-01, -7.1533e-02,\n",
      "        -2.5195e-01, -2.4768e-01,  8.9645e-03, -5.6427e-02, -7.6233e-02,\n",
      "         8.1970e-02,  1.1887e-02,  1.7136e-02, -1.5419e-02,  1.2238e-01,\n",
      "        -7.2083e-02, -1.9556e-01, -5.4535e-02,  3.5801e-03, -3.0566e-01,\n",
      "         1.4397e-02, -3.6652e-02, -4.3640e-02, -7.4524e-02,  3.6194e-02,\n",
      "         1.2238e-02,  3.7048e-02,  4.3579e-02, -1.8402e-02, -1.9861e-01,\n",
      "        -1.2891e-01,  1.7444e-01,  9.4681e-03,  6.9580e-02,  2.4933e-02,\n",
      "         3.8109e-03, -3.1719e-03, -5.1117e-02, -6.5186e-02,  8.5266e-02,\n",
      "         4.1748e-02, -2.2217e-02, -9.9976e-02, -2.4011e-01, -3.5797e-02,\n",
      "        -4.0741e-02, -1.2341e-01,  1.1243e-01, -1.7881e-03,  4.3793e-03,\n",
      "        -2.1561e-02,  1.5793e-02, -5.9692e-02,  4.4800e-02, -3.9032e-02,\n",
      "         2.0422e-01,  9.6321e-04,  1.3843e-01, -1.0815e-01, -7.0923e-02,\n",
      "         2.4567e-02, -5.6229e-03, -1.8829e-02, -1.4587e-02, -5.6976e-02,\n",
      "        -1.0492e-01, -1.7102e-01, -1.4673e-01,  4.5252e-04,  9.3811e-02,\n",
      "        -1.8823e-01,  3.5736e-02,  4.1748e-02,  2.0398e-01, -5.5023e-02,\n",
      "         2.4963e-01, -4.5166e-02,  2.5439e-01, -2.0996e-01, -1.6455e-01,\n",
      "         7.8430e-03,  6.2294e-03, -3.1586e-02, -1.1444e-03, -3.2910e-01,\n",
      "        -5.0537e-02, -3.0493e-01, -1.4624e-01, -2.5928e-01, -8.3435e-02,\n",
      "        -1.4636e-01, -1.8823e-01,  3.2471e-02, -5.2063e-02,  5.0629e-02,\n",
      "         6.4514e-02,  5.4321e-02,  2.1240e-01,  3.1421e-01, -3.3960e-01,\n",
      "         3.0859e-01,  2.6245e-01,  2.9639e-01, -3.1641e-01,  9.7809e-03,\n",
      "         2.0538e-02,  9.4177e-02, -2.2900e-01,  5.0000e-01, -2.3291e-01,\n",
      "         7.5500e-02, -1.3550e-02, -5.8716e-02,  2.1606e-02,  7.4043e-03,\n",
      "         1.3680e-02, -2.6188e-03, -1.0925e-02,  5.3253e-02,  1.1945e-01,\n",
      "        -1.5198e-01,  3.0396e-01,  2.0129e-01, -2.2461e-01, -2.1667e-01,\n",
      "         2.5293e-01, -3.3521e-01,  2.8366e-02, -2.0020e-02, -8.2731e-04,\n",
      "        -1.3660e-01, -3.5828e-02, -4.8413e-01, -3.5522e-02,  2.2534e-01,\n",
      "        -2.2437e-01,  1.6907e-01, -6.2073e-02, -2.6894e-03, -2.6270e-01,\n",
      "         2.2308e-02,  4.0550e-03,  1.4221e-02, -2.3718e-01, -4.1309e-01,\n",
      "         2.1204e-01, -1.5491e-01, -1.5930e-01,  2.1790e-02,  3.1372e-01,\n",
      "        -3.9978e-02, -2.5391e-01, -2.9640e-03,  1.0452e-02,  4.3396e-02,\n",
      "         1.4099e-01, -6.9763e-02, -1.7261e-01, -2.6025e-01,  2.2449e-01,\n",
      "         2.6367e-01, -1.0811e-02,  2.5098e-01,  4.9744e-02,  1.3626e-02,\n",
      "        -3.3264e-02,  1.6614e-01, -4.4769e-02,  1.2354e-01,  7.2510e-02,\n",
      "         3.8544e-02,  2.0093e-01, -2.2717e-01, -1.9312e-01,  2.5708e-01,\n",
      "        -3.1647e-02, -3.8116e-02,  4.4250e-02, -9.1553e-03,  4.4556e-03,\n",
      "        -7.8087e-03,  1.1279e-01, -1.7249e-01, -1.0156e-01,  1.4636e-01,\n",
      "         1.8970e-01, -4.8004e-02, -2.5439e-01, -4.4220e-02, -3.4027e-02,\n",
      "         3.3131e-03, -1.2299e-02, -1.0175e-01,  8.8623e-02,  2.8854e-02,\n",
      "         1.2311e-01,  2.3193e-01, -1.8005e-02, -1.7151e-02,  2.2507e-02,\n",
      "        -1.9089e-02,  2.2400e-02,  2.4918e-02,  2.5543e-02, -2.3621e-02,\n",
      "        -8.5205e-02, -1.5369e-01, -1.9263e-01, -1.7163e-01,  3.3716e-01,\n",
      "         3.5449e-01,  3.3643e-01,  3.0853e-02, -2.6291e-02, -3.3875e-02,\n",
      "        -2.6520e-02, -4.7363e-02,  8.5510e-02, -2.1436e-01, -6.3843e-02,\n",
      "         5.2307e-02,  2.7026e-01, -2.9053e-01, -9.8419e-03,  1.2405e-02,\n",
      "        -4.5967e-03, -3.7537e-02, -4.6967e-02, -5.2063e-02, -7.3700e-03,\n",
      "        -1.2476e-01,  2.0862e-01, -2.2607e-01,  5.5634e-02, -3.8757e-02,\n",
      "        -2.6709e-01,  2.4612e-02, -1.6260e-03,  3.8395e-03, -6.2599e-03,\n",
      "         6.1066e-02, -9.0637e-02, -2.4048e-02, -1.1633e-01,  1.5894e-01,\n",
      "        -1.1702e-03,  2.4834e-03,  3.1323e-01, -7.8583e-03, -7.3731e-05,\n",
      "        -5.7831e-02,  7.2098e-04,  6.5369e-02, -1.2115e-02, -1.5784e-01,\n",
      "        -1.4502e-01, -2.6050e-01,  2.5928e-01,  2.9956e-01, -1.6235e-02,\n",
      "         3.3295e-02,  1.4629e-03, -1.3107e-02, -1.2217e-03,  2.7676e-03,\n",
      "         1.3477e-01, -1.2054e-01, -2.0764e-01, -2.8101e-01,  2.5000e-01,\n",
      "        -3.1464e-02,  2.4902e-01,  3.9673e-02, -5.2452e-03,  1.1368e-02,\n",
      "        -3.5324e-03,  1.3818e-01,  1.3745e-01,  2.7637e-01,  1.9604e-01,\n",
      "        -6.9885e-02,  1.5488e-02,  2.9102e-01,  1.0052e-01, -2.3251e-03,\n",
      "        -1.1360e-02, -5.6671e-02,  3.0594e-02, -8.2626e-03,  1.0138e-01,\n",
      "        -1.1360e-02,  1.2683e-01, -1.9458e-01, -1.7188e-01, -2.5903e-01,\n",
      "         2.6221e-01,  1.1505e-02,  1.6937e-02,  1.0109e-02, -9.5844e-04,\n",
      "        -2.0493e-02,  1.6464e-02, -1.5857e-01, -7.0007e-02, -1.5808e-01,\n",
      "        -1.8469e-01,  9.4318e-04,  1.0199e-01,  1.9974e-02,  3.1158e-02,\n",
      "        -1.0841e-02,  2.6169e-02, -3.7659e-02,  5.5771e-03,  2.0096e-02,\n",
      "         8.3084e-03,  7.1411e-02, -1.3123e-01, -2.4622e-01, -2.6184e-02,\n",
      "         1.2337e-02,  9.8511e-02,  1.8311e-01, -1.0156e-01, -6.7627e-02,\n",
      "         2.5955e-02, -3.4302e-02,  6.3660e-02,  2.3071e-02, -1.7517e-01,\n",
      "        -3.3966e-02,  3.1348e-01,  8.0490e-03,  5.0621e-03,  1.1548e-01,\n",
      "        -1.7334e-01,  2.1912e-01,  2.3047e-01,  2.1631e-01, -2.9984e-02,\n",
      "         1.8933e-01, -2.8198e-01, -1.1375e-02,  2.0966e-02, -1.8082e-02,\n",
      "         2.1576e-02,  4.2267e-02, -1.0193e-02,  9.9609e-02,  1.2596e-02,\n",
      "        -1.6895e-01,  2.8931e-01,  1.9434e-01,  3.8971e-02,  2.7368e-01,\n",
      "        -2.8101e-01,  7.2517e-03,  1.7151e-02,  4.0222e-02,  2.8320e-02,\n",
      "         3.6438e-02,  4.1840e-02,  8.5815e-02, -7.1045e-02, -3.3661e-02,\n",
      "         1.2311e-01, -1.4990e-01,  1.9177e-01,  4.9622e-02,  3.2135e-02,\n",
      "         1.1200e-02,  9.6741e-02,  2.6886e-02, -1.1963e-02, -3.1738e-02,\n",
      "        -1.2886e-02, -1.5918e-01, -3.6804e-02,  2.0032e-01,  2.7368e-01,\n",
      "         6.7871e-02,  6.1432e-02,  9.8328e-02,  9.6313e-02, -6.7200e-02,\n",
      "         9.6283e-03, -2.0459e-01, -1.6882e-01, -2.7344e-01,  2.9175e-01,\n",
      "         2.9443e-01, -8.2520e-02, -4.1351e-02, -1.6373e-02,  1.6815e-02,\n",
      "        -4.1199e-02, -4.6478e-02,  1.8250e-01, -1.9604e-01,  2.3047e-01,\n",
      "        -1.2854e-01, -1.3171e-01,  1.1551e-02,  3.0786e-01,  8.2275e-02,\n",
      "        -2.4994e-02,  3.9886e-02,  3.0426e-02,  7.7744e-03, -4.2511e-02,\n",
      "        -3.0182e-02,  5.1331e-02, -1.5503e-02, -4.6295e-02,  9.7290e-02,\n",
      "         1.4417e-01,  3.4790e-02,  7.5073e-02, -2.4438e-01, -1.4465e-01,\n",
      "         1.9745e-02,  3.7933e-02, -1.4820e-03, -2.2850e-03,  8.8196e-03,\n",
      "        -6.9458e-02, -9.1736e-02,  3.9825e-03,  9.3918e-03, -7.8186e-02,\n",
      "        -2.0157e-02, -1.6687e-01, -3.6255e-01,  1.8433e-01, -1.5076e-01,\n",
      "         1.6333e-01,  5.8411e-02,  9.3811e-02,  2.8809e-01, -1.8417e-02,\n",
      "        -6.8321e-03,  4.6204e-02, -8.3351e-04,  2.7710e-02, -6.2012e-02,\n",
      "         1.2573e-01,  9.0027e-02,  2.0105e-01,  2.5781e-01,  2.7246e-01,\n",
      "        -3.7048e-02, -2.6807e-01,  4.7798e-03,  4.2816e-02,  3.2745e-02,\n",
      "         1.6467e-01, -4.4434e-02, -3.7720e-02, -6.5857e-02,  9.6191e-02,\n",
      "         5.0079e-02, -2.4277e-02, -3.7170e-02, -1.9214e-01, -9.6893e-03,\n",
      "         4.2725e-02,  5.1788e-02, -3.6163e-02, -3.9856e-02,  2.9602e-02,\n",
      "         1.3939e-02,  4.6600e-02, -1.7090e-01, -2.4890e-01, -2.2632e-01,\n",
      "         1.3672e-01,  7.0557e-02, -5.1460e-03,  1.0887e-02, -1.4026e-01],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.self_attn.out_proj.weight Parameter containing:\n",
      "tensor([[-0.0394,  0.1011,  0.1742,  ...,  0.0601,  0.0140, -0.0045],\n",
      "        [-0.1042,  0.0557, -0.0096,  ...,  0.0515,  0.0674, -0.0153],\n",
      "        [ 0.1372, -0.0352, -0.0198,  ...,  0.0699, -0.0421,  0.0487],\n",
      "        ...,\n",
      "        [-0.0885, -0.0024, -0.0937,  ...,  0.0717,  0.1617, -0.1224],\n",
      "        [ 0.1267, -0.1790, -0.0782,  ...,  0.0695,  0.0371,  0.0068],\n",
      "        [-0.0271, -0.0821,  0.0432,  ...,  0.0273,  0.1317, -0.0173]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.self_attn.out_proj.bias Parameter containing:\n",
      "tensor([-2.9659e-03,  1.1673e-02, -9.7412e-02,  1.2537e-01, -7.8247e-02,\n",
      "        -8.0139e-02,  1.4465e-01, -2.4506e-02, -1.5393e-01, -2.2522e-02,\n",
      "        -1.1969e-01,  1.9592e-01, -2.8915e-02, -4.8157e-02,  9.3031e-04,\n",
      "         3.5431e-02,  3.4454e-02, -2.8259e-02, -1.5393e-01, -1.4136e-01,\n",
      "        -4.4769e-02, -1.1194e-01, -1.2207e-01, -1.7303e-02, -4.4098e-02,\n",
      "         8.6288e-03, -9.7595e-02,  6.5430e-02, -5.5542e-03, -5.2368e-02,\n",
      "         1.4185e-01, -1.7853e-02,  6.6101e-02,  4.3854e-02, -1.3110e-01,\n",
      "        -8.2764e-02,  1.1908e-01, -7.4585e-02, -1.3391e-01, -7.5134e-02,\n",
      "         5.6519e-02, -5.4565e-02,  4.5227e-02,  4.9347e-02, -4.5868e-02,\n",
      "        -2.1240e-02,  8.6523e-01, -2.0325e-02, -2.5284e-02, -1.6760e-01,\n",
      "        -1.1009e-02, -2.0706e-02,  6.7566e-02, -2.0645e-02,  7.8369e-02,\n",
      "        -9.1187e-02, -4.6814e-02, -1.4656e-02, -1.0419e-01, -4.0985e-02,\n",
      "        -6.5346e-03, -8.3313e-02,  1.0419e-01,  3.5950e-02,  1.0669e-01,\n",
      "        -9.1431e-02, -9.1187e-02, -6.8970e-03,  6.9214e-02,  1.0193e-01,\n",
      "        -9.5947e-02,  6.6345e-02,  1.3489e-01, -1.2610e-01,  1.3977e-01,\n",
      "         2.9175e-02,  3.6346e-02,  5.7739e-02,  4.4922e-02,  5.3864e-02,\n",
      "         5.9448e-02,  4.3518e-02, -1.4229e-02, -6.5735e-02,  7.0068e-02,\n",
      "        -6.1340e-02, -2.3636e-02, -1.4091e-02,  5.0354e-02, -1.5541e-02,\n",
      "         2.0142e-02,  9.0698e-02, -1.2524e-01, -7.7515e-03, -1.1700e-01,\n",
      "         1.0938e-01,  8.6365e-02, -1.8799e-01,  6.4758e-02, -4.5990e-02,\n",
      "        -4.5837e-02,  8.3557e-02, -3.1677e-02,  6.2134e-02, -1.2276e-02,\n",
      "        -2.3376e-02, -7.7026e-02,  5.3101e-02,  1.1664e-01, -2.2614e-02,\n",
      "        -1.0858e-01, -5.5481e-02, -2.4734e-02,  3.4302e-02, -7.7026e-02,\n",
      "         3.9635e-03,  1.0738e-03,  2.2290e-01, -7.3792e-02, -1.2537e-01,\n",
      "        -1.9974e-02,  1.3599e-01, -1.5930e-01, -1.2230e-02, -4.5166e-03,\n",
      "        -3.9185e-02, -1.5457e-02, -5.8929e-02,  2.9861e-02, -2.4445e-02,\n",
      "        -1.1163e-01, -2.1708e-04, -1.5076e-02, -2.0264e-02,  5.3009e-02,\n",
      "         5.8350e-02, -6.2042e-02,  1.8445e-01,  1.2341e-01,  1.9470e-02,\n",
      "         1.0480e-01, -8.5754e-02,  6.9809e-03,  4.3884e-02, -2.4384e-02,\n",
      "        -6.0616e-03,  4.3671e-02,  1.5488e-02,  1.1951e-01,  6.9702e-02,\n",
      "        -3.4851e-02,  1.4717e-02, -7.7148e-02, -2.6947e-02,  1.3831e-01,\n",
      "         5.3253e-02, -1.2854e-01, -5.3314e-02,  5.2521e-02,  9.4360e-02,\n",
      "         9.2651e-02,  5.1239e-02,  5.3955e-02,  1.8127e-02, -8.8562e-02,\n",
      "        -1.1700e-01,  8.0627e-02, -8.8257e-02, -3.2654e-02,  1.2390e-02,\n",
      "         1.0742e-01,  3.6001e-05,  9.3384e-02, -7.1838e-02, -1.1438e-01,\n",
      "         1.0254e-01,  1.2596e-02, -4.1931e-02, -5.9601e-02,  9.2590e-02,\n",
      "        -9.8724e-03, -8.7952e-02,  3.2196e-03, -2.8793e-02,  4.4373e-02,\n",
      "        -3.2288e-02, -1.9379e-02,  1.2000e-01, -9.0210e-02,  1.1330e-02,\n",
      "         3.0762e-02,  1.0431e-01,  7.7332e-02,  1.0910e-02, -7.9346e-02,\n",
      "        -2.6520e-02, -9.0576e-02,  2.5964e-04, -4.1962e-02,  1.1200e-01,\n",
      "         3.1708e-02,  6.6040e-02,  5.9174e-02, -2.6535e-02, -4.8889e-02,\n",
      "         1.1774e-01, -2.7783e-01,  1.1676e-01, -4.0588e-02,  3.9917e-02,\n",
      "        -3.2593e-02, -1.4539e-01,  6.0181e-02, -3.1250e-02, -5.9853e-03,\n",
      "        -3.3569e-02,  1.8469e-01, -8.1116e-02, -2.8229e-02,  2.3174e-03,\n",
      "        -1.2183e-01,  4.8523e-02,  1.2848e-02, -5.7281e-02,  5.7007e-02,\n",
      "        -6.5979e-02,  1.4392e-01,  9.6436e-03,  7.4463e-02,  2.3727e-02,\n",
      "        -4.6326e-02, -1.7700e-01,  1.0864e-01,  7.0984e-02, -5.5511e-02,\n",
      "         3.2288e-02, -3.0167e-02,  1.0109e-02,  9.5520e-02, -2.8992e-02,\n",
      "         2.9968e-02, -3.3550e-03,  7.0679e-02, -7.3181e-02, -8.3694e-03,\n",
      "        -7.9224e-02,  5.7861e-02, -7.0190e-02, -2.1948e-01,  1.9798e-03,\n",
      "        -7.6172e-02, -9.5947e-02, -6.9702e-02,  1.1871e-01, -5.5450e-02,\n",
      "        -4.0405e-02, -1.3501e-01, -8.7891e-02, -2.9716e-03, -1.2769e-01,\n",
      "         8.3435e-02, -2.1149e-02, -5.8380e-02,  1.3336e-02,  1.1749e-01,\n",
      "        -6.5186e-02, -4.1809e-03, -3.9764e-02,  1.1926e-01,  5.6122e-02,\n",
      "        -6.1920e-02, -1.2772e-02,  9.3933e-02, -1.0608e-01, -5.2094e-02,\n",
      "         1.1548e-01,  5.5237e-02, -8.7463e-02, -8.4534e-02, -4.6295e-02,\n",
      "        -1.4807e-01,  2.1988e-02,  3.2867e-02, -2.6184e-02, -6.0944e-02,\n",
      "        -8.5510e-02,  2.5620e-02,  1.6699e-01,  2.1729e-02, -1.4612e-01,\n",
      "         9.0698e-02,  1.0529e-02,  1.5063e-01, -3.2104e-01,  7.0251e-02,\n",
      "        -1.2646e-01,  6.9397e-02,  6.4087e-02,  2.6062e-02, -8.6548e-02,\n",
      "         9.6558e-02, -1.2988e-01,  1.7737e-01,  4.6265e-02,  6.0883e-02,\n",
      "         6.6162e-02,  7.6485e-03, -1.7090e-01,  8.6609e-02, -2.0844e-02,\n",
      "        -1.2317e-01, -3.1555e-02, -1.5869e-01,  4.6295e-02,  2.0111e-02,\n",
      "        -9.1858e-02,  1.4368e-01,  1.8005e-01,  3.6774e-02, -8.7280e-02,\n",
      "        -1.0913e-01,  5.6335e-02, -1.7731e-02, -1.1566e-02, -1.7456e-01,\n",
      "        -9.8389e-02,  8.6243e-02,  8.2520e-02,  3.8239e-02,  3.8635e-02,\n",
      "        -7.5806e-02,  1.0931e-01,  9.8877e-02,  1.2550e-02, -1.4381e-02,\n",
      "         1.6296e-01,  2.2729e-01,  7.4463e-02, -6.6345e-02,  5.4962e-02,\n",
      "         3.9551e-02, -4.5990e-02,  2.0947e-01,  1.4099e-01,  5.7892e-02,\n",
      "         1.2192e-02,  9.1797e-02, -3.2990e-02,  2.2510e-01,  2.7893e-02,\n",
      "         1.0559e-01,  5.5847e-02,  2.7832e-02, -1.0626e-01, -5.4077e-02,\n",
      "        -1.8616e-01,  1.4587e-01,  1.6504e-01,  6.3171e-02,  1.1664e-01,\n",
      "         3.6713e-02, -2.1072e-02,  2.8244e-02, -1.2048e-01,  1.9629e-01,\n",
      "        -1.0498e-01, -4.6722e-02,  6.8359e-02, -7.3547e-02,  1.2610e-01,\n",
      "        -1.4075e-01, -3.2330e-03, -1.6125e-01, -1.8213e-01, -1.0510e-01,\n",
      "        -9.6558e-02,  5.4962e-02,  3.2272e-03,  3.5629e-03,  2.9297e-02,\n",
      "         1.8570e-02, -5.2124e-02,  1.6211e-01, -8.9111e-02,  7.3242e-02,\n",
      "        -9.3689e-02,  3.2349e-03, -2.8351e-02,  1.3220e-01, -5.7404e-02,\n",
      "        -5.9967e-02, -5.0110e-02, -5.5603e-02,  1.2756e-01, -1.5308e-01,\n",
      "        -1.0797e-01, -2.0584e-02,  1.3391e-01,  1.0522e-01, -3.0289e-02,\n",
      "         1.0388e-01, -8.5754e-02, -3.7384e-02,  1.7868e-02,  1.1859e-01,\n",
      "        -3.5156e-02,  7.7576e-02,  7.6721e-02, -1.7578e-02, -3.5614e-02,\n",
      "         1.2207e-01, -4.0405e-02,  4.3182e-03, -8.4412e-02, -5.2948e-02,\n",
      "         1.5442e-01, -1.0553e-01,  2.3632e-03,  4.4861e-02, -1.1243e-01,\n",
      "         1.3145e-02,  1.0803e-01, -8.8501e-02, -8.6288e-03, -3.9856e-02,\n",
      "         1.1145e-01,  4.6692e-02,  5.4291e-02, -1.0086e-02, -4.3457e-02,\n",
      "        -4.8401e-02,  2.4628e-02, -1.0468e-01, -5.2738e-04, -1.3293e-01,\n",
      "         6.6467e-02,  1.0413e-01,  9.0027e-02, -9.7168e-02,  3.5400e-02,\n",
      "         7.7881e-02, -1.0919e-01,  9.5093e-02,  5.4474e-02, -4.9347e-02,\n",
      "         1.4435e-02, -7.3669e-02,  1.5479e-01, -2.0142e-01, -1.1328e-01,\n",
      "        -1.8387e-02, -5.4108e-02,  1.3855e-02, -1.6434e-02, -1.6931e-01,\n",
      "         4.2206e-02, -1.1761e-01, -4.2694e-02, -5.7068e-02, -3.3905e-02,\n",
      "        -3.2764e-01,  1.3721e-01, -7.6599e-02,  4.4670e-03,  1.0553e-01,\n",
      "        -8.7280e-02, -1.0394e-01, -6.7024e-03,  1.9348e-02,  3.6957e-02,\n",
      "        -8.4595e-02,  3.8177e-02, -2.2163e-03,  9.1370e-02, -4.1199e-02,\n",
      "        -1.3953e-01,  2.4216e-02,  7.3364e-02,  6.8542e-02,  3.0197e-02],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.self_attn_layer_norm.weight Parameter containing:\n",
      "tensor([0.6055, 0.5801, 0.5908, 0.5405, 0.6509, 0.5586, 0.5571, 0.5586, 0.5571,\n",
      "        0.5625, 0.5776, 0.5371, 0.5659, 0.5679, 0.6016, 0.5840, 0.5547, 0.5752,\n",
      "        0.5068, 0.5820, 0.5962, 0.6938, 0.5659, 0.5352, 0.5649, 0.5078, 0.5269,\n",
      "        0.5864, 0.5845, 0.6040, 0.5928, 0.5908, 0.6260, 0.5815, 0.5640, 0.6011,\n",
      "        0.5884, 0.5376, 0.5654, 0.6064, 0.5991, 0.5835, 0.6089, 0.5928, 0.5054,\n",
      "        0.6362, 0.3506, 0.5723, 0.6143, 0.6494, 0.5840, 0.6040, 0.5674, 0.5679,\n",
      "        0.5454, 0.5806, 0.5195, 0.5806, 0.4834, 0.6167, 0.5649, 0.5928, 0.5498,\n",
      "        0.5674, 0.5400, 0.5225, 0.3035, 0.5718, 0.5420, 0.5693, 0.5757, 0.5488,\n",
      "        0.5347, 0.5640, 0.5713, 0.6006, 0.5581, 0.5957, 0.5830, 0.6250, 0.5610,\n",
      "        0.6133, 0.5420, 0.6055, 0.6367, 0.6113, 0.5435, 0.5679, 0.5996, 0.5986,\n",
      "        0.5708, 0.5737, 0.6245, 0.5835, 0.6162, 0.5322, 0.5410, 0.5527, 0.5913,\n",
      "        0.5991, 0.5991, 0.5879, 0.5806, 0.6060, 0.5293, 0.5850, 0.5649, 0.6123,\n",
      "        0.6167, 0.6216, 0.5977, 0.6304, 0.6309, 0.5962, 0.5566, 0.5610, 0.5869,\n",
      "        0.5181, 0.6284, 0.5679, 0.5581, 0.5146, 0.5947, 0.6001, 0.5527, 0.5811,\n",
      "        0.5767, 0.5752, 0.5317, 0.5439, 0.5405, 0.6152, 0.6025, 0.5933, 0.5620,\n",
      "        0.5298, 0.5859, 0.4807, 0.5171, 0.5879, 0.5405, 0.5728, 0.5566, 0.6011,\n",
      "        0.5474, 0.6157, 0.5928, 0.5776, 0.4673, 0.5952, 0.5903, 0.5708, 0.5410,\n",
      "        0.5850, 0.5137, 0.5708, 0.5830, 0.5806, 0.6216, 0.5747, 0.5913, 0.5508,\n",
      "        0.5840, 0.5527, 0.5581, 0.5635, 0.6133, 0.5361, 0.5498, 0.5566, 0.4844,\n",
      "        0.6187, 0.5337, 0.5938, 0.4575, 0.5332, 0.6138, 0.6162, 0.5679, 0.5630,\n",
      "        0.6372, 0.5737, 0.5684, 0.6187, 0.5918, 0.6055, 0.6045, 0.5308, 0.5830,\n",
      "        0.6016, 0.5913, 0.5791, 0.6240, 0.5371, 0.5894, 0.5522, 0.6240, 0.5386,\n",
      "        0.5889, 0.4932, 0.5928, 0.4968, 0.5176, 0.5776, 0.5459, 0.5640, 0.4702,\n",
      "        0.6060, 0.5967, 0.5942, 0.5762, 0.5684, 0.5269, 0.6094, 0.5537, 0.5654,\n",
      "        0.5605, 0.5840, 0.5967, 0.5859, 0.5591, 0.5615, 0.5425, 0.5625, 0.5615,\n",
      "        0.5913, 0.5591, 0.6021, 0.6294, 0.6094, 0.5942, 0.5635, 0.6089, 0.5337,\n",
      "        0.5713, 0.5493, 0.5415, 0.6152, 0.5513, 0.6250, 0.5132, 0.6167, 0.5439,\n",
      "        0.5381, 0.5811, 0.5024, 0.5967, 0.6079, 0.6006, 0.5098, 0.5859, 0.5811,\n",
      "        0.5479, 0.5967, 0.5952, 0.4927, 0.5347, 0.5703, 0.5464, 0.5767, 0.5879,\n",
      "        0.5996, 0.6172, 0.5962, 0.5439, 0.5552, 0.6338, 0.5850, 0.6309, 0.6309,\n",
      "        0.5791, 0.5781, 0.5620, 0.5850, 0.5093, 0.5767, 0.5840, 0.6128, 0.5137,\n",
      "        0.5210, 0.5684, 0.5596, 0.5825, 0.5737, 0.6509, 0.5391, 0.5415, 0.4893,\n",
      "        0.5703, 0.5322, 0.5815, 0.5967, 0.5479, 0.4160, 0.6108, 0.5034, 0.6138,\n",
      "        0.6011, 0.5508, 0.3481, 0.5693, 0.6069, 0.5903, 0.6050, 0.5840, 0.5830,\n",
      "        0.5518, 0.5078, 0.6099, 0.4822, 0.5210, 0.5669, 0.5854, 0.6289, 0.6006,\n",
      "        0.5068, 0.5620, 0.5708, 0.6147, 0.5991, 0.5752, 0.5757, 0.6060, 0.5181,\n",
      "        0.6138, 0.5137, 0.6240, 0.6587, 0.6069, 0.5415, 0.5903, 0.5093, 0.5605,\n",
      "        0.6099, 0.5972, 0.5547, 0.5181, 0.6157, 0.6177, 0.5801, 0.5918, 0.5752,\n",
      "        0.4824, 0.5938, 0.5786, 0.6001, 0.5659, 0.6138, 0.4570, 0.5771, 0.5845,\n",
      "        0.6104, 0.5405, 0.4924, 0.5444, 0.5688, 0.5591, 0.6108, 0.5859, 0.5947,\n",
      "        0.5840, 0.5781, 0.5518, 0.5449, 0.5942, 0.5708, 0.5645, 0.6035, 0.6030,\n",
      "        0.5596, 0.5630, 0.6177, 0.5840, 0.5483, 0.5981, 0.6284, 0.6099, 0.5537,\n",
      "        0.5767, 0.6099, 0.5811, 0.5854, 0.4849, 0.5288, 0.5674, 0.6265, 0.6304,\n",
      "        0.6143, 0.5483, 0.6309, 0.6274, 0.6064, 0.6147, 0.5488, 0.5176, 0.5166,\n",
      "        0.6113, 0.5356, 0.5347, 0.5596, 0.5483, 0.5781, 0.5859, 0.5879, 0.5273,\n",
      "        0.6235, 0.5796, 0.5967, 0.5610, 0.6304, 0.5571, 0.6001, 0.5669, 0.5503,\n",
      "        0.5825, 0.5093, 0.5903, 0.5742, 0.6182, 0.6196, 0.6304, 0.5557, 0.6172,\n",
      "        0.6221, 0.5938, 0.5635, 0.5776, 0.6006, 0.5732, 0.5776, 0.6367, 0.5840,\n",
      "        0.5156, 0.5933, 0.5723, 0.6011, 0.6001, 0.5928, 0.5513, 0.5762, 0.5391,\n",
      "        0.5376, 0.6748, 0.5815, 0.5928, 0.5601, 0.5522, 0.5503, 0.5605, 0.5801,\n",
      "        0.5688, 0.5625, 0.5869, 0.5874, 0.6587, 0.5688, 0.6025, 0.5796, 0.5073,\n",
      "        0.6133, 0.2551, 0.5698, 0.6025, 0.6104, 0.5630, 0.6133, 0.5576, 0.5586,\n",
      "        0.5850, 0.5806, 0.6025, 0.5195, 0.5767, 0.5752, 0.5693, 0.5859, 0.6372,\n",
      "        0.5283, 0.5874, 0.5869], requires_grad=True)\n",
      "esm_model.layers.10.self_attn_layer_norm.bias Parameter containing:\n",
      "tensor([ 3.3356e-02, -6.9771e-03,  8.2169e-03, -3.2379e-02, -2.5223e-02,\n",
      "         9.3384e-03,  5.6152e-02, -2.6871e-02,  3.5156e-02, -1.9867e-02,\n",
      "         3.8544e-02, -2.5833e-02,  5.4535e-02, -4.1260e-02, -8.9722e-02,\n",
      "        -7.3425e-02, -1.2070e-02,  2.3285e-02, -1.7380e-02, -3.0273e-02,\n",
      "         5.1544e-02,  5.3436e-02,  4.4518e-03,  8.0795e-03, -2.0630e-02,\n",
      "        -4.2877e-02, -5.8670e-03,  3.4485e-02,  5.4718e-02,  7.4272e-03,\n",
      "         1.7136e-02,  4.1870e-02,  5.4382e-02,  4.5662e-03,  5.0262e-02,\n",
      "        -5.3291e-03, -3.0243e-02, -6.5460e-03,  2.0007e-01,  3.0106e-02,\n",
      "        -2.7039e-02,  4.5013e-03, -2.3174e-03,  4.7791e-02,  2.0798e-02,\n",
      "        -8.8272e-03,  8.4180e-01,  6.4270e-02,  9.3536e-03,  3.7781e-02,\n",
      "         4.4983e-02, -6.8970e-03,  3.6530e-02,  4.9225e-02, -5.5084e-02,\n",
      "        -4.1351e-02,  1.6190e-02,  3.5187e-02,  3.4058e-02, -1.1833e-02,\n",
      "         9.0942e-03, -3.9787e-03, -6.4087e-03,  6.0883e-02,  4.1771e-03,\n",
      "         9.7198e-03,  1.9360e-01,  3.3447e-02, -1.2417e-03,  1.5137e-02,\n",
      "         3.3600e-02,  1.2550e-02, -1.6861e-02,  1.9241e-02, -1.5526e-02,\n",
      "        -1.1215e-02,  2.1286e-02,  7.4219e-02, -5.3680e-02,  1.7502e-02,\n",
      "         7.6599e-02, -3.5370e-02,  7.9498e-03,  1.1902e-02,  6.6662e-04,\n",
      "        -1.9012e-02,  1.1482e-02,  4.1016e-02,  5.0293e-02,  6.4026e-02,\n",
      "         1.5221e-02,  2.0813e-02,  7.3204e-03, -2.7115e-02,  8.6609e-02,\n",
      "         3.5370e-02, -3.4119e-02, -1.6388e-02,  1.0522e-01,  4.2145e-02,\n",
      "         2.8152e-02,  3.8177e-02,  2.9907e-02, -1.6052e-02,  3.4485e-02,\n",
      "        -4.8309e-02, -8.4000e-03, -1.6418e-02,  7.5500e-02,  4.9530e-02,\n",
      "         2.8503e-02, -3.7018e-02,  3.4237e-03,  2.4292e-02, -1.1391e-02,\n",
      "        -2.4017e-02, -5.8556e-03, -4.7211e-02,  2.9053e-02,  3.0457e-02,\n",
      "         6.6833e-02, -6.1655e-04,  8.6914e-02,  5.0232e-02, -2.0279e-02,\n",
      "         3.5553e-02,  5.7068e-03,  4.2694e-02,  1.1520e-03,  4.6906e-02,\n",
      "        -1.4429e-03,  2.2011e-03, -4.0527e-02,  4.6143e-02,  2.0874e-02,\n",
      "         6.8245e-03,  1.0735e-02, -3.8269e-02, -2.5772e-02,  3.3875e-02,\n",
      "         2.3712e-02,  1.5587e-02,  5.6580e-02, -7.7705e-03,  1.8433e-02,\n",
      "        -7.0000e-03, -1.1589e-02,  1.2604e-02, -9.0759e-02, -5.9906e-02,\n",
      "         8.3130e-02,  6.0608e-02,  5.2612e-02,  5.1605e-02, -1.1719e-02,\n",
      "         1.0338e-02,  1.0248e-01, -5.2986e-03,  3.5217e-02, -5.4703e-03,\n",
      "         1.5129e-02,  9.4452e-03,  2.9922e-02, -2.4063e-02,  3.1128e-02,\n",
      "         3.2196e-02,  2.9694e-02,  4.0741e-02,  1.6449e-02,  3.8971e-02,\n",
      "         3.5614e-02,  2.8656e-02,  4.8401e-02,  6.7177e-03,  3.0563e-02,\n",
      "         7.9773e-02,  5.7098e-02, -3.6812e-03,  3.4210e-02,  1.5747e-02,\n",
      "         4.5471e-03,  9.0698e-02,  1.5747e-02,  6.4011e-03,  5.2567e-03,\n",
      "         2.7100e-02,  2.2430e-02, -3.3325e-02,  2.4643e-02,  4.6417e-02,\n",
      "         1.0712e-02,  2.1973e-03,  8.0078e-02,  5.4565e-02,  7.8491e-02,\n",
      "         2.9419e-02,  2.8000e-02, -9.2163e-03, -3.3142e-02,  5.0507e-02,\n",
      "         1.4488e-02,  1.6443e-01,  1.8738e-02,  5.1666e-02,  1.5495e-02,\n",
      "        -1.1497e-02,  4.4769e-02,  1.1894e-02,  4.4060e-03,  3.5339e-02,\n",
      "         8.7097e-02,  3.2883e-03,  2.6215e-02, -1.5808e-02,  1.6037e-02,\n",
      "        -4.4434e-02, -2.7130e-02,  2.5528e-02, -1.4839e-02,  2.3087e-02,\n",
      "         8.4839e-03,  3.4851e-02,  1.5900e-02,  1.5421e-03,  3.1235e-02,\n",
      "         3.1433e-02, -2.8732e-02, -2.7771e-02,  7.2266e-02,  4.1077e-02,\n",
      "        -1.3367e-01,  4.4098e-02,  4.0924e-02, -5.0354e-03,  2.5314e-02,\n",
      "         2.5009e-02, -7.2815e-02,  1.2978e-02, -3.0807e-02,  4.4403e-02,\n",
      "        -1.8555e-02, -1.5305e-02,  6.8237e-02,  3.5522e-02, -2.9556e-02,\n",
      "         4.4250e-03,  1.2352e-02,  8.1360e-02, -4.4189e-02,  1.9958e-02,\n",
      "        -1.1368e-02, -9.2163e-03,  3.9276e-02, -4.8790e-03,  2.8061e-02,\n",
      "         4.8126e-02, -5.3528e-02, -6.8817e-03, -2.3178e-02,  1.6661e-03,\n",
      "         1.1307e-02,  6.0310e-03,  2.7145e-02, -1.5610e-02,  9.0393e-02,\n",
      "         1.7792e-02,  2.8839e-02,  3.2043e-02, -7.2670e-03, -4.4037e-02,\n",
      "         4.1687e-02,  5.9509e-02,  3.5645e-02,  6.2866e-02,  6.8016e-03,\n",
      "         1.5610e-02, -1.1635e-02,  4.0665e-03, -1.4458e-02, -1.4542e-02,\n",
      "        -2.5681e-02,  2.2919e-02, -5.1605e-02,  4.6600e-02,  4.0192e-02,\n",
      "         3.1204e-02,  3.4088e-02, -1.1688e-02,  3.9139e-03,  5.7251e-02,\n",
      "        -4.9530e-02,  2.4200e-02, -1.4359e-02,  2.9007e-02, -2.3785e-03,\n",
      "        -9.6191e-02, -1.5701e-02, -2.1988e-02,  1.4961e-02, -6.1989e-03,\n",
      "        -5.5878e-02,  4.1962e-03, -3.6285e-02, -4.0955e-02,  5.2246e-02,\n",
      "        -5.9296e-02,  1.9531e-02, -6.6071e-03,  4.0863e-02,  2.8030e-02,\n",
      "         3.9246e-02, -3.1052e-02,  4.2236e-02,  3.1891e-02, -3.3630e-02,\n",
      "        -2.3071e-02,  4.8637e-03, -1.4488e-02, -3.5339e-02,  1.2505e-02,\n",
      "         9.2896e-02, -6.5735e-02,  2.5467e-02, -5.8136e-02,  5.7953e-02,\n",
      "         6.8703e-03, -5.9586e-03, -1.4374e-02,  1.8768e-02,  1.0857e-02,\n",
      "        -5.1208e-02,  1.4076e-02,  6.9641e-02, -2.0126e-02,  2.0401e-02,\n",
      "         1.0849e-02, -5.5756e-02, -8.4839e-03,  4.2267e-03,  4.2267e-02,\n",
      "         8.0872e-02,  6.6833e-02, -4.5197e-02, -9.6664e-03,  5.6854e-02,\n",
      "        -1.1940e-02, -3.4882e-02,  2.3148e-02, -5.8899e-02,  1.2421e-02,\n",
      "         1.1490e-02, -1.8890e-02,  9.3765e-03, -9.0790e-03,  8.2932e-03,\n",
      "         7.3364e-02,  5.3955e-02,  2.6337e-02,  6.1035e-02,  3.5522e-02,\n",
      "         2.0081e-02,  7.5256e-02,  4.2755e-02,  1.9714e-02,  6.9580e-03,\n",
      "         4.2206e-02,  1.9791e-02, -9.5139e-03,  3.1403e-02, -2.4094e-02,\n",
      "         6.4941e-02,  9.3460e-03, -7.5607e-03,  5.8380e-02,  4.7424e-02,\n",
      "         4.6967e-02,  4.1229e-02, -1.1124e-02,  3.8086e-02,  4.8889e-02,\n",
      "         2.4246e-02,  7.7881e-02,  1.4778e-02, -1.9608e-02, -3.9917e-02,\n",
      "         3.3855e-03,  6.3248e-03,  2.1820e-02,  3.9276e-02,  4.9561e-02,\n",
      "         4.8096e-02,  2.9633e-02,  3.8513e-02,  1.8280e-02,  5.3741e-02,\n",
      "        -9.8419e-03, -1.6037e-02, -4.0771e-02,  3.8330e-02,  7.4890e-02,\n",
      "        -2.3865e-02, -7.4959e-03,  1.7136e-02,  1.6565e-03,  4.9858e-03,\n",
      "         3.4943e-02, -4.5166e-02,  6.2469e-02,  4.2633e-02,  4.3732e-02,\n",
      "        -4.7684e-06, -3.5248e-02,  3.2234e-04,  2.8763e-02, -1.1093e-02,\n",
      "        -2.9922e-02, -2.2888e-02,  2.2766e-02,  2.0828e-02, -1.2932e-02,\n",
      "         6.6406e-02, -6.4583e-03, -7.5531e-03,  8.8440e-02,  3.3379e-04,\n",
      "        -4.5227e-02,  1.3222e-02,  6.7749e-02,  4.0955e-02, -5.9814e-03,\n",
      "        -1.2787e-02,  7.9712e-02,  3.7964e-02,  1.2878e-02, -2.9404e-02,\n",
      "        -6.7949e-04,  2.4124e-02, -6.5552e-02, -4.6463e-03,  6.5735e-02,\n",
      "        -9.2163e-03,  1.8402e-02,  7.2083e-02,  1.5221e-02,  2.9984e-02,\n",
      "         9.7351e-02,  7.2205e-02,  1.5602e-02, -4.8065e-03, -1.6388e-02,\n",
      "         1.9547e-02,  1.1292e-02, -1.9791e-02, -3.4454e-02,  4.2023e-02,\n",
      "        -6.3133e-03,  2.6947e-02,  3.2043e-02, -2.0943e-03, -5.2223e-03,\n",
      "        -1.6003e-01,  6.5979e-02,  2.7740e-02, -8.6731e-02,  1.2367e-02,\n",
      "         3.8940e-02,  1.8005e-02, -1.3687e-02,  8.8501e-03,  1.7212e-02,\n",
      "        -3.1013e-03,  7.7576e-02,  3.4760e-02,  3.5797e-02, -5.2872e-03,\n",
      "        -3.1738e-02,  3.9856e-02,  1.4244e-02, -2.2354e-03, -4.2084e-02],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.fc1.weight Parameter containing:\n",
      "tensor([[ 0.0397, -0.0590,  0.1331,  ..., -0.0302, -0.1350,  0.0076],\n",
      "        [ 0.0724, -0.1838, -0.0940,  ...,  0.0597,  0.0161,  0.0363],\n",
      "        [ 0.2217,  0.0322,  0.0714,  ...,  0.0876,  0.2861, -0.0514],\n",
      "        ...,\n",
      "        [ 0.1250,  0.0056,  0.1251,  ..., -0.1211, -0.0711, -0.1065],\n",
      "        [-0.0308, -0.0573,  0.0745,  ..., -0.0289, -0.0139, -0.1353],\n",
      "        [-0.0182,  0.0565,  0.0017,  ..., -0.0945,  0.0778,  0.0282]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.fc1.bias Parameter containing:\n",
      "tensor([-0.0225, -0.0502, -0.2272,  ..., -0.0690, -0.0733, -0.0333],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.fc2.weight Parameter containing:\n",
      "tensor([[-0.0417,  0.1606, -0.2539,  ...,  0.1327, -0.2224, -0.0042],\n",
      "        [-0.0203,  0.1768,  0.3052,  ..., -0.1337, -0.0630, -0.0448],\n",
      "        [-0.1846,  0.1182,  0.2700,  ..., -0.0669,  0.0493, -0.1170],\n",
      "        ...,\n",
      "        [ 0.0328,  0.1752,  0.1130,  ...,  0.1268,  0.0462,  0.2379],\n",
      "        [-0.0324, -0.0522, -0.5088,  ...,  0.1061, -0.1305,  0.0937],\n",
      "        [-0.0121, -0.0562,  0.1687,  ...,  0.1356, -0.0170,  0.0240]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.fc2.bias Parameter containing:\n",
      "tensor([ 2.0386e-02, -2.1515e-02, -2.4338e-02,  9.5947e-02, -8.4717e-02,\n",
      "        -4.4403e-02,  1.0968e-01, -1.2848e-02, -9.1614e-02, -6.0455e-02,\n",
      "        -4.1351e-02,  1.0150e-01,  3.9154e-02, -5.4810e-02,  8.8730e-03,\n",
      "        -1.8494e-02,  3.3081e-02,  6.7078e-02, -9.9121e-02, -7.2327e-02,\n",
      "        -8.4610e-03,  7.2975e-03, -6.0577e-02, -7.4997e-03,  7.9163e-02,\n",
      "        -6.2513e-04, -4.6967e-02, -6.8726e-02, -7.1297e-03, -4.4983e-02,\n",
      "         8.2458e-02, -2.6611e-02, -3.3691e-02,  1.2787e-02, -4.8157e-02,\n",
      "         8.2626e-03,  8.8318e-02,  4.8943e-03, -1.4458e-02, -1.1597e-01,\n",
      "         5.8777e-02,  7.7820e-03,  1.4725e-02,  1.1969e-01, -1.7166e-03,\n",
      "        -3.0487e-02,  6.6211e-01, -2.5986e-02,  1.6144e-02, -6.6711e-02,\n",
      "         2.7588e-02,  1.1387e-03,  5.3589e-02,  4.0855e-03, -2.4567e-02,\n",
      "        -2.0950e-02, -4.8157e-02, -7.1106e-02, -5.7831e-02, -4.2389e-02,\n",
      "         3.5248e-02, -4.6204e-02,  3.1338e-03,  9.9548e-02,  1.0986e-01,\n",
      "         1.0262e-03, -1.3538e-01,  3.8849e-02,  3.0701e-02,  9.9976e-02,\n",
      "        -3.2135e-02, -1.9394e-02,  5.6274e-02, -5.9906e-02,  5.8502e-02,\n",
      "         2.8366e-02,  7.5493e-03,  4.9591e-02,  1.8082e-02,  7.5073e-02,\n",
      "        -4.5074e-02,  5.1178e-02,  5.3558e-02, -3.9795e-02,  8.5999e-02,\n",
      "        -2.2827e-02,  2.0584e-02, -1.2683e-01, -2.4078e-02, -2.2797e-02,\n",
      "         4.7272e-02,  1.6235e-02, -5.5298e-02,  5.4896e-05, -6.8604e-02,\n",
      "         2.1408e-02,  5.1056e-02, -1.3293e-01,  1.3618e-02,  1.1101e-02,\n",
      "         4.1275e-03,  1.5961e-02,  1.5381e-02, -1.1218e-04,  2.4384e-02,\n",
      "        -4.3091e-02, -9.6588e-03,  1.5045e-02,  8.5510e-02,  1.8600e-02,\n",
      "        -1.1859e-01, -2.8564e-02, -3.1403e-02,  5.9418e-02, -8.3069e-02,\n",
      "        -3.9276e-02,  1.0577e-01,  3.8605e-02, -1.4404e-01, -6.9885e-02,\n",
      "         5.2979e-02,  5.9540e-02, -1.1853e-01, -2.3590e-02,  3.5645e-02,\n",
      "        -7.2327e-02, -1.8463e-02, -4.5074e-02, -4.1077e-02,  2.2079e-02,\n",
      "        -2.1912e-02,  5.3528e-02,  2.5375e-02, -2.6291e-02, -3.9032e-02,\n",
      "        -3.0609e-02, -9.3994e-02,  8.4839e-02,  4.1931e-02,  3.2135e-02,\n",
      "         5.3833e-02, -6.1768e-02,  1.2283e-02,  1.8539e-02,  2.8706e-03,\n",
      "         1.9440e-02,  2.9114e-02,  5.5511e-02,  9.0149e-02,  1.0181e-01,\n",
      "        -3.1616e-02,  5.2719e-03, -3.9886e-02, -2.7275e-03,  2.6199e-02,\n",
      "         2.2537e-02, -9.0576e-02, -2.8412e-02,  6.0272e-02,  1.5762e-02,\n",
      "         3.2928e-02, -1.0071e-02, -1.3008e-02, -4.0793e-04,  1.9852e-02,\n",
      "        -3.6407e-02, -2.8534e-03,  4.0192e-02, -4.0100e-02,  1.7639e-02,\n",
      "        -3.8891e-03, -4.4525e-02,  9.1797e-02,  1.8845e-02, -4.2175e-02,\n",
      "         8.1970e-02,  4.2969e-02, -4.9866e-02, -4.0314e-02, -6.7749e-02,\n",
      "        -1.9684e-02, -9.9373e-04,  6.3293e-02,  2.1622e-02, -1.2093e-02,\n",
      "         1.0691e-03,  3.7140e-02,  9.8145e-02,  2.0676e-02, -3.7292e-02,\n",
      "         4.4617e-02, -2.5864e-02,  1.2726e-02, -7.4219e-02, -2.1286e-02,\n",
      "         1.1749e-02,  5.3368e-03,  4.8706e-02, -2.9312e-02,  8.3618e-02,\n",
      "        -4.1016e-02, -3.9398e-02,  1.3191e-02, -4.2152e-03, -8.2855e-03,\n",
      "         7.4951e-02, -1.4062e-01,  5.7404e-02, -1.3779e-02, -2.0370e-02,\n",
      "        -3.4218e-03, -1.0895e-01,  4.3457e-02,  8.5754e-02,  8.8024e-04,\n",
      "         7.5745e-02,  1.1072e-01,  1.7197e-02, -8.8989e-02, -4.2358e-02,\n",
      "        -9.7107e-02,  1.1757e-02,  8.8196e-03,  1.4111e-01,  2.0538e-02,\n",
      "        -2.8824e-02,  5.8258e-02,  1.0586e-03,  1.6922e-02, -2.5436e-02,\n",
      "        -4.6806e-03, -1.3989e-01,  9.5581e-02,  7.2266e-02, -1.0083e-01,\n",
      "         4.6654e-03,  2.2537e-02, -1.4420e-02, -4.1504e-02,  1.6678e-02,\n",
      "        -3.6926e-02, -4.4586e-02,  1.8219e-02, -3.4851e-02, -8.5983e-03,\n",
      "        -7.4890e-02,  1.9730e-02, -4.3396e-02, -5.0079e-02,  7.5684e-02,\n",
      "        -3.5583e-02, -4.3335e-02,  1.0498e-02, -1.8341e-02,  3.9825e-02,\n",
      "         5.0446e-02, -8.3252e-02, -1.1316e-01,  1.5701e-02,  3.2288e-02,\n",
      "         3.3295e-02, -1.2280e-01, -3.9642e-02, -1.0262e-02,  5.5481e-02,\n",
      "        -8.5632e-02,  5.0781e-02,  1.2596e-02,  6.0669e-02, -7.1228e-02,\n",
      "        -6.0486e-02, -1.0651e-02, -2.7374e-02, -9.8419e-04,  9.7885e-03,\n",
      "         7.7332e-02,  7.0251e-02, -3.5065e-02, -5.1544e-02, -9.8511e-02,\n",
      "        -3.8338e-03,  3.0807e-02,  3.9734e-02,  5.4169e-02, -1.4351e-02,\n",
      "         2.0468e-04,  8.2703e-03,  1.1566e-01,  3.3600e-02, -6.6101e-02,\n",
      "         3.7537e-02,  6.1066e-02,  4.6509e-02, -1.7029e-01, -4.7821e-02,\n",
      "        -1.0168e-01,  6.6589e-02,  2.9541e-02, -5.1636e-02, -1.9714e-01,\n",
      "        -7.6866e-03, -5.6091e-02,  7.7209e-02, -2.5742e-02,  1.2505e-02,\n",
      "         8.1543e-02, -3.6713e-02, -1.2256e-01,  6.8726e-02,  4.4342e-02,\n",
      "        -9.4910e-02, -5.5969e-02, -4.7363e-02, -4.1748e-02,  1.3260e-02,\n",
      "        -5.5084e-02,  2.9556e-02,  1.2384e-01,  2.8580e-02, -7.5562e-02,\n",
      "        -7.9712e-02,  2.8229e-02, -1.4069e-02,  4.5700e-03, -1.1151e-01,\n",
      "        -6.9092e-02,  4.8615e-02, -4.5044e-02, -2.6520e-02, -9.9564e-03,\n",
      "        -2.1988e-02,  7.1777e-02, -1.1692e-03, -3.7506e-02, -7.9117e-03,\n",
      "         1.2646e-01,  1.1469e-01,  1.6006e-02,  2.2629e-02,  5.9570e-02,\n",
      "         5.9906e-02, -2.4475e-02,  1.1151e-01,  4.3518e-02, -4.3549e-02,\n",
      "        -2.4429e-02,  4.3182e-02, -5.6061e-02,  9.9854e-02, -1.3779e-02,\n",
      "         1.0992e-01, -3.5703e-05, -9.7656e-03, -1.7725e-01, -1.0852e-01,\n",
      "        -1.0260e-01,  9.7290e-02,  1.9470e-02, -2.6321e-03,  4.7180e-02,\n",
      "         5.0735e-03, -3.4928e-04,  4.0955e-02, -9.1125e-02,  7.1045e-02,\n",
      "        -1.0803e-02, -8.8272e-03,  3.7445e-02, -2.9068e-02,  1.0223e-01,\n",
      "        -4.6539e-02,  1.6876e-02, -2.8427e-02, -1.1163e-01, -3.4637e-02,\n",
      "        -2.7954e-02,  5.5817e-02, -2.2812e-03,  1.3527e-02, -3.4058e-02,\n",
      "         2.0599e-03,  2.2736e-02,  1.6772e-01, -8.5678e-03, -3.0914e-02,\n",
      "        -2.1881e-02,  1.6846e-02, -1.5823e-02, -1.3168e-02, -3.9482e-03,\n",
      "         2.1332e-02,  1.4832e-02,  9.6416e-04,  2.4536e-02, -4.5013e-02,\n",
      "        -7.0435e-02, -9.0942e-03,  7.8979e-02,  1.6174e-02,  4.1718e-02,\n",
      "        -1.8204e-02, -2.3605e-02, -4.7302e-02,  3.8696e-02,  1.6815e-02,\n",
      "        -5.9265e-02,  9.4421e-02,  1.5228e-02,  3.8147e-02, -2.5146e-02,\n",
      "         3.6041e-02, -5.2277e-02,  5.0720e-02, -2.9984e-02,  8.0795e-03,\n",
      "         7.8613e-02, -4.1840e-02,  1.5991e-02,  4.9652e-02, -5.0354e-02,\n",
      "         5.7983e-02,  4.2664e-02, -1.8661e-02, -5.9906e-02, -8.8013e-02,\n",
      "         2.5543e-02, -1.0455e-01,  1.8692e-03, -9.8633e-02, -2.4948e-02,\n",
      "        -1.3451e-02, -7.7248e-03,  5.8472e-02, -3.7708e-03, -8.4717e-02,\n",
      "         1.4496e-02,  3.9597e-03,  7.5134e-02, -3.7354e-02,  4.5471e-02,\n",
      "         1.5572e-02, -4.0741e-03,  1.1584e-01, -8.5999e-02,  2.3689e-03,\n",
      "        -2.1423e-02, -2.1210e-02,  2.5696e-02, -1.6150e-01, -5.2429e-02,\n",
      "        -4.6570e-02,  4.6967e-02,  5.5298e-02, -7.8369e-02, -2.6794e-02,\n",
      "        -2.8351e-02, -7.3181e-02,  1.5942e-01,  1.4145e-02,  4.8370e-02,\n",
      "        -2.2522e-01,  7.2266e-02, -2.5391e-02,  2.9297e-02,  4.7699e-02,\n",
      "        -1.6281e-02, -7.4005e-03,  1.9867e-02, -2.7008e-02, -4.0680e-02,\n",
      "         3.0823e-03, -1.8501e-03,  5.0316e-03,  2.5803e-02,  1.8677e-02,\n",
      "        -2.7237e-02,  7.1716e-03, -2.4506e-02,  5.1941e-02,  2.8030e-02],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.10.final_layer_norm.weight Parameter containing:\n",
      "tensor([0.6792, 0.7222, 0.7183, 0.6548, 0.7764, 0.6445, 0.6724, 0.7056, 0.6597,\n",
      "        0.7114, 0.6753, 0.6665, 0.6782, 0.6973, 0.6714, 0.7520, 0.6821, 0.7080,\n",
      "        0.6558, 0.6582, 0.6875, 0.7100, 0.6699, 0.6768, 0.7236, 0.6372, 0.6816,\n",
      "        0.7178, 0.6152, 0.7041, 0.6968, 0.7222, 0.6807, 0.7144, 0.6782, 0.7114,\n",
      "        0.6782, 0.6997, 0.7437, 0.7036, 0.7334, 0.7354, 0.7178, 0.7036, 0.6724,\n",
      "        0.6802, 0.3892, 0.7207, 0.7319, 0.7344, 0.6592, 0.7021, 0.6748, 0.7183,\n",
      "        0.7378, 0.6743, 0.6455, 0.7031, 0.6553, 0.7910, 0.6782, 0.6802, 0.6855,\n",
      "        0.7334, 0.7202, 0.7061, 0.4429, 0.7266, 0.7021, 0.6294, 0.6968, 0.6948,\n",
      "        0.6787, 0.7583, 0.7290, 0.7031, 0.6831, 0.6689, 0.6772, 0.7139, 0.7300,\n",
      "        0.7065, 0.6821, 0.7163, 0.7080, 0.7412, 0.6646, 0.6543, 0.7437, 0.7002,\n",
      "        0.6670, 0.6943, 0.7339, 0.7275, 0.7290, 0.6309, 0.6650, 0.5981, 0.7285,\n",
      "        0.7207, 0.6997, 0.6890, 0.6567, 0.7061, 0.6445, 0.6914, 0.6729, 0.7139,\n",
      "        0.7334, 0.7383, 0.7100, 0.7104, 0.7305, 0.7046, 0.6958, 0.6685, 0.6924,\n",
      "        0.6841, 0.7383, 0.6528, 0.6802, 0.6885, 0.7422, 0.6953, 0.7012, 0.6914,\n",
      "        0.6177, 0.7158, 0.6870, 0.6597, 0.6763, 0.7437, 0.7041, 0.7119, 0.6895,\n",
      "        0.6890, 0.7603, 0.6641, 0.6890, 0.7393, 0.6675, 0.7212, 0.6885, 0.7095,\n",
      "        0.6851, 0.6953, 0.6836, 0.7271, 0.6587, 0.6646, 0.6831, 0.7305, 0.7090,\n",
      "        0.6860, 0.6523, 0.7192, 0.7817, 0.6689, 0.7305, 0.7236, 0.7324, 0.6577,\n",
      "        0.6943, 0.6553, 0.7075, 0.7349, 0.7095, 0.7246, 0.7031, 0.6138, 0.6626,\n",
      "        0.7456, 0.6665, 0.7207, 0.6235, 0.6509, 0.7256, 0.7070, 0.7056, 0.6626,\n",
      "        0.7515, 0.7520, 0.6885, 0.7207, 0.6890, 0.7173, 0.7061, 0.6943, 0.7153,\n",
      "        0.7271, 0.7036, 0.7197, 0.7339, 0.6729, 0.7163, 0.6650, 0.7544, 0.6675,\n",
      "        0.6880, 0.6240, 0.7031, 0.6465, 0.7153, 0.6948, 0.6777, 0.7163, 0.7241,\n",
      "        0.7021, 0.6973, 0.6758, 0.7285, 0.6899, 0.6968, 0.6870, 0.6836, 0.7046,\n",
      "        0.7344, 0.7344, 0.7261, 0.6904, 0.6650, 0.7222, 0.6963, 0.7085, 0.6260,\n",
      "        0.7622, 0.6919, 0.7104, 0.6851, 0.7378, 0.7065, 0.6621, 0.7188, 0.7061,\n",
      "        0.6587, 0.6392, 0.6431, 0.7241, 0.6855, 0.7344, 0.6318, 0.7158, 0.6133,\n",
      "        0.7104, 0.7046, 0.6689, 0.6782, 0.7515, 0.7388, 0.9907, 0.7065, 0.7275,\n",
      "        0.7144, 0.7266, 0.7900, 0.6655, 0.6479, 0.6592, 0.6548, 0.7397, 0.6694,\n",
      "        0.6895, 0.6899, 0.6758, 0.6743, 0.6680, 0.7671, 0.7427, 0.7412, 0.7603,\n",
      "        0.7744, 0.6899, 0.6797, 0.6953, 0.6479, 0.6685, 0.6753, 0.7300, 0.6826,\n",
      "        0.7212, 0.7109, 0.6831, 0.7314, 0.7085, 0.7417, 0.6758, 0.6641, 0.6382,\n",
      "        0.7114, 0.6211, 0.6733, 0.7310, 0.6338, 0.5273, 0.6924, 0.6650, 0.7036,\n",
      "        0.6538, 0.7114, 0.4749, 0.7007, 0.7100, 0.7144, 0.7188, 0.6479, 0.6636,\n",
      "        0.6641, 0.6626, 0.7544, 0.6587, 0.6206, 0.7212, 0.6978, 0.7402, 0.7358,\n",
      "        0.6758, 0.6743, 0.7056, 0.6885, 0.7090, 0.7456, 0.6958, 0.7383, 0.6704,\n",
      "        0.6875, 0.6450, 0.7529, 0.7529, 0.7095, 0.6914, 0.7554, 0.6929, 0.7358,\n",
      "        0.6821, 0.7466, 0.6792, 0.7173, 0.6904, 0.7339, 0.6904, 0.7314, 0.6836,\n",
      "        0.6646, 0.7388, 0.7134, 0.7080, 0.6421, 0.7168, 0.6069, 0.7012, 0.6631,\n",
      "        0.6978, 0.6870, 0.6812, 0.6982, 0.7612, 0.7036, 0.7134, 0.7241, 0.7026,\n",
      "        0.6597, 0.6572, 0.6768, 0.6909, 0.7241, 0.6846, 0.6323, 0.7388, 0.7197,\n",
      "        0.7217, 0.7168, 0.6997, 0.6538, 0.6763, 0.7437, 0.7324, 0.7261, 0.7031,\n",
      "        0.7812, 0.7100, 0.7217, 0.7031, 0.6738, 0.6602, 0.7134, 0.7466, 0.7188,\n",
      "        0.7490, 0.7153, 0.7080, 0.7686, 0.7393, 0.6875, 0.6753, 0.6763, 0.6460,\n",
      "        0.7031, 0.6987, 0.6650, 0.7954, 0.7241, 0.7280, 0.7310, 0.7520, 0.7363,\n",
      "        0.7358, 0.7178, 0.7271, 0.7300, 0.7002, 0.6938, 0.6714, 0.6636, 0.7007,\n",
      "        0.7358, 0.6558, 0.7075, 0.7090, 0.7427, 0.7217, 0.7515, 0.6733, 0.7104,\n",
      "        0.7222, 0.7280, 0.7412, 0.7197, 0.7285, 0.7393, 0.6729, 0.7002, 0.7295,\n",
      "        0.7295, 0.7007, 0.6953, 0.7480, 0.7476, 0.6865, 0.6719, 0.6719, 0.6719,\n",
      "        0.6782, 0.7915, 0.6914, 0.7178, 0.6709, 0.7227, 0.7271, 0.6538, 0.6743,\n",
      "        0.7217, 0.7163, 0.6812, 0.6992, 0.7666, 0.6958, 0.7544, 0.7002, 0.5820,\n",
      "        0.7134, 0.3655, 0.6719, 0.6895, 0.7529, 0.6841, 0.7271, 0.6592, 0.7061,\n",
      "        0.6992, 0.6450, 0.6836, 0.7100, 0.6567, 0.6943, 0.7031, 0.7129, 0.7295,\n",
      "        0.6704, 0.7109, 0.6807], requires_grad=True)\n",
      "esm_model.layers.10.final_layer_norm.bias Parameter containing:\n",
      "tensor([-1.4824e-02,  1.8997e-02, -1.1559e-02, -4.7699e-02,  1.2825e-02,\n",
      "         1.7731e-02, -2.0950e-02, -1.6575e-03,  1.4801e-02, -2.9846e-02,\n",
      "         4.1901e-02,  8.9661e-02, -1.0841e-02, -1.9455e-03, -1.7975e-02,\n",
      "         2.7939e-02,  1.7578e-02, -4.2267e-02, -5.8411e-02,  1.2085e-02,\n",
      "         4.0222e-02,  2.6138e-02,  8.3740e-02, -2.0828e-02, -8.5876e-02,\n",
      "         5.4474e-02, -8.3008e-02,  4.9805e-02,  9.3201e-02, -3.8727e-02,\n",
      "        -1.8219e-02,  2.7420e-02, -2.7802e-02,  4.8790e-03,  2.8397e-02,\n",
      "         3.1952e-02, -2.6184e-02, -5.0446e-02,  1.9627e-03, -4.3793e-02,\n",
      "        -8.7524e-02, -8.7769e-02,  3.4210e-02, -3.3360e-03, -8.9844e-02,\n",
      "         3.5610e-03, -4.6631e-02, -1.0315e-02, -4.7852e-02, -4.4647e-02,\n",
      "         6.6895e-02,  1.7593e-02,  2.3453e-02, -9.6802e-02,  8.2764e-02,\n",
      "        -2.7939e-02, -3.4943e-02,  7.7087e-02, -9.1858e-02, -2.4506e-02,\n",
      "        -4.1656e-02,  3.2990e-02,  1.3550e-02, -1.2305e-01, -4.1016e-02,\n",
      "        -1.1914e-01,  2.1191e-01, -8.5571e-02,  1.0291e-01, -9.0408e-03,\n",
      "        -2.9663e-02,  1.0034e-01,  6.9519e-02, -5.4626e-02,  4.1260e-02,\n",
      "         8.4473e-02,  9.5825e-02, -4.6611e-05, -6.5247e-02,  3.1605e-03,\n",
      "         1.0620e-01,  1.7090e-02, -7.6111e-02,  6.6757e-03,  1.5686e-02,\n",
      "        -3.3283e-04, -3.8849e-02,  1.1023e-01,  9.3323e-02, -2.0752e-02,\n",
      "         5.0697e-03,  2.2278e-02,  3.1311e-02,  1.9653e-02,  4.4556e-02,\n",
      "         3.1708e-02,  1.2909e-02,  1.8036e-02,  1.2189e-01, -2.3773e-02,\n",
      "         6.9084e-03,  5.2460e-02,  1.6815e-02,  5.1025e-02,  6.5796e-02,\n",
      "        -2.6345e-04, -8.5144e-02,  4.0841e-04,  2.9037e-02, -4.7272e-02,\n",
      "         9.7778e-02,  4.2267e-02,  4.7028e-02, -5.0507e-02, -5.5664e-02,\n",
      "         8.5754e-02, -4.5990e-02,  4.0030e-04,  1.0455e-01, -1.7151e-02,\n",
      "        -4.0802e-02,  5.3986e-02,  5.2094e-02,  1.0582e-02, -3.0151e-02,\n",
      "        -1.5495e-02, -1.8250e-02, -6.6589e-02,  5.4840e-02, -6.2408e-02,\n",
      "        -9.3689e-02, -2.7817e-02, -3.3295e-02, -4.7668e-02,  7.5256e-02,\n",
      "         7.1411e-02,  8.2397e-02,  5.5969e-02,  1.2073e-01,  6.7810e-02,\n",
      "         1.1177e-02, -9.2041e-02,  7.2876e-02,  5.7495e-02, -1.6281e-02,\n",
      "        -3.3539e-02, -9.1782e-03,  5.2765e-02,  5.5817e-02, -4.2267e-02,\n",
      "         3.7079e-02, -7.9163e-02, -8.0933e-02, -1.1948e-02,  2.6917e-02,\n",
      "         3.1097e-02, -1.2347e-01,  7.1411e-03,  1.1658e-02,  7.0435e-02,\n",
      "        -6.3095e-03, -4.0092e-03,  9.8938e-02, -2.0462e-02, -9.3262e-02,\n",
      "         5.6686e-03, -4.6631e-02, -1.5466e-01, -4.4739e-02,  3.6591e-02,\n",
      "         1.7957e-01,  1.9363e-02, -2.0706e-02, -2.5864e-02, -8.1848e-02,\n",
      "        -8.3389e-03, -1.6037e-02,  3.2471e-02, -3.9612e-02,  7.2815e-02,\n",
      "         1.0078e-02, -5.3955e-02, -7.8430e-02, -4.8798e-02,  3.6133e-02,\n",
      "         3.0533e-02, -4.3640e-02,  2.4689e-02, -1.0480e-01,  2.7603e-02,\n",
      "        -5.4901e-02,  1.5527e-01,  3.3020e-02,  1.1041e-01, -1.8097e-02,\n",
      "        -8.1100e-03, -2.5864e-02, -7.0984e-02,  1.2878e-01,  6.1920e-02,\n",
      "         7.2050e-04,  2.3041e-02,  5.8502e-02, -1.6571e-02, -6.1371e-02,\n",
      "         5.5237e-02, -9.9304e-02, -5.4871e-02,  2.4216e-02,  2.0294e-02,\n",
      "         5.6686e-03,  8.0688e-02,  2.9358e-02, -7.6599e-02, -5.1239e-02,\n",
      "        -5.4047e-02, -9.8633e-02, -5.0232e-02,  9.7412e-02,  8.4686e-03,\n",
      "        -4.6844e-02,  3.7720e-02,  1.6907e-01, -6.3538e-02,  2.5238e-02,\n",
      "        -2.4700e-04,  2.4155e-02, -1.5945e-02,  4.4830e-02,  5.9326e-02,\n",
      "        -3.8818e-02, -2.0264e-02, -1.1334e-01,  1.0406e-01, -8.5876e-02,\n",
      "         2.1713e-02, -8.0383e-02,  6.8970e-02,  4.3823e-02, -2.4094e-02,\n",
      "        -7.2823e-03,  5.2826e-02,  3.1021e-02, -3.6407e-02, -6.1493e-02,\n",
      "        -5.4962e-02,  7.4158e-02, -4.3427e-02, -2.8877e-03, -7.7026e-02,\n",
      "        -2.4918e-02,  4.2633e-02, -7.8369e-02,  7.5500e-02,  6.8130e-03,\n",
      "        -7.9712e-02,  6.8481e-02,  2.3788e-02, -2.4307e-02, -1.1066e-01,\n",
      "         4.2816e-02,  9.0149e-02,  3.8548e-03,  5.8746e-04, -1.0284e-02,\n",
      "         1.3611e-01, -4.7394e-02, -5.2872e-03, -2.1149e-02,  1.0803e-01,\n",
      "        -6.1493e-02,  3.8910e-02,  7.6721e-02,  5.2567e-03, -6.9458e-02,\n",
      "        -5.8060e-03, -2.2095e-02, -3.1403e-02, -1.2146e-01,  1.4148e-01,\n",
      "        -3.9154e-02,  2.2507e-02, -8.8257e-02, -8.3801e-02,  4.2786e-02,\n",
      "        -1.0925e-02,  4.3152e-02,  5.0842e-02, -6.9397e-02,  3.2501e-02,\n",
      "        -5.3711e-02, -1.2238e-01,  5.1346e-03,  6.8855e-03,  1.4929e-01,\n",
      "         1.3321e-02,  1.1959e-03,  3.9124e-02,  1.3525e-01,  1.0663e-01,\n",
      "        -3.1967e-03, -4.5380e-02,  2.6215e-02,  6.8420e-02,  3.6469e-02,\n",
      "        -5.7098e-02,  3.6774e-02, -6.8176e-02,  2.2858e-02, -1.6235e-01,\n",
      "        -3.7567e-02,  2.0203e-02, -5.7404e-02,  4.6021e-02, -5.5786e-02,\n",
      "         3.4409e-03,  1.1212e-01,  2.4765e-02, -2.1076e-03,  7.9224e-02,\n",
      "        -3.5248e-02,  2.5269e-02,  3.7933e-02, -5.8098e-03,  8.9783e-02,\n",
      "        -1.0691e-03, -2.8839e-02,  1.1682e-01,  6.0547e-02,  1.4685e-01,\n",
      "        -7.6294e-02,  1.4099e-01,  3.8452e-02,  7.6538e-02,  2.7069e-02,\n",
      "        -7.4036e-02,  1.2854e-01, -7.1764e-05,  3.1174e-02, -5.6915e-02,\n",
      "         9.3689e-03,  1.2311e-01,  4.8676e-02,  3.4576e-02,  8.1665e-02,\n",
      "         6.0638e-02,  4.2450e-02, -4.6844e-03,  1.1444e-01, -5.4993e-02,\n",
      "        -4.8615e-02,  4.8553e-02,  7.3425e-02, -9.0820e-02,  1.3281e-01,\n",
      "        -1.0876e-01,  5.6793e-02,  9.4177e-02, -3.6804e-02,  1.8723e-02,\n",
      "         6.3049e-02,  2.7725e-02, -1.3943e-03, -4.7638e-02,  8.0383e-02,\n",
      "         1.6464e-02, -4.0314e-02,  1.2703e-02,  5.7640e-03,  6.9763e-02,\n",
      "        -3.3150e-03,  1.1148e-03,  2.4490e-02, -3.0502e-02, -6.1096e-02,\n",
      "         2.0203e-02, -3.1372e-02,  5.2765e-02, -8.9172e-02,  1.0132e-01,\n",
      "         3.2410e-02, -2.4170e-02,  1.6724e-01, -2.1210e-02,  2.2629e-02,\n",
      "        -4.0466e-02, -2.0477e-02, -1.1757e-02,  1.3403e-01, -2.0081e-02,\n",
      "        -2.7512e-02, -2.9785e-02, -2.4979e-02, -5.1613e-03, -4.9347e-02,\n",
      "        -6.2408e-02,  2.7679e-02,  6.7810e-02,  1.0941e-02, -6.4270e-02,\n",
      "         7.0129e-02, -9.5581e-02, -3.7781e-02, -9.6207e-03,  5.3406e-02,\n",
      "         5.9692e-02, -4.6349e-04,  7.3608e-02, -1.4458e-02,  1.2665e-02,\n",
      "         5.4108e-02,  4.0924e-02,  4.2023e-02, -4.4708e-02, -1.8051e-02,\n",
      "         1.2469e-01,  3.8971e-02, -8.4534e-02, -1.6699e-01, -4.7684e-07,\n",
      "         1.5326e-03,  4.2038e-03, -5.1208e-02,  6.1279e-02,  4.2297e-02,\n",
      "         1.1603e-01,  1.6406e-01,  3.6560e-02,  1.4417e-01, -2.3453e-02,\n",
      "         5.1910e-02,  4.7668e-02, -1.5942e-01,  2.4368e-02,  2.3376e-02,\n",
      "         7.6538e-02, -1.8477e-05, -5.0110e-02, -2.5272e-03, -8.6670e-03,\n",
      "         2.9312e-02, -1.5056e-04, -5.3101e-02,  7.2815e-02,  3.6285e-02,\n",
      "         1.0938e-01,  1.7563e-02,  1.9211e-02, -3.0155e-03,  5.9090e-03,\n",
      "         4.0741e-02, -1.0254e-01, -4.2938e-02,  3.7659e-02, -5.4047e-02,\n",
      "         5.4565e-02, -8.5571e-02, -9.8511e-02, -3.6896e-02, -9.4482e-02,\n",
      "         1.7590e-01,  9.6619e-02,  8.1787e-03,  2.4643e-02,  2.7557e-02,\n",
      "        -1.1017e-02,  2.3651e-02, -3.3112e-02, -4.9400e-03,  6.1157e-02,\n",
      "        -8.8684e-02,  1.2659e-01, -2.4933e-02,  4.8492e-02, -8.7341e-02,\n",
      "        -3.2898e-02, -6.6162e-02,  8.0627e-02, -5.9174e-02, -5.7800e-02],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.self_attn.k_proj.weight Parameter containing:\n",
      "tensor([[-0.0037,  0.0052, -0.1417,  ..., -0.1179,  0.0158, -0.0928],\n",
      "        [ 0.0569,  0.0364,  0.0700,  ..., -0.1206,  0.0595, -0.0394],\n",
      "        [-0.0325, -0.0765,  0.0137,  ...,  0.2434, -0.0532, -0.1044],\n",
      "        ...,\n",
      "        [-0.0345, -0.0377,  0.0301,  ...,  0.0086, -0.0754,  0.0564],\n",
      "        [-0.0571,  0.0038, -0.0834,  ..., -0.0578,  0.0155, -0.0766],\n",
      "        [-0.0743,  0.0639,  0.1138,  ...,  0.0715,  0.1545,  0.1215]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.self_attn.k_proj.bias Parameter containing:\n",
      "tensor([-9.8755e-02, -3.2684e-02,  1.1896e-01,  1.9080e-01, -4.6234e-02,\n",
      "        -5.4883e-01,  1.4008e-02,  1.7939e+00, -4.4873e-01,  8.8672e-01,\n",
      "         1.1641e+00,  8.5889e-01,  2.9816e-02, -2.0752e-02,  9.4238e-02,\n",
      "         8.9294e-02,  4.2188e-01,  2.7490e-01,  1.2461e+00, -7.1228e-02,\n",
      "        -3.9526e-01, -4.0869e-01, -5.4102e-01,  2.9541e-02,  9.3262e-02,\n",
      "        -4.8535e-01, -5.0720e-02, -9.6558e-02,  1.4246e-01,  2.9277e+00,\n",
      "         4.7344e+00,  6.0664e+00,  5.1575e-02,  5.0659e-02,  3.0391e+00,\n",
      "         2.6343e-01, -4.1724e-01,  1.9501e-02, -6.3330e-01,  1.1660e+00,\n",
      "        -2.3125e+00,  6.8359e-01, -7.6953e-01,  8.7744e-01,  3.1860e-01,\n",
      "         4.0259e-01,  6.3965e-01, -2.3669e-01, -1.4717e-02, -3.3765e-01,\n",
      "        -1.2866e-01, -2.6932e-02,  2.0547e+00, -3.7070e+00,  4.9531e+00,\n",
      "         6.1758e+00,  8.1104e-01,  2.0190e-01,  1.4736e+00, -7.8369e-01,\n",
      "         5.1709e-01,  6.1890e-02,  7.1680e-01, -1.1543e+00,  3.3789e-01,\n",
      "        -2.0032e-01, -4.4214e-01,  4.1040e-01, -6.2891e-01, -1.5488e+00,\n",
      "        -1.4668e+00, -1.7852e+00,  1.5393e-01,  5.6305e-02, -5.7281e-02,\n",
      "         1.8726e-01,  1.5332e-01, -3.1934e+00,  4.9453e+00, -6.4531e+00,\n",
      "         2.2186e-02, -1.3369e+00, -5.7910e-01,  1.5439e+00, -1.4170e+00,\n",
      "         2.5342e-01,  7.7441e-01,  1.1875e+00, -2.2129e+00,  3.9502e-01,\n",
      "        -1.2537e-01,  6.4404e-01, -1.0781e+00,  2.1719e+00,  1.7061e+00,\n",
      "         1.7383e-01, -1.3474e-02,  3.3508e-02,  1.4697e-01, -6.0822e-02,\n",
      "         1.0254e-01,  1.8358e-03, -3.7524e-01, -6.3965e-01,  6.9336e-02,\n",
      "         1.0283e+00, -1.6104e+00, -1.0020e+00,  5.2094e-02,  6.1707e-02,\n",
      "        -9.4986e-03, -3.0731e-02, -7.4234e-03, -2.9663e-01,  2.4683e-01,\n",
      "        -1.1774e-01, -4.0869e-01, -5.0879e-01,  4.2450e-02, -5.0537e-02,\n",
      "         1.3916e-01,  7.4219e-01,  2.0886e-01,  5.2979e-01,  1.3340e+00,\n",
      "        -4.5044e-02,  2.2900e-01, -5.2578e+00,  2.0239e-01,  9.9414e-01,\n",
      "        -3.9551e-01,  9.7803e-01,  1.2832e+00, -4.9347e-02,  6.6455e-01,\n",
      "         8.6182e-01, -3.9771e-01, -2.4883e+00, -3.7988e+00,  2.4194e-01,\n",
      "         1.4832e-02, -1.4258e+00, -2.0195e+00,  7.9541e-01,  8.7463e-02,\n",
      "         4.4495e-02, -4.1870e-02,  7.0251e-02,  1.7310e-01,  2.9517e-01,\n",
      "         1.1151e-01, -2.6831e-01, -5.5420e-01, -3.3765e-01, -8.0261e-02,\n",
      "        -1.9434e-01,  7.5562e-02,  7.5195e-02, -1.3220e-01,  2.1042e-02,\n",
      "         1.1688e-01, -9.1980e-02, -3.8965e-01,  4.7900e-01,  9.9915e-02,\n",
      "         1.3049e-01,  2.5195e-01, -6.3037e-01,  1.5198e-02, -1.1497e-02,\n",
      "        -1.8530e-03, -1.8295e-02, -2.7832e-02,  5.5939e-02, -6.6345e-02,\n",
      "         3.5461e-02, -2.1667e-01, -3.3081e-02, -1.6465e+00, -3.0384e-03,\n",
      "         5.9296e-02,  3.6030e-03, -4.9927e-02, -3.0060e-02,  3.6957e-02,\n",
      "        -2.9388e-02, -1.0780e-02,  4.0466e-02,  3.2257e-02, -5.5664e-01,\n",
      "        -7.8857e-01, -2.0098e+00, -5.0842e-02, -2.3758e-02, -8.2825e-02,\n",
      "         2.7905e-01,  7.4316e-01,  2.1448e-01, -2.3083e-01,  3.4590e+00,\n",
      "        -2.0837e-01, -2.6978e-01, -1.0376e-01,  1.2262e-01,  2.7893e-02,\n",
      "        -4.5105e-02,  2.7930e-01, -3.0859e-01,  2.6636e-01,  1.7012e+00,\n",
      "         3.1816e+00, -4.8169e-01, -3.8916e-01,  1.7432e-01, -4.4995e-01,\n",
      "         7.3975e-02, -2.1038e-03,  9.6588e-03,  2.0721e-02,  4.5624e-02,\n",
      "         5.0812e-02,  3.6316e-02,  2.0679e-01,  3.9711e-03, -8.6487e-02,\n",
      "        -2.8174e-01,  5.0537e-01, -2.4255e-01, -9.2926e-03,  2.0706e-02,\n",
      "         2.6443e-02, -4.8920e-02, -4.9286e-02,  5.8289e-02, -3.0563e-02,\n",
      "        -3.0045e-02,  1.6650e-01,  8.3008e-02,  4.8169e-01,  1.0078e+00,\n",
      "        -8.5144e-02,  6.2500e-02, -7.4524e-02, -3.9600e-01,  5.8154e-01,\n",
      "        -1.6251e-02, -6.8945e-01, -3.9551e+00, -7.8613e-01, -1.8726e-01,\n",
      "         7.1338e-01, -3.5034e-01, -5.8014e-02, -7.0557e-02,  2.4414e-01,\n",
      "         1.4270e-01, -5.0928e-01,  1.6680e+00, -1.5605e+00,  2.0374e-01,\n",
      "        -6.4746e-01, -9.7363e-01, -1.4001e-01, -1.5930e-02,  4.7302e-03,\n",
      "         2.8732e-02, -1.4847e-02, -1.8036e-02, -2.7847e-02,  3.5004e-02,\n",
      "         8.5815e-02, -4.6509e-02,  2.7588e-01,  6.2012e-02, -2.4668e+00,\n",
      "        -1.2482e-01, -6.3049e-02, -6.3820e-03, -5.8044e-02, -6.9031e-02,\n",
      "        -1.3329e-02,  5.0720e-02,  4.5204e-03,  1.6089e-01, -6.5918e-02,\n",
      "        -4.1870e-01,  7.8583e-03, -2.0234e+00, -3.1616e-02,  3.8391e-02,\n",
      "         2.2659e-02, -4.9835e-02, -5.2795e-02,  1.1591e-01,  2.2217e-01,\n",
      "         4.4800e-01, -5.7316e-04,  3.0347e-01, -2.0776e-01, -2.7002e-01,\n",
      "         1.6296e-02, -2.0561e-03,  5.7281e-02, -2.2675e-02, -7.0190e-02,\n",
      "         2.9556e-02, -9.8083e-02, -9.6924e-02, -6.9397e-02, -1.5430e-01,\n",
      "         4.6655e-01,  4.5142e-01,  2.7954e-02,  2.2354e-02,  1.4636e-01,\n",
      "        -1.1681e-02, -2.3608e-01,  1.7236e-01, -1.4709e-01, -1.6328e+00,\n",
      "        -8.8135e-02,  3.0396e-01,  6.7749e-02, -1.8457e-01, -1.1551e-02,\n",
      "         1.3107e-02,  1.6357e-02, -1.0620e-01,  1.3477e-01,  3.5474e-01,\n",
      "        -1.2451e+00, -1.3647e-01, -1.4014e-01,  1.5210e-01,  1.0773e-02,\n",
      "         8.0957e-01, -1.8936e-02, -2.5845e-03,  2.7054e-02,  1.2161e-02,\n",
      "         7.5317e-02, -5.7159e-02, -6.9519e-02, -1.0345e-01, -5.4102e-01,\n",
      "        -2.0825e-01, -7.3535e-01,  6.3086e-01, -5.6696e-04,  4.0710e-02,\n",
      "        -2.6245e-02,  7.8369e-02,  4.7760e-02, -7.4280e-02, -5.2277e-02,\n",
      "         1.4246e-01, -1.2573e-01,  3.6831e-03,  8.2397e-02,  2.1179e-01,\n",
      "         2.7344e-02,  2.1683e-02,  1.4404e-01, -2.7002e-01, -7.5830e-01,\n",
      "        -9.0381e-01,  5.0537e-02,  1.8274e-01, -2.1240e-01,  5.5389e-02,\n",
      "         9.3457e-01, -4.9225e-02, -1.7090e-01, -2.8687e-01,  3.2935e-01,\n",
      "        -3.8599e-01, -3.2471e-01,  2.0605e-01,  2.5684e+00, -3.5957e+00,\n",
      "         2.0398e-01,  2.8394e-01, -1.7139e+00, -2.3621e-01, -3.3016e-03,\n",
      "        -1.7822e-01,  4.8389e-01,  7.8223e-01,  3.3350e-01,  3.2324e+00,\n",
      "         5.1562e+00,  6.6562e+00,  3.3301e-01,  4.1943e-01,  1.1797e+00,\n",
      "         8.7988e-01, -8.7219e-02, -6.0333e-02, -1.9763e-01, -3.2690e-01,\n",
      "        -1.8516e+00, -2.7783e-01, -7.3486e-02, -9.7412e-02,  8.5547e-01,\n",
      "        -5.0146e-01,  7.8369e-01,  3.1104e-01,  4.3907e-03,  1.4807e-01,\n",
      "         2.3547e-01, -2.4902e-01,  2.7344e-01,  2.1741e-01, -4.7109e+00,\n",
      "        -6.2266e+00,  4.9683e-02,  6.7188e-01,  1.6357e-01, -7.8418e-01,\n",
      "        -1.5222e-01,  1.5884e-02, -1.2611e-02,  3.7744e-01,  1.4922e+00,\n",
      "        -2.8477e+00,  2.3621e-02, -5.8136e-02,  8.2568e-01, -9.5898e-01,\n",
      "         1.1847e-01,  2.0532e-01, -5.3613e-01,  5.7617e-01,  5.6348e-01,\n",
      "         3.1641e-01,  3.1323e-01, -2.2217e-01, -4.3945e+00, -4.5664e+00,\n",
      "         2.8540e-01, -2.0547e+00,  3.3281e+00, -7.8186e-02,  8.0029e-01,\n",
      "         5.2734e-01,  6.5430e-01, -1.1670e+00, -1.7129e+00,  2.4629e+00,\n",
      "        -7.1338e-01,  1.4141e+00, -1.3782e-01,  5.0146e-01, -9.3115e-01,\n",
      "        -1.0957e+00, -1.0352e+00,  2.6855e-02, -3.8696e-01, -7.3926e-01,\n",
      "        -4.3762e-02, -1.0420e+00,  2.5840e+00, -4.7656e+00, -2.3657e-01,\n",
      "        -3.2080e-01, -2.1035e+00, -5.3125e-01,  3.5339e-02,  2.1240e-01,\n",
      "         3.0908e-01,  3.8110e-01, -1.3154e+00, -1.6777e+00,  9.4922e-01,\n",
      "         1.3535e+00,  1.3318e-01, -1.1416e+00, -1.7021e+00,  7.7441e-01],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.self_attn.v_proj.weight Parameter containing:\n",
      "tensor([[ 0.0326,  0.1560, -0.3662,  ..., -0.1714, -0.0984,  0.1399],\n",
      "        [ 0.1669,  0.0220, -0.0041,  ...,  0.1373,  0.0416,  0.0977],\n",
      "        [-0.1914,  0.2122, -0.0768,  ..., -0.0021,  0.0701,  0.0876],\n",
      "        ...,\n",
      "        [ 0.0407,  0.3745,  0.0121,  ..., -0.0090,  0.0668, -0.0916],\n",
      "        [-0.2471,  0.0062, -0.0161,  ...,  0.2203, -0.0463,  0.3457],\n",
      "        [-0.1906,  0.0710, -0.0962,  ..., -0.1000,  0.0844,  0.0109]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.self_attn.v_proj.bias Parameter containing:\n",
      "tensor([ 2.5040e-02,  1.2482e-02, -3.4973e-02, -1.3382e-02,  2.5120e-03,\n",
      "         2.3102e-02,  1.0307e-02, -1.2331e-03, -1.6159e-02, -5.3940e-03,\n",
      "         1.6357e-02, -1.8082e-02,  3.9093e-02, -8.6365e-03, -1.1658e-02,\n",
      "         1.9150e-02, -1.4923e-02, -3.3813e-02, -1.2909e-02, -5.3787e-03,\n",
      "        -2.9087e-03,  2.7275e-03,  7.3433e-03, -3.8567e-03, -8.8654e-03,\n",
      "         1.5305e-02, -1.9064e-03, -2.8473e-02, -1.4023e-02, -8.1329e-03,\n",
      "         2.0294e-03, -1.4603e-02,  6.4316e-03,  5.5161e-03,  6.0501e-03,\n",
      "         1.0323e-02, -2.2079e-02,  1.3763e-02, -2.5978e-03, -2.1114e-03,\n",
      "        -1.8738e-02, -1.8911e-03, -3.7659e-02, -1.1902e-02,  4.6539e-03,\n",
      "         1.7578e-02,  1.1894e-02,  3.3169e-03,  2.4567e-03,  1.0727e-02,\n",
      "        -1.4374e-02, -1.2093e-02,  1.1253e-02, -3.5744e-03, -1.6113e-02,\n",
      "        -1.4328e-02, -2.6264e-03,  5.8365e-03,  1.0666e-02,  1.3687e-02,\n",
      "         5.0583e-03, -1.3687e-02, -4.3640e-03,  1.0109e-02,  9.1934e-03,\n",
      "        -1.1139e-02,  6.3057e-03,  1.8509e-02, -4.5105e-02, -2.5604e-02,\n",
      "         8.0466e-06,  3.1872e-03, -1.8066e-02,  2.9388e-02, -1.3313e-02,\n",
      "         1.4137e-02,  6.8207e-03, -6.1569e-03,  1.3443e-02,  1.4481e-02,\n",
      "        -3.2654e-03, -9.2850e-03,  4.6463e-03, -2.9495e-02, -1.6144e-02,\n",
      "        -1.2417e-03,  2.1652e-02, -1.1436e-02,  2.0584e-02,  1.4069e-02,\n",
      "        -2.4826e-02, -7.5188e-03,  2.9488e-03, -3.1147e-03, -2.3102e-02,\n",
      "         1.4648e-02, -1.5556e-02, -1.6174e-02,  1.9241e-02,  9.9030e-03,\n",
      "        -1.9333e-02,  1.3399e-03, -2.3708e-03,  5.9395e-03,  1.0170e-02,\n",
      "        -1.3885e-02, -8.6670e-03, -1.0384e-02, -4.1504e-03, -1.4595e-02,\n",
      "         7.3433e-04,  3.3051e-02,  1.1604e-02,  1.2871e-02,  2.6550e-02,\n",
      "         3.1948e-03, -3.2043e-04, -1.2535e-02,  9.4986e-03, -1.4973e-03,\n",
      "         2.0767e-02, -4.6143e-02, -1.6815e-02, -2.8778e-02, -2.0020e-02,\n",
      "        -7.5607e-03,  4.9734e-04,  3.4546e-02,  2.2552e-02,  1.5778e-02,\n",
      "        -1.4404e-02, -1.4954e-03,  3.1311e-02, -2.0538e-02, -2.4815e-03,\n",
      "        -2.4445e-02, -1.1604e-02,  2.8671e-02,  9.4299e-03,  2.1118e-02,\n",
      "        -1.2032e-02,  6.7368e-03,  3.1036e-02,  1.0239e-02, -2.5589e-02,\n",
      "        -2.4765e-02,  1.3733e-02,  3.0334e-02,  9.1400e-03, -1.0834e-02,\n",
      "        -2.2400e-02, -1.5305e-02,  3.3447e-02, -1.9852e-02,  1.6661e-03,\n",
      "         3.0918e-03,  6.9466e-03, -1.5625e-02, -8.8577e-03, -1.1818e-02,\n",
      "         1.1589e-02,  1.5167e-02,  2.7485e-03,  2.0752e-02,  1.4915e-03,\n",
      "        -1.3107e-02,  2.5146e-02, -1.0971e-02, -2.6184e-02, -2.1927e-02,\n",
      "         5.6000e-02, -1.4847e-02,  4.1428e-03, -2.6337e-02, -2.1729e-02,\n",
      "        -3.0746e-03, -4.7112e-03,  2.0233e-02, -3.5706e-03, -1.1200e-02,\n",
      "        -2.0248e-02, -3.2196e-02,  7.8735e-03, -7.5874e-03,  2.4929e-03,\n",
      "        -1.4038e-02, -1.5764e-03,  3.1952e-02, -5.4993e-02, -5.7411e-03,\n",
      "        -1.8417e-02,  4.9820e-03,  2.9938e-02, -3.1677e-02, -4.9629e-03,\n",
      "         8.2550e-03, -3.1986e-03, -7.3853e-03, -1.6037e-02,  1.6737e-03,\n",
      "         4.7493e-03,  5.5328e-02, -3.4363e-02,  3.1021e-02,  5.4474e-03,\n",
      "        -3.7231e-03,  1.2253e-02, -1.0963e-02, -5.8632e-03, -4.5662e-03,\n",
      "         1.3000e-02, -3.7323e-02, -3.6621e-03,  1.4580e-02,  5.4932e-02,\n",
      "         3.4714e-03, -2.2385e-02, -8.5115e-04, -3.0613e-04, -9.0332e-03,\n",
      "         2.2755e-03, -9.5367e-03,  2.9736e-03,  4.2877e-03,  3.3905e-02,\n",
      "         2.0401e-02,  1.7075e-02,  8.5983e-03, -2.4292e-02, -4.7913e-03,\n",
      "         4.7302e-03, -3.4561e-03, -2.4834e-03,  6.6795e-03,  3.4637e-03,\n",
      "        -3.8671e-04,  8.6365e-03,  8.8348e-03, -1.5915e-02, -1.7700e-02,\n",
      "        -1.1787e-02, -1.7273e-02,  9.3842e-03, -5.0385e-02, -6.0005e-03,\n",
      "        -3.2692e-03,  5.2414e-03,  2.5291e-03,  2.3148e-02,  3.9635e-03,\n",
      "         9.3689e-03,  8.6670e-03, -4.4136e-03, -3.1128e-02,  3.4733e-03,\n",
      "        -5.9624e-03, -2.5528e-02,  5.7953e-02, -1.1387e-03, -1.1353e-02,\n",
      "         2.8976e-02,  8.8043e-03, -2.4048e-02,  5.3314e-02,  1.8585e-02,\n",
      "        -6.1989e-03, -1.7578e-02, -1.0597e-02,  1.3283e-02,  2.9404e-02,\n",
      "        -8.9111e-03, -8.0872e-03,  1.4984e-02,  6.4514e-02, -8.0948e-03,\n",
      "        -9.5062e-03, -2.5391e-02,  2.3788e-02, -2.1774e-02,  1.4198e-02,\n",
      "        -2.8244e-02, -2.0111e-02,  1.7075e-02, -1.7761e-02,  2.1400e-03,\n",
      "         1.0422e-02,  4.4403e-02,  2.1423e-02, -4.7493e-03,  6.6910e-03,\n",
      "         1.8066e-02,  6.9771e-03, -8.2169e-03,  2.4261e-03, -7.8659e-03,\n",
      "         9.0714e-03,  1.9592e-02,  1.4372e-03,  2.8687e-03,  2.5787e-02,\n",
      "         1.1459e-02,  2.7599e-03, -7.5111e-03,  8.9312e-04,  1.5579e-02,\n",
      "         1.0986e-03,  1.2711e-02,  2.7145e-02,  1.0521e-02, -1.2238e-02,\n",
      "         2.0218e-03,  2.6302e-03, -8.5220e-03, -2.0248e-02,  5.6839e-03,\n",
      "        -5.9090e-03, -8.7585e-03, -7.1983e-03,  5.4016e-03,  1.2589e-03,\n",
      "         5.0049e-03,  2.9812e-03, -1.9119e-02, -1.1706e-04,  7.9956e-03,\n",
      "         8.3084e-03,  1.8585e-02,  7.4158e-03,  1.6373e-02, -2.1744e-03,\n",
      "        -1.4633e-02, -1.4236e-02, -3.1967e-03, -3.9215e-03,  2.5902e-03,\n",
      "         2.9049e-03, -1.5396e-02, -1.7532e-02,  2.7313e-02, -5.4588e-03,\n",
      "         1.1787e-02,  3.1433e-02, -1.7059e-02,  3.4363e-02,  3.0411e-02,\n",
      "        -2.8515e-03, -8.3237e-03,  2.1118e-02, -2.1011e-02, -2.2507e-02,\n",
      "         7.8964e-03, -3.0121e-02, -3.0117e-03,  2.7328e-02, -4.7546e-02,\n",
      "         1.9484e-03, -8.3008e-03,  2.4261e-02, -2.0096e-02, -1.5488e-02,\n",
      "        -1.0061e-03, -1.6937e-02,  7.5684e-03, -2.4891e-03,  3.9978e-02,\n",
      "        -2.0538e-02, -2.7817e-02,  1.1289e-04,  4.8828e-03, -7.1716e-03,\n",
      "         4.4342e-02,  2.1942e-02,  2.9831e-03,  7.6103e-03, -1.0765e-02,\n",
      "         2.1553e-03,  8.4639e-06, -4.4067e-02, -6.0501e-03,  1.1444e-02,\n",
      "        -3.9764e-02,  5.2948e-03, -1.7624e-02,  1.2627e-03, -5.2246e-02,\n",
      "         5.3131e-02,  2.4857e-02,  1.7212e-02, -4.2023e-02,  2.9541e-02,\n",
      "         1.2352e-02,  1.6144e-02, -1.0078e-02, -1.7136e-02,  2.9030e-03,\n",
      "        -2.7893e-02, -1.6724e-02,  1.1627e-02,  4.0497e-02, -2.2644e-02,\n",
      "         2.0466e-03,  4.6120e-03, -8.5678e-03,  3.1708e-02, -4.5547e-03,\n",
      "         3.8605e-02, -1.2184e-02,  5.6763e-03,  2.8610e-02, -2.9007e-02,\n",
      "         1.6953e-02,  5.7556e-02, -1.4526e-02, -1.5841e-03, -1.2123e-02,\n",
      "        -8.4381e-03,  2.2156e-02, -6.0921e-03,  6.0272e-03,  1.3466e-02,\n",
      "        -3.1647e-02,  2.8854e-02, -2.3041e-03,  2.0172e-02, -6.6910e-03,\n",
      "         4.1595e-02, -1.6235e-02, -6.0425e-03, -2.0733e-03,  1.7105e-02,\n",
      "        -1.6815e-02,  2.2964e-02,  8.7357e-03, -2.1729e-02,  1.3411e-04,\n",
      "         2.8519e-02,  3.7327e-03, -1.2154e-02,  9.9030e-03,  1.2779e-02,\n",
      "        -1.7960e-02,  7.8354e-03,  3.3226e-03, -1.7517e-02, -2.5269e-02,\n",
      "         1.9943e-02,  1.3863e-02,  6.0654e-03, -1.7166e-02,  8.2092e-03,\n",
      "         8.5526e-03, -2.3460e-03,  1.5038e-02, -3.5980e-02, -5.2948e-03,\n",
      "        -3.3417e-02,  1.4198e-02, -2.7451e-02,  3.7174e-03, -5.2567e-03,\n",
      "        -3.7354e-02, -1.9054e-03, -1.6983e-02, -2.3407e-02, -7.4434e-04,\n",
      "        -1.0696e-02,  1.2302e-03, -1.1490e-02,  2.3407e-02,  6.5727e-03,\n",
      "         4.0627e-04,  1.7593e-02,  2.4700e-03, -2.4567e-02,  2.3712e-02,\n",
      "         1.7410e-02, -3.0365e-02, -5.6152e-03,  4.0619e-02, -1.4086e-03],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.self_attn.q_proj.weight Parameter containing:\n",
      "tensor([[ 0.0309,  0.1046,  0.1205,  ..., -0.0392, -0.0074,  0.0516],\n",
      "        [ 0.0663, -0.0903, -0.1622,  ...,  0.0555,  0.0708,  0.0084],\n",
      "        [ 0.0528, -0.1440,  0.0686,  ...,  0.2776,  0.0590,  0.0644],\n",
      "        ...,\n",
      "        [ 0.1356, -0.0917, -0.1301,  ...,  0.1143, -0.0486,  0.1869],\n",
      "        [ 0.0724,  0.0271, -0.0259,  ..., -0.0624,  0.1423, -0.0470],\n",
      "        [-0.0387, -0.0768,  0.0441,  ...,  0.0675,  0.0094, -0.0291]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.self_attn.q_proj.bias Parameter containing:\n",
      "tensor([-1.0460e-02, -5.5908e-02,  8.2153e-02,  2.1704e-01, -1.6403e-02,\n",
      "        -1.9263e-01,  5.2582e-02,  3.0713e-01,  6.2103e-02, -7.7209e-02,\n",
      "         1.0443e-01,  8.2764e-02,  6.3538e-02, -7.2266e-02,  1.1847e-01,\n",
      "         1.1505e-01,  1.9824e-01,  1.2018e-01,  2.5659e-01,  3.3752e-02,\n",
      "        -5.0385e-02, -1.1734e-02, -1.5686e-02, -9.7595e-02, -3.3234e-02,\n",
      "         1.4954e-01,  2.0520e-01, -1.6125e-01,  8.8928e-02,  2.5537e-01,\n",
      "         2.6416e-01,  2.4292e-01, -2.6413e-02,  1.4969e-02,  1.3855e-01,\n",
      "        -2.0462e-02, -2.7979e-01, -1.3196e-01, -1.1523e-01,  1.9800e-01,\n",
      "        -2.5220e-01,  8.2642e-02,  2.1992e-03, -9.3918e-03, -1.7868e-02,\n",
      "         9.1629e-03, -1.0414e-03, -6.3019e-03, -1.9104e-01,  4.2206e-02,\n",
      "         1.2378e-01, -7.8491e-02,  2.3035e-01, -2.1912e-01,  2.1179e-01,\n",
      "         2.3767e-01, -3.2196e-02,  4.4830e-02,  1.4030e-02,  2.6871e-02,\n",
      "        -1.9360e-01,  1.8738e-01,  2.1057e-01, -1.9885e-01,  1.2146e-02,\n",
      "        -1.5762e-02, -2.6779e-02,  2.1133e-02, -8.8453e-04, -2.0859e-02,\n",
      "        -3.3691e-02, -8.4656e-02,  1.7859e-01,  5.6915e-02, -3.8666e-02,\n",
      "         4.4739e-02, -9.5901e-03, -2.1460e-01,  2.0813e-01, -2.1985e-01,\n",
      "        -6.6662e-04,  9.5520e-03,  1.2413e-02,  8.9111e-02,  2.8247e-01,\n",
      "         1.7487e-02,  2.3962e-01,  2.3242e-01, -2.4670e-01, -1.6754e-02,\n",
      "         6.3705e-03,  2.8717e-02, -2.3300e-02,  2.2156e-02,  7.0312e-02,\n",
      "         4.1580e-03,  1.0239e-02,  2.5620e-02,  9.8633e-02, -1.1316e-01,\n",
      "         4.0985e-02, -1.0933e-02, -1.8750e-01, -2.3730e-01, -8.3542e-03,\n",
      "        -6.8626e-03,  1.4185e-01,  1.7798e-01,  1.2255e-03,  3.5156e-02,\n",
      "        -2.8473e-02, -4.4403e-02, -5.1819e-02, -1.6602e-01,  1.2085e-01,\n",
      "         2.2232e-02, -1.8295e-02,  5.6885e-02, -5.5313e-03, -3.9276e-02,\n",
      "         4.0015e-01,  3.1665e-01,  1.2231e-01,  1.9824e-01,  2.7490e-01,\n",
      "        -2.7374e-02, -1.2512e-02, -2.7173e-01, -1.1581e-02,  3.3081e-02,\n",
      "        -1.0605e-03,  9.2468e-02,  2.5610e-01, -1.4319e-01,  3.2056e-01,\n",
      "         2.7637e-01, -5.4596e-02, -2.2180e-01, -2.6587e-01, -3.0563e-02,\n",
      "         5.7487e-03, -6.1646e-02, -9.7839e-02,  2.3865e-01,  4.9652e-02,\n",
      "         4.6844e-02, -4.7638e-02,  1.7319e-03,  1.4185e-01,  1.3171e-01,\n",
      "         8.3679e-02, -1.2915e-01, -5.6152e-02, -7.5035e-03, -1.6953e-02,\n",
      "         1.0388e-01, -1.3222e-02, -1.0635e-02, -4.0253e-02,  4.4922e-02,\n",
      "         8.7646e-02, -5.8502e-02, -2.2839e-01,  1.6040e-01,  7.9727e-03,\n",
      "         8.0322e-02, -1.0565e-01,  2.5903e-01,  2.1927e-02,  1.6205e-02,\n",
      "        -3.5858e-02,  3.2410e-02,  7.3853e-02,  5.3162e-02, -1.7505e-01,\n",
      "        -4.4952e-02,  4.0588e-02, -3.8940e-02,  1.9177e-01,  6.1249e-02,\n",
      "        -9.5520e-03, -1.0681e-02,  1.4427e-02,  7.0374e-02,  4.9713e-02,\n",
      "        -2.1881e-02,  5.9479e-02,  1.8347e-01, -6.0081e-03, -1.0107e-01,\n",
      "         1.2183e-01,  1.8774e-01, -7.6172e-02, -1.8692e-02, -8.5754e-02,\n",
      "         1.9824e-01,  2.2974e-01,  1.6737e-04, -1.1467e-02,  1.8445e-01,\n",
      "         1.3901e-02, -4.6600e-02, -2.2873e-02,  2.7695e-02,  2.9724e-02,\n",
      "        -1.3879e-01,  1.8127e-01, -1.3782e-01,  6.7566e-02,  2.0349e-01,\n",
      "         2.0874e-01, -2.6657e-02, -4.6906e-02, -6.0196e-03,  2.2292e-05,\n",
      "         2.0798e-02,  1.1002e-02, -3.4271e-02, -3.5675e-02, -3.6194e-02,\n",
      "         1.8967e-02,  1.2488e-01,  2.2354e-02,  1.6211e-01,  1.9974e-02,\n",
      "        -9.7656e-02,  1.0002e-02,  1.2311e-01, -1.0968e-01,  1.0388e-01,\n",
      "        -5.1765e-03, -5.0446e-02, -9.3628e-02,  9.1736e-02,  1.2733e-02,\n",
      "        -9.0149e-02, -7.5417e-03,  3.7506e-02, -1.1115e-01, -2.8174e-01,\n",
      "        -9.2773e-03,  1.1511e-01, -6.4270e-02, -2.6318e-01,  1.9775e-01,\n",
      "        -4.3274e-02, -1.2769e-01, -2.9688e-01,  2.1439e-02,  7.2632e-02,\n",
      "         5.4565e-02,  1.8631e-02,  6.0547e-02, -1.0699e-01,  2.5049e-01,\n",
      "         1.4502e-01, -1.7566e-01,  2.8394e-01, -2.1265e-01,  3.4760e-02,\n",
      "        -2.3590e-02, -2.1973e-02,  2.2018e-02, -2.7802e-02, -7.4158e-03,\n",
      "         3.1311e-02, -1.3069e-02,  1.6876e-02, -2.2774e-03,  3.7109e-02,\n",
      "         5.4474e-02, -1.1299e-02, -1.4502e-01, -1.1237e-01,  1.4014e-01,\n",
      "        -1.7288e-02, -2.7802e-02, -1.9516e-02, -1.2878e-02, -1.0132e-02,\n",
      "        -1.2222e-02, -1.5869e-02, -2.1332e-02,  1.6370e-01,  4.8553e-02,\n",
      "         2.7173e-01, -1.1346e-01,  5.7037e-02,  1.1139e-02,  4.3060e-02,\n",
      "         2.0844e-02, -4.4678e-02, -7.4646e-02,  1.6602e-01,  1.7029e-01,\n",
      "         2.5610e-01, -7.0679e-02,  8.7204e-03,  2.9968e-02,  7.2449e-02,\n",
      "        -3.6449e-03,  2.1729e-02,  6.4392e-02, -4.9713e-02, -1.0950e-01,\n",
      "         6.6162e-02, -6.9519e-02, -2.4643e-02, -3.8025e-02,  7.0679e-02,\n",
      "         9.4238e-02, -2.6489e-01, -1.2619e-02, -6.4697e-02, -9.6130e-02,\n",
      "         3.6377e-02, -1.0468e-01,  4.4373e-02, -8.0261e-03, -3.5132e-01,\n",
      "        -7.9498e-03,  5.8075e-02,  5.4718e-02, -1.2115e-01, -6.5193e-03,\n",
      "         2.5162e-02,  2.0996e-02,  9.8572e-02, -1.7746e-02,  3.3386e-02,\n",
      "        -2.3792e-01, -2.2354e-02,  4.6310e-03, -3.8666e-02,  1.7868e-02,\n",
      "         9.2163e-02,  3.0136e-02, -2.6794e-02, -7.4844e-03, -2.3712e-02,\n",
      "         4.2358e-02,  1.3733e-02, -1.8835e-01, -1.3928e-01,  1.0803e-01,\n",
      "        -3.1433e-02,  9.6191e-02,  7.3975e-02,  3.4088e-02,  4.4312e-02,\n",
      "         8.4900e-02, -4.4220e-02, -3.9215e-02, -1.2305e-01,  1.6336e-03,\n",
      "         2.0874e-01, -4.4785e-03,  1.4209e-01,  7.6721e-02, -1.5210e-01,\n",
      "         1.1749e-01,  7.9041e-02,  8.4473e-02, -1.5247e-01, -2.2534e-01,\n",
      "        -2.6050e-01, -5.4169e-04,  7.7553e-03,  3.8776e-03,  2.7634e-02,\n",
      "         6.4026e-02, -8.1787e-03, -4.0192e-02, -1.8555e-01,  2.6709e-01,\n",
      "        -2.3376e-01, -1.5222e-01,  2.0401e-02,  2.7612e-01, -2.8955e-01,\n",
      "         1.8967e-02, -2.4292e-02, -3.2349e-02, -1.0590e-02, -5.1666e-02,\n",
      "        -1.3428e-01,  2.5049e-01,  2.6685e-01,  3.5583e-02,  2.1899e-01,\n",
      "         2.6978e-01,  1.8896e-01, -2.5520e-03,  3.8147e-02,  6.2378e-02,\n",
      "         1.3025e-01,  1.9653e-02, -1.5732e-02, -1.1761e-01, -9.1248e-02,\n",
      "        -2.5391e-01, -5.1910e-02, -1.5068e-02,  1.6434e-02,  5.9326e-02,\n",
      "        -2.9587e-02, -6.7940e-03, -7.2327e-03,  7.3486e-02, -3.2867e-02,\n",
      "         1.9318e-02, -2.1729e-01, -2.7588e-02,  1.1772e-02, -2.3987e-01,\n",
      "        -2.4316e-01,  1.4137e-02,  1.5358e-02,  3.0838e-02,  1.1230e-02,\n",
      "         7.7271e-02,  1.0779e-01,  1.9873e-01,  2.0935e-01,  2.2388e-01,\n",
      "        -2.1436e-01, -1.8311e-02,  6.4880e-02,  5.5573e-02, -2.7512e-02,\n",
      "        -1.8692e-02,  7.6437e-04, -5.4248e-01,  1.7847e-01,  1.9531e-01,\n",
      "         1.0498e-01,  5.5481e-02, -3.8815e-03, -2.5757e-01, -2.4182e-01,\n",
      "         3.0777e-02, -7.1533e-02,  2.5806e-01, -2.0809e-03,  7.3792e-02,\n",
      "         2.8955e-01,  2.8955e-01, -3.2812e-01, -3.1372e-01,  2.6172e-01,\n",
      "        -7.0312e-02,  5.9540e-02,  2.6264e-03,  1.2751e-03, -5.2155e-02,\n",
      "        -4.2877e-02,  2.6660e-01, -1.8274e-01, -2.4268e-01, -2.6123e-01,\n",
      "         1.4488e-02, -9.8816e-02,  2.1289e-01, -2.4792e-01, -1.5656e-02,\n",
      "        -1.7685e-02, -2.2839e-01, -2.4506e-02,  3.2861e-01,  9.3567e-02,\n",
      "         7.8857e-02,  6.2500e-02, -2.4475e-01, -2.0972e-01,  5.8380e-02,\n",
      "         5.9235e-02,  8.7433e-03, -4.4617e-02, -5.6488e-02,  9.5367e-03],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.self_attn.out_proj.weight Parameter containing:\n",
      "tensor([[-0.0478, -0.0045,  0.0905,  ...,  0.0832, -0.0648,  0.2607],\n",
      "        [-0.0835,  0.0445, -0.0538,  ..., -0.1197,  0.2147,  0.0696],\n",
      "        [ 0.1426,  0.0967, -0.0292,  ...,  0.0126, -0.0820, -0.0816],\n",
      "        ...,\n",
      "        [-0.0942, -0.2356, -0.1405,  ...,  0.1965, -0.1906,  0.4128],\n",
      "        [-0.1139, -0.1477, -0.2981,  ..., -0.0184, -0.1378, -0.1039],\n",
      "        [ 0.0773,  0.0646,  0.1026,  ..., -0.0150, -0.2096,  0.0764]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.self_attn.out_proj.bias Parameter containing:\n",
      "tensor([-4.3793e-02,  2.9236e-02,  2.7618e-02,  9.2102e-02, -8.3801e-02,\n",
      "        -2.2141e-02,  8.4412e-02,  6.9313e-03, -1.2671e-01, -4.4128e-02,\n",
      "        -8.8013e-02,  1.1407e-01, -1.8692e-02,  2.2564e-03,  3.9154e-02,\n",
      "         6.1218e-02,  5.2979e-02,  8.4591e-04, -1.1981e-01, -1.5295e-01,\n",
      "        -5.8319e-02, -5.6793e-02, -5.9021e-02, -7.9102e-02, -6.3477e-02,\n",
      "         2.8320e-02, -6.6895e-02, -1.5251e-02,  4.0863e-02, -1.3867e-01,\n",
      "         3.0853e-02,  3.6011e-02,  4.7119e-02,  7.2754e-02, -9.0027e-02,\n",
      "        -1.0278e-01,  1.0919e-01, -8.5510e-02, -1.1102e-01, -7.1045e-02,\n",
      "         4.7272e-02, -7.4463e-02, -2.7634e-02,  7.8918e-02, -8.5022e-02,\n",
      "        -8.3557e-02,  7.2119e-01, -5.4108e-02, -5.5206e-02, -1.5247e-01,\n",
      "         1.4633e-02,  7.3814e-03,  2.2415e-02, -6.0608e-02,  1.0223e-01,\n",
      "        -9.4360e-02, -1.6846e-01, -7.6111e-02, -1.1542e-01, -4.2847e-02,\n",
      "         1.2619e-02, -1.4587e-01,  8.2703e-02, -5.3139e-03,  5.0079e-02,\n",
      "        -1.0803e-01,  4.0131e-03,  7.3608e-02,  3.6499e-02,  1.3293e-01,\n",
      "        -2.6962e-02,  7.9895e-02,  1.2457e-01, -1.1816e-01,  9.6069e-02,\n",
      "         4.6997e-02,  8.9844e-02,  3.1921e-02, -1.7462e-03,  4.2023e-02,\n",
      "         1.9470e-02,  5.9174e-02, -4.2603e-02, -4.4281e-02,  2.0215e-01,\n",
      "        -3.6560e-02, -7.8003e-02, -7.5623e-02,  7.4219e-02, -6.6719e-03,\n",
      "         1.7834e-01,  3.5126e-02, -5.1819e-02,  3.6621e-02, -1.1017e-01,\n",
      "         9.1736e-02,  1.2408e-01, -1.4832e-01,  7.0801e-02, -3.8666e-02,\n",
      "         7.7820e-02,  2.8595e-02, -2.9488e-03,  6.3721e-02,  4.4403e-02,\n",
      "         4.8492e-02, -8.5571e-02,  8.6121e-02,  1.3440e-01,  4.0192e-02,\n",
      "        -7.1655e-02, -1.3000e-02, -5.9265e-02, -4.8828e-02, -5.6671e-02,\n",
      "        -1.0277e-02,  5.4260e-02,  1.0620e-01, -2.6413e-02, -1.9482e-01,\n",
      "        -2.5604e-02,  1.1261e-01, -7.9712e-02, -7.4402e-02, -4.7028e-02,\n",
      "        -4.1809e-02, -3.6011e-02, -1.0278e-01,  7.4402e-02, -4.0466e-02,\n",
      "        -8.7646e-02,  4.6143e-02,  8.5571e-02, -1.0089e-01, -8.0948e-03,\n",
      "         7.3303e-02, -1.2427e-01,  1.9604e-01,  1.2030e-01,  3.9597e-03,\n",
      "         1.2573e-01, -1.0815e-01, -6.0486e-02,  5.4199e-02, -1.6647e-02,\n",
      "        -8.4167e-02,  2.2339e-02,  5.2612e-02,  1.5295e-01,  9.8755e-02,\n",
      "        -1.0327e-01, -3.4821e-02, -8.2947e-02, -6.2256e-02,  1.7114e-01,\n",
      "        -5.8777e-02, -1.6895e-01, -1.0132e-01,  9.9365e-02,  1.1499e-01,\n",
      "         5.8777e-02, -6.8909e-02,  1.4062e-01, -1.5068e-02, -1.1646e-01,\n",
      "        -1.2354e-01, -4.4006e-02, -1.3928e-01, -1.7029e-01,  4.0314e-02,\n",
      "         1.3086e-01,  4.4067e-02,  2.7649e-02,  3.9482e-03, -5.8319e-02,\n",
      "         2.6352e-02,  5.8655e-02,  5.6152e-03, -9.8633e-02,  7.3303e-02,\n",
      "        -4.9377e-02, -3.5839e-03, -6.1874e-03, -5.9814e-02,  1.2085e-02,\n",
      "        -6.0394e-02, -9.2957e-02,  2.0862e-01, -1.0956e-01,  2.8397e-02,\n",
      "        -3.5187e-02,  8.8318e-02,  1.3452e-01, -6.6101e-02, -4.9133e-02,\n",
      "        -7.2754e-02, -8.8013e-02, -1.8921e-02,  3.2288e-02,  1.1444e-01,\n",
      "         1.0162e-01, -1.7532e-02, -9.8114e-03, -3.0365e-02, -2.4643e-02,\n",
      "         1.1707e-01, -2.1582e-01,  1.5894e-01, -7.7820e-02, -3.7628e-02,\n",
      "        -4.7913e-02, -9.9182e-02,  1.5808e-01, -6.6101e-02,  6.5613e-03,\n",
      "         2.0332e-03,  6.4270e-02, -1.0205e-01,  4.0436e-03, -9.1858e-03,\n",
      "        -1.2927e-01,  1.1467e-02,  1.1768e-01,  1.7746e-02,  9.4788e-02,\n",
      "        -1.1023e-01,  1.3953e-01,  5.6885e-02,  9.2087e-03, -5.0125e-03,\n",
      "         8.7036e-02, -2.0862e-01,  6.1462e-02,  9.5886e-02, -1.6919e-01,\n",
      "         2.4124e-02, -5.1918e-03,  7.2449e-02,  1.3391e-01, -4.2450e-02,\n",
      "        -1.4801e-03,  9.4452e-03, -7.4615e-03, -1.7700e-01, -7.3853e-02,\n",
      "        -1.9928e-02,  4.6295e-02, -4.5013e-02, -9.0271e-02,  4.8584e-02,\n",
      "        -8.3069e-02, -1.4270e-01, -9.5276e-02,  2.9755e-02,  7.1182e-03,\n",
      "        -4.1389e-03, -7.9956e-02, -1.1279e-01,  1.0170e-02, -1.0828e-01,\n",
      "         9.1187e-02, -8.5632e-02, -6.1615e-02,  1.8585e-02,  1.5419e-02,\n",
      "        -5.5573e-02,  3.8879e-02, -1.7990e-02,  5.4016e-02,  8.8440e-02,\n",
      "        -2.1118e-02, -4.9194e-02,  3.9337e-02, -1.3306e-01, -8.4717e-02,\n",
      "         1.1981e-01,  4.9957e-02, -1.0339e-01, -6.4270e-02,  2.2934e-02,\n",
      "        -1.1743e-01,  1.0272e-01,  3.7441e-03, -2.4628e-02,  1.9791e-02,\n",
      "        -4.5319e-02,  7.2327e-02,  2.7734e-01,  2.3331e-02, -1.1755e-01,\n",
      "         4.1962e-02,  3.0045e-02,  1.6809e-01, -2.1033e-01,  2.9800e-02,\n",
      "        -1.1627e-01,  1.3696e-01,  1.2366e-01, -2.8858e-03, -2.4878e-01,\n",
      "         1.2036e-01, -2.9205e-02,  1.7432e-01,  9.4299e-03,  6.3171e-02,\n",
      "         1.5678e-03,  1.4122e-02, -1.0968e-01,  6.1218e-02, -9.9670e-02,\n",
      "        -8.9844e-02, -3.8544e-02, -3.9337e-02, -5.7343e-02, -5.4657e-02,\n",
      "        -3.6713e-02,  9.8267e-02,  1.0571e-01, -1.7014e-02, -1.1040e-02,\n",
      "        -1.1273e-01,  4.6173e-02,  2.0081e-02,  5.8197e-02, -1.2177e-01,\n",
      "        -1.5503e-01,  1.4246e-01,  7.9102e-02,  1.9028e-02,  8.8196e-02,\n",
      "         6.9946e-02,  1.2152e-01,  5.1880e-02,  5.2795e-02,  3.3325e-02,\n",
      "         4.0894e-02,  1.4636e-01,  1.4502e-01, -1.3939e-02, -4.2358e-02,\n",
      "         5.0278e-03,  7.0129e-02,  2.4597e-01,  2.2449e-01,  3.2196e-02,\n",
      "         9.5093e-02,  6.1859e-02, -6.8787e-02,  1.7700e-01,  1.7838e-02,\n",
      "         1.2085e-01,  9.3506e-02,  5.1483e-02, -1.5100e-01, -2.7710e-02,\n",
      "        -7.2815e-02,  1.1249e-01,  7.2693e-02, -7.1899e-02,  4.4098e-02,\n",
      "         2.3529e-02, -6.9275e-02,  8.5022e-02, -4.1870e-02,  1.2311e-01,\n",
      "        -3.0426e-02, -3.8361e-02,  1.1823e-01, -4.0405e-02,  2.3584e-01,\n",
      "        -1.2988e-01,  2.0889e-02, -9.9548e-02, -1.3635e-01, -8.6060e-02,\n",
      "        -9.0210e-02,  1.1749e-02, -8.7280e-02, -5.8472e-02,  6.9771e-03,\n",
      "         2.4551e-02,  7.4402e-02,  1.1694e-01, -6.0822e-02,  3.9032e-02,\n",
      "        -1.1554e-01,  1.4572e-02, -1.9714e-02,  1.1432e-01, -2.0355e-02,\n",
      "         2.1591e-02, -1.5612e-03, -2.5589e-02,  6.8787e-02, -1.1481e-01,\n",
      "        -6.8970e-02,  2.4826e-02,  2.0789e-01,  3.3264e-02, -8.6609e-02,\n",
      "        -2.3209e-02, -2.1606e-02, -1.2222e-02, -2.9724e-02,  2.2614e-02,\n",
      "        -9.3689e-02,  2.3242e-01,  7.2083e-02,  1.1765e-02, -6.4697e-02,\n",
      "         1.1188e-01, -6.2927e-02,  2.4521e-02, -1.4026e-01, -4.6051e-02,\n",
      "         1.4111e-01, -7.4959e-03, -8.6731e-02,  2.3819e-02, -1.2585e-01,\n",
      "        -6.0043e-03,  6.9946e-02, -9.5276e-02,  4.3243e-02,  2.4063e-02,\n",
      "         8.1665e-02,  2.2034e-02, -5.1208e-02, -2.0538e-02, -2.1255e-02,\n",
      "        -2.6123e-02, -1.3924e-02, -5.0262e-02, -3.2959e-02, -5.6335e-02,\n",
      "         6.9199e-03,  1.1383e-01,  1.6467e-01, -2.3193e-02,  8.9966e-02,\n",
      "         7.4341e-02, -7.5134e-02,  1.5244e-02, -1.2749e-02,  5.2765e-02,\n",
      "         1.0553e-01, -6.3782e-02,  1.3855e-01, -1.7590e-01, -1.2744e-01,\n",
      "        -3.8055e-02, -8.4595e-02,  9.6680e-02,  5.3644e-04, -1.0883e-01,\n",
      "         3.3302e-03, -7.4097e-02,  1.9638e-02, -7.1167e-02,  3.7750e-02,\n",
      "        -3.5669e-01,  2.6138e-02, -7.4768e-02,  7.7438e-03,  7.0557e-02,\n",
      "        -1.5930e-01, -4.5807e-02, -8.5144e-03,  3.8433e-03,  3.7018e-02,\n",
      "        -5.3101e-02,  4.7424e-02,  1.1463e-03,  9.3079e-02, -5.9906e-02,\n",
      "        -4.5990e-02, -3.7598e-02,  2.3300e-02,  3.3569e-02,  5.9509e-02],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.self_attn_layer_norm.weight Parameter containing:\n",
      "tensor([0.6543, 0.6499, 0.5972, 0.5591, 0.7002, 0.5596, 0.5811, 0.6318, 0.5635,\n",
      "        0.6279, 0.5752, 0.5396, 0.6372, 0.5850, 0.6284, 0.5786, 0.6196, 0.5923,\n",
      "        0.4866, 0.6431, 0.6431, 0.7051, 0.6274, 0.5737, 0.5767, 0.4897, 0.5327,\n",
      "        0.6162, 0.5776, 0.6670, 0.6567, 0.6357, 0.6294, 0.6362, 0.5483, 0.6211,\n",
      "        0.6333, 0.5581, 0.5278, 0.6807, 0.6206, 0.5635, 0.6714, 0.6084, 0.5386,\n",
      "        0.6450, 0.3872, 0.6094, 0.6147, 0.6396, 0.5918, 0.6509, 0.5840, 0.5840,\n",
      "        0.5576, 0.5967, 0.5586, 0.5996, 0.5010, 0.6343, 0.5532, 0.6025, 0.5381,\n",
      "        0.6143, 0.5903, 0.5664, 0.3513, 0.5693, 0.5747, 0.5698, 0.5811, 0.5488,\n",
      "        0.5298, 0.6147, 0.5654, 0.6089, 0.5483, 0.6460, 0.5996, 0.6245, 0.6025,\n",
      "        0.6694, 0.5479, 0.6729, 0.6665, 0.6211, 0.5728, 0.5962, 0.6504, 0.6934,\n",
      "        0.5840, 0.5557, 0.6172, 0.6494, 0.6123, 0.5527, 0.5464, 0.5073, 0.6899,\n",
      "        0.6196, 0.6260, 0.6157, 0.5659, 0.5737, 0.5781, 0.6226, 0.5771, 0.6572,\n",
      "        0.6455, 0.6729, 0.6606, 0.6465, 0.6265, 0.6191, 0.6118, 0.5957, 0.6172,\n",
      "        0.5474, 0.6904, 0.5728, 0.6147, 0.6196, 0.6294, 0.6631, 0.6445, 0.6064,\n",
      "        0.5806, 0.6128, 0.5781, 0.5874, 0.5454, 0.7227, 0.6206, 0.6099, 0.5957,\n",
      "        0.5547, 0.6372, 0.5146, 0.5029, 0.6499, 0.5742, 0.5649, 0.6152, 0.6650,\n",
      "        0.5742, 0.6162, 0.6436, 0.6357, 0.4697, 0.6094, 0.6260, 0.5859, 0.5796,\n",
      "        0.6226, 0.5278, 0.6538, 0.5527, 0.5713, 0.6333, 0.6147, 0.6177, 0.5688,\n",
      "        0.5552, 0.5454, 0.5723, 0.6421, 0.6221, 0.5054, 0.5947, 0.5718, 0.5083,\n",
      "        0.7212, 0.5312, 0.6274, 0.4441, 0.6167, 0.6450, 0.6362, 0.6230, 0.5869,\n",
      "        0.6616, 0.5605, 0.5596, 0.6406, 0.6147, 0.6714, 0.5981, 0.5747, 0.5640,\n",
      "        0.5845, 0.6216, 0.6055, 0.7231, 0.5835, 0.6680, 0.5469, 0.6733, 0.5757,\n",
      "        0.5698, 0.5688, 0.5957, 0.5625, 0.5811, 0.6270, 0.5859, 0.6265, 0.4336,\n",
      "        0.6494, 0.6670, 0.6426, 0.6187, 0.5786, 0.6152, 0.6172, 0.5825, 0.5532,\n",
      "        0.6040, 0.5859, 0.5874, 0.6230, 0.5522, 0.6377, 0.5469, 0.5718, 0.5420,\n",
      "        0.6313, 0.5991, 0.5952, 0.6606, 0.6714, 0.6084, 0.5571, 0.6240, 0.5293,\n",
      "        0.5645, 0.5645, 0.5747, 0.6768, 0.5835, 0.6816, 0.5923, 0.6265, 0.5566,\n",
      "        0.5674, 0.6030, 0.5649, 0.6240, 0.5894, 0.6162, 0.6763, 0.6230, 0.6299,\n",
      "        0.5518, 0.6011, 0.6558, 0.5269, 0.5176, 0.5781, 0.5610, 0.5659, 0.5693,\n",
      "        0.6396, 0.6118, 0.5620, 0.5840, 0.5713, 0.6646, 0.6016, 0.6704, 0.6001,\n",
      "        0.6401, 0.6060, 0.5581, 0.5942, 0.5171, 0.5732, 0.6274, 0.6743, 0.5967,\n",
      "        0.5557, 0.5776, 0.5840, 0.6123, 0.5615, 0.6709, 0.5732, 0.5259, 0.5405,\n",
      "        0.5981, 0.5244, 0.6152, 0.5752, 0.5664, 0.3862, 0.6475, 0.5386, 0.6021,\n",
      "        0.6099, 0.6157, 0.3459, 0.5215, 0.6416, 0.5859, 0.6450, 0.5830, 0.6323,\n",
      "        0.5889, 0.5430, 0.6587, 0.5381, 0.5259, 0.6548, 0.6006, 0.6553, 0.6660,\n",
      "        0.5513, 0.5273, 0.6196, 0.6699, 0.5962, 0.6035, 0.5884, 0.6880, 0.5874,\n",
      "        0.6172, 0.5034, 0.7090, 0.7485, 0.6235, 0.5718, 0.6196, 0.4993, 0.6167,\n",
      "        0.6465, 0.6987, 0.5303, 0.4780, 0.6455, 0.6875, 0.5732, 0.6611, 0.6284,\n",
      "        0.5039, 0.6245, 0.6235, 0.6226, 0.6216, 0.6646, 0.4302, 0.5923, 0.6060,\n",
      "        0.6548, 0.5693, 0.5381, 0.5742, 0.5459, 0.6094, 0.6001, 0.5957, 0.6274,\n",
      "        0.6128, 0.5791, 0.5693, 0.6421, 0.6680, 0.6045, 0.6025, 0.6445, 0.6128,\n",
      "        0.5933, 0.6064, 0.6133, 0.5771, 0.5522, 0.6147, 0.6543, 0.6802, 0.6113,\n",
      "        0.6255, 0.6055, 0.6694, 0.5542, 0.5186, 0.5718, 0.5498, 0.6455, 0.7139,\n",
      "        0.6313, 0.5293, 0.7051, 0.6821, 0.5840, 0.5620, 0.5874, 0.4839, 0.5518,\n",
      "        0.6309, 0.5303, 0.6235, 0.5884, 0.6113, 0.5889, 0.6372, 0.6865, 0.5718,\n",
      "        0.6626, 0.6406, 0.6460, 0.6396, 0.6660, 0.5815, 0.6240, 0.5684, 0.5278,\n",
      "        0.6465, 0.5757, 0.6235, 0.5659, 0.6304, 0.6289, 0.6533, 0.5952, 0.6196,\n",
      "        0.6445, 0.6450, 0.6094, 0.6230, 0.6709, 0.6597, 0.5952, 0.6558, 0.6089,\n",
      "        0.5078, 0.6250, 0.6118, 0.6250, 0.6367, 0.6367, 0.5996, 0.6313, 0.5181,\n",
      "        0.5679, 0.7188, 0.5542, 0.6226, 0.6187, 0.6001, 0.5962, 0.5684, 0.6040,\n",
      "        0.6729, 0.5786, 0.5913, 0.6431, 0.6611, 0.5737, 0.6582, 0.5762, 0.5132,\n",
      "        0.6543, 0.2773, 0.5835, 0.6211, 0.6372, 0.5566, 0.6128, 0.5605, 0.5713,\n",
      "        0.6494, 0.5654, 0.5786, 0.5859, 0.5728, 0.6108, 0.6143, 0.6484, 0.6709,\n",
      "        0.5693, 0.6699, 0.6230], requires_grad=True)\n",
      "esm_model.layers.11.self_attn_layer_norm.bias Parameter containing:\n",
      "tensor([ 1.9369e-03,  1.1742e-02, -2.2202e-03, -6.1920e-02, -3.1708e-02,\n",
      "        -1.3489e-02,  3.7231e-02, -5.2063e-02,  6.0852e-02, -1.2672e-02,\n",
      "         2.5146e-02,  1.4870e-02,  2.9938e-02,  1.3638e-03, -3.4973e-02,\n",
      "         1.4481e-02,  9.8877e-03,  2.1095e-03, -1.3557e-02, -3.3234e-02,\n",
      "         3.2593e-02, -1.6266e-02,  2.7069e-02, -2.1194e-02, -6.0158e-03,\n",
      "         1.2817e-02, -2.3315e-02,  3.7537e-02,  1.9073e-02,  3.8116e-02,\n",
      "         2.8427e-02,  6.4514e-02, -1.6312e-02,  4.6005e-03,  4.8065e-02,\n",
      "         3.5889e-02, -6.8331e-04,  7.2937e-03,  2.1155e-01,  1.4069e-02,\n",
      "         2.3941e-02,  8.9417e-03,  3.0014e-02,  4.6539e-02, -1.1047e-02,\n",
      "        -2.0477e-02,  1.0693e+00,  2.3102e-02, -2.2141e-02,  1.3062e-02,\n",
      "        -7.1068e-03, -3.6377e-02,  2.0996e-02,  6.2561e-03, -4.9896e-02,\n",
      "         6.5041e-03, -9.2087e-03,  5.2399e-02, -2.5269e-02,  6.1302e-03,\n",
      "        -1.8311e-02,  9.1705e-03,  3.6926e-02,  2.2964e-02,  2.1896e-02,\n",
      "         3.4515e-02,  1.2793e-01,  4.1046e-02, -2.7895e-05, -3.4302e-02,\n",
      "         1.5579e-02,  8.5388e-02, -2.5883e-03, -1.8845e-02,  3.7262e-02,\n",
      "        -1.2222e-02,  2.9922e-02,  3.4546e-02, -4.7913e-03,  9.5189e-05,\n",
      "         2.2202e-02, -7.3486e-02, -8.4305e-04,  2.9556e-02, -3.5431e-02,\n",
      "         8.7662e-03,  2.5009e-02,  4.1107e-02,  2.7695e-02,  5.2460e-02,\n",
      "         3.2074e-02, -6.8359e-03,  2.0187e-02, -2.4216e-02,  2.7893e-02,\n",
      "         5.2429e-02, -1.6129e-02,  6.2828e-03,  3.3661e-02, -3.5133e-03,\n",
      "        -2.5253e-02,  4.7119e-02, -1.2070e-02, -2.2171e-02,  2.0630e-02,\n",
      "        -3.2074e-02, -9.5444e-03, -1.2703e-03,  2.9800e-02, -8.6899e-03,\n",
      "         3.8086e-02, -8.9874e-03,  9.5367e-03,  2.5314e-02, -1.0864e-02,\n",
      "        -1.4030e-02, -2.8976e-02, -6.5041e-03,  2.5345e-02,  1.6464e-02,\n",
      "         3.7140e-02, -6.4507e-03,  2.0477e-02,  5.3680e-02,  2.1942e-02,\n",
      "         3.8574e-02,  5.0079e-02,  1.3992e-02,  5.1231e-03,  2.2263e-02,\n",
      "         3.1647e-02, -1.2825e-02,  4.7058e-02,  1.1497e-02,  3.7292e-02,\n",
      "        -3.7415e-02,  2.3712e-02, -5.1849e-02,  4.6051e-02,  4.1382e-02,\n",
      "         2.7084e-02,  1.9440e-02,  7.0618e-02,  2.3438e-02,  1.9974e-02,\n",
      "        -2.2415e-02,  4.1351e-02,  2.9373e-02, -8.0139e-02,  4.8828e-03,\n",
      "         7.8735e-02,  1.3107e-02,  2.8362e-03, -2.0569e-02, -2.4734e-02,\n",
      "         1.4343e-02,  7.4158e-02,  3.5248e-03,  2.7863e-02,  8.9340e-03,\n",
      "         1.8982e-02,  2.2293e-02, -3.6499e-02, -5.5298e-02,  4.9515e-03,\n",
      "        -6.5851e-04,  3.5782e-03,  5.7312e-02,  3.8086e-02,  1.1765e-02,\n",
      "         1.0002e-02,  1.3435e-02,  5.8716e-02,  8.8882e-03,  2.1713e-02,\n",
      "         5.5054e-02,  6.7368e-03, -1.7914e-02,  3.5126e-02,  3.4119e-02,\n",
      "         5.8517e-03,  5.1086e-02, -5.1392e-02, -1.7349e-02,  3.6102e-02,\n",
      "         5.0240e-03,  1.2291e-02, -4.8096e-02,  8.2153e-02,  1.3628e-03,\n",
      "        -3.3783e-02,  9.0866e-03,  4.5868e-02,  3.2166e-02,  5.5847e-02,\n",
      "         9.6970e-03, -5.2834e-03, -5.4626e-02, -3.0365e-02,  5.5481e-02,\n",
      "         1.9440e-02,  1.2183e-01,  1.6068e-02,  2.1591e-02,  4.3640e-03,\n",
      "         4.0169e-03,  1.7639e-02,  3.7594e-03,  5.4291e-02,  3.2440e-02,\n",
      "         4.4861e-02, -3.2990e-02,  2.3743e-02, -2.9831e-02,  1.6449e-02,\n",
      "         4.2419e-03, -5.4565e-02, -3.5248e-02, -3.1433e-02,  2.7054e-02,\n",
      "         2.0706e-02,  1.3332e-03,  2.7847e-02,  1.1467e-02,  1.1810e-02,\n",
      "         2.8015e-02, -6.4888e-03,  6.6872e-03, -1.8906e-02,  2.8351e-02,\n",
      "        -4.5074e-02, -1.8959e-03,  2.9678e-02, -1.1932e-02,  6.3660e-02,\n",
      "         6.4278e-03, -7.1716e-02,  7.9803e-03,  1.7136e-02,  1.2138e-02,\n",
      "        -5.3741e-02, -3.9917e-02,  3.8635e-02,  6.2225e-02, -3.8055e-02,\n",
      "        -3.6591e-02,  2.9449e-02,  7.7087e-02, -3.7537e-02, -3.9062e-03,\n",
      "        -4.8157e-02,  3.2654e-02,  7.9895e-02,  1.4198e-02,  5.8685e-02,\n",
      "        -1.5869e-02, -6.0425e-02, -4.0039e-02, -5.3894e-02,  2.4506e-02,\n",
      "         3.7048e-02, -2.5063e-03,  1.2543e-02,  3.1815e-03,  3.1403e-02,\n",
      "        -9.6359e-03, -3.0502e-02,  5.2643e-04, -6.7596e-03, -3.8666e-02,\n",
      "        -4.6051e-02,  5.8563e-02,  3.6041e-02,  9.5062e-03,  6.1941e-04,\n",
      "        -1.1696e-02,  3.9276e-02, -2.5528e-02, -6.7200e-02,  4.3701e-02,\n",
      "        -3.2463e-03,  3.7018e-02, -1.2367e-02,  1.6922e-02, -1.1425e-03,\n",
      "        -2.0279e-02,  5.7251e-02, -1.3742e-03, -1.5671e-02,  5.8350e-02,\n",
      "        -1.9592e-02,  2.1561e-02, -1.6190e-02,  8.1177e-02,  2.2476e-02,\n",
      "        -7.8430e-02, -1.6861e-02,  1.1429e-02,  4.1199e-02, -6.3477e-02,\n",
      "        -3.8528e-03,  2.1240e-02, -3.6652e-02,  1.2611e-02, -1.9493e-03,\n",
      "        -2.7557e-02, -2.8534e-03,  3.6713e-02,  2.0676e-02,  2.1133e-02,\n",
      "         4.9210e-03,  1.2383e-02, -2.2858e-02, -3.6163e-03,  1.2512e-02,\n",
      "         8.9645e-03,  9.0027e-03,  1.9867e-02,  1.6357e-02,  3.4729e-02,\n",
      "         1.8173e-02, -4.3243e-02, -2.2144e-03, -1.0857e-02,  5.1056e-02,\n",
      "         5.3131e-02,  1.0925e-02, -6.2943e-03,  3.2196e-02,  1.5686e-02,\n",
      "        -4.8828e-02, -4.5197e-02,  2.3834e-02, -7.1030e-03,  1.6823e-03,\n",
      "        -1.2352e-02, -4.8187e-02,  2.6932e-02, -4.1351e-03,  3.4698e-02,\n",
      "         5.1239e-02,  2.3987e-02, -5.2155e-02, -7.2289e-03,  3.7323e-02,\n",
      "         1.8982e-02, -3.9764e-02,  6.0059e-02, -7.3303e-02,  7.5562e-02,\n",
      "         2.3823e-03, -2.1683e-02,  8.8882e-03,  4.6539e-02,  3.2104e-02,\n",
      "         6.4758e-02,  3.2227e-02, -1.1246e-02,  2.7267e-02,  4.6768e-03,\n",
      "         1.5823e-02,  7.0435e-02,  3.7201e-02,  2.6840e-02, -8.9951e-03,\n",
      "         1.9272e-02,  2.0889e-02,  1.9623e-02, -1.3573e-02, -3.1189e-02,\n",
      "         4.6570e-02, -1.2245e-02,  9.1705e-03,  6.4636e-02,  9.0256e-03,\n",
      "         1.5373e-02,  4.5700e-03, -9.1324e-03,  4.5685e-02,  1.5930e-02,\n",
      "        -5.9700e-03, -1.5430e-03,  1.5213e-02, -1.3817e-02, -1.7639e-02,\n",
      "        -7.8049e-03, -1.1269e-02,  3.4576e-02,  2.4826e-02,  8.8272e-03,\n",
      "        -2.4261e-02,  3.4515e-02,  3.0792e-02,  5.4382e-02,  3.2074e-02,\n",
      "        -1.4046e-02, -2.5635e-02, -2.1317e-02,  4.9988e-02,  6.2683e-02,\n",
      "         4.6295e-02, -2.0264e-02, -3.0613e-03, -2.8777e-04,  1.6012e-03,\n",
      "        -3.2253e-03, -5.0385e-02,  4.7760e-02,  2.2446e-02,  7.8659e-03,\n",
      "        -2.6550e-02,  5.8975e-03, -1.6495e-02,  4.3518e-02,  4.4739e-02,\n",
      "        -1.4748e-02,  5.0621e-03,  1.5144e-02, -1.7883e-02,  2.0523e-03,\n",
      "         4.7180e-02,  3.6125e-03, -1.1490e-02,  7.4280e-02, -2.4429e-02,\n",
      "        -2.5803e-02,  3.1586e-02,  5.5450e-02,  4.9072e-02,  3.3234e-02,\n",
      "        -1.1223e-02,  5.0049e-02,  4.7180e-02,  2.8595e-02,  6.5498e-03,\n",
      "         1.2177e-02,  3.8361e-02, -2.9388e-02, -1.7300e-03,  7.8430e-02,\n",
      "        -2.8076e-02,  4.2664e-02,  3.3813e-02,  1.3718e-02, -4.7028e-02,\n",
      "         8.4778e-02,  1.1395e-01,  7.9041e-03, -1.3565e-02, -2.1805e-02,\n",
      "         3.2593e-02,  3.0289e-02, -3.5126e-02, -5.9570e-02,  2.0294e-03,\n",
      "        -2.7409e-03,  3.2867e-02,  6.5689e-03, -2.1790e-02, -1.9257e-02,\n",
      "        -4.1229e-02,  7.0923e-02,  9.5825e-02, -7.7576e-02, -2.1973e-03,\n",
      "         5.2765e-02,  4.0070e-02,  1.2085e-02,  1.1398e-02,  6.8970e-03,\n",
      "        -2.3239e-02,  3.8452e-02,  3.1052e-02,  2.4918e-02, -4.3106e-03,\n",
      "        -4.6326e-02,  5.0720e-02,  1.8585e-02, -4.6295e-02, -3.0319e-02],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.fc1.weight Parameter containing:\n",
      "tensor([[-0.0653, -0.0770, -0.0497,  ..., -0.2012,  0.0026, -0.1191],\n",
      "        [-0.0171, -0.0796,  0.0082,  ..., -0.0023, -0.0267, -0.0238],\n",
      "        [-0.2152,  0.1779,  0.0366,  ..., -0.1120, -0.0365,  0.2642],\n",
      "        ...,\n",
      "        [-0.1381, -0.1920,  0.1523,  ..., -0.1401,  0.0080,  0.0390],\n",
      "        [ 0.0911, -0.1445, -0.0562,  ...,  0.1324, -0.1182,  0.0088],\n",
      "        [ 0.1205, -0.1201, -0.0898,  ..., -0.1172,  0.0039,  0.0244]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.fc1.bias Parameter containing:\n",
      "tensor([-0.0596, -0.0617, -0.0571,  ..., -0.0550, -0.0746, -0.0540],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.fc2.weight Parameter containing:\n",
      "tensor([[ 0.0180, -0.1259,  0.0156,  ...,  0.0870, -0.2913, -0.1492],\n",
      "        [-0.2720, -0.2712,  0.0936,  ...,  0.2944,  0.1915,  0.0351],\n",
      "        [-0.0652, -0.2384, -0.0234,  ..., -0.1193, -0.2322,  0.1998],\n",
      "        ...,\n",
      "        [-0.1819, -0.0915,  0.2145,  ..., -0.0173,  0.0631,  0.1440],\n",
      "        [-0.0080,  0.1757,  0.1548,  ..., -0.1042,  0.2142, -0.0630],\n",
      "        [ 0.0223, -0.1172, -0.0118,  ...,  0.0699,  0.2035, -0.0864]],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.fc2.bias Parameter containing:\n",
      "tensor([-9.6985e-02,  6.7078e-02,  1.0236e-01,  2.5085e-02, -9.5459e-02,\n",
      "        -1.3725e-02,  1.1218e-01,  1.9989e-02, -6.5491e-02, -1.5472e-02,\n",
      "        -8.4900e-02,  1.1469e-01,  1.9951e-03,  7.9041e-03,  1.0002e-02,\n",
      "         2.5864e-02,  7.1838e-02, -1.0077e-01, -1.6190e-02, -1.7224e-01,\n",
      "         2.4612e-02, -3.7872e-02, -5.8563e-02, -7.3608e-02, -5.6885e-02,\n",
      "         2.2888e-02, -4.9622e-02, -5.3650e-02,  2.0355e-02, -1.2952e-01,\n",
      "         5.3711e-02,  5.5786e-02,  4.4495e-02,  8.6487e-02, -1.1307e-02,\n",
      "        -6.8970e-02,  1.3379e-01, -3.5645e-02, -8.8196e-03, -9.0149e-02,\n",
      "         6.3095e-03, -6.2744e-02,  3.0930e-02,  8.7524e-02,  7.7095e-03,\n",
      "        -1.0553e-01,  8.7598e-01, -1.1604e-02, -3.3783e-02, -1.0559e-01,\n",
      "        -1.0376e-02, -2.7344e-02,  4.0710e-02,  7.3730e-02,  4.6143e-02,\n",
      "        -4.8553e-02, -1.5771e-01, -1.0413e-01, -5.9937e-02, -4.6509e-02,\n",
      "         6.5186e-02, -3.6285e-02,  6.4270e-02,  4.5288e-02,  5.9387e-02,\n",
      "        -4.1473e-02, -1.3586e-01,  5.7648e-02,  3.1300e-03,  1.2781e-01,\n",
      "        -1.0056e-02,  7.8735e-02, -4.5258e-02, -5.8899e-02, -5.5389e-02,\n",
      "         1.6357e-02,  1.2390e-01,  7.8003e-02, -2.1343e-03,  3.2959e-02,\n",
      "         4.0894e-02,  5.3894e-02, -2.0477e-02, -9.5886e-02,  1.8445e-01,\n",
      "        -7.0435e-02, -5.7068e-02, -2.1935e-03, -5.6305e-03,  4.2053e-02,\n",
      "         6.0272e-02, -8.0643e-03, -7.4707e-02,  2.1000e-03, -5.6732e-02,\n",
      "         2.9144e-03,  1.6370e-01, -1.7224e-01,  5.0125e-03, -2.9572e-02,\n",
      "         5.1880e-02,  6.4087e-02, -1.1574e-02,  9.9121e-02,  5.0110e-02,\n",
      "         5.7335e-03, -3.7048e-02,  3.8696e-02,  1.1542e-01,  7.6660e-02,\n",
      "        -7.9590e-02,  9.8206e-02, -7.4585e-02,  6.1890e-02, -1.0168e-01,\n",
      "         6.9946e-02,  4.9042e-02,  3.7262e-02, -2.4078e-02, -2.0325e-01,\n",
      "        -5.9113e-02,  5.6793e-02, -1.2207e-02, -2.3163e-02, -4.1779e-02,\n",
      "         3.0270e-03, -9.4360e-02, -3.3600e-02,  7.0557e-02, -2.7435e-02,\n",
      "        -9.2529e-02,  1.4221e-02,  3.4424e-02, -8.5083e-02,  1.6937e-02,\n",
      "         9.1492e-02, -1.3184e-01,  1.3831e-01,  3.5004e-02,  4.1931e-02,\n",
      "         9.5032e-02, -8.1299e-02, -4.1901e-02,  8.7402e-02,  1.9974e-02,\n",
      "        -9.3445e-02,  9.8267e-03,  6.5552e-02,  9.2041e-02,  3.9978e-02,\n",
      "        -1.2769e-01, -3.2593e-02, -1.5576e-01, -1.3574e-01,  2.0764e-01,\n",
      "        -5.9692e-02, -5.1117e-02, -1.1011e-01,  5.6213e-02,  9.2773e-02,\n",
      "         5.7648e-02, -3.5095e-02,  5.7190e-02,  3.6652e-02, -7.4768e-02,\n",
      "        -7.2815e-02, -1.3351e-02, -2.2568e-02, -1.5332e-01, -4.2145e-02,\n",
      "        -1.3138e-02,  3.9795e-02,  3.2043e-02, -2.5528e-02, -6.6711e-02,\n",
      "         1.1426e-01,  1.7914e-02,  6.7688e-02, -5.1880e-02, -1.4534e-02,\n",
      "        -4.3907e-03,  5.8929e-02, -3.6285e-02,  8.7402e-02,  3.8361e-02,\n",
      "        -6.8848e-02, -8.4229e-03,  6.8848e-02, -3.5126e-02,  1.9699e-02,\n",
      "        -7.6660e-02,  3.5919e-02,  7.5562e-02, -4.5471e-02, -6.1035e-03,\n",
      "        -1.0327e-01, -1.1243e-01, -6.5063e-02,  1.8738e-02,  1.0699e-01,\n",
      "         8.4412e-02, -2.8000e-02, -3.6983e-03, -3.4119e-02, -2.4628e-02,\n",
      "         1.2903e-01, -9.6680e-02,  1.4929e-01, -3.5400e-02, -5.8655e-02,\n",
      "        -3.1586e-02, -7.6477e-02,  1.2537e-01, -4.0131e-02,  5.5145e-02,\n",
      "         2.7420e-02, -1.0361e-02, -1.6907e-01, -4.4403e-02, -3.4973e-02,\n",
      "        -1.1304e-01, -9.3079e-02,  7.4646e-02,  1.5594e-02,  9.8328e-02,\n",
      "        -3.0792e-02,  1.6235e-01,  4.3213e-02, -8.0490e-03, -1.5974e-03,\n",
      "         5.4016e-02, -1.3940e-01,  9.3079e-02,  6.1859e-02, -1.5234e-01,\n",
      "         1.8339e-03, -4.4861e-03,  4.3579e-02,  6.7444e-02, -8.7097e-02,\n",
      "         1.0475e-02, -1.7197e-02,  9.6802e-02, -8.3496e-02, -7.3303e-02,\n",
      "        -2.6871e-02,  1.8539e-02, -4.6051e-02, -4.2328e-02,  4.9255e-02,\n",
      "        -3.8940e-02, -4.4769e-02, -6.2408e-02,  3.5461e-02,  5.5908e-02,\n",
      "         5.1804e-03, -1.3708e-01, -7.3181e-02, -3.5126e-02, -5.0598e-02,\n",
      "         4.1870e-02, -1.4748e-02, -6.3538e-02, -2.4307e-02, -5.1056e-02,\n",
      "        -6.7932e-02,  2.9541e-02, -1.8625e-03,  5.2765e-02,  6.4201e-03,\n",
      "        -1.7456e-02, -5.6152e-02, -6.8970e-02, -1.6785e-01, -9.0759e-02,\n",
      "         1.4355e-01,  9.7275e-03, -7.9224e-02, -1.2152e-01, -3.4241e-02,\n",
      "        -1.3171e-01,  7.7759e-02,  6.1531e-03, -2.5131e-02,  1.2866e-01,\n",
      "        -8.6670e-03,  5.9601e-02,  2.1094e-01,  6.8970e-02, -1.4417e-01,\n",
      "         2.4307e-02,  1.1963e-01,  1.4355e-01, -2.0776e-01,  3.5858e-02,\n",
      "        -8.3008e-02,  8.8867e-02,  1.9507e-01,  3.9886e-02, -1.4648e-01,\n",
      "         6.8420e-02, -2.8183e-02,  7.4280e-02,  5.0293e-02,  5.1697e-02,\n",
      "        -5.4718e-02,  5.6946e-02, -7.0686e-03,  5.8655e-02,  8.6517e-03,\n",
      "        -6.6589e-02, -3.3966e-02, -8.3435e-02, -3.0716e-02, -5.7251e-02,\n",
      "        -2.2263e-02,  4.9652e-02,  4.5197e-02,  4.7241e-02, -2.5711e-02,\n",
      "        -6.0944e-02, -1.3039e-02,  3.2288e-02,  5.3497e-02, -7.9102e-02,\n",
      "        -1.2708e-01,  5.5847e-02,  5.7587e-02, -1.9760e-03,  1.2372e-01,\n",
      "         4.9896e-02,  4.6570e-02,  8.7097e-02,  1.4111e-01,  1.2985e-02,\n",
      "         7.0923e-02,  1.6016e-01,  1.4124e-01,  8.5754e-02, -2.9236e-02,\n",
      "        -5.0842e-02,  2.3514e-02,  1.7871e-01,  1.0773e-01,  6.0577e-02,\n",
      "         9.2468e-02,  8.4045e-02, -3.7201e-02,  1.3586e-01,  6.1615e-02,\n",
      "         1.1578e-01,  7.5867e-02,  9.1187e-02, -3.5339e-02, -9.5520e-02,\n",
      "        -4.4464e-02,  7.7881e-02,  3.8300e-02, -8.9111e-02,  4.1504e-02,\n",
      "         5.6000e-02, -9.1797e-02,  8.4961e-02, -4.7058e-02,  8.1726e-02,\n",
      "        -9.8328e-02, -6.9885e-02,  7.4768e-03,  4.4785e-03,  2.0447e-01,\n",
      "        -1.1023e-01,  1.2611e-02, -9.7351e-02, -1.1365e-01, -7.7698e-02,\n",
      "        -7.3120e-02,  4.6875e-02, -1.5271e-01,  1.4511e-02, -2.0508e-02,\n",
      "         2.5299e-02,  6.9084e-03,  3.7689e-02, -8.4717e-02, -5.8014e-02,\n",
      "        -6.7200e-02,  2.9022e-02,  6.9458e-02,  2.8290e-02, -4.6959e-03,\n",
      "         2.5726e-02,  1.2915e-01,  1.5659e-03,  5.6000e-02, -4.2572e-02,\n",
      "        -4.7058e-02,  2.5925e-02,  2.4329e-01,  3.2288e-02, -2.9892e-02,\n",
      "        -3.8361e-02,  7.9346e-02, -6.6650e-02, -1.4351e-02, -1.1864e-02,\n",
      "        -7.4524e-02,  2.1619e-01,  4.6997e-02, -3.3020e-02, -6.2042e-02,\n",
      "         6.7078e-02, -6.1920e-02,  5.8075e-02, -2.3056e-02, -9.1614e-02,\n",
      "         1.3550e-01, -6.5308e-02, -1.0138e-01, -8.3313e-02, -9.1919e-02,\n",
      "         1.8489e-04,  2.9984e-02, -1.5198e-01,  7.4097e-02,  1.2842e-01,\n",
      "         9.1370e-02, -7.0457e-03, -7.1472e-02, -9.4849e-02, -2.6291e-02,\n",
      "        -2.7740e-02,  1.7975e-02,  8.3008e-03, -8.6548e-02,  3.0411e-02,\n",
      "         8.1909e-02,  1.4783e-01,  1.6418e-01, -4.4037e-02,  1.0583e-01,\n",
      "         1.2711e-02, -8.1726e-02,  7.4707e-02, -2.7206e-02,  4.0527e-02,\n",
      "         1.5698e-01, -2.4719e-03,  1.3696e-01, -1.3159e-01, -1.5015e-01,\n",
      "         2.8534e-02, -1.0352e-01,  5.3406e-02,  1.0231e-02, -1.3831e-01,\n",
      "        -1.6663e-02, -8.0750e-02,  3.2135e-02, -1.2878e-01,  1.1528e-02,\n",
      "        -4.5508e-01,  1.4458e-02, -9.7839e-02,  3.1638e-04,  8.9844e-02,\n",
      "        -1.3879e-01, -5.6274e-02, -6.5979e-02, -1.9104e-02, -4.7974e-02,\n",
      "        -3.4729e-02, -1.8982e-02, -3.7445e-02, -1.6968e-02, -8.0505e-02,\n",
      "        -8.3435e-02,  9.5978e-03,  7.6675e-03,  4.9835e-02,  1.0022e-01],\n",
      "       requires_grad=True)\n",
      "esm_model.layers.11.final_layer_norm.weight Parameter containing:\n",
      "tensor([0.6802, 0.7178, 0.7739, 0.6865, 0.7163, 0.6318, 0.6704, 0.7075, 0.6650,\n",
      "        0.7075, 0.6880, 0.6309, 0.7100, 0.6880, 0.6758, 0.7485, 0.6738, 0.7349,\n",
      "        0.6963, 0.6699, 0.7041, 0.6968, 0.7031, 0.6445, 0.7246, 0.6309, 0.7183,\n",
      "        0.7153, 0.6592, 0.7188, 0.7012, 0.6997, 0.6890, 0.7046, 0.7109, 0.6772,\n",
      "        0.6987, 0.7568, 0.7646, 0.6924, 0.7002, 0.7246, 0.6606, 0.7095, 0.6875,\n",
      "        0.6943, 0.3191, 0.6807, 0.7002, 0.6992, 0.6787, 0.7461, 0.6719, 0.7046,\n",
      "        0.7427, 0.6943, 0.6577, 0.7290, 0.6665, 0.7446, 0.6816, 0.7710, 0.6748,\n",
      "        0.7139, 0.7378, 0.6699, 0.4436, 0.7495, 0.6753, 0.6260, 0.7114, 0.7358,\n",
      "        0.7017, 0.7222, 0.7295, 0.7568, 0.6992, 0.6836, 0.6567, 0.7129, 0.7329,\n",
      "        0.7251, 0.7251, 0.7207, 0.7075, 0.6821, 0.6733, 0.6899, 0.7637, 0.7207,\n",
      "        0.7046, 0.6782, 0.7275, 0.7095, 0.7646, 0.6567, 0.6816, 0.6338, 0.6948,\n",
      "        0.7041, 0.7271, 0.6709, 0.6831, 0.7090, 0.6724, 0.7085, 0.7227, 0.6714,\n",
      "        0.7646, 0.6885, 0.6851, 0.7026, 0.7153, 0.6875, 0.6934, 0.7070, 0.6816,\n",
      "        0.7046, 0.7812, 0.6309, 0.6899, 0.7080, 0.6748, 0.7261, 0.6772, 0.7095,\n",
      "        0.6157, 0.7461, 0.6890, 0.7129, 0.7266, 0.7432, 0.7261, 0.7173, 0.7192,\n",
      "        0.6890, 0.7188, 0.7202, 0.6929, 0.6973, 0.7017, 0.7144, 0.7158, 0.7217,\n",
      "        0.6841, 0.7354, 0.6714, 0.7002, 0.7095, 0.7148, 0.6689, 0.7104, 0.7485,\n",
      "        0.7129, 0.6924, 0.7007, 0.7446, 0.6738, 0.7036, 0.7056, 0.7144, 0.6470,\n",
      "        0.6328, 0.6997, 0.6885, 0.6934, 0.6787, 0.7222, 0.7681, 0.6206, 0.7139,\n",
      "        0.7070, 0.6367, 0.7085, 0.5776, 0.6646, 0.7490, 0.6982, 0.6870, 0.6968,\n",
      "        0.7261, 0.7217, 0.6973, 0.6841, 0.6978, 0.7070, 0.7051, 0.7314, 0.7417,\n",
      "        0.7324, 0.7129, 0.7119, 0.7461, 0.6748, 0.6484, 0.6729, 0.7100, 0.6982,\n",
      "        0.6304, 0.6592, 0.7183, 0.6924, 0.5879, 0.7051, 0.6084, 0.7202, 0.6738,\n",
      "        0.7104, 0.6982, 0.6748, 0.7197, 0.7183, 0.6704, 0.6953, 0.6777, 0.6650,\n",
      "        0.7227, 0.7144, 0.6821, 0.7075, 0.6733, 0.7583, 0.6924, 0.6895, 0.6016,\n",
      "        0.7285, 0.7202, 0.7300, 0.7124, 0.7070, 0.7070, 0.7109, 0.6753, 0.6597,\n",
      "        0.7012, 0.6328, 0.6523, 0.7178, 0.7007, 0.7217, 0.6162, 0.7148, 0.6431,\n",
      "        0.7378, 0.7051, 0.6475, 0.6738, 0.7085, 0.7456, 0.6641, 0.6870, 0.7236,\n",
      "        0.7261, 0.7393, 0.7290, 0.6812, 0.6733, 0.6714, 0.6890, 0.7524, 0.7461,\n",
      "        0.6973, 0.7300, 0.6753, 0.6782, 0.6738, 0.7305, 0.7100, 0.7373, 0.7344,\n",
      "        0.7363, 0.7412, 0.7158, 0.7188, 0.6685, 0.6865, 0.7070, 0.7246, 0.7148,\n",
      "        0.7251, 0.6909, 0.7046, 0.6914, 0.7129, 0.7227, 0.7051, 0.6973, 0.6245,\n",
      "        0.6782, 0.6084, 0.6816, 0.6992, 0.6968, 0.4866, 0.6899, 0.7041, 0.6924,\n",
      "        0.7134, 0.7085, 0.5244, 0.7095, 0.6875, 0.7271, 0.7056, 0.6597, 0.6855,\n",
      "        0.6973, 0.7012, 0.7026, 0.7202, 0.6562, 0.7324, 0.6938, 0.7119, 0.7544,\n",
      "        0.6060, 0.6733, 0.7007, 0.6987, 0.7412, 0.7227, 0.7100, 0.6973, 0.7065,\n",
      "        0.7227, 0.6509, 0.7207, 0.7202, 0.7080, 0.6782, 0.7119, 0.6704, 0.7524,\n",
      "        0.6562, 0.6582, 0.6890, 0.6528, 0.6968, 0.7109, 0.7090, 0.7192, 0.6772,\n",
      "        0.6997, 0.7603, 0.7227, 0.7524, 0.6450, 0.7109, 0.6001, 0.6855, 0.6851,\n",
      "        0.7192, 0.6714, 0.6860, 0.6729, 0.7495, 0.7598, 0.6987, 0.7583, 0.7070,\n",
      "        0.6851, 0.6978, 0.6484, 0.6621, 0.6904, 0.6709, 0.6685, 0.7734, 0.7354,\n",
      "        0.7778, 0.7300, 0.6860, 0.6958, 0.6650, 0.7485, 0.7427, 0.7388, 0.6665,\n",
      "        0.7134, 0.6865, 0.6553, 0.6953, 0.6841, 0.6050, 0.6899, 0.7153, 0.7212,\n",
      "        0.6982, 0.6753, 0.7051, 0.6899, 0.7993, 0.6929, 0.6855, 0.6733, 0.6606,\n",
      "        0.6880, 0.6860, 0.6558, 0.7700, 0.6689, 0.7168, 0.7031, 0.7734, 0.5322,\n",
      "        0.7012, 0.7065, 0.6987, 0.6694, 0.7236, 0.7109, 0.6636, 0.6021, 0.7681,\n",
      "        0.6772, 0.6733, 0.7002, 0.6958, 0.7412, 0.7061, 0.7393, 0.6914, 0.7212,\n",
      "        0.6982, 0.7578, 0.7422, 0.7173, 0.7212, 0.7100, 0.6890, 0.7007, 0.6758,\n",
      "        0.7227, 0.7275, 0.6807, 0.7466, 0.7290, 0.7319, 0.6763, 0.6802, 0.7075,\n",
      "        0.6353, 0.7588, 0.7324, 0.7148, 0.7085, 0.6797, 0.7495, 0.6719, 0.7500,\n",
      "        0.6963, 0.7119, 0.6724, 0.6953, 0.7563, 0.6802, 0.7285, 0.6650, 0.5664,\n",
      "        0.6987, 0.2969, 0.6812, 0.7153, 0.7397, 0.6870, 0.7412, 0.6265, 0.7256,\n",
      "        0.7017, 0.6265, 0.6846, 0.7373, 0.6650, 0.7007, 0.7280, 0.7290, 0.6943,\n",
      "        0.6611, 0.6577, 0.6753], requires_grad=True)\n",
      "esm_model.layers.11.final_layer_norm.bias Parameter containing:\n",
      "tensor([ 0.0193,  0.0717, -0.1190,  0.0403, -0.0216,  0.0389, -0.0451,  0.1059,\n",
      "         0.0250, -0.0401,  0.0183,  0.0434, -0.0240, -0.0115, -0.0038,  0.0861,\n",
      "         0.0130, -0.0683, -0.1328,  0.0718,  0.0100,  0.0454,  0.0629,  0.0125,\n",
      "        -0.0659,  0.1013, -0.0633,  0.0370,  0.1498,  0.0192, -0.0224,  0.0082,\n",
      "        -0.0147, -0.0327, -0.0148,  0.0676, -0.0187, -0.0216, -0.0819, -0.0328,\n",
      "        -0.0421, -0.0804, -0.0154,  0.0498, -0.2025,  0.0107, -0.0076, -0.0699,\n",
      "        -0.0505, -0.0925,  0.0029,  0.0349,  0.0413, -0.1671,  0.0482, -0.0036,\n",
      "        -0.0598,  0.1700, -0.1165,  0.0199, -0.1083, -0.0683, -0.0169, -0.0671,\n",
      "        -0.0737, -0.0492,  0.2986, -0.0244,  0.1169, -0.0786,  0.0100,  0.1672,\n",
      "         0.1334, -0.1278,  0.1432,  0.1284,  0.0696,  0.0352, -0.0549, -0.0026,\n",
      "         0.0742,  0.0682, -0.0869, -0.0116,  0.0189,  0.0718, -0.0478, -0.0196,\n",
      "         0.1169,  0.0812,  0.0544, -0.0050, -0.1016,  0.0353,  0.0652,  0.1375,\n",
      "         0.0051,  0.0614,  0.1272, -0.0453,  0.0527,  0.0662,  0.0022, -0.1141,\n",
      "         0.1326,  0.0607, -0.1123,  0.0205,  0.1331, -0.0162,  0.1522, -0.0604,\n",
      "         0.0421, -0.0544, -0.0477,  0.0967, -0.0193,  0.1455,  0.1354,  0.1484,\n",
      "        -0.0399,  0.0610, -0.0065, -0.0212, -0.0153, -0.0183,  0.0312, -0.0853,\n",
      "         0.0864, -0.0830, -0.0659,  0.0374,  0.0543, -0.0010,  0.0578,  0.0634,\n",
      "         0.0543,  0.0844,  0.1595,  0.1019, -0.0055, -0.0364,  0.1451,  0.0723,\n",
      "        -0.0168, -0.0267,  0.0418,  0.0948,  0.0753,  0.0346,  0.0007, -0.0753,\n",
      "        -0.0626, -0.0349,  0.0794,  0.0778, -0.1400,  0.0294,  0.0316,  0.1259,\n",
      "         0.0340, -0.0439,  0.0051, -0.1161, -0.0083,  0.0029, -0.0488, -0.1198,\n",
      "        -0.0859,  0.0464,  0.2625,  0.0222,  0.0406,  0.0059, -0.0246, -0.0912,\n",
      "        -0.0175, -0.0068, -0.0360,  0.1100, -0.0201, -0.1023, -0.1439, -0.0534,\n",
      "         0.0492,  0.0703, -0.1237,  0.0563, -0.0453,  0.0127, -0.0302,  0.1012,\n",
      "         0.0324,  0.1265,  0.0012, -0.0128, -0.0090, -0.1062,  0.0160,  0.0977,\n",
      "         0.0448, -0.0163,  0.0946, -0.0740,  0.0082,  0.0901, -0.1372, -0.0206,\n",
      "         0.0141,  0.0522, -0.0157,  0.0375,  0.0817, -0.1098, -0.1157, -0.0321,\n",
      "        -0.0325, -0.0878,  0.0792, -0.0032, -0.0600,  0.0349,  0.1495, -0.1143,\n",
      "        -0.0296, -0.0014, -0.0008,  0.0142, -0.0091,  0.1349, -0.0200, -0.1049,\n",
      "        -0.0699,  0.1064, -0.0713, -0.0272, -0.0526,  0.0229,  0.0591, -0.0362,\n",
      "        -0.0403,  0.0990, -0.0246, -0.0077, -0.0760, -0.1873,  0.0615, -0.0401,\n",
      "        -0.0139,  0.0101, -0.0868,  0.0021, -0.0423,  0.1146, -0.0179, -0.1079,\n",
      "        -0.0163,  0.0033, -0.1010, -0.1447,  0.0879,  0.0507,  0.0374,  0.0380,\n",
      "         0.0163,  0.0956, -0.0657,  0.0532,  0.0055,  0.0571, -0.1448, -0.0257,\n",
      "         0.1298, -0.0361, -0.0288,  0.0327,  0.0749, -0.0599, -0.1907,  0.1893,\n",
      "         0.0241,  0.0499,  0.0073, -0.0513,  0.0248, -0.0743,  0.1013, -0.0179,\n",
      "        -0.0491,  0.1165, -0.0809, -0.1390,  0.0317,  0.1218,  0.1667, -0.0104,\n",
      "         0.0439,  0.0665,  0.2262,  0.0048,  0.0712, -0.0555,  0.0338,  0.1044,\n",
      "         0.0428,  0.0772,  0.0042, -0.1091,  0.0808, -0.2216, -0.0040, -0.0390,\n",
      "        -0.0936,  0.0242, -0.0502, -0.0204,  0.0323,  0.0378,  0.0174,  0.2397,\n",
      "        -0.0613,  0.1125,  0.0638, -0.0388,  0.0549,  0.0515, -0.0665,  0.1024,\n",
      "         0.0916,  0.0504, -0.1142,  0.1445, -0.0235,  0.0189,  0.0620, -0.0549,\n",
      "         0.0688,  0.0603,  0.0072, -0.0676,  0.0399,  0.1075,  0.0785,  0.1058,\n",
      "         0.0794,  0.1595, -0.0016,  0.0854,  0.1633, -0.0964,  0.0326, -0.0335,\n",
      "         0.0402, -0.0705,  0.1993, -0.1361,  0.0797, -0.0016, -0.0306, -0.0221,\n",
      "         0.0201, -0.0152,  0.0543, -0.0980,  0.0262,  0.0234, -0.0192,  0.1248,\n",
      "        -0.0363,  0.0901, -0.0367, -0.0164, -0.0483, -0.0198, -0.0423, -0.0035,\n",
      "        -0.1295,  0.1111, -0.0227,  0.0726,  0.0570, -0.0412,  0.1837,  0.0371,\n",
      "         0.0556, -0.0681,  0.0266, -0.0461,  0.1399, -0.0254, -0.0673, -0.1390,\n",
      "         0.0042,  0.0256, -0.0916, -0.0225, -0.0139,  0.0875,  0.0264, -0.1212,\n",
      "         0.1346, -0.1226, -0.0442, -0.0346,  0.0457,  0.0756,  0.0266,  0.0846,\n",
      "         0.0551, -0.0025,  0.1024,  0.0097, -0.0415, -0.0886,  0.0271,  0.0417,\n",
      "         0.1237, -0.0311, -0.1199, -0.0062,  0.0368,  0.0048, -0.0717,  0.1061,\n",
      "         0.0642,  0.0638,  0.2213,  0.0822,  0.1135,  0.0115,  0.0169, -0.0387,\n",
      "        -0.1241, -0.0140,  0.0772,  0.0622,  0.0325,  0.0486,  0.0884, -0.0565,\n",
      "         0.0820,  0.0450, -0.0520,  0.0963,  0.0433,  0.0345,  0.1213,  0.0674,\n",
      "         0.0068, -0.0392,  0.0679, -0.0482, -0.0097,  0.0660, -0.0374,  0.0628,\n",
      "        -0.0981, -0.0868,  0.0491, -0.0861,  0.2961,  0.1306,  0.0562,  0.0354,\n",
      "         0.0411,  0.0612,  0.0155, -0.0337,  0.1028,  0.0922, -0.1058,  0.1198,\n",
      "        -0.0417,  0.1414, -0.0742,  0.0263, -0.0418,  0.0218, -0.0190, -0.0621],\n",
      "       requires_grad=True)\n",
      "linear1.weight Parameter containing:\n",
      "tensor([[ 0.0187,  0.0362,  0.0432,  ...,  0.0221, -0.0409,  0.0176],\n",
      "        [-0.0065, -0.0048,  0.0422,  ..., -0.0376, -0.0138, -0.0278],\n",
      "        [ 0.0414,  0.0087,  0.0126,  ...,  0.0204, -0.0002,  0.0254],\n",
      "        ...,\n",
      "        [-0.0078,  0.0379, -0.0222,  ..., -0.0131, -0.0454,  0.0269],\n",
      "        [-0.0125,  0.0444, -0.0099,  ...,  0.0006,  0.0349,  0.0084],\n",
      "        [-0.0434, -0.0022,  0.0333,  ..., -0.0356,  0.0086,  0.0332]],\n",
      "       requires_grad=True)\n",
      "linear1.bias Parameter containing:\n",
      "tensor([ 0.0078, -0.0408, -0.0025, -0.0039, -0.0238, -0.0330,  0.0396, -0.0039,\n",
      "        -0.0185, -0.0357,  0.0409, -0.0077, -0.0228, -0.0276,  0.0287,  0.0171,\n",
      "        -0.0449,  0.0061, -0.0359, -0.0436, -0.0205, -0.0201,  0.0269, -0.0432,\n",
      "        -0.0014, -0.0317,  0.0005, -0.0292,  0.0120,  0.0248, -0.0185,  0.0220,\n",
      "        -0.0277,  0.0369,  0.0279,  0.0243, -0.0414, -0.0229,  0.0406, -0.0055,\n",
      "         0.0019,  0.0404,  0.0089, -0.0331,  0.0318, -0.0027, -0.0288,  0.0218,\n",
      "         0.0310, -0.0332,  0.0352, -0.0286, -0.0017, -0.0123,  0.0345, -0.0303,\n",
      "         0.0061, -0.0341,  0.0328, -0.0350, -0.0071,  0.0141,  0.0201, -0.0345,\n",
      "        -0.0032,  0.0345, -0.0342,  0.0082, -0.0280, -0.0028,  0.0183, -0.0427,\n",
      "        -0.0170, -0.0159,  0.0401,  0.0451, -0.0012, -0.0303, -0.0118, -0.0212,\n",
      "        -0.0106, -0.0025, -0.0451,  0.0086,  0.0149, -0.0242, -0.0273,  0.0233,\n",
      "        -0.0222,  0.0229, -0.0039,  0.0426,  0.0196, -0.0093, -0.0247,  0.0008,\n",
      "        -0.0037,  0.0261, -0.0050,  0.0050, -0.0081, -0.0187, -0.0143, -0.0333,\n",
      "        -0.0111,  0.0442,  0.0030, -0.0238,  0.0249, -0.0154, -0.0255, -0.0253,\n",
      "        -0.0040, -0.0370, -0.0091, -0.0139,  0.0352,  0.0240,  0.0289, -0.0331,\n",
      "         0.0069,  0.0354, -0.0365, -0.0209,  0.0299,  0.0394,  0.0412,  0.0436,\n",
      "         0.0085,  0.0400, -0.0221,  0.0233, -0.0420, -0.0037, -0.0061,  0.0242,\n",
      "         0.0070,  0.0057, -0.0312, -0.0429,  0.0241, -0.0343, -0.0412, -0.0128,\n",
      "         0.0034,  0.0086, -0.0345,  0.0053, -0.0082,  0.0030, -0.0209,  0.0096,\n",
      "         0.0004, -0.0299,  0.0224, -0.0190, -0.0116,  0.0441, -0.0204,  0.0281,\n",
      "         0.0188,  0.0311, -0.0278, -0.0071, -0.0281, -0.0370, -0.0432,  0.0435,\n",
      "         0.0167, -0.0157, -0.0138,  0.0035, -0.0325, -0.0090,  0.0061,  0.0071,\n",
      "        -0.0098,  0.0291, -0.0178, -0.0173,  0.0094,  0.0177,  0.0224, -0.0103,\n",
      "         0.0435, -0.0222, -0.0309,  0.0010, -0.0065,  0.0216,  0.0015,  0.0115,\n",
      "         0.0028, -0.0449,  0.0029, -0.0148, -0.0142,  0.0302, -0.0336, -0.0284,\n",
      "        -0.0188,  0.0299, -0.0113, -0.0184, -0.0150,  0.0075,  0.0187,  0.0401,\n",
      "        -0.0119,  0.0307, -0.0124,  0.0383, -0.0449,  0.0084,  0.0161,  0.0129,\n",
      "        -0.0405, -0.0064, -0.0198,  0.0009,  0.0177,  0.0083,  0.0279, -0.0430,\n",
      "        -0.0138, -0.0168, -0.0314, -0.0404,  0.0357, -0.0088,  0.0065,  0.0277,\n",
      "        -0.0267,  0.0161, -0.0251,  0.0162,  0.0216,  0.0200, -0.0310, -0.0405],\n",
      "       requires_grad=True)\n",
      "ln1.weight Parameter containing:\n",
      "tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1.], requires_grad=True)\n",
      "ln1.bias Parameter containing:\n",
      "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "       requires_grad=True)\n",
      "linear2.weight Parameter containing:\n",
      "tensor([[ 6.0920e-02,  1.9312e-02,  2.9248e-02,  ...,  7.4972e-03,\n",
      "          6.3248e-02,  4.2909e-02],\n",
      "        [ 3.8461e-02, -2.9992e-02, -1.2980e-02,  ..., -4.0450e-03,\n",
      "         -6.2549e-02, -4.2506e-02],\n",
      "        [ 4.5822e-02, -6.4185e-02,  2.5783e-02,  ...,  3.9622e-02,\n",
      "         -8.4575e-05,  5.8394e-02],\n",
      "        ...,\n",
      "        [-4.1781e-02, -1.3316e-02, -5.6668e-02,  ..., -3.6490e-03,\n",
      "         -3.5591e-02, -3.3764e-02],\n",
      "        [ 8.9659e-03,  1.5633e-02, -2.2341e-02,  ..., -1.6982e-02,\n",
      "          2.4664e-02,  1.7883e-02],\n",
      "        [-3.1773e-02,  2.2630e-02, -4.0015e-03,  ...,  6.4147e-02,\n",
      "          2.6460e-02,  3.8456e-02]], requires_grad=True)\n",
      "linear2.bias Parameter containing:\n",
      "tensor([ 0.0160, -0.0415,  0.0020,  0.0552, -0.0437,  0.0609, -0.0526, -0.0228,\n",
      "        -0.0223,  0.0519,  0.0132,  0.0211, -0.0294, -0.0407, -0.0062,  0.0580,\n",
      "        -0.0625, -0.0610,  0.0025, -0.0469,  0.0419,  0.0111,  0.0497,  0.0323,\n",
      "         0.0439, -0.0089, -0.0300, -0.0502,  0.0220, -0.0344, -0.0552,  0.0607,\n",
      "         0.0020,  0.0196, -0.0062, -0.0185, -0.0196,  0.0641, -0.0418,  0.0241,\n",
      "        -0.0156, -0.0197,  0.0322,  0.0058,  0.0580,  0.0276, -0.0503, -0.0443,\n",
      "        -0.0247,  0.0422, -0.0022, -0.0338,  0.0178,  0.0131, -0.0457,  0.0131,\n",
      "         0.0263,  0.0406, -0.0622,  0.0569,  0.0521, -0.0603,  0.0564,  0.0456,\n",
      "         0.0397, -0.0120, -0.0531,  0.0543,  0.0279,  0.0615, -0.0221,  0.0599,\n",
      "         0.0222,  0.0032,  0.0052,  0.0627,  0.0644, -0.0292,  0.0477,  0.0030,\n",
      "         0.0583,  0.0163, -0.0304, -0.0525, -0.0269,  0.0340, -0.0433,  0.0256,\n",
      "         0.0222, -0.0478, -0.0315,  0.0423,  0.0257,  0.0018, -0.0615, -0.0049,\n",
      "        -0.0557, -0.0494,  0.0313,  0.0263, -0.0449, -0.0216, -0.0309,  0.0142,\n",
      "         0.0571, -0.0414, -0.0092, -0.0362, -0.0493,  0.0459, -0.0270, -0.0458,\n",
      "         0.0413,  0.0505,  0.0163, -0.0420, -0.0050, -0.0605,  0.0095, -0.0195],\n",
      "       requires_grad=True)\n",
      "ln2.weight Parameter containing:\n",
      "tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], requires_grad=True)\n",
      "ln2.bias Parameter containing:\n",
      "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "       requires_grad=True)\n",
      "linear3.weight Parameter containing:\n",
      "tensor([[ 0.0374, -0.0530, -0.0471,  0.0188, -0.0700, -0.0324,  0.0860, -0.0899,\n",
      "         -0.0236, -0.0043, -0.0026, -0.0195, -0.0698,  0.0677,  0.0882,  0.0875,\n",
      "         -0.0238, -0.0571, -0.0718, -0.0428,  0.0260, -0.0604,  0.0488, -0.0757,\n",
      "          0.0453, -0.0199,  0.0648,  0.0702,  0.0626, -0.0132, -0.0720, -0.0885,\n",
      "         -0.0097,  0.0737, -0.0570, -0.0063,  0.0115,  0.0427, -0.0778,  0.0322,\n",
      "         -0.0393, -0.0364,  0.0049,  0.0755,  0.0809, -0.0696, -0.0886, -0.0747,\n",
      "         -0.0656,  0.0818,  0.0392,  0.0230, -0.0240,  0.0540, -0.0841, -0.0642,\n",
      "         -0.0516, -0.0596, -0.0373, -0.0357, -0.0706, -0.0633, -0.0228, -0.0809,\n",
      "          0.0529,  0.0331,  0.0014,  0.0542,  0.0268, -0.0519, -0.0124,  0.0552,\n",
      "          0.0789, -0.0130,  0.0498, -0.0078, -0.0173, -0.0704,  0.0577,  0.0863,\n",
      "          0.0154, -0.0404,  0.0745,  0.0769,  0.0625,  0.0385, -0.0294, -0.0857,\n",
      "         -0.0841,  0.0034, -0.0393, -0.0804, -0.0619, -0.0340, -0.0642, -0.0292,\n",
      "         -0.0775, -0.0804, -0.0528, -0.0869,  0.0540,  0.0675, -0.0489, -0.0478,\n",
      "         -0.0718,  0.0038, -0.0334,  0.0794,  0.0815,  0.0523, -0.0695,  0.0428,\n",
      "         -0.0912,  0.0389,  0.0065, -0.0176,  0.0687,  0.0188, -0.0887,  0.0207]],\n",
      "       requires_grad=True)\n",
      "linear3.bias Parameter containing:\n",
      "tensor([-0.0657], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "# [i for i in filter(lambda p:p.requires_grad, ion.parameters())]\n",
    "for i, j in ion.named_parameters():\n",
    "    if j.requires_grad:\n",
    "        print(i, j)\n",
    "    # print(i, j.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.4999, 0.5560],\n",
       "        [0.5555, 0.4149],\n",
       "        [0.4432, 0.0545]])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target = torch.rand(3, 2, requires_grad=False)\n",
    "target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4671/4671 [32:40<00:00,  2.38it/s, bce=0.281] \n",
      "100%|██████████| 4671/4671 [32:40<00:00,  2.38it/s, bce=0.623] \n"
     ]
    }
   ],
   "source": [
    "device = \"cuda:0\"\n",
    "ion.train() \n",
    "ion = ion.to(device)\n",
    "optimizer = optim.Adam(filter(lambda p:p.requires_grad, ion.parameters()), lr=0.005)\n",
    "bce = nn.BCELoss()\n",
    "for epoch in range(2):\n",
    "    pbar = tqdm(trainloader)\n",
    "    for i, (s, labels) in enumerate(pbar):\n",
    "        s = s.to(device)\n",
    "        labels = labels.to(device)\n",
    "        pre = ion(s) \n",
    "        loss = bce(pre.squeeze(), labels.float())\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        pbar.set_postfix(bce=loss.item())\n",
    "    torch.save(model.state_dict(), \"./checkpoints/try606/%d.pt\"%epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1557it [05:48,  4.47it/s]\n"
     ]
    }
   ],
   "source": [
    "device = \"cuda:2\"\n",
    "ion.eval() \n",
    "predict = [] \n",
    "truth = [] \n",
    "valloader = ds.val_dataloader()\n",
    "with torch.no_grad():\n",
    "    ion = ion.to(device)\n",
    "    for i, (s, labels, _) in tqdm(enumerate(valloader)):\n",
    "        s = s.to(device) \n",
    "        truth.append(labels)\n",
    "        pre, _ = ion(s) \n",
    "        predict.append(pre.squeeze().cpu().numpy()) \n",
    "predict = np.concatenate(predict, axis=0)\n",
    "truth = np.concatenate(truth, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     non-ion       0.97      0.93      0.95      7905\n",
      "         ion       0.95      0.98      0.96     10775\n",
      "\n",
      "    accuracy                           0.95     18680\n",
      "   macro avg       0.96      0.95      0.95     18680\n",
      "weighted avg       0.96      0.95      0.95     18680\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "predict_label = (predict>0.5).astype(int) \n",
    "print(classification_report(truth, predict_label, target_names=[\"non-ion\", \"ion\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "seqs = {}\n",
    "seqs[\"virus_seq\"] = utils.readDataset(\"/home/tyfei/datasets/ion_channel/Known_Virus_Ion_pro/all_Known_Virus_Ion_pro.rename_0.99.fasta\", trunc=1200)\n",
    "filtered = []\n",
    "labels = [] \n",
    "\n",
    "for i, tt in seqs.items():\n",
    "    labels.extend([i]*len(tt))\n",
    "    for name, seq in tt:\n",
    "        if \"*\" in seq:\n",
    "            filtered.append((name, seq[:-1].upper()))\n",
    "        else:\n",
    "            filtered.append((name, seq.upper()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 37/37 [00:06<00:00,  5.44it/s]\n"
     ]
    }
   ],
   "source": [
    "batch_labels, _, batch_tokens = batch_converter(filtered) \n",
    "allres = []\n",
    "batch_tokens = batch_tokens.to(\"cuda:0\")\n",
    "for i in tqdm(range(0,len(batch_tokens), 10)):\n",
    "    with torch.no_grad():\n",
    "        results = model(batch_tokens[i:i+10])\n",
    "        # token_representations = results[\"representations\"][12].cpu().numpy()\n",
    "        allres.append(pre.squeeze().cpu().numpy())\n",
    "allres = np.concatenate(allres, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([111.,  37.,   0.,   0.,   0.,   0.,   0.,   0.,   0., 148.]),\n",
       " array([3.06374772e-04, 1.00096077e-01, 1.99885786e-01, 2.99675494e-01,\n",
       "        3.99465173e-01, 4.99254882e-01, 5.99044561e-01, 6.98834300e-01,\n",
       "        7.98623979e-01, 8.98413718e-01, 9.98203397e-01]),\n",
       " <BarContainer object of 10 artists>)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhqUlEQVR4nO3de3CU5f2/8fdCwhKYJJJQdlkJEDrxCKINQg1asEAY5FCHsWihiC3OwCBI5GQyeAhOTQRrjBLBgaFAxQhTK5SpBwitBjBWIUBVcEQlQhC2GTRuEogJJPfvD4b9/tbEw+Ju9t5wvWaeme6z9y6f3EX38snurMMYYwQAAGCRDpEeAAAA4NsIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWiYn0ABejublZJ06cUHx8vBwOR6THAQAAP4IxRrW1tfJ4POrQ4fuvkURloJw4cUIpKSmRHgMAAFyEyspK9erV63vXRGWgxMfHSzr/AyYkJER4GgAA8GPU1NQoJSXF/zr+faIyUC78WichIYFAAQAgyvyYt2fwJlkAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFgnJtIDAADQ3vXNfjXSIwTt8yfGRvTP5woKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsE3Sg7Ny5U+PHj5fH45HD4dCWLVu+c+2MGTPkcDhUWFgYcL6hoUFz5sxR9+7d1bVrV02YMEHHjx8PdhQAANBOBR0op0+f1sCBA1VUVPS967Zs2aJ3331XHo+nxX1ZWVnavHmzNm7cqN27d6uurk7jxo1TU1NTsOMAAIB2KCbYB4wZM0Zjxoz53jVffPGFZs+erW3btmns2LEB9/l8Pq1Zs0YvvPCCRo4cKUnasGGDUlJStGPHDo0ePTrYkQAAQDsT8vegNDc3a+rUqVq4cKGuvfbaFveXl5fr7NmzyszM9J/zeDzq37+/ysrKWn3OhoYG1dTUBBwAAKD9CnmgLF26VDExMbr//vtbvd/r9apTp07q1q1bwHmXyyWv19vqY/Lz85WYmOg/UlJSQj02AACwSEgDpby8XM8884zWrVsnh8MR1GONMd/5mJycHPl8Pv9RWVkZinEBAIClQhoou3btUlVVlXr37q2YmBjFxMTo6NGjmj9/vvr27StJcrvdamxsVHV1dcBjq6qq5HK5Wn1ep9OphISEgAMAALRfIQ2UqVOn6v3339eBAwf8h8fj0cKFC7Vt2zZJUnp6umJjY1VSUuJ/3MmTJ/Xhhx8qIyMjlOMAAIAoFfSneOrq6vTpp5/6b1dUVOjAgQNKSkpS7969lZycHLA+NjZWbrdbV155pSQpMTFR06dP1/z585WcnKykpCQtWLBAAwYM8H+qBwAAXNqCDpS9e/fq1ltv9d+eN2+eJGnatGlat27dj3qOp59+WjExMZo0aZLq6+s1YsQIrVu3Th07dgx2HAAA0A45jDEm0kMEq6amRomJifL5fLwfBQBgvb7Zr0Z6hKB9/sTYH14UpGBev/kuHgAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYJ2gA2Xnzp0aP368PB6PHA6HtmzZ4r/v7NmzevDBBzVgwAB17dpVHo9Hd999t06cOBHwHA0NDZozZ466d++url27asKECTp+/PhP/mEAAED7EHSgnD59WgMHDlRRUVGL+86cOaN9+/bp4Ycf1r59+/TKK6/o8OHDmjBhQsC6rKwsbd68WRs3btTu3btVV1encePGqamp6eJ/EgAA0G7EBPuAMWPGaMyYMa3el5iYqJKSkoBzy5cv1+DBg3Xs2DH17t1bPp9Pa9as0QsvvKCRI0dKkjZs2KCUlBTt2LFDo0ePvogfAwAAtCdhfw+Kz+eTw+HQZZddJkkqLy/X2bNnlZmZ6V/j8XjUv39/lZWVtfocDQ0NqqmpCTgAAED7FdZA+eabb5Sdna3JkycrISFBkuT1etWpUyd169YtYK3L5ZLX6231efLz85WYmOg/UlJSwjk2AACIsLAFytmzZ3XXXXepublZK1as+MH1xhg5HI5W78vJyZHP5/MflZWVoR4XAABYJCyBcvbsWU2aNEkVFRUqKSnxXz2RJLfbrcbGRlVXVwc8pqqqSi6Xq9XnczqdSkhICDgAAED7FfJAuRAnn3zyiXbs2KHk5OSA+9PT0xUbGxvwZtqTJ0/qww8/VEZGRqjHAQAAUSjoT/HU1dXp008/9d+uqKjQgQMHlJSUJI/HozvuuEP79u3TP//5TzU1NfnfV5KUlKROnTopMTFR06dP1/z585WcnKykpCQtWLBAAwYM8H+qBwAAXNqCDpS9e/fq1ltv9d+eN2+eJGnatGnKzc3V1q1bJUnXX399wOPefPNNDR8+XJL09NNPKyYmRpMmTVJ9fb1GjBihdevWqWPHjhf5YwAAgPbEYYwxkR4iWDU1NUpMTJTP5+P9KAAA6/XNfjXSIwTt8yfGhvw5g3n95rt4AACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgnZhID2AjvhYbAIDI4goKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsE7QgbJz506NHz9eHo9HDodDW7ZsCbjfGKPc3Fx5PB7FxcVp+PDhOnjwYMCahoYGzZkzR927d1fXrl01YcIEHT9+/Cf9IAAAoP0IOlBOnz6tgQMHqqioqNX7ly1bpoKCAhUVFWnPnj1yu90aNWqUamtr/WuysrK0efNmbdy4Ubt371ZdXZ3GjRunpqami/9JAABAuxET7APGjBmjMWPGtHqfMUaFhYVavHixJk6cKElav369XC6XiouLNWPGDPl8Pq1Zs0YvvPCCRo4cKUnasGGDUlJStGPHDo0ePfon/DgAAKA9COl7UCoqKuT1epWZmek/53Q6NWzYMJWVlUmSysvLdfbs2YA1Ho9H/fv396/5toaGBtXU1AQcAACg/QppoHi9XkmSy+UKOO9yufz3eb1ederUSd26dfvONd+Wn5+vxMRE/5GSkhLKsQEAgGXC8ikeh8MRcNsY0+Lct33fmpycHPl8Pv9RWVkZslkBAIB9QhoobrdbklpcCamqqvJfVXG73WpsbFR1dfV3rvk2p9OphISEgAMAALRfIQ2U1NRUud1ulZSU+M81NjaqtLRUGRkZkqT09HTFxsYGrDl58qQ+/PBD/xoAAHBpC/pTPHV1dfr000/9tysqKnTgwAElJSWpd+/eysrKUl5entLS0pSWlqa8vDx16dJFkydPliQlJiZq+vTpmj9/vpKTk5WUlKQFCxZowIAB/k/1AACAS1vQgbJ3717deuut/tvz5s2TJE2bNk3r1q3TokWLVF9fr1mzZqm6ulpDhgzR9u3bFR8f73/M008/rZiYGE2aNEn19fUaMWKE1q1bp44dO4bgRwIAANHOYYwxkR4iWDU1NUpMTJTP5wvL+1H6Zr8a8ucMt8+fGBvpEQAA34HXlfOCef3mu3gAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1Qh4o586d00MPPaTU1FTFxcWpX79+euyxx9Tc3OxfY4xRbm6uPB6P4uLiNHz4cB08eDDUowAAgCgV8kBZunSpnn/+eRUVFemjjz7SsmXL9OSTT2r58uX+NcuWLVNBQYGKioq0Z88eud1ujRo1SrW1taEeBwAARKGQB8o777yj3/zmNxo7dqz69u2rO+64Q5mZmdq7d6+k81dPCgsLtXjxYk2cOFH9+/fX+vXrdebMGRUXF4d6HAAAEIVCHig333yz/vWvf+nw4cOSpP/+97/avXu3brvtNklSRUWFvF6vMjMz/Y9xOp0aNmyYysrKWn3OhoYG1dTUBBwAAKD9ign1Ez744IPy+Xy66qqr1LFjRzU1Nenxxx/X7373O0mS1+uVJLlcroDHuVwuHT16tNXnzM/P15IlS0I9KgAAsFTIr6Bs2rRJGzZsUHFxsfbt26f169frz3/+s9avXx+wzuFwBNw2xrQ4d0FOTo58Pp//qKysDPXYAADAIiG/grJw4UJlZ2frrrvukiQNGDBAR48eVX5+vqZNmya32y3p/JWUnj17+h9XVVXV4qrKBU6nU06nM9SjAgAAS4X8CsqZM2fUoUPg03bs2NH/MePU1FS53W6VlJT4729sbFRpaakyMjJCPQ4AAIhCIb+CMn78eD3++OPq3bu3rr32Wu3fv18FBQX64x//KOn8r3aysrKUl5entLQ0paWlKS8vT126dNHkyZNDPQ4AAIhCIQ+U5cuX6+GHH9asWbNUVVUlj8ejGTNm6JFHHvGvWbRokerr6zVr1ixVV1dryJAh2r59u+Lj40M9DgAAiEIOY4yJ9BDBqqmpUWJionw+nxISEkL+/H2zXw35c4bb50+MjfQIAIDvwOvKecG8fvNdPAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOuEJVC++OIL/f73v1dycrK6dOmi66+/XuXl5f77jTHKzc2Vx+NRXFychg8froMHD4ZjFAAAEIVCHijV1dUaOnSoYmNj9frrr+vQoUN66qmndNlll/nXLFu2TAUFBSoqKtKePXvkdrs1atQo1dbWhnocAAAQhWJC/YRLly5VSkqK1q5d6z/Xt29f//82xqiwsFCLFy/WxIkTJUnr16+Xy+VScXGxZsyYEeqRAABAlAn5FZStW7dq0KBB+u1vf6sePXrohhtu0OrVq/33V1RUyOv1KjMz03/O6XRq2LBhKisrC/U4AAAgCoU8UI4cOaKVK1cqLS1N27Zt08yZM3X//ffrr3/9qyTJ6/VKklwuV8DjXC6X/75va2hoUE1NTcABAADar5D/iqe5uVmDBg1SXl6eJOmGG27QwYMHtXLlSt19993+dQ6HI+BxxpgW5y7Iz8/XkiVLQj0qAACwVMivoPTs2VPXXHNNwLmrr75ax44dkyS53W5JanG1pKqqqsVVlQtycnLk8/n8R2VlZajHBgAAFgl5oAwdOlQff/xxwLnDhw+rT58+kqTU1FS53W6VlJT4729sbFRpaakyMjJafU6n06mEhISAAwAAtF8h/xXPAw88oIyMDOXl5WnSpEl67733tGrVKq1atUrS+V/tZGVlKS8vT2lpaUpLS1NeXp66dOmiyZMnh3ocAAAQhUIeKDfeeKM2b96snJwcPfbYY0pNTVVhYaGmTJniX7No0SLV19dr1qxZqq6u1pAhQ7R9+3bFx8eHehwAABCFQh4okjRu3DiNGzfuO+93OBzKzc1Vbm5uOP54AAAQ5fguHgAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYJ2wB0p+fr4cDoeysrL854wxys3NlcfjUVxcnIYPH66DBw+GexQAABAlwhooe/bs0apVq3TdddcFnF+2bJkKCgpUVFSkPXv2yO12a9SoUaqtrQ3nOAAAIEqELVDq6uo0ZcoUrV69Wt26dfOfN8aosLBQixcv1sSJE9W/f3+tX79eZ86cUXFxcbjGAQAAUSRsgXLfffdp7NixGjlyZMD5iooKeb1eZWZm+s85nU4NGzZMZWVlrT5XQ0ODampqAg4AANB+xYTjSTdu3Kh9+/Zpz549Le7zer2SJJfLFXDe5XLp6NGjrT5ffn6+lixZEvpBAQCAlUJ+BaWyslJz587Vhg0b1Llz5+9c53A4Am4bY1qcuyAnJ0c+n89/VFZWhnRmAABgl5BfQSkvL1dVVZXS09P955qamrRz504VFRXp448/lnT+SkrPnj39a6qqqlpcVbnA6XTK6XSGelQAAGCpkF9BGTFihD744AMdOHDAfwwaNEhTpkzRgQMH1K9fP7ndbpWUlPgf09jYqNLSUmVkZIR6HAAAEIVCfgUlPj5e/fv3DzjXtWtXJScn+89nZWUpLy9PaWlpSktLU15enrp06aLJkyeHehwAABCFwvIm2R+yaNEi1dfXa9asWaqurtaQIUO0fft2xcfHR2IcAABgmTYJlLfeeivgtsPhUG5urnJzc9vijwcAAFEmIldQEHp9s1+N9AhB+/yJsZEeAQBgKb4sEAAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYJ2QB0p+fr5uvPFGxcfHq0ePHrr99tv18ccfB6wxxig3N1cej0dxcXEaPny4Dh48GOpRAABAlAp5oJSWluq+++7Tf/7zH5WUlOjcuXPKzMzU6dOn/WuWLVumgoICFRUVac+ePXK73Ro1apRqa2tDPQ4AAIhCMaF+wjfeeCPg9tq1a9WjRw+Vl5frV7/6lYwxKiws1OLFizVx4kRJ0vr16+VyuVRcXKwZM2aEeiQAABBlwv4eFJ/PJ0lKSkqSJFVUVMjr9SozM9O/xul0atiwYSorK2v1ORoaGlRTUxNwAACA9iusgWKM0bx583TzzTerf//+kiSv1ytJcrlcAWtdLpf/vm/Lz89XYmKi/0hJSQnn2AAAIMLCGiizZ8/W+++/r5deeqnFfQ6HI+C2MabFuQtycnLk8/n8R2VlZVjmBQAAdgj5e1AumDNnjrZu3aqdO3eqV69e/vNut1vS+SspPXv29J+vqqpqcVXlAqfTKafTGa5RAQCAZUJ+BcUYo9mzZ+uVV17Rv//9b6Wmpgbcn5qaKrfbrZKSEv+5xsZGlZaWKiMjI9TjAACAKBTyKyj33XefiouL9Y9//EPx8fH+95UkJiYqLi5ODodDWVlZysvLU1pamtLS0pSXl6cuXbpo8uTJoR4HAABEoZAHysqVKyVJw4cPDzi/du1a3XPPPZKkRYsWqb6+XrNmzVJ1dbWGDBmi7du3Kz4+PtTjAACAKBTyQDHG/OAah8Oh3Nxc5ebmhvqPBwAA7QDfxQMAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALBORANlxYoVSk1NVefOnZWenq5du3ZFchwAAGCJiAXKpk2blJWVpcWLF2v//v265ZZbNGbMGB07dixSIwEAAEtELFAKCgo0ffp03Xvvvbr66qtVWFiolJQUrVy5MlIjAQAAS8RE4g9tbGxUeXm5srOzA85nZmaqrKysxfqGhgY1NDT4b/t8PklSTU1NWOZrbjgTludFoHD9/wcAtonG15Vw/Dv6wnMaY35wbUQC5dSpU2pqapLL5Qo473K55PV6W6zPz8/XkiVLWpxPSUkJ24wIv8TCSE8AAPgu4fx3dG1trRITE793TUQC5QKHwxFw2xjT4pwk5eTkaN68ef7bzc3N+uqrr5ScnNzq+p+ipqZGKSkpqqysVEJCQkifG/+HfW4b7HPbYa/bBvvcdsKx18YY1dbWyuPx/ODaiARK9+7d1bFjxxZXS6qqqlpcVZEkp9Mpp9MZcO6yyy4L54hKSEjgL38bYJ/bBvvcdtjrtsE+t51Q7/UPXTm5ICJvku3UqZPS09NVUlIScL6kpEQZGRmRGAkAAFgkYr/imTdvnqZOnapBgwbppptu0qpVq3Ts2DHNnDkzUiMBAABLRCxQ7rzzTn355Zd67LHHdPLkSfXv31+vvfaa+vTpE6mRJJ3/ddKjjz7a4ldKCC32uW2wz22HvW4b7HPbifReO8yP+awPAABAG+K7eAAAgHUIFAAAYB0CBQAAWIdAAQAA1rkkA2XFihVKTU1V586dlZ6erl27dn3v+tLSUqWnp6tz587q16+fnn/++TaaNLoFs8+vvPKKRo0apZ/97GdKSEjQTTfdpG3btrXhtNEr2L/PF7z99tuKiYnR9ddfH94B25Fg97qhoUGLFy9Wnz595HQ69fOf/1x/+ctf2mja6BXsPr/44osaOHCgunTpop49e+oPf/iDvvzyyzaaNjrt3LlT48ePl8fjkcPh0JYtW37wMW3+WmguMRs3bjSxsbFm9erV5tChQ2bu3Lmma9eu5ujRo62uP3LkiOnSpYuZO3euOXTokFm9erWJjY01L7/8chtPHl2C3ee5c+eapUuXmvfee88cPnzY5OTkmNjYWLNv3742njy6BLvPF3z99demX79+JjMz0wwcOLBtho1yF7PXEyZMMEOGDDElJSWmoqLCvPvuu+btt99uw6mjT7D7vGvXLtOhQwfzzDPPmCNHjphdu3aZa6+91tx+++1tPHl0ee2118zixYvN3//+dyPJbN68+XvXR+K18JILlMGDB5uZM2cGnLvqqqtMdnZ2q+sXLVpkrrrqqoBzM2bMML/85S/DNmN7EOw+t+aaa64xS5YsCfVo7crF7vOdd95pHnroIfPoo48SKD9SsHv9+uuvm8TERPPll1+2xXjtRrD7/OSTT5p+/foFnHv22WdNr169wjZje/NjAiUSr4WX1K94GhsbVV5erszMzIDzmZmZKisra/Ux77zzTov1o0eP1t69e3X27NmwzRrNLmafv625uVm1tbVKSkoKx4jtwsXu89q1a/XZZ5/p0UcfDfeI7cbF7PXWrVs1aNAgLVu2TJdffrmuuOIKLViwQPX19W0xclS6mH3OyMjQ8ePH9dprr8kYo//97396+eWXNXbs2LYY+ZIRidfCiH6bcVs7deqUmpqaWnwhocvlavHFhRd4vd5W1587d06nTp1Sz549wzZvtLqYff62p556SqdPn9akSZPCMWK7cDH7/Mknnyg7O1u7du1STMwl9Y//T3Ixe33kyBHt3r1bnTt31ubNm3Xq1CnNmjVLX331Fe9D+Q4Xs88ZGRl68cUXdeedd+qbb77RuXPnNGHCBC1fvrwtRr5kROK18JK6gnKBw+EIuG2MaXHuh9a3dh6Bgt3nC1566SXl5uZq06ZN6tGjR7jGazd+7D43NTVp8uTJWrJkia644oq2Gq9dCebvdHNzsxwOh1588UUNHjxYt912mwoKCrRu3TquovyAYPb50KFDuv/++/XII4+ovLxcb7zxhioqKvhetzBo69fCS+o/obp3766OHTu2KPGqqqoWZXiB2+1udX1MTIySk5PDNms0u5h9vmDTpk2aPn26/va3v2nkyJHhHDPqBbvPtbW12rt3r/bv36/Zs2dLOv8iaoxRTEyMtm/frl//+tdtMnu0uZi/0z179tTll18e8NXyV199tYwxOn78uNLS0sI6czS6mH3Oz8/X0KFDtXDhQknSddddp65du+qWW27Rn/70J65yh0gkXgsvqSsonTp1Unp6ukpKSgLOl5SUKCMjo9XH3HTTTS3Wb9++XYMGDVJsbGzYZo1mF7PP0vkrJ/fcc4+Ki4v5/fGPEOw+JyQk6IMPPtCBAwf8x8yZM3XllVfqwIEDGjJkSFuNHnUu5u/00KFDdeLECdXV1fnPHT58WB06dFCvXr3COm+0uph9PnPmjDp0CHwp69ixo6T/+y98/HQReS0M29tvLXXhI2xr1qwxhw4dMllZWaZr167m888/N8YYk52dbaZOnepff+GjVQ888IA5dOiQWbNmDR8z/hGC3efi4mITExNjnnvuOXPy5En/8fXXX0fqR4gKwe7zt/Epnh8v2L2ura01vXr1MnfccYc5ePCgKS0tNWlpaebee++N1I8QFYLd57Vr15qYmBizYsUK89lnn5ndu3ebQYMGmcGDB0fqR4gKtbW1Zv/+/Wb//v1GkikoKDD79+/3f5zbhtfCSy5QjDHmueeeM3369DGdOnUyv/jFL0xpaan/vmnTpplhw4YFrH/rrbfMDTfcYDp16mT69u1rVq5c2cYTR6dg9nnYsGFGUotj2rRpbT94lAn27/P/j0AJTrB7/dFHH5mRI0eauLg406tXLzNv3jxz5syZNp46+gS7z88++6y55pprTFxcnOnZs6eZMmWKOX78eBtPHV3efPPN7/13rg2vhQ5juAYGAADsckm9BwUAAEQHAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1/h/psDONhcrfLAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "allres\n",
    "import matplotlib.pyplot as plt\n",
    "plt.hist(allres)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}