IDBMS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "18280ced",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'tensorflow_text'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Input \u001b[1;32mIn [3]\u001b[0m, in \u001b[0;36m<cell line: 9>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmatplotlib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mticker\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mticker\u001b[39;00m\n\u001b[0;32m      8\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mtf\u001b[39;00m\n\u001b[1;32m----> 9\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtensorflow_text\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mtf_text\u001b[39;00m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow_text'"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import typing\n",
    "from typing import Any, Tuple\n",
    "import einops\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "import tensorflow as tf\n",
    "import tensorflow_text as tf_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "18f0daa2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "^C\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "94419fc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ShapeChecker():\n",
    "  def __init__(self):\n",
    "    # Keep a cache of every axis-name seen\n",
    "    self.shapes = {}\n",
    "\n",
    "  def __call__(self, tensor, names, broadcast=False):\n",
    "    if not tf.executing_eagerly():\n",
    "      return\n",
    "\n",
    "    parsed = einops.parse_shape(tensor, names)\n",
    "\n",
    "    for name, new_dim in parsed.items():\n",
    "      old_dim = self.shapes.get(name, None)\n",
    "\n",
    "      if (broadcast and new_dim == 1):\n",
    "        continue\n",
    "\n",
    "      if old_dim is None:\n",
    "        # If the axis name is new, add its length to the cache.\n",
    "        self.shapes[name] = new_dim\n",
    "        continue\n",
    "\n",
    "      if new_dim != old_dim:\n",
    "        raise ValueError(f\"Shape mismatch for dimension: '{name}'\\n\"\n",
    "                         f\"    found: {new_dim}\\n\"\n",
    "                         f\"    expected: {old_dim}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "27fbb50d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data():\n",
    "  text = pd.read_csv(\"train.csv\")\n",
    "  context = np.array(text.iloc[ : , 0 ].tolist())\n",
    "  target =  np.array(text.iloc[ : , 1 ].tolist())\n",
    "  return target, context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "3268377c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Which report includes a Circuit of Tripoli?\n"
     ]
    }
   ],
   "source": [
    "target_raw, context_raw = load_data()\n",
    "print(context_raw[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "50a6b147",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SELECT Report FROM table WHERE Circuit = tripoli\n"
     ]
    }
   ],
   "source": [
    "print(target_raw[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "83fd12d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "BUFFER_SIZE = len(context_raw)\n",
    "BATCH_SIZE = 64\n",
    "\n",
    "is_train = np.random.uniform(size=(len(target_raw),)) < 0.8\n",
    "\n",
    "train_raw = (\n",
    "    tf.data.Dataset\n",
    "    .from_tensor_slices((context_raw[is_train], target_raw[is_train]))\n",
    "    .shuffle(BUFFER_SIZE)\n",
    "    .batch(BATCH_SIZE))\n",
    "val_raw = (\n",
    "    tf.data.Dataset\n",
    "    .from_tensor_slices((context_raw[~is_train], target_raw[~is_train]))\n",
    "    .shuffle(BUFFER_SIZE)\n",
    "    .batch(BATCH_SIZE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "f4b7f24d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[b'Which Election had 2nd Member George Alexander Morrison, a 1st Party of Unionist, and 3rd Member of John Buchan?'\n",
      " b'When fr2.0 9 is the series who is the winning team?'\n",
      " b'Name the number of awardees for english and hindi'\n",
      " b'For the 1967 Cleveland Browns season what is the total number of times they played the Minnesota Vikings and had an atttendance smaller than 68,431?'\n",
      " b'Who is the manufacturer when 150.088 is the average speed (mph)?'], shape=(5,), dtype=string)\n",
      "\n",
      "tf.Tensor(\n",
      "[b'SELECT Election FROM table WHERE 2nd Member = george alexander morrison AND 1st Party = unionist AND 3rd Member = john buchan'\n",
      " b'SELECT Winning team FROM table WHERE Series = FR2.0 9'\n",
      " b'SELECT COUNT Awardee(s) FROM table WHERE Language = English and Hindi'\n",
      " b'SELECT COUNT Week FROM table WHERE Opponent = minnesota vikings AND Attendance < 68,431'\n",
      " b'SELECT Manufacturer FROM table WHERE Average Speed (mph) = 150.088'], shape=(5,), dtype=string)\n"
     ]
    }
   ],
   "source": [
    "for example_context_strings, example_target_strings in train_raw.take(1):\n",
    "  print(example_context_strings[:5])\n",
    "  print()\n",
    "  print(example_target_strings[:5])\n",
    "  break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "9e17f7df",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tf_lower_and_split_punct(text):\n",
    "  # Split accented characters.\n",
    "  text = tf_text.normalize_utf8(text, 'NFKD')\n",
    "  text = tf.strings.lower(text)\n",
    "  # Keep space, a to z, and select punctuation.\n",
    "  text = tf.strings.regex_replace(text, '[^ a-z.?!,¿]', '')\n",
    "  # Add spaces around punctuation.\n",
    "  text = tf.strings.regex_replace(text, '[.?!,¿]', r' \\0 ')\n",
    "  # Strip whitespace.\n",
    "  text = tf.strings.strip(text)\n",
    "\n",
    "  text = tf.strings.join(['[START]', text, '[END]'], separator=' ')\n",
    "  return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "d25e0c8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "max_vocab_size = 5000\n",
    "\n",
    "context_text_processor = tf.keras.layers.TextVectorization(\n",
    "    standardize=tf_lower_and_split_punct,\n",
    "    max_tokens=max_vocab_size,\n",
    "    ragged=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "691dc41e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['', '[UNK]', 'the', '[START]', '[END]', '?', 'what', 'is', 'of', ',']"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "context_text_processor.adapt(train_raw.map(lambda context, target: context))\n",
    "\n",
    "# Here are the first 10 words from the vocabulary:\n",
    "context_text_processor.get_vocabulary()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "8728419a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " '[UNK]',\n",
       " 'from',\n",
       " 'table',\n",
       " 'select',\n",
       " '[START]',\n",
       " '[END]',\n",
       " 'where',\n",
       " 'and',\n",
       " '.']"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_text_processor = tf.keras.layers.TextVectorization(\n",
    "    standardize=tf_lower_and_split_punct,\n",
    "    max_tokens=max_vocab_size,\n",
    "    ragged=True)\n",
    "\n",
    "target_text_processor.adapt(train_raw.map(lambda context, target: target))\n",
    "target_text_processor.get_vocabulary()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "524b4463",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[3, 20, 242, 32, 153, 297, 719, 1417, 1, 9, 10, 112, 144, 8, 4313, 9, 12,\n",
       "  240, 297, 8, 239, 1, 5, 4]                                              ,\n",
       " [3, 14, 4027, 19, 7, 2, 99, 24, 7, 2, 142, 21, 5, 4],\n",
       " [3, 22, 2, 25, 8, 1, 13, 299, 12, 2131, 4]]>"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_tokens = context_text_processor(example_context_strings)\n",
    "example_tokens[:3, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "6ee1fe29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[START] which election had nd member george alexander [UNK] , a st party of unionist , and rd member of john [UNK] ? [END]'"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "context_vocab = np.array(context_text_processor.get_vocabulary())\n",
    "tokens = context_vocab[example_tokens[0].numpy()]\n",
    "' '.join(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "4ee21983",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Mask')"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAfdElEQVR4nO3de5Sd1Xnf8e8zc0b3yyAhidHoBkGAMTcbcbHJMtiCGGOnqBccO66rOriKW5raXXYbbCcEvNos3GY5tDWrsYJdFIwvGJsFTWs7QjV2sizuyNyxMAEhNEigu9D1nHn6x3mVnPfdr3JenTln3rNnfp+1tM7sffb7vlujrT17nrMv5u6IiEh8esqugIiItEYduIhIpNSBi4hESh24iEik1IGLiERKHbiISKTUgXeQmV1mZpvLrodIbMzsATP7ZNn16HbqwAsys30Nf4bN7EBD+mMl1+3vGnvyQ2O4oW6bzewuM7ugzDrK2GNmL5vZYTM7MZO/wczczJaUVLVxQx14Qe4+7egfYBPwmw15d5Zdv4wtST2nAxcDzwN/bWbLy62WjEF/C3z0aMLMzgYml1ed8UUd+AiZ2UQzu8XMtiR/bjGzicco++/M7FkzW5Bc9ydmtsnMtprZn5nZ5KTcZcnI+bNmts3MhszsE8dbN6/b7O43ALcBX07ub2b2p8m9d5vZk2Z21ki+DzJu3QH8i4b0SuAvjibM7INm9oSZ7TGzV83sxob3JpnZN81su5ntMrNHzGxe9gFmNpC00c918i8SI3XgI/dF6qPc84BzgQuBP8gWMrM/BP4lcKm7b6bemZ6WXHcqMAjc0HDJScDMJP9a4FYzO2EE9fwB8E4zmwr8BvCe5Pn9wG8B20dwbxm/HgRmmNnbzKyXelv6ZsP7b1Hv4PuBDwL/2sxWJO+tpN7GFwKzgU8BBxpvnoRhfgp81d3/pGN/i0ipAx+5jwFfcvdt7v4GcBPw8Yb3zcy+ArwfeK+7v2FmBvwr4N+7+w533wv8MfCRhuuOJPc94u7/F9gHnD6Cem4BjPp/pCPUwytnAObuz7n70AjuLePb0VH4FdTDda8dfcPdH3D3p9x92N2fBL4NXJq8fYR6x32qu9fc/TF339Nw3zOBB4A/cvfVo/D3iE6l7AqMAfOBVxrSryR5R/UDq4DfcvfdSd4cYArwWL0vB+qda2/DddvdvdqQ3g9MG0E9BwEHdrn7/zOzrwK3AovM7B7gc5n/PCJF3QH8DDiZhvAJgJldBNwMnAVMACYC32u4biHwHTPrpz5y/6K7H0ne/xjwInB3h+sfLY3AR24LsLghvSjJO2on8CHgf5nZJUnem9R/VXy7u/cnf2YmHzx2yj8GHnf3twDc/b+7+/nA26mHUv5DB58tY5i7v0L9w8yrqIfqGn0LuA9Y6O4zgT+jPlgh+e3yJnc/E3g39f8njfH0G6n/X/lWEp6RDHXgI/dt4A/MbE4yneoG0jFA3P0B6qOJe8zsIncfBv4c+FMzmwtgZoNm9v52Viz5sHLQzP4I+CTwhST/AjO7yMz6qMcoDwK1dj5bxp1rgfcdHSA0mA7scPeDZnYh8NtH3zCz95rZ2UnnvId6SKWxHR4BrgGmAneYmfqrDH1DRu4/AY8CTwJPAY8neSnuvhb4BHCfmZ0P/D71Xw8fNLM9wP2MLMbdaL6Z7aMeN38EOBu4zN3/Knl/BvUfIDuph3y2A/qASFrm7r9y90dz3vo3wJfMbC/1wc1dDe+dRD08sgd4jvqHldnBz2HgnwBzgW+oE08zHeggIhIn/TQTEYmUOnARkUipAxcRiZQ6cBGRSI3qQp4JNtEnMXX0Hvj3i2TqCnxgaxNztjE5ciTI8uHh5vfqDaeuei09W896wp+hRe7N1Jz9gt46EOaNI3vZ+aa7zynj2SfO6vUlC/vKeLQAv3xyStlV6Khjte1R7cAnMZWL2rEhXt5MIg87Pauk/0N5NeyIsyqLfy289dDWIK+2b1/Te/X2h1uX1HbsTKV7JocNb3j//qb3trPODvL84aeaXjeW3e93v9K8VGcsWdjHwz9eVNbjx733zz+37Cp01LHatkIoIiKRUgcuIhKpODazyoZMcsIlebLx5iKqG3913NccSzZckqdIuCTPeA+XSHuM9dDDWKcRuIhIpNSBi4hEqutCKEWm3lXmnBiUqb7xZnizTKilMn8gvG5L+hyDjV+7MChz2nVPhLfOzGipDM4PylRf2xLkiXSTH2/5RanPVwhnZDQCFxGJlDpwEZFIqQMXEYlUoRh4cl7dbdTPtXPgd4AXgO8CS4CXgQ+7e/N5c034cPPl7vvPPznIe/1dpwV5i7/0UCqdjXdDGHNf+rsPh3VqWiPFu2M1mm07BopJx6XoCPy/AT9y9zOAc6mfnnE9sM7dlwLrkrRIbNS2JVpNO3AzmwG8B/g61I84cvddwNXAmqTYGmBFZ6oo0hlq2xK7IiGUU4A3qJ+qfi7wGPBpYJ67DwG4+9DRw3mzzGwVsApgEuHGTT0TJjStwPDhw6n0xLWPB2WW/CTcCW44M/2wZ3K4g9/wgfQOfq9875ygzOJrnmxax7y/R95K0FZWh0rHtK1tLxrsuhm5LenktEKFZ9qvSAilArwT+J/u/g7qp5gX/pXS3Ve7+zJ3X9ZHzlatIuVpW9ueMztcvyDSaUU68M3AZnc/+ong3dQb/VYzGwBIXrd1pooiHaO2LVFr+nufu79uZq+a2enu/gKwHHg2+bMSuDl5vbeVCmTDIz0TJwVliuzrnReayM4w6ZkV7s/dczAd1ikSLsmT/XtI9+t0246RwhxxKRq4+z3gTjObALwEfIL66P0uM7sW2ARc05kqinSU2rZEq1AH7u4bgGU5b7XheB2R8qhtS8y0ElNEJFJdN/dp+NDBMDNzoEM2Jg6wb8X5Qd7Uux9MpatbXm+pTj1TwumPlpk2WNu1q6V7i3STVqcRKnZeDo3ARUQipQ5cRCRSXRdCyZOdDugXnBmUyYZLAHpnzkxnzJ8X3nvXnlS6OhSGWfLOrbRDh3LrmiqTczhFNhyUNyVSJDYKvZRDI3ARkUipAxcRiZQ6cBGRSJUfA7fmP0N6581Jpavrw3hb3lS/2u7d6YxsGujNLK+vXh6u6ajc/2hYqUy9K/NmB0WqW8MtNPb+MH0Yxa4Hw43uFt708/B5ImNQXuxccfHiNAIXEYmUOnARkUiVH0Lx4aZFsudNBtMDAeufEeQNvxJO/8uq7Ugfddi3K5wemHcmZnb6X164JM/0D7yYTvPiMUqKdAeFNLqXRuAiIpFSBy4iEqnyQygtqO3ZG2bmzDAJZrgUCNf4o0+1WCuR7qZQyNijEbiISKTUgYuIREoduIhIpKKMgfdkDlOAYxwEUSDm3dvfn0rrYAYZq4rsGKg4eVw0AhcRiZQ6cBGRSEUZQhk+fDjIm7u+P8jbsSIdaslbLVkkZFI5/dQgr/qCVlDK2FP0YAaFWrqDRuAiIpFSBy4iEil14CIikSoUAzezl4G9QA2ouvsyM5sFfBdYArwMfNjddx7rHse+efpnSN5BwNmd/yqZQxgAdnwovHVt1/bmj7/w7HTG488HZfLi3dnph3n1ru7I+XYUmNoYeFdOvDHnUAs5fh1t22NYq4cYd8p4jckfzwj8ve5+nrsfPbLmemCduy8F1iVpkRipbUuURhJCuRpYk3y9Blgx4tqIdAe1bYlC0WmEDvyVmTnwNXdfDcxz9yEAdx8ys/BwR8DMVgGrACYRnlsZPKhWC/Ky4Qp6LChTfXNHkNczeVL63vvDAx784dZ2HxzVFZsKl3RSW9r2osEoZ+SO29DDWFG01V3i7luShrzWzMJA8TEk/yFWA8ywWXmH24iUqS1te9m5k9S2ZdQVCqG4+5bkdRtwD3AhsNXMBgCS12Jniol0EbVtiVnTEbiZTQV63H1v8vVvAF8C7gNWAjcnr/e2o0I9fWGVsuGKvDMx8xy89O2p9IQfPtJyvWTsGe22XTaFS8aeIiGUecA9Zna0/Lfc/Udm9ghwl5ldC2wCrulcNUU6Qm1bota0A3f3l4DgR7e7bweWd6JSIqNBbVtip5WYIiKRKn/uU2ZlYt5Og8ElpwyGmRvCyQMtxbyzByFDa6snj3Gv6vJ3ptKV+x8NylTmD6Sv2TLU2vNFGnTb6smRUDy/TiNwEZFIqQMXEYlU+SGUFgw/8WxL12268d1B3sk/SO9RVHvyuZbunSsn9JIXMslSyES6icIV3UsjcBGRSKkDFxGJlDpwEZFIlR8Dz061KzBlr3LG0jBzX7jTYHXza6n0oht/HpSp5U0bFBGJgHovEZFIqQMXEYlU+SGUIiGTuXNS6drGl8Lb5BwEYZW+dJnM2ZpFny8yFmg64NijEbiISKTUgYuIRKr8EEoB1W1vpNKVeeERhX/7qVODvIU3hbNOmhletzDI61n+apBXKDzTornr+1Ppbe/a1bZ7y/g1ljazatVYCyNpBC4iEil14CIikVIHLiISqShi4FnVreEh4Sf/eW9YroV758W787Qz5p2lmLeM1FiL9Uo+jcBFRCKlDlxEJFJRhlByz5oscghCgfMut/1eeOjD3P8RTkfsfftpqXTtmV82f77IKBntKYMK2ZRDI3ARkUipAxcRiZQ6cBGRSBWOgZtZL/Ao8Jq7f8jMZgHfBZYALwMfdvedx75D+1iPBXkebkYYKrDzYF68O49i3mNDN7XrMih2HbfjGYF/Gmg8sv16YJ27LwXWJWmR2KhdS7QKdeBmtgD4IHBbQ/bVwJrk6zXAirbWTKTD1K4ldkVDKLcA/xGY3pA3z92HANx9yMzCLQIBM1sFrAKYxJTWa9qgZ+aMMPNwuDKytm9f+rqJk3Julg7HDB84UKwO2XvlhXUOHw7zhj2bUeh5Wfs+cnGQN+07Dx73fXqnTQvyst+3osr+nrTgFlps15Bu24sG45yRqx0K49A7kJ/fdARuZh8Ctrn7Y6082N1Xu/syd1/Wx8RWbiHSdiNt15Bu23Nmh1s5iHRakWHDJcA/MrOrgEnADDP7JrDVzAaSUcoAEG5QItK91K4lek07cHf/PPB5ADO7DPicu/9zM/uvwErg5uT13s5VM214T/grft7mUj1T0iGb4f3721eHQwfbdq9WtBIuydNquCRP2d+T49GN7brTNOMkZhtzc0cyD/xm4Aoz2whckaRFYqd2LdE4rk9e3P0B4IHk6+3A8vZXSWR0qV1LrLQSU0QkUuXPfcrbITArM62s6GEKwweax2Rt2dnpjN5w6tuP7rkjyGs1nmi96dkKXguXkG66Mb0j4qIbj/9wZgg/A4Dwc4D9/zRnOuKLe8LrfvFsS3WQ7qEpg63p5s8ONAIXEYmUOnARkUiVH0LJ6JkcrpbM/tpvlb6gTO40wnPOSKV3nTUzKDPjzvVN63TVaZfk5LY2/S4vZJLVasgkq8i0ySnfD6cjjto6SIlWN4cVxhONwEVEIqUOXEQkUuWHUDIzTFpdLdkzYUKQl505MSPnQ/hsOCYvFFNktWLu83M2bmpF3r2DDaAoPjtHZKRimNEyHsI8GoGLiERKHbiISKTUgYuIRKr8GHgLcmO9veF+zL3Tp6fStb17i92rBe2Kd4/2vWX8GA8x4fFGI3ARkUipAxcRiVTXhVCymz1BuHrxtS+8Oyiz4L88EuRlQyZ20TnhvR96MpX+4ZYngjIfmP+O/MqKRKTVqX8KvXQvjcBFRCKlDlxEJFLqwEVEItV1MfAiu/UtXvNyeN0ZpwR5tadfSJfJxLvzKN4t44ni23HTCFxEJFLqwEVEIlV+CKXAmZiVk+am0jsvXRyU6T0YHkMw5enmj8/u9Dd8pBqUObDigiBv8j0PHfe9IdxFMG8laO+sE1Lpj64P/yLfWX5hkFfd/FrTOok0imFXwVaNh/CQRuAiIpFSBy4iEqnyQyje/ATG6tDrqfSMu7YHZfatOL/pfb63OTz/8ZoFFze9rki4JE+rm1DVduxMpb95+mBOKYVLpP3GQ9hhLGk6AjezSWb2sJn9wsyeMbObkvxZZrbWzDYmryc0u5dIN1HbltgVCaEcAt7n7ucC5wFXmtnFwPXAOndfCqxL0iIxUduWqDXtwL3u6KGQfckfB64G1iT5a4AVnaigSKeobUvsCsXAzawXeAw4FbjV3R8ys3nuPgTg7kNmNvcY164CVgFMYkpbKp23WnPq3WF8O+uaheEuhr0z04c+WCX8ltR27wnyDl6ZXrE58S8fbvp86T7tatuLBsv/OKkd2jmtUPH0zis0C8Xda+5+HrAAuNDMzir6AHdf7e7L3H1ZHxNbrKZIZ7Srbc+ZHW6DLNJpxzWN0N13AQ8AVwJbzWwAIHnd1u7KiYwWtW2JUdPf+8xsDnDE3XeZ2WTgcuDLwH3ASuDm5PXelmqQWYlZGZgXFKluGUpn5Ew9rF6+LMgb+tShVHrxxzYGZQ7e3Z9KT742DM/49nDa4sT/82iQl7Xxa+FqydOuSx8YceSy8NfMQ/3pf5Y3zw1/zi7+w58HeVbpS6Xbdd7nWNXxtj3Olb3KczyEcIoE7gaANUmssAe4y93/0szWA3eZ2bXAJuCaDtZTpBPUtiVqTTtwd38SCPZYdfftwPJOVEpkNKhtS+y0lF5EJFLlz33KxLNr295sekllXs6srvvDmPTC+9Pp6nvCwxomXJk+5KGa2S3wmApsAbD0d8Ophdm7V3Lqnf1HmXp3wSop5i0dMB5iybHSCFxEJFLqwEVEIlV+CCUjb5Vl77RpqfTwzl1BGesNF1Jk79XzsyeCMgd/Mz3Vb+L/1opKGb8ULomLRuAiIpFSBy4iEqnyQyjZMzFzZnd45pzKvIMSXvtCuFHV4B+HqxWzFDKRsUChj/FJI3ARkUipAxcRiZQ6cBGRSJUfAy+wonH40MFUumfipKBMXry7MufE9H327AvKZO8tEqN27fynWHpcNAIXEYmUOnARkUiVHkLJrqD0vM2kMmGWops2+dxZqfTm3zktKDP/y+nQy+ufCacjDnz1kSDv1d+/IJVefOuzQZnarl1FqinSNXQmZlw0AhcRiZQ6cBGRSKkDFxGJVOkx8LzdB1uSXZIP1J75ZSq9YOPLQZnsJMaTbgmnI+Yd8bDgP6fLDefshpind/r0VLq2d2+h60RiUySerjj5yGgELiISKXXgIiKRKj2Ekg199PSFVcruPpgXdrFKX5DXM3lqKl3b91YrNaS3vz/Iy04RzKvT1pwpifNyQjQi44HCJe2nEbiISKTUgYuIRKr8EEpmleVw5vAGCDevyjvQIW915qbr0qslB29uLXzR6opKhUukmyiEMfY0HYGb2UIz+4mZPWdmz5jZp5P8WWa21sw2Jq8ndL66Iu2jti2xKxJCqQKfdfe3ARcD15nZmcD1wDp3XwqsS9IiMVHblqg17cDdfcjdH0++3gs8BwwCVwNrkmJrgBUdqqNIR6htS+yOKwZuZkuAdwAPAfPcfQjq/xHMbO4xrlkFrAKYxJTmD8k54CF76ELelEGvhtcFMe+c1ZrBbog5sXT/9fOCvAPz0nH5Kd9/MCgj8Rhp2140WP7HSc20c6fBLMXXy1F4FoqZTQO+D3zG3fcUvc7dV7v7Mndf1sfEVuoo0lHtaNtzZhfbSkGknQp14GbWR72B3+nuP0iyt5rZQPL+ALCtM1UU6Ry1bYlZ09/7zMyArwPPuftXGt66D1gJ3Jy83tuRGuYoeqDDjk+mV0LOui2c1nf5hvSAa+1Z04Iy9jcbgrwCwSDpct3YtruRwiPdq0jg7hLg48BTZrYhyfsC9cZ9l5ldC2wCrulIDUU6R21bota0A3f3vwHsGG8vb291REaP2rbETkvpRUQiVfrcp2AaX86ufr3T0nHp2r59QZnK7NlBXhDzzplGmI155021+sCSC4K8vOX8IiKjSSNwEZFIqQMXEYlU6SGUQuxYnzP9ver27W151BW//Ykgr+fIhuYX5oRn8laVisSm1RWcmn7YeRqBi4hESh24iEikogih1PbuTWfkhCsqiwaDvOorr6Yz8kIa2TM5f7rheKsHwK6VFwV5J9zxcJCXN8tGZCwqEnpRmGVkNAIXEYmUOnARkUipAxcRiVTpMXAf9lS6smRxWOitt1LJ6htvBkWCeDeEsfKcGPjuj6dj1zP/Yv0xavoP6789vM7zYvVnLE2lq89vbOl5ImNBXpxccfHiNAIXEYmUOnARkUiVHkLJyg2FZEIflQU5UwY3vxbkVU5Jh2NqL28KyvTfmZ7q50GJEcgJ2ShkIt1M4Yu4aAQuIhIpdeAiIpFSBy4iEqnyY+BFduzLTMfLi3fnqb2yOf2onGXsw5e+M5Xu+enjhe4tIlI2jcBFRCKlDlxEJFLlh1CKyIRZrNIXFskJjxTZ+a9IyCT3edUjTa9rWd7hEEEFdFiEtF+rhzd0o/EwJVIjcBGRSKkDFxGJVPkhlEy4wHrC8y+zoZCeqVOCMrXdu4O8ypwTU+m8TbCC6vT2hs/PCZdUBk5K33vo9ab3LkzhEYnMeAhXdKOmI3Az+4aZbTOzpxvyZpnZWjPbmLye0NlqirSf2rbErkgI5Xbgykze9cA6d18KrEvSIrG5HbVtiVjTDtzdfwbsyGRfDaxJvl4DrGhvtUQ6T21bYtdqDHyeuw8BuPuQmc09VkEzWwWsAphEGLvOxnu9wJm/fvhwoUoWiXkH9y546HBbY97STVpq24sGy/84qRWKXcet47NQ3H21uy9z92V9TOz040RGTWPbnjM7/PBbpNNa7cC3mtkAQPK6rX1VEimV2rZEo9Xf++4DVgI3J6/3tq1GBfSc0B/k5YVVgumHEyYEZYYz19WWnx/eJ2dqY2Xto82qKXEqtW2PtlZXXir00h2KTCP8NrAeON3MNpvZtdQb9xVmthG4IkmLREVtW2LXdATu7h89xlvL21wXkVGlti2x01J6EZFIRTn3aXhXuGzecuLbfuBA+rqcOHl2p8G+9c+F9zkcLqXvnT+QSm+/bVpQZuZVOsBYxqYisXPFyTtPI3ARkUipAxcRiVTpIZTs7n+9s8K9g7IrKocPHAxvlLODX3Y3wtqOneFlmZ0Gix7UUN0ylErPvKrQZSJdRWGOuGkELiISKXXgIiKRKj2Ekl0tmbcBVXYFZd5skrxzJAsd4JCZhZIXQumdFs4wqe3b1/Teuc/LhIzyNs96659dnErv/rXw7zb/yz9v6fkijWI9A1OhnzqNwEVEIqUOXEQkUurARUQiVXoMPIhd50wHzMa8KwsGgzLVza+19PhszDvvUGObHsbAK7PT0x2rr7xa7HkFDoyYeveD6XShO4uURzHpcmgELiISKXXgIiKRKj+EkhMyybKLzkml3zgzDCrMuiM8OKXoqsp0dTzIq76ecyhLpt6906fnlAnvNZzZYCsvpFI5Y2kqveP8E4MyM+5cHz4vZyplWKd0vfNCRkXPBRU5KtbpiK3qlpCRRuAiIpFSBy4iEil14CIikSo/Bl5gGqE//HQqfcJDOWUKPKoyd06QV932Rir96g0XB2UW3tR82Xpt794CNSim+nz6IIgZzxc8GKLA5wnBJYp3S4Nuie1KMRqBi4hESh24iEikyg+hZH7tv+ypA0GRB86e3NKteyZOSqWrb2xvek1euKRnypQgLzhUooXwhUi3aXU6oEIv5dAIXEQkUurARUQiVX4IJTML5a8vnZ9TZncq2TN5UlCkZ1q4OnN4wdxUes/bZgRlclc0Zu+zf3/4vEx4ht7wZ2Hedas2vpRKr156StPn550TapXwny47oybvurxzQUVGKtaVmLGHfkY0AjezK83sBTN70cyub1elRMqmti0xaLkDN7Ne4FbgA8CZwEfN7Mx2VUykLGrbEouRjMAvBF5095fc/TDwHeDq9lRLpFRq2xKFkcTAB4HGUww2AxdlC5nZKmBVkjx0v9+dXlaZXULZfKYfvFUwb2sm/ViBe+c7EUifkHwwv2Az95+azXm8+UVFvif514X1jkOr9V7cpue31LZ7BzY+nS0TgfHWRjIKrnJun7a27ZF04JaTF6xod/fVwGoAM3vU3ZeN4JmlUL1HVxfUW227y6nedSMJoWwGFjakFwBbRlYdka6gti1RGEkH/giw1MxONrMJwEeA+9pTLZFSqW1LFFoOobh71cz+LfBjoBf4hrs/0+Sy1a0+r2Sq9+gqtd5q21FQvQHznGO/RESk+2kpvYhIpNSBi4hEalQ68JiWJZvZN8xsm5k93ZA3y8zWmtnG5DXcZKRkZrbQzH5iZs+Z2TNm9ukkv6vrbmaTzOxhM/tFUu+bkvyurvdRsbRttevRNVrtuuMdeITLkm8HrszkXQ+sc/elwLok3W2qwGfd/W3AxcB1yfe52+t+CHifu58LnAdcaWYX0/31jq1t347a9WganXbt7h39A7wL+HFD+vPA5zv93BHWeQnwdEP6BWAg+XoAeKHsOhb4O9wLXBFT3YEp1JemXhRDvWNr22rXpdW5Y+16NEIoecuSB0fhue00z92HAJLXuU3Kl8rMlgDvAB4igrqbWa+ZbQC2AWvdPYp6E3/bjuF7/HfUrkOj0YEXWpYs7WFm04DvA59x9z1l16cId6+5+3nUVzxeaGZnlVylotS2R4nadb7R6MDHwrLkrWY2AJC8biu5PrnMrI96I7/T3X+QZEdRdwB33wU8QD1WG0O9Y2/bMXyP1a7/AaPRgY+FZcn3ASuTr1dSj8N1FTMz4OvAc+7+lYa3urruZjbHzPqTrycDlwPP0+X1TsTetrv+e6x23cQoBfGvAn4J/Ar4YtkfKjSp67eBIeAI9RHWtcBs6p8Yb0xeZ5Vdz5x6/zr1X9+fBDYkf67q9roD5wBPJPV+Grghye/qejfUP4q2rXY96vUelXatpfQiIpHSSkwRkUipAxcRiZQ6cBGRSKkDFxGJlDpwEZFIqQMXEYmUOnARkUj9f3DP88PH5BLOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(1, 2, 1)\n",
    "plt.pcolormesh(example_tokens.to_tensor())\n",
    "plt.title('Token IDs')\n",
    "\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.pcolormesh(example_tokens.to_tensor() != 0)\n",
    "plt.title('Mask')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "e308ef79",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_text(context, target):\n",
    "  context = context_text_processor(context).to_tensor()\n",
    "  target = target_text_processor(target)\n",
    "  targ_in = target[:,:-1].to_tensor()\n",
    "  targ_out = target[:,1:].to_tensor()\n",
    "  return (context, targ_in), targ_out\n",
    "\n",
    "\n",
    "train_ds = train_raw.map(process_text, tf.data.AUTOTUNE)\n",
    "val_ds = val_raw.map(process_text, tf.data.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "d03a75b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  3   6   7   2 835 109 447  31  13 714]\n",
      "\n",
      "[ 5  4 17 68  2  3  7 64  8 25]\n",
      "[ 4 17 68  2  3  7 64  8 25  6]\n"
     ]
    }
   ],
   "source": [
    "for (ex_context_tok, ex_tar_in), ex_tar_out in train_ds.take(1):\n",
    "  print(ex_context_tok[0, :10].numpy()) \n",
    "  print()\n",
    "  print(ex_tar_in[0, :10].numpy()) \n",
    "  print(ex_tar_out[0, :10].numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "248b75ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "UNITS = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "41cdb67b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.layers.Layer):\n",
    "  def __init__(self, text_processor, units):\n",
    "    super(Encoder, self).__init__()\n",
    "    self.text_processor = text_processor\n",
    "    self.vocab_size = text_processor.vocabulary_size()\n",
    "    self.units = units\n",
    "\n",
    "    # The embedding layer converts tokens to vectors\n",
    "    self.embedding = tf.keras.layers.Embedding(self.vocab_size, units,\n",
    "                                               mask_zero=True)\n",
    "\n",
    "    # The RNN layer processes those vectors sequentially.\n",
    "    self.rnn = tf.keras.layers.Bidirectional(\n",
    "        merge_mode='sum',\n",
    "        layer=tf.keras.layers.GRU(units,\n",
    "                            # Return the sequence and state\n",
    "                            return_sequences=True,\n",
    "                            recurrent_initializer='glorot_uniform'))\n",
    "\n",
    "  def call(self, x):\n",
    "    shape_checker = ShapeChecker()\n",
    "    shape_checker(x, 'batch s')\n",
    "\n",
    "    # 2. The embedding layer looks up the embedding vector for each token.\n",
    "    x = self.embedding(x)\n",
    "    shape_checker(x, 'batch s units')\n",
    "\n",
    "    # 3. The GRU processes the sequence of embeddings.\n",
    "    x = self.rnn(x)\n",
    "    shape_checker(x, 'batch s units')\n",
    "\n",
    "    # 4. Returns the new sequence of embeddings.\n",
    "    return x\n",
    "\n",
    "  def convert_input(self, texts):\n",
    "    texts = tf.convert_to_tensor(texts)\n",
    "    if len(texts.shape) == 0:\n",
    "      texts = tf.convert_to_tensor(texts)[tf.newaxis]\n",
    "    context = self.text_processor(texts).to_tensor()\n",
    "    context = self(context)\n",
    "    return context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "b59ad247",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context tokens, shape (batch, s): (64, 25)\n",
      "Encoder output, shape (batch, s, units): (64, 25, 256)\n"
     ]
    }
   ],
   "source": [
    "# Encode the input sequence.\n",
    "encoder = Encoder(context_text_processor, UNITS)\n",
    "ex_context = encoder(ex_context_tok)\n",
    "\n",
    "print(f'Context tokens, shape (batch, s): {ex_context_tok.shape}')\n",
    "print(f'Encoder output, shape (batch, s, units): {ex_context.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "2d75f2d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CrossAttention(tf.keras.layers.Layer):\n",
    "  def __init__(self, units, **kwargs):\n",
    "    super().__init__()\n",
    "    self.mha = tf.keras.layers.MultiHeadAttention(key_dim=units, num_heads=1, **kwargs)\n",
    "    self.layernorm = tf.keras.layers.LayerNormalization()\n",
    "    self.add = tf.keras.layers.Add()\n",
    "\n",
    "  def call(self, x, context):\n",
    "    shape_checker = ShapeChecker()\n",
    "\n",
    "    shape_checker(x, 'batch t units')\n",
    "    shape_checker(context, 'batch s units')\n",
    "\n",
    "    attn_output, attn_scores = self.mha(\n",
    "        query=x,\n",
    "        value=context,\n",
    "        return_attention_scores=True)\n",
    "\n",
    "    shape_checker(x, 'batch t units')\n",
    "    shape_checker(attn_scores, 'batch heads t s')\n",
    "\n",
    "    # Cache the attention scores for plotting later.\n",
    "    attn_scores = tf.reduce_mean(attn_scores, axis=1)\n",
    "    shape_checker(attn_scores, 'batch t s')\n",
    "    self.last_attention_weights = attn_scores\n",
    "\n",
    "    x = self.add([x, attn_output])\n",
    "    x = self.layernorm(x)\n",
    "\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "0824d757",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context sequence, shape (batch, s, units): (64, 25, 256)\n",
      "Target sequence, shape (batch, t, units): (64, 19, 256)\n",
      "Attention result, shape (batch, t, units): (64, 19, 256)\n",
      "Attention weights, shape (batch, t, s):    (64, 19, 25)\n"
     ]
    }
   ],
   "source": [
    "attention_layer = CrossAttention(UNITS)\n",
    "\n",
    "# Attend to the encoded tokens\n",
    "embed = tf.keras.layers.Embedding(target_text_processor.vocabulary_size(),\n",
    "                                  output_dim=UNITS, mask_zero=True)\n",
    "ex_tar_embed = embed(ex_tar_in)\n",
    "\n",
    "result = attention_layer(ex_tar_embed, ex_context)\n",
    "\n",
    "print(f'Context sequence, shape (batch, s, units): {ex_context.shape}')\n",
    "print(f'Target sequence, shape (batch, t, units): {ex_tar_embed.shape}')\n",
    "print(f'Attention result, shape (batch, t, units): {result.shape}')\n",
    "print(f'Attention weights, shape (batch, t, s):    {attention_layer.last_attention_weights.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "3adb8f79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.        , 1.        , 1.        , 1.        , 1.        ,\n",
       "       0.99999994, 1.        , 1.        , 1.        , 0.99999994,\n",
       "       1.        , 1.        , 1.        , 1.        , 1.        ,\n",
       "       1.        , 1.        , 1.        , 1.        ], dtype=float32)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_layer.last_attention_weights[0].numpy().sum(axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "2c86b534",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAan0lEQVR4nO3dfbRddX3n8fcnNyHEhEASSEjCQ5BmUahIxAwigk+IRXQVnGlcZeg02LRRqw5SXRI7OEMtbUNnRrQdF06qSAYBQcQBu1wgzfA40ESwEKABA07kKU8kxCQ85uE7f+ydenK5N/fce/f+7XN+9/Na665zzj7n7N933/u93/u737MfFBGYmVleRjUdgJmZVc/F3cwsQy7uZmYZcnE3M8uQi7uZWYZc3M3MMuTiXjNJ35T05abj6Iuk0yQ90eZr3yvp2bpjMgOQdKekP2o6jm6WZXEvE+NFSWN7LV8j6QMtj2dJCkmjKxr3fEn3ti6LiE9GxF9Usf6qRcQ9EXFMFeuSdJWkS6tYl3WH8vfpdUkH91r+UPl7Nauh0IwMi3uZUKcBAfxOs9GYZe//AefueSDpeGBcc+HYHtkVd+APgH8CrgLm71ko6WrgCOBHkrZL+iJwd/n0lnLZO8vX/qGkVeXs/zZJR7asJyR9UtLq8vlvqHAs8E3gneW6tpSv32tGK+mPJT0pabOkWyTNGGjdvTdQ0v6SXtkzY5J0saSdkiaWjy+V9LXy/lhJ/03S05LWl22iceVze7VaJJ0o6Z8lbZP0fUnX956NS/q8pA2S1kr6eLlsIXAe8MVy239ULr9I0nPl+p6QdHr7P0brEldT/M7tMR/4X3seSPpwmVNbJT0j6ZKW5/aX9F1JmyRtkfRTSdN6DyBpuqSVkr5Q54ZkJyKy+gKeBP4EeDuwA5jW8twa4AMtj2dRzPBHtyw7p1zHscBo4GLgvpbnA/gH4CCKPxYbgTPL584H7u0Vz1XApeX99wMvACcCY4G/A+5uZ919bOfdwL8r7/8EeAr4UMtzHy3vfw24BZgMHAD8CPjr8rn3As+W9/cDfglcAIwB/i3wekvs7wV2Al8pnz8LeBmY1Hs7y8fHAM8AM1q+10c3nR/+qvR3bQ3wAeCJ8velp/yZH1nm8qwyb46nmEi+FVgPnFO+/xNlPr6pfO/bgYnlc3cCf1Su4+fAwqa3t9u+spq5SzqVIrFuiIgHKQrevx/kaj5BUfxWRcRO4K+AOa2zd2BxRGyJiKeBO4A5ba77PODKiPhZRLwGfIlipj9rCOu+C3hP+XnBW4G/LR/vD/wb4J5y1v/HwIURsTkitpXb83t9rO9kij9mfxsROyLiJmBFr9fsAL5SPv9jYDtFEe/LLoo/YMdJGhMRayLiqf6+MdbV9szezwAeB57b80RE3BkRj0TE7ohYCVwHvKd8egcwBfiNiNgVEQ9GxNaW9R5HUeT/S0QsSbAdWcmquFP8S/iTiHihfHwtLa2ZNh0JfL38N3ELsBkQMLPlNeta7r8MTGhz3TMoZscARMR2YNMQ130XxazoROAR4HaKX5qTgSfL78EhFLOiB1u259ZyeV+xPRfltKn0TK/XbCr/4A0YX0Q8CXwOuATYIOl7rS0oy8rVFJOo82lpyQBIeoekOyRtlPQr4JPAwS3vuw34nqTnJf2NpDEtbz+P4g/FjXVvQI6yKe5lH/ljFLPXdZLWARcCJ0g6oXxZ71Ng9nVKzGeAT0TEQS1f4yLivjbCGOgUm89T/PHYE/N4ipnLc/2+o3/3UcyaPwrcFRH/QtHK+TBF4YeiBfQK8Fst23JgRPRVkNcCM3v1+A8fRDxv2PaIuDYi9vw3FcBlg1ifdYmI+CXFB6tnATf1evpairbg4RFxIMXnUirftyMi/jwijgNOAT7C3v37Syhy+FpJPbVuRIayKe4UvfJdFP/KzSm/jgXu4dcJsx54c8t7NgK7ey37JvAlSb8FIOlASfPajGE9cJik/fp5/lrg45LmqNhN86+A5RGxps31/6uIeBl4EPg0vy7m91G0le4qX7Mb+HvgcklTy+2ZKem3+1jl/RTfv89IGi3pbOCkQYS01/dW0jGS3l9u56sUf2R2DWJ91l0WAO+PiJd6LT8A2BwRr0o6iZY2qaT3STq+LNxbKdo0rTmyA5gHjAeulpRTvapdTt+s+cB3IuLpiFi35wv4H8B5ZW/6r4GLyxbFF8oC+ZfA/y2XnRwRP6SYYX5P0lbgUeBDbcbwf4DHgHWSXuj9ZEQsA74M/IBipnw0ffe/23UXxYebK1oeH8Cv9wICuIjiA+J/KrfnH+mjTx4Rr1N8iLoA2AL8PsWHu6+1Gcu3KfrrWyT9b4p++2KKmdc6YCrwZ+1vmnWTiHgqIh7o46k/Ab4iaRvwn4EbWp47lKLlshVYRZG/3+213j15ORW40gW+fdq7xWr2a5KWA9+MiO80HYuZDY7/Ctq/kvQeSYeWbZn5FHvh3Np0XGY2eJUcdm/ZOIbi3+YJFLuR/m5ErG02JDMbCrdlzMwy5LaMmVmGkrZleiaMj9GTJ6cc0noZ+0zvPdXysY0XX4iIvg7Qqt3Bk3ti1uFjBn6h1ebnK9/UdAi1GUpuJy3uoydPZubnL0w5pPVy9IX3Nx1Cbf4xbvzlwK+qx6zDx7DitiOaGt6A355xwsAv6lJDyW23ZczMMuTibmaWoaRtGY3ZzejD8u351unIeSubDsGsFjm3U5rkmbuZWYZc3M3MMpR2V8itozjg1nZPfW6tNi84pekQOt+3fNrvbnTb8w83HULH65k++Pd45m5mliEXdzOzDLm4m5llqK2eu6SDgG8Bb6G4XNofUlzx/HqKq5OvAT4WES/uaz0Tp27nzM/eM/Rou9CKOb46WCerKrdHGu++mNrqQb+j3Zn714FbI+I3gRMorpqyCFgWEbOBZeVjs27j3LYsDVjcJU0E3k1xGTUi4vWI2AKcDSwtX7aU4hqmZl3DuW05a6ct82aKC0l/R9IJFBdlvgCYtudCDhGxds8FmHuTtBBYCDBu2gQe335oJYF3i4n39r1866kb0wZifakst4+YObKue7Ov3RfdsukM7bRlRgMnAldExNuAlxjEv6kRsSQi5kbE3LEHjRtimGa1qCy3D5niz1ass7RT3J8Fno2I5eXjGyl+IdZLmg5Q3m6oJ0Sz2ji3LVsD/i8ZEeskPSPpmIh4Ajgd+Jfyaz6wuLy9eaB1vfqrsTz+o9nDDDkTF+X7fZhx2X1Nh9CWKnPbfi3XI067rd3UbqPws8A1kvYDfgF8nGLWf4OkBcDTwLx6QjSrlXPbstRWcY+Ih4C5fTx1eqXRmCXm3LZc+QhVM7MMJd1/a9QBOxn/Hu8CuC8HnjX4I9HMukG39ay7nWfuZmYZcnE3M8tQ0rbMrl2j2PKSD2Taly3ff2ufy30NVet2/e0i6XZNPTxzNzPLkIu7mVmGXNzNzDKUtOc+af+XmTf7n1MO2ThfrMNy5D555/PM3cwsQy7uZmYZStqW2bx9Atfcf0rKIZt3RTPDzv7U8oFfZDZETZ750S2h9njmbmaWIRd3M7MMpb3w46iAcTuTDpm72ec/2HQIZrVw+2V4PHM3M8uQi7uZWYZc3M3MMpS05z56zC6mTt2acsiu44t1WK7cQ0/LM3czswy5uJuZZcjF3cwsQy7uZmYZcnE3M8uQi7uZWYba2hVS0hpgG7AL2BkRcyVNBq4HZgFrgI9FxIv7Ws/EMa9yxozHhxNv1/HFOjpbVbk90ni3xs43mJn7+yJiTkTMLR8vApZFxGxgWfnYrBs5ty07w2nLnA0sLe8vBc4ZdjRmncG5bV2v3SNUA/iJpAD+Z0QsAaZFxFqAiFgraWpfb5S0EFgIMPrASdx042kVhN1FLm46gPocdul9TYdQhUpy+4iZaU+w2rQmL9ZRt1xaTu1m5Lsi4vkyyW+X1HbjvPxlWQIwbsbhMYQYzepUSW7PPWF/57Z1lLbaMhHxfHm7AfghcBKwXtJ0gPJ2Q11BmtXFuW25GnDmLmk8MCoitpX3Pwh8BbgFmA8sLm9vHmhdMW43u07YPryIR6gj561sOoTsVJnbNnS5tEE6TTttmWnADyXtef21EXGrpJ8CN0haADwNzKsvTLNaOLctWwMW94j4BfCGP60RsQk4vY6gzFJwblvOfISqmVmGku6/NW7MDo6fvjblkB1r66kbmw7BrBbuoXcGz9zNzDLk4m5mlqGkbZlXto/l4Xtnpxyycy3urO/DUYuyONrUOkCnHb06UttEnrmbmWXIxd3MLEMu7mZmGUrac9dOGOezdHSkdX96Su1jHPpV9/Utvbo/A+jUnr5n7mZmGXJxNzPLUNorDIyCnW9KOqLVKJOLdZjtpVPbLIPlmbuZWYZc3M3MMpS0LbN7NLw2xVcja9LRF97fdAhmtcilnVIVz9zNzDLk4m5mliEXdzOzDCXtuU8+YDvnvu/elEM2bsWcnqZDMKuc+9udzzN3M7MMubibmWUoaVtm88vjueahk1IO2byr6l397PMfrHcAsz6kuCCHWz/D45m7mVmGXNzNzDLk4m5mlqG2e+6SeoAHgOci4iOSJgPXA7OANcDHIuLFfa5kt+CVtCeizN3qK97R5/LZn1qeOJLuVEleWy366+u7F9+ewczcLwBWtTxeBCyLiNnAsvKxWbdxXluW2irukg4DPgx8q2Xx2cDS8v5S4JxKIzOrmfPactbuzP1rwBeB3S3LpkXEWoDydmpfb5S0UNIDkh7YtX37cGI1q9rXGGJew965vXHTrloDNRusAYu7pI8AGyJiSDtUR8SSiJgbEXN7JkwYyirMKjfcvIa9c/uQKT7NhHWWdj7dfBfwO5LOAvYHJkr6LrBe0vSIWCtpOrChzkDNKua8tqwNWNwj4kvAlwAkvRf4QkT8vqT/CswHFpe3Nw842qiAcTuHEa715iNUh6bSvLZaeK+Y4RnOfu6LgTMkrQbOKB+bdTvntWVhUDudR8SdwJ3l/U3A6dWHZJaW89py5CNUzcwylPZwUR+hWjkfoWq58hGqw+OZu5lZhlzczcwylLRHcsTEF/j6B69KOWQylx99bNMhmFXOLZDu5Zm7mVmGXNzNzDKUtC3zzIsH8x+/vyDlkOl0+aEuRy26r+kQrAOluFZq3UZqa8kzdzOzDLm4m5llyMXdzCxDSXvu4ya8xgmnrk45ZMfaeurGpkMwq8VI7XF3Gs/czcwy5OJuZpahpG2Z13f38MzWg1IO2bl+fFCfiw88y20r624+4Vdn8MzdzCxDLu5mZhlycTczy1DSnvuMsVu45JhbUg6ZjM8KaTlyn7x7eeZuZpYhF3czswwlbcu8sOMAvr323SmHTGbivYN7vY9QtW4wlLNCupXTGTxzNzPLkIu7mVmGkrZl9u/ZwW9OWJdyyMatmNPTdAhmlXPrpfMNOHOXtL+kFZIelvSYpD8vl0+WdLuk1eXtpPrDNauOc9ty1k5b5jXg/RFxAjAHOFPSycAiYFlEzAaWlY/Nuolz27I1YHGPwvby4ZjyK4CzgaXl8qXAOXUEaFYX57blrK2eu6Qe4EHgN4BvRMRySdMiYi1ARKyVNLWf9y4EFgLsN2ESt/7dadVE3i0yvR54R/rWjYN+S1W5fcTMpB9fNS6HC2d3k57pg39PW3vLRMSuiJgDHAacJOkt7Q4QEUsiYm5EzB29//jBR2hWo6py+5Ap/uDcOsugdoWMiC3AncCZwHpJ0wHK2w1VB2eWinPbcjPg/5KSDgF2RMQWSeOADwCXAbcA84HF5e3NA61r4tTtnPnZe4YXcZfxrpCdq8rcHmm8K2Rqg7+ITzuNwunA0rI3OQq4ISL+QdL9wA2SFgBPA/MGPbpZs5zblq0Bi3tErATe1sfyTcDpdQRlloJz23Lm0w+YmWUo6f5br+4aw+PbD005ZOP6O1ukzwpp3Wxfu0K6H98ZPHM3M8uQi7uZWYaStmVe3zSWNd+ZnXLIzrXA34fKDeEIVauej16tXm1HqJqZWXdxcTczy1DStsyxMzay4i+uSDlk5bwngOXIed3pBn+EqmfuZmYZcnE3M8uQi7uZWYaS9txXvTKJkx/63ZRDVu/H9a7+wLMG31szG64Uuy+6r5+WZ+5mZhlycTczy1DStsyul0ezeeUhKYfsOpsXN/P9OWrRfY2MayNHU0eujtR2kGfuZmYZcnE3M8uQi7uZWYaS9tzfMmUjK/7Apx8w6zTO6/x45m5mliEXdzOzDCVtyzy3cxxf3vCWlENW7qSHdg3q9Svm9NQUiVl1Brubots4nc8zdzOzDLm4m5llKGlbZvO2CVx3x6kph2ze5U0HsLejL7y/6RAsA514nVS3ivY24Mxd0uGS7pC0StJjki4ol0+WdLuk1eXtpPrDNauOc9ty1k5bZifw+Yg4FjgZ+LSk44BFwLKImA0sKx+bdRPntmVrwOIeEWsj4mfl/W3AKmAmcDawtHzZUuCcmmI0q4Vz23I2qJ67pFnA24DlwLSIWAvFL4mkqf28ZyGwEGD0wQcy+rCXhhXwSHXkvJVNh5C14eb2ETOTfnyVFffK69H23jKSJgA/AD4XEVvbfV9ELImIuRExt2fi+KHEaFarKnL7kCk+nsE6S1vFXdIYiuS/JiJuKhevlzS9fH46sKGeEM3q49y2XLWzt4yAbwOrIuKrLU/dAswv788Hbq4+PLP6OLctZ+00Ct8F/AfgEUkPlcv+DFgM3CBpAfA0MK+WCM3q49y2bA1Y3CPiXkD9PH16teGYpePctpz59ANmZhlKuv9W7BavvzQm5ZDZWH3V2/tcPvv8BxNHYlat/k5l4F0kh8czdzOzDLm4m5llKO1hdbvEqBf3Szpk7p66/J2Der3PCmndwhcQGR7P3M3MMuTibmaWobRtmZ5g96TXkw6ZO+8tY7lym2V4PHM3M8uQi7uZWYZc3M3MMpS2575b8IovalCl1Ve8o8/lsz+1PHEkZtXykavD45m7mVmGXNzNzDKUtEei0bvZ7+BXUg6ZDV9D1XLlNks9PHM3M8uQi7uZWYZc3M3MMpS05z5uzA6On7425ZAda+upG5sOwawW7qF3Bs/czcwy5OJuZpahpG2ZV7aP5eF7Z6ccsnMt7qzvw1GL7ms6BMvEYC+yUbeR2ibyzN3MLEMu7mZmGUralolRsHtspBwyHfWzXaG0cQzA11C1XI3U9kt/Bpy5S7pS0gZJj7Ysmyzpdkmry9tJ9YZpVj3ntuWsnbbMVcCZvZYtApZFxGxgWfnYrNtchXPbMjVgcY+Iu4HNvRafDSwt7y8Fzqk2LLP6ObctZ0P9QHVaRKwFKG+n9vdCSQslPSDpgV3bXxricF0g1PdXfxR9f1nThpTbGzftShagWTtq31smIpZExNyImNszYXzdw5kl05rbh0zpaTocs70MtbivlzQdoLzdUF1IZo1yblsWhror5C3AfGBxeXtzO28atRPGbuqsXQOb08z34dmLT6lsXYddmuVRrUPKbWteVUfG5rJLZTu7Ql4H3A8cI+lZSQsoEv8MSauBM8rHZl3FuW05G3DmHhHn9vPU6RXHYpaUc9ty5tMPmJllKOnpB3aPhtemNLC7X5ecGiAFn37AcpVLr7wqnrmbmWXIxd3MLENJ2zIas5vRh2V8lGqNjpy3sukQzGrhdko9PHM3M8uQi7uZWYbSXqxj5yhef2FcyiGzsfqKdwzq9bM/tbymSMyqNdgjS93GaY9n7mZmGXJxNzPLkIu7mVmGkvbcJ0/YznnvzPJMgv1aMcfn+bb8uO/d+TxzNzPLkIu7mVmGkrZlNm+bwHV3nJpyyOZd3nQAe/OJw6wKVV0Yo0puFe3NM3czswy5uJuZZcjF3cwsQ0l77qN2wLh1I+8CGZ3k+Yuqu0B2f2ZcNrJ2d7XOUPfnAN3W0/fM3cwsQy7uZmYZStqWOWjyds45956UQzbOR6hajrqtRTESeeZuZpYhF3czswwlbcu8+KsJXH/raSmHbN7ipgNoz1GLvIeLta8Tj1Dtz0htIQ1r5i7pTElPSHpS0qKqgjJrmnPbut2Qi7ukHuAbwIeA44BzJR1XVWBmTXFuWw6GM3M/CXgyIn4REa8D3wPOriYss0Y5t63rDafnPhN4puXxs8AbruIsaSGwsHz42lMX/emjwxhzqA4GXmhg3CbHHtS4TzU4doWOqWg9Q8rtnumrR1Jud9Hv1OqGxq3UoHN7OMW9r/MIxBsWRCwBlgBIeiAi5g5jzCFpatwmxx6p21zVqvpY5tzugHGbHLvpbR7se4bTlnkWOLzl8WHA88NYn1mncG5b1xtOcf8pMFvSUZL2A34PuKWasMwa5dy2rjfktkxE7JT0GeA2oAe4MiIeG+BtS4Y63jA1NW6TY3ubh8i53dHjNjl2V22zIt7QSjQzsy7n0w+YmWXIxd3MLENJinuTh3JLWiPpEUkPVbirXH9jXSlpg6RHW5ZNlnS7pNXl7aRE414i6blyux+SdFYN4x4u6Q5JqyQ9JumCcnmKbe5v7Nq3u1cc2ed2U3m9j7Gzze1K8zoiav2i+EDqKeDNwH7Aw8BxdY/bMv4a4OBEY70bOBF4tGXZ3wCLyvuLgMsSjXsJ8IWat3c6cGJ5/wDg5xSH66fY5v7Grn27W2IYEbndVF7vY+xsc7vKvE4xcx8xh3JHxN3A5l6LzwaWlveXAuckGrd2EbE2In5W3t8GrKI4ujPFNvc3dkojIrebyut9jF27pnK7yrxOUdz7OpQ75S9hAD+R9GB5uHhq0yJiLRQ/OGBqwrE/I2ll+a9tLf827yFpFvA2YDmJt7nX2JBuu0dybjeZ1zACcnu4eZ2iuLd1KHeN3hURJ1Kc4e/Tkt6dcOwmXQEcDcwB1gL/va6BJE0AfgB8LiK21jVOm2Mn226c203JPreryOsUxb3RQ7kj4vnydgPwQ4p/pVNaL2k6QHm7IcWgEbE+InZFxG7g76lpuyWNoUjCayLipnJxkm3ua+xU210aybndSF5D/rldVV6nKO6NHcotabykA/bcBz4IpD5z3y3A/PL+fODmFIPuScDSR6lhuyUJ+DawKiK+2vJU7dvc39gptrvFSM7tRvIa8s7tSvO6zk+cWz4BPoviU9+ngP+UYsxy3DdT7MHwMPBY3WMD11H8y7SDYla3AJgCLKM47+gyYHKica8GHgFWUiTk9BrGPZWiDbESeKj8OivRNvc3du3b3SuO7HO7qbzex9jZ5naVee3TD5iZZchHqJqZZcjF3cwsQy7uZmYZcnE3M8uQi7uZWYZc3M3MMuTibmaWof8PxZb4bp3ebYEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "attention_weights = attention_layer.last_attention_weights\n",
    "mask=(ex_context_tok != 0).numpy()\n",
    "\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.pcolormesh(mask*attention_weights[:, 0, :])\n",
    "plt.title('Attention weights')\n",
    "\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.pcolormesh(mask)\n",
    "plt.title('Mask');\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "21272e03",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.layers.Layer):\n",
    "  @classmethod\n",
    "  def add_method(cls, fun):\n",
    "    setattr(cls, fun.__name__, fun)\n",
    "    return fun\n",
    "\n",
    "  def __init__(self, text_processor, units):\n",
    "    super(Decoder, self).__init__()\n",
    "    self.text_processor = text_processor\n",
    "    self.vocab_size = text_processor.vocabulary_size()\n",
    "    self.word_to_id = tf.keras.layers.StringLookup(\n",
    "        vocabulary=text_processor.get_vocabulary(),\n",
    "        mask_token='', oov_token='[UNK]')\n",
    "    self.id_to_word = tf.keras.layers.StringLookup(\n",
    "        vocabulary=text_processor.get_vocabulary(),\n",
    "        mask_token='', oov_token='[UNK]',\n",
    "        invert=True)\n",
    "    self.start_token = self.word_to_id('[START]')\n",
    "    self.end_token = self.word_to_id('[END]')\n",
    "\n",
    "    self.units = units\n",
    "\n",
    "\n",
    "    # 1. The embedding layer converts token IDs to vectors\n",
    "    self.embedding = tf.keras.layers.Embedding(self.vocab_size,\n",
    "                                               units, mask_zero=True)\n",
    "\n",
    "    # 2. The RNN keeps track of what's been generated so far.\n",
    "    self.rnn = tf.keras.layers.GRU(units,\n",
    "                                   return_sequences=True,\n",
    "                                   return_state=True,\n",
    "                                   recurrent_initializer='glorot_uniform')\n",
    "\n",
    "    # 3. The RNN output will be the query for the attention layer.\n",
    "    self.attention = CrossAttention(units)\n",
    "\n",
    "    # 4. This fully connected layer produces the logits for each\n",
    "    # output token.\n",
    "    self.output_layer = tf.keras.layers.Dense(self.vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "0e75f738",
   "metadata": {},
   "outputs": [],
   "source": [
    "@Decoder.add_method\n",
    "def call(self,\n",
    "         context, x,\n",
    "         state=None,\n",
    "         return_state=False):  \n",
    "  shape_checker = ShapeChecker()\n",
    "  shape_checker(x, 'batch t')\n",
    "  shape_checker(context, 'batch s units')\n",
    "\n",
    "  # 1. Lookup the embeddings\n",
    "  x = self.embedding(x)\n",
    "  shape_checker(x, 'batch t units')\n",
    "\n",
    "  # 2. Process the target sequence.\n",
    "  x, state = self.rnn(x, initial_state=state)\n",
    "  shape_checker(x, 'batch t units')\n",
    "\n",
    "  # 3. Use the RNN output as the query for the attention over the context.\n",
    "  x = self.attention(x, context)\n",
    "  self.last_attention_weights = self.attention.last_attention_weights\n",
    "  shape_checker(x, 'batch t units')\n",
    "  shape_checker(self.last_attention_weights, 'batch t s')\n",
    "\n",
    "  # Step 4. Generate logit predictions for the next token.\n",
    "  logits = self.output_layer(x)\n",
    "  shape_checker(logits, 'batch t target_vocab_size')\n",
    "\n",
    "  if return_state:\n",
    "    return logits, state\n",
    "  else:\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "6aa2bd8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder = Decoder(target_text_processor, UNITS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "ee4b4de7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "encoder output shape: (batch, s, units) (64, 25, 256)\n",
      "input target tokens shape: (batch, t) (64, 19)\n",
      "logits shape shape: (batch, target_vocabulary_size) (64, 19, 5000)\n"
     ]
    }
   ],
   "source": [
    "logits = decoder(ex_context, ex_tar_in)\n",
    "\n",
    "print(f'encoder output shape: (batch, s, units) {ex_context.shape}')\n",
    "print(f'input target tokens shape: (batch, t) {ex_tar_in.shape}')\n",
    "print(f'logits shape shape: (batch, target_vocabulary_size) {logits.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "18aebcb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "@Decoder.add_method\n",
    "def get_initial_state(self, context):\n",
    "  batch_size = tf.shape(context)[0]\n",
    "  start_tokens = tf.fill([batch_size, 1], self.start_token)\n",
    "  done = tf.zeros([batch_size, 1], dtype=tf.bool)\n",
    "  embedded = self.embedding(start_tokens)\n",
    "  return start_tokens, done, self.rnn.get_initial_state(embedded)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "3c0dc77a",
   "metadata": {},
   "outputs": [],
   "source": [
    "@Decoder.add_method\n",
    "def tokens_to_text(self, tokens):\n",
    "  words = self.id_to_word(tokens)\n",
    "  result = tf.strings.reduce_join(words, axis=-1, separator=' ')\n",
    "  result = tf.strings.regex_replace(result, '^ *\\[START\\] *', '')\n",
    "  result = tf.strings.regex_replace(result, ' *\\[END\\] *$', '')\n",
    "  return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "91ce9175",
   "metadata": {},
   "outputs": [],
   "source": [
    "@Decoder.add_method\n",
    "def get_next_token(self, context, next_token, done, state, temperature = 0.0):\n",
    "  logits, state = self(\n",
    "    context, next_token,\n",
    "    state = state,\n",
    "    return_state=True) \n",
    "\n",
    "  if temperature == 0.0:\n",
    "    next_token = tf.argmax(logits, axis=-1)\n",
    "  else:\n",
    "    logits = logits[:, -1, :]/temperature\n",
    "    next_token = tf.random.categorical(logits, num_samples=1)\n",
    "\n",
    "  # If a sequence produces an `end_token`, set it `done`\n",
    "  done = done | (next_token == self.end_token)\n",
    "  # Once a sequence is done it only produces 0-padding.\n",
    "  next_token = tf.where(done, tf.constant(0, dtype=tf.int64), next_token)\n",
    "\n",
    "  return next_token, done, state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "6fe70ffe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([b'representative heart fresno kathy poll commonwealth fabian dec team male',\n",
       "       b'the martinez cf after im giant ice grizzlies saints contested',\n",
       "       b'nashville boysgirls winstonsalem clubs dcsf elector anthony ga andretti king'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Setup the loop variables.\n",
    "next_token, done, state = decoder.get_initial_state(ex_context)\n",
    "tokens = []\n",
    "\n",
    "for n in range(10):\n",
    "  # Run one step.\n",
    "  next_token, done, state = decoder.get_next_token(\n",
    "      ex_context, next_token, done, state, temperature=1.0)\n",
    "  # Add the token to the output.\n",
    "  tokens.append(next_token)\n",
    "\n",
    "# Stack all the tokens together.\n",
    "tokens = tf.concat(tokens, axis=-1) # (batch, t)\n",
    "\n",
    "# Convert the tokens back to a a string\n",
    "result = decoder.tokens_to_text(tokens)\n",
    "result[:3].numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "92baf381",
   "metadata": {},
   "outputs": [],
   "source": [
    "class NLtoSQL(tf.keras.Model):\n",
    "  @classmethod\n",
    "  def add_method(cls, fun):\n",
    "    setattr(cls, fun.__name__, fun)\n",
    "    return fun\n",
    "\n",
    "  def __init__(self, units,\n",
    "               context_text_processor,\n",
    "               target_text_processor):\n",
    "    super().__init__()\n",
    "    # Build the encoder and decoder\n",
    "    encoder = Encoder(context_text_processor, units)\n",
    "    decoder = Decoder(target_text_processor, units)\n",
    "\n",
    "    self.encoder = encoder\n",
    "    self.decoder = decoder\n",
    "\n",
    "  def call(self, inputs):\n",
    "    context, x = inputs\n",
    "    context = self.encoder(context)\n",
    "    logits = self.decoder(context, x)\n",
    "\n",
    "    #TODO(b/250038731): remove this\n",
    "    try:\n",
    "      # Delete the keras mask, so keras doesn't scale the loss+accuracy. \n",
    "      del logits._keras_mask\n",
    "    except AttributeError:\n",
    "        pass\n",
    "    return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "efe30e78",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context tokens, shape: (batch, s, units) (64, 25)\n",
      "Target tokens, shape: (batch, t) (64, 19)\n",
      "logits, shape: (batch, t, target_vocabulary_size) (64, 19, 5000)\n"
     ]
    }
   ],
   "source": [
    "model = NLtoSQL(UNITS, context_text_processor, target_text_processor)\n",
    "\n",
    "logits = model((ex_context_tok, ex_tar_in))\n",
    "\n",
    "print(f'Context tokens, shape: (batch, s, units) {ex_context_tok.shape}')\n",
    "print(f'Target tokens, shape: (batch, t) {ex_tar_in.shape}')\n",
    "print(f'logits, shape: (batch, t, target_vocabulary_size) {logits.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "e77785a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def masked_loss(y_true, y_pred):\n",
    "    # Calculate the loss for each item in the batch.\n",
    "    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(\n",
    "        from_logits=True, reduction='none')\n",
    "    loss = loss_fn(y_true, y_pred)\n",
    "\n",
    "    # Mask off the losses on padding.\n",
    "    mask = tf.cast(y_true != 0, loss.dtype)\n",
    "    loss *= mask\n",
    "\n",
    "    # Return the total.\n",
    "    return tf.reduce_sum(loss)/tf.reduce_sum(mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "7e4282e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def masked_acc(y_true, y_pred):\n",
    "    # Calculate the loss for each item in the batch.\n",
    "    y_pred = tf.argmax(y_pred, axis=-1)\n",
    "    y_pred = tf.cast(y_pred, y_true.dtype)\n",
    "\n",
    "    match = tf.cast(y_true == y_pred, tf.float32)\n",
    "    mask = tf.cast(y_true != 0, tf.float32)\n",
    "\n",
    "    return tf.reduce_sum(match)/tf.reduce_sum(mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "75841db6",
   "metadata": {},
   "outputs": [],
   "source": [
    "@NLtoSQL.add_method\n",
    "def translate(self,\n",
    "              texts, *,\n",
    "              max_length=50,\n",
    "              temperature=0.0):\n",
    "  # Process the input texts\n",
    "  context = self.encoder.convert_input(texts)\n",
    "  batch_size = tf.shape(texts)[0]\n",
    "\n",
    "  # Setup the loop inputs\n",
    "  tokens = []\n",
    "  attention_weights = []\n",
    "  next_token, done, state = self.decoder.get_initial_state(context)\n",
    "\n",
    "  for _ in range(max_length):\n",
    "    # Generate the next token\n",
    "    next_token, done, state = self.decoder.get_next_token(\n",
    "        context, next_token, done,  state, temperature)\n",
    "\n",
    "    # Collect the generated tokens\n",
    "    tokens.append(next_token)\n",
    "    attention_weights.append(self.decoder.last_attention_weights)\n",
    "\n",
    "    if tf.executing_eagerly() and tf.reduce_all(done):\n",
    "      break\n",
    "\n",
    "  # Stack the lists of tokens and attention weights.\n",
    "  tokens = tf.concat(tokens, axis=-1)   # t*[(batch 1)] -> (batch, t)\n",
    "  self.last_attention_weights = tf.concat(attention_weights, axis=1)  # t*[(batch 1 s)] -> (batch, t s)\n",
    "\n",
    "  result = self.decoder.tokens_to_text(tokens)\n",
    "  return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "1858a28b",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='adam',\n",
    "              loss=masked_loss, \n",
    "              metrics=[masked_acc, masked_loss])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "3c505079",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'expected_loss': 8.517193, 'expected_acc': 0.0002}"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_size = 1.0 * target_text_processor.vocabulary_size()\n",
    "\n",
    "{\"expected_loss\": tf.math.log(vocab_size).numpy(),\n",
    " \"expected_acc\": 1/vocab_size}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "1febfdca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20/20 [==============================] - 16s 219ms/step - loss: 8.5416 - masked_acc: 7.4627e-05 - masked_loss: 8.5416\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'loss': 8.541553497314453,\n",
       " 'masked_acc': 7.462686335202307e-05,\n",
       " 'masked_loss': 8.541553497314453}"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(val_ds, steps=20, return_dict=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "c94dd977",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/25\n",
      "100/100 [==============================] - 52s 519ms/step - loss: 0.3880 - masked_acc: 0.9078 - masked_loss: 0.3880 - val_loss: 0.6691 - val_masked_acc: 0.8646 - val_masked_loss: 0.6691\n",
      "Epoch 2/25\n",
      "100/100 [==============================] - 70s 700ms/step - loss: 0.4139 - masked_acc: 0.9025 - masked_loss: 0.4139 - val_loss: 0.6886 - val_masked_acc: 0.8602 - val_masked_loss: 0.6886\n",
      "Epoch 3/25\n",
      "100/100 [==============================] - 76s 758ms/step - loss: 0.4176 - masked_acc: 0.9030 - masked_loss: 0.4176 - val_loss: 0.6608 - val_masked_acc: 0.8686 - val_masked_loss: 0.6608\n",
      "Epoch 4/25\n",
      "100/100 [==============================] - 79s 791ms/step - loss: 0.4215 - masked_acc: 0.9024 - masked_loss: 0.4215 - val_loss: 0.6514 - val_masked_acc: 0.8652 - val_masked_loss: 0.6514\n",
      "Epoch 5/25\n",
      "100/100 [==============================] - 83s 833ms/step - loss: 0.4326 - masked_acc: 0.8978 - masked_loss: 0.4326 - val_loss: 0.6698 - val_masked_acc: 0.8630 - val_masked_loss: 0.6698\n",
      "Epoch 6/25\n",
      "100/100 [==============================] - 78s 787ms/step - loss: 0.4203 - masked_acc: 0.9013 - masked_loss: 0.4203 - val_loss: 0.6568 - val_masked_acc: 0.8716 - val_masked_loss: 0.6568\n",
      "Epoch 7/25\n",
      "100/100 [==============================] - 80s 800ms/step - loss: 0.4200 - masked_acc: 0.9025 - masked_loss: 0.4200 - val_loss: 0.6360 - val_masked_acc: 0.8751 - val_masked_loss: 0.6360\n",
      "Epoch 8/25\n",
      "100/100 [==============================] - 80s 800ms/step - loss: 0.3037 - masked_acc: 0.9231 - masked_loss: 0.3038 - val_loss: 0.6158 - val_masked_acc: 0.8751 - val_masked_loss: 0.6158\n",
      "Epoch 9/25\n",
      "100/100 [==============================] - 80s 795ms/step - loss: 0.3036 - masked_acc: 0.9235 - masked_loss: 0.3036 - val_loss: 0.6663 - val_masked_acc: 0.8707 - val_masked_loss: 0.6663\n",
      "Epoch 10/25\n",
      "100/100 [==============================] - 84s 845ms/step - loss: 0.3197 - masked_acc: 0.9185 - masked_loss: 0.3197 - val_loss: 0.6588 - val_masked_acc: 0.8695 - val_masked_loss: 0.6588\n",
      "Epoch 11/25\n",
      "100/100 [==============================] - 81s 810ms/step - loss: 0.3261 - masked_acc: 0.9194 - masked_loss: 0.3261 - val_loss: 0.6471 - val_masked_acc: 0.8745 - val_masked_loss: 0.6471\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(\n",
    "    train_ds.repeat(), \n",
    "    epochs=25,\n",
    "    steps_per_epoch = 100,\n",
    "    validation_data=val_ds,\n",
    "    validation_steps = 20,\n",
    "    callbacks=[\n",
    "        tf.keras.callbacks.EarlyStopping(patience=3)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "84ba88e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x22b9d7dc0d0>"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEICAYAAABS0fM3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAp0ElEQVR4nO3deXxV9Z3/8dcne0JIAoQQIOygiGxqwG2Ka6tOVWqtFddqHf2h1bb+fjpq+2jHaWv3dWawjKO4jLZq1c4wo9W21krdCRTEpSIgSFgT9iUh2+f3xzkhl5ANyMlNct7Px+M+7tnuvZ8b8fs+53vO+V5zd0REJL5Skl2AiIgkl4JARCTmFAQiIjGnIBARiTkFgYhIzCkIRERiLtIgMLNzzewDM1thZne2sP52M1sSPt4xs3oz6x9lTSIiciCL6j4CM0sFlgOfBMqBhcBl7v5eK9tfANzq7me29b6FhYU+cuTITq5WRKR3W7RoUaW7D2xpXVqEnzsdWOHuqwDM7HFgJtBiEACXAb9u701HjhxJWVlZpxUpIhIHZramtXVRdg0NBdYmzJeHyw5iZjnAucDTEdYjIiItiDIIrIVlrfVDXQC86u5bW3wjsxvMrMzMyioqKjqtQBERiTYIyoFhCfMlwPpWtp1FG91C7n6fu5e6e+nAgS12cYmIyGGK8hzBQmCcmY0C1hE09pc338jM8oHTgCsjrCU5avbAxndg49vQZyCMPh2yC5JdlYjIASILAnevM7ObgReAVGCeu79rZrPD9XPDTS8Cfu/ue6KqpUvs2wUb3oYNS2HDkuC5cjl4Q9M2lgol02Ds2TD2LBg8FVJ0K4eIJFdkl49GpbS01JN+1VDV9mAvf/2SpoZ/y0r2nwLpOzho5AdPgSFToXgS7CiHFX8MHuv/GmyXMwDGnBkEw5gzIbcoGd9GRGLAzBa5e2mL6xQE7di7NWjoExv9baub1ucPCxr8xoZ/8BToO6jt99xdAateghUvwsoXYU94AnzwFBhzVhAMw6ZDanokX0lE4kdB0FG7NweN/folYffO27Dj46b1/UY2NfaNDX+fwiP7zIaG4OhixR+DYFj7Jng9ZPSF0ac1dSMVDD+yzxGRWFMQNOcOuzaEe/hLm/b2dyVc1NR/TNCts7/RnwzZ/Y7sczuiegd8tKApGHaEt2IUHtUUCiNOhfTs6GsRkV5DQQBQ8QEs+01To79nc7jCgkZ2f6M/BYonQ1ZeJ1Z9mNyDE86N5xZWvwr1+yAtKwiDsWcHj8JxYC3dthETNXtg8/vBkdXGd4IwLRgO/UYER3H9RkJeCaRGeZGcSPemIAD44Hfw+BUwcHzTSdzBU2DQRMjM7fQ6I1GzF9a81hQMWz4MlucPD44Uxp4Fo07rHiEWhcYjucZLcje9E0xvWcH+E/WZeZDTPzg531DX9FpLhfySpnAoSAiJfiODE/dxDtOeZO9WeOs+2LUx+G+d3T/475cTPmf3C56z8vXfNIGCAKBuX3ApZ2/qUtm2JjjZvOJFWPVnqNkNKWkw7MQwGM6GQZN65iWq9bXBUdymd2DjsqZHVcLN5wUjgiuyiicFgV48MVhmBg31sHNd8Dfathq2h8+N8/uPCEPpfcJQSAiIxrAoGA4ZOV30xaVVVdvhjXvh9XuDf+s5A6BqW3BOrSWW2hQKB4RE43RCgDROZxdASmpXfqsuoyCIg7oaKH+r6Whh47JgeWY+5A2G3EHho6hpuu+gpunsfsnbe9q7NaHBfwc2LYPNf4OG2mB9WhYUHRM29pODBn/QscEe3+Gq2QPbPz4wHPaHxWqo3Xvg9rmDEo4imoVF3pBe23h0C/t2wRtz4fV/Dbr9jrkATr8r+DfQ0AD7dgY7CHsbH1sS5re0vK6+ppUPsyAMsvs3C4z+TYHRZyDkDw2OxHP695ijDgVBHO3aBCv/BOvKgkPo3Zth96bgUVd98PYp6c2CoujgsGhcdrhHVQ0NsO2jpr37xq6dneVN2+QOatq7L54cTA8Y27X9++6wpzIhHD5KCIw1Qb2JNwqmpEPBsCAYiifD0BNg6PGQN7THNBLdUs2eoAvo1X8JGu+jzoMz7gq6dI+Ee/DezUOiKgyK5tON8813DgDSsoMux4JhwXP+sPBREjzyhkJaxpHV20kUBNLEPdiDSgyG3ZubhUX4vKeCFscJzMw/MDD6Fh8cIDmFQT/9pmUJe/rvQm14A7mlBifpiyeGDX/YxdMTbqqrrw2u5tqWcASxfU1wU+Hm95uOZHIHNYXCkOOD56648qynq62Csnnwys+Cf4Njz4Yzvhb8LZNd196tQbfijnXBv+8da4PH9rXBfPMuRyz4/yMxHAqGN03nD+uycxkKAjk89XWwt7JZWGxqFhhhgNTsbv19MvMO7McvngQDj4H0rK77Ll2ltjo40lm3GNYtCh6NJ/UhuCy5MRyGnhD8LXrTeasjUbcPFj0Mf/lJ8O9q1Glwxtdh+InJrqzjaquDc1M7wmBoDIgdH4fP5Qd3S2X0bXZUURJ0OzUuyy3ulCNiBYFEb9/uYG+oMTD2VARDbRRPCvaA4txFUrU9uEFx3aIgIMrLgoYOgpP7g44NjxhOCB4Dj47XOYe6GljyKCz4cdCIDj8Fzvw6jPy7ZFfW+Roagv83mofD9rVNRxdV2w58jaUG56Hyh8HUy+D4qw/ro9sKAl1YLZ0jMzd49B+d7Eq6n+yCYOTZ0ac3Ldu5/sCjhneehkUPBuvS+wSXNw9NCIf8Yb0vTOtrYenjsOCHwYn7kmkwc07wd+pt37VRSkpw3q3vIChppatr3+4gEPeHQ3nTc92+SMrSEYFId9DQENwPsT4hHDYua+pGyClsCoXGrqWc/smt+XA11Ac3d/75+8GJ+CHHBV1AY8/uvQHQDeiIQKS7S0mBgUcFjymzgmV1+4IT7I1dSusWwYe/Z/8J/H4jg1Bo7FYaPBky+iTrG7SvoQHe+20QAJXLg3tcZv0ajj5PAZBkCgKR7iotM+weOr5pWfXOhPMNi+DjN4NuJQBLgcKjgz3sIccFrxs0Mfkn5Rsa4G//C3/+Hmx+L7hQ4POPwPgLeubNjr2QgkCkJ8nKg1EzgkejXRuD37hofHz4e1j6q2BdSlpwM15jOAw5DoqO7Zpr291h+fPw0j1BN9eAcXDxA3DsZxUA3YzOEYj0Nu7BycbGYFi3OHiu3h6sT80IjhQSw2Hg+M67ac89GPbkpXuCcx79RsHpd8LEz2ngvyTSOQKRODFruh79mAuCZe7BjW+JRw7LfgNlDwTr07KDS32HHt8UDgPGHtplrO7w0cvw0neD39XIHw4X/ltwzkM/stSt6YhAJK4aGmDrqgPDYcPSpru/M3LDkXoTjhz6j275xO6a1+BP98CaV4JhFWbcBlOv7DbDK4iOCESkJSkpUDg2eEy+JFjWUB9c0ZMYDgvvbxqfKis/+KGmxmDIyodXfx6MfptbDOf9CE74QnCiW3oMBYGINElJDU4uFx0DUy8PltXXQsXfDgyH1+c0janUZyCc810o/aKGy+ihFAQi0rbU9KZBARuHN2i8x2HH2uBGsO58/4K0K9JruMzsXDP7wMxWmNmdrWxzupktMbN3zezlKOsRkU7SeI/DhJkKgV4gsiMCM0sF5gCfBMqBhWY2393fS9imALgXONfdPzazHjAGsYhI7xLlEcF0YIW7r3L3GuBxYGazbS4HnnH3jwHcvflg3iIiErEog2AosDZhvjxclugooJ+Z/dnMFplZi+OrmtkNZlZmZmUVFRURlSsiEk9RBkFLo0g1v2khDTgB+DRwDvANMzvqoBe53+fupe5eOnDgwM6vVEQkxqK8aqgcGJYwXwKsb2GbSnffA+wxswXAFGB5hHWJiEiCKI8IFgLjzGyUmWUAs4D5zbb5b+ATZpZmZjnAicD7EdYkIiLNRHZE4O51ZnYz8AKQCsxz93fNbHa4fq67v29mzwNvAw3A/e7+TlQ1iYjIwTTWkIhIDLQ11pAGBRcRiTkFgYhIzCkIRERiTkEgIhJzCgIRkZhTEIiIxJyCQEQk5hQEIiIxpyAQEYk5BYGISMwpCEREYk5BICIScwoCEZGYUxCIiMScgkBEJOYUBCIiMacgEBGJOQWBiEjMKQhERGJOQSAiEnMKAhGRmFMQiIjEXKRBYGbnmtkHZrbCzO5sYf3pZrbDzJaEj29GWY+IiBwsLao3NrNUYA7wSaAcWGhm8939vWab/sXdz4+qDhERaVuURwTTgRXuvsrda4DHgZkRfp6IiByGKINgKLA2Yb48XNbcyWa21Mx+Z2bHtvRGZnaDmZWZWVlFRUUUtYqIxFaUQWAtLPNm84uBEe4+BfhX4L9aeiN3v8/dS929dODAgZ1bpYhIzEUZBOXAsIT5EmB94gbuvtPdd4fTzwHpZlYYYU0iItJMlEGwEBhnZqPMLAOYBcxP3MDMis3MwunpYT1bIqxJRESaieyqIXevM7ObgReAVGCeu79rZrPD9XOBzwE3mlkdUAXMcvfm3UciIhIh62ntbmlpqZeVlSW7DBGRHsXMFrl7aUvrdGexiEjMKQhERGJOQSAiEnMKAhGRmFMQiIjEnIJARCTmFAQiIjGnIBARiTkFgYhIzCkIRERiTkEgIhJzCgIRkZhTEIiIxJyCQEQk5iL7PQKRqDQ0OJV79rFhezUbdlSzYUcVG3ZUs357FRt3VLOvroGhBdmU9MtmaL9sSvrl7J/Oy0pPdvki3Y6CQLoVd2fLnho2hg170NCHjf32atbvqGLTzmpq6w/8HY2MtBQG52cxOD+LnMwMVlTs5s/LN1Nd23DAdnlZaQcEQ+N0Sb9sSgpyyMtOI/zRPJHYUBBIl3F3tu+tZX3YqG/YWc2G/Y19U6NfU3dg452eahTnZzE4P5sTRvRjcH42QwqC+cbGv3+fjIMa8MZQWbetivJtVZRv28u67cH06i17eGVFJXtr6g94Td/MtDAgDgyJoQXBdEFOuoJCeh0FgRyS2voGqmrrqa6pp6o2fITT1bX17K1pmt+8c1+zRr7qoD301BSjOC9ozCeXFHDOsVlh4x409sX5WRT2ySQl5dAbXzOjMDeTwtxMpgwrOGh9YzA1D4nybXsp31bFG6u2sntf3QGv6ZORetCRRGNIDCnIJiMtBTOwhBoMwmVGY4Y0zjdNN9tWYSNdSEHQSzQ0OPvqGvY3yI3P1bX1VNU07G+0q2vq2VtTR1VtwrbNG/WEhr15Q9+8S6YtKQZFfbMYXJDFhMF5nDW+iOL8LIYUBHvyQwqyKczNJPUwGvnOYGb065NBvz4ZTCrJP2i9u7Ozqo7y7XvDgAgDI5wuW72VndV1LbxzZ9YYPnNwqLA/QODvxg7k3iuOJyNN13/IoVMQdIHq2nq27a2hqqae6rAB3re/sW44oNGubr68pp7quvoDXtu0bdP8vmbdKR2VlZ5Cdnoq2empZGWkkpMRTPfNSqOobybZ4XxWeur+6ZyMcD5hWeL6xuX9ctJJS+25DZOZkZ+TTn5OPscOOTgoAHZW1+4Phg07qvZ3azX+FLjjuIOHyxrng20Spput9/BNvJ33woMafv3WWu56Zhk/vmSyjibkkHU4CMzsFGBk4mvc/ZEIaurxauoaWFq+nVdXVPLaii38de22Q9qTTk2xsHFNISu9qdHNSk+hb1YaA/tm7p9vbISbtkvZ3yhnpgXb5GSkNTXaCY11ZlrKYXW5SJO8rHTyBqdzzOC8pNZR1DeLX7z4IaMH9uFLZ4xNai3S83QoCMzsP4ExwBKg8eyaA20GgZmdC/wCSAXud/fvt7LdNOAN4FJ3f6pDlXcjDQ3Oext28trKSl5buYW3PtrK3pp6zGDikHy++HejGDmgTyuNe+PeeMr+6fQevBctyfHVs8exessefvTCB4wYkMP5k4ckuyTpQTp6RFAKTHD3Du/WmlkqMAf4JFAOLDSz+e7+Xgvb/QB4oaPvnWzuzuote4M9/pWVvL5yC9v21gIwZmAfLj6+hFPHDuCk0QMoyMlIcrUSB2bGDy6ezLptVfzfJ5cypCCb44f3S3ZZ0kN0NAjeAYqBDYfw3tOBFe6+CsDMHgdmAu812+4W4Glg2iG8d5fbtLOa11ZW8uqKLby2opL1O6oBGJyfxZnjB3Hq2AGcMqaQ4vysJFcqcZWVnsq/X3UCF937Gjc8UsZvbzqVYf1zkl2W9AAdDYJC4D0zewvY17jQ3S9s4zVDgbUJ8+XAiYkbmNlQ4CLgTLpZEOzYW8vrq7aEjX8lKyv2AFCQk87Jowdw4xmFnDpmAKMK++jknHQbA3IzmXfNNC6691Wue3ghT914iu6mlnZ1NAjuPoz3bql1bN619HPgDnevb6sxNbMbgBsAhg8ffhiltK+qpp6yNVuDPf6VlbyzbgcNDtnpqUwb1Z/Plw7j1LGFTBicpxOs0q2NLcpl7pUn8IV5b/Glxxbz4DXTevTVWxI962i3v5mNAMa5+x/NLAdIdfddbWx/MnC3u58Tzt8F4O7fS9jmI5oCoxDYC9zg7v/V2vuWlpZ6WVlZh2puS119A0vLd/DaikpeXVnJ4jXbqalvIC3FOG54AaeMKeTUsYVMHVaga7OlR3pi4cfc8fQyrjhxON/5zEQducacmS1y99KW1nX0qqHrCfbI+xNcPTQUmAuc1cbLFgLjzGwUsA6YBVyeuIG7j0r4jIeA/20rBI5EQ4PzwaZd4Qne4MqexrtGJwzO4wunjOCUsYVMH9mfPpm6vUJ6vkunDeejyr3MfXklowr78A+fGJ3skqSb6miL9yWCk79vArj7h2ZW1NYL3L3OzG4muBooFZjn7u+a2exw/dzDL/vQPb24nNufehuAkQNyuHDqEE4dU8jJYwbQv4+u7JHe6R/POZrVlXu457n3GTGgD5+cMCjZJUk31KGuITN7091PNLO/uvtxZpYGLHb3ydGXeKDD7RratLOal5dXcOrYQoYWZEdQmUj3VFVTz6X3vc6Hm3bzm9knM3Foy3dJS+/WVtdQRzu/XzazrwHZZvZJ4DfA/3RWgV1hUF4Wny8dphCQ2MnOSOX+q0vpl5POdQ8vZGN46bNIo44GwZ1ABbAM+D/Ac+7+9ciqEpFOVZSXxQPXTGPPvnque3ghe/ZFO1ie9CwdDYK73f0/3P0Sd/8cMM/MHouyMBHpXMcMzuNfLz+O9zfs5CuP/5X6ho6PfyW9W0eDYHjj5Z9mlgE8A3wYWVUiEokzji7i7guP5Y/vb+a7z72f7HKkm+joVUPXAo+FYXAG8Dt3/1l0ZYlIVK4+eSSrKvbwwCsfMbKwD1edNCLZJUmStRkEZnZ8wuwvgH8HXiU4eXy8uy+OsjgRicY3zp/Ax1v3cvf8dxneP4fTjhqY7JIkidq8fNTMXmrjte7uZ3Z+SW3rrDuLReJu9746Lpn7OuVb9/LUjadwdHHfZJckEWrr8tEODzHRXSgIRDrP+u1VfGbOq6SnpvBfXzqVgX0zk12SROSI7yMws3wz+6mZlYWPn5iZ7koR6eGGFGTzwBemsXVPDf/wSBnVtfXtv0h6nY5eNTQP2AV8PnzsBB6MqigR6TqTSvL5+aypvF2+nf/35FIadFlp7HQ0CMa4+z+5+6rw8c+ARrAS6SXOObaYu84bz7PLNvCTP3yQ7HKki3U0CKrM7O8aZ8zsVKAqmpJEJBmu/8RoLps+jDkvreQ3ZWvbf4H0Gh29j2A28EjCeYFtwBeiKUlEksHM+NbMiazdWsXXfruMkn45nDxmQLLLki7Q0SOCne4+BZgMTHb34wjOGYhIL5KemsKcK45nxIA+zH50ESsrdie7JOkCHQ2CpwHcfae77wyXPRVNSSKSTPnZ6cHPW6YYX3xoIVv31CS7JIlYm0FgZuPN7GIg38w+m/C4BsjqkgpFpMsN65/DfVefwIYd1cz+z0Xsq9Nlpb1Ze0cERwPnAwXABQmP44HrI61MRJLqhBH9+fElU3hr9VbuenoZPe3m097C3dlbU8eGHVWRHZ21d7I4B7gNuM/dX4+kAhHpti6cMoQ1lXv4yR+WM6qwD7ecNS7ZJfVI9Q3OzqpadlbXsqOqlp1VdeyoCqf3L2ucD9btSlhfWx+E8E2nj+Efzx3f6fW1FwQjCH6NLN3MXgR+B7zl2jUQiY2bzxzLR2EYjCjsw4VThiS7pKSpqqmnfNtetic23FW17Kiq29+gJzbqu8JGfXc7PwSUlmLkZaeTn51OXlYaednplPTLDufD5dlpTB5aEMn3ajMI3P37wPfNrC9wNvBFYK6ZvQ88D7zg7psiqUxEugUz43sXT6J8WxW3/WYpQwuyOGFE/2SXFald1bWs2LybDzfvZkX4+HDzLsq3VdHabnCfjNSwwU4PG/Kc/Q34gQ16+kHLczJSMbOu/ZIJDmvQOTObAJwHfMrdz+n0qtqgQedEkmPbnhouuvdVdlXX8dubTmX4gJxkl3TEtu6p4cNNu1hRsZsPN+1mZfi8cWfT7zpnpKUwurAP4wb1ZVxRLiML+9Av58CGPS8rjbTUjl6EmRyHPfqomV3p7o+G06e6+6sJ625293/r9GrboSAQSZ5VFbu56N7XGNg3k6dvPIX87PRkl9Qud2fzrn18uGk3Kzbv4sNwT3/l5t1sSTj5mpORyriiXMYU5TKuqC9ji3IZV5TLsP45pKYkb2+9sxxJECx29+ObT7c038rrzyX4QZtU4P6wqylx/Uzg20ADUAd81d1faes9FQQiyfXGqi1c9cCbnDhqAA9eO430brIn3NDgrNtedUBXTmPXzq7qpj76/Ox0xhXlMjZ8jBsUNPqD87JI6QUNfmvaCoL2ThZbK9MtzTf/0FRgDvBJoBxYaGbz3f29hM1eBOa7u5vZZOBJoPNPiYtIpzlp9AC+e9Ekbn/qbe56ZhmXnFCyf11iP3dil/cBDckBLUdHtrcWl2/cWX1Ao79y8x6qEobRLszNZFxRLp+ZOpRxg5oa/oG5mUntj++O2gsCb2W6pfnmpgMr3H0VgJk9DswE9geBuyfev96nA+8pIt3AJaXDWL1lD3NeWslTi8qTWsuQ/CzGDurLiScO2N+dM7Yol4KcjKTW1ZO0FwTjzextgiAeE04Tzrc3DPVQIHEIw3LgxOYbmdlFwPeAIuDTHSlaRJLvtk8dzTnHFjddGpmwG9c46Qcsa5rxFrf1g5Yd/L6+//UDcjMZW5RLbmZHx86U1rT3F5wCDOLABh2C+wvWt/Palo69Dtrjd/ffAr81sxkE5wvOPuiNzG4AbgAYPnx4Ox8rIl3BzJhcUpDsMqQTtHeW52cEI4+uSXwAe8N1bSkHhiXMl9BGeLj7AoKjjsIW1t3n7qXuXjpw4MB2PlZERA5Fe0Ew0t3fbr7Q3cuAke28diEwzsxGmVkGMAuYn7iBmY218KyNmR0PZABbOli7iIh0gva6htoaYTS7rRe6e52Z3Qy8QHD56Dx3f9fMZofr5wIXA1ebWS3BL55dquErRES6VntBsNDMrnf3/0hcaGbXAYvae3N3fw54rtmyuQnTPwB+0PFyRUSks7UXBF8lOJF7BU0NfylBF85FEdYlIiJdpL1B5zYBp5jZGcDEcPGz7v6nyCsTEZEu0aELcN39JeCliGsREZEk6B6DhIiISNIoCEREYk5BICIScwoCEZGYUxCIiMScgkBEJOYUBCIiMacgEBGJOQWBiEjMKQhERGJOQSAiEnMKAhGRmFMQiIjEnIJARCTmFAQiIjGnIBARiTkFgYhIzCkIRERiTkEgIhJzkQaBmZ1rZh+Y2Qozu7OF9VeY2dvh4zUzmxJlPSIicrDIgsDMUoE5wHnABOAyM5vQbLOPgNPcfTLwbeC+qOoREZGWRXlEMB1Y4e6r3L0GeByYmbiBu7/m7tvC2TeAkgjrERGRFkQZBEOBtQnz5eGy1lwH/K6lFWZ2g5mVmVlZRUVFJ5YoIiJRBoG1sMxb3NDsDIIguKOl9e5+n7uXunvpwIEDO7FEERFJi/C9y4FhCfMlwPrmG5nZZOB+4Dx33xJhPSIi0oIojwgWAuPMbJSZZQCzgPmJG5jZcOAZ4Cp3Xx5hLSIi0orIjgjcvc7MbgZeAFKBee7+rpnNDtfPBb4JDADuNTOAOncvjaomERE5mLm32G3fbZWWlnpZWVmyyxAR6VHMbFFrO9q6s1hEJOYUBCIiMacgEBGJOQWBiEjMKQhERGJOQSAiEnMKAhGRmFMQiIjEnIJARCTmFAQiIjGnIBARiTkFgYhIzCkIRERiTkEgIhJzCgIRkZhTEIiIxJyCQEQk5qL88fouU1tbS3l5OdXV1ckupVvLysqipKSE9PT0ZJciIt1IrwiC8vJy+vbty8iRIwl/+1iacXe2bNlCeXk5o0aNSnY5ItKN9IquoerqagYMGKAQaIOZMWDAAB01ichBekUQAAqBDtDfSERa0muCINlyc3OTXYKIyGGJNAjM7Fwz+8DMVpjZnS2sH29mr5vZPjO7LcpaRESkZZEFgZmlAnOA84AJwGVmNqHZZluBLwM/jqqOrubu3H777UycOJFJkybxxBNPALBhwwZmzJjB1KlTmThxIn/5y1+or6/nmmuu2b/tz372syRXLyJxFOVVQ9OBFe6+CsDMHgdmAu81buDum4HNZvbpzvrQf/6fd3lv/c7OejsAJgzJ458uOLZD2z7zzDMsWbKEpUuXUllZybRp05gxYwa/+tWvOOecc/j6179OfX09e/fuZcmSJaxbt4533nkHgO3bt3dq3SIiHRFl19BQYG3CfHm47JCZ2Q1mVmZmZRUVFZ1SXFReeeUVLrvsMlJTUxk0aBCnnXYaCxcuZNq0aTz44IPcfffdLFu2jL59+zJ69GhWrVrFLbfcwvPPP09eXl6yyxeRGIryiKClS1T8cN7I3e8D7gMoLS1t8z06uuceFfeWy5sxYwYLFizg2Wef5aqrruL222/n6quvZunSpbzwwgvMmTOHJ598knnz5nVxxSISd1EeEZQDwxLmS4D1EX5etzBjxgyeeOIJ6uvrqaioYMGCBUyfPp01a9ZQVFTE9ddfz3XXXcfixYuprKykoaGBiy++mG9/+9ssXrw42eWLSAxFeUSwEBhnZqOAdcAs4PIIP69buOiii3j99deZMmUKZsYPf/hDiouLefjhh/nRj35Eeno6ubm5PPLII6xbt45rr72WhoYGAL73ve8luXoRiSNrrSujU97c7O+BnwOpwDx3v8fMZgO4+1wzKwbKgDygAdgNTHD3Vs/2lpaWellZ2QHL3n//fY455phovkQvo7+VSDyZ2SJ3L21pXaRjDbn7c8BzzZbNTZjeSNBlJCIiSaI7i0VEYk5BICIScwoCEZGYUxCIiMScgkBEJOYUBCIiMacgSIK2frtg9erVTJw4sQurEZG4UxCIiMRcr/jx+gP87k7YuKxz37N4Epz3/VZX33HHHYwYMYKbbroJgLvvvhszY8GCBWzbto3a2lq+853vMHPmzEP62Orqam688UbKyspIS0vjpz/9KWeccQbvvvsu1157LTU1NTQ0NPD0008zZMgQPv/5z1NeXk59fT3f+MY3uPTSS4/oa4tIPPS+IEiCWbNm8dWvfnV/EDz55JM8//zz3HrrreTl5VFZWclJJ53EhRdeeEi/GzxnzhwAli1bxt/+9jc+9alPsXz5cubOnctXvvIVrrjiCmpqaqivr+e5555jyJAhPPvsswDs2LGj87+oiPRKvS8I2thzj8pxxx3H5s2bWb9+PRUVFfTr14/Bgwdz6623smDBAlJSUli3bh2bNm2iuLi4w+/7yiuvcMsttwAwfvx4RowYwfLlyzn55JO55557KC8v57Of/Szjxo1j0qRJ3Hbbbdxxxx2cf/75fOITn4jq64pIL6NzBJ3kc5/7HE899RRPPPEEs2bN4rHHHqOiooJFixaxZMkSBg0aRHV19SG9Z2sDAl5++eXMnz+f7OxszjnnHP70pz9x1FFHsWjRIiZNmsRdd93Ft771rc74WiISA73viCBJZs2axfXXX09lZSUvv/wyTz75JEVFRaSnp/PSSy+xZs2aQ37PGTNm8Nhjj3HmmWeyfPlyPv74Y44++mhWrVrF6NGj+fKXv8yqVat4++23GT9+PP379+fKK68kNzeXhx56qPO/pIj0SgqCTnLssceya9cuhg4dyuDBg7niiiu44IILKC0tZerUqYwfP/6Q3/Omm25i9uzZTJo0ibS0NB566CEyMzN54oknePTRR0lPT6e4uJhvfvObLFy4kNtvv52UlBTS09P55S9/GcG3FJHeKNLfI4iCfo/gyOhvJRJPbf0egc4RiIjEnLqGkmTZsmVcddVVByzLzMzkzTffTFJFIhJXCoIkmTRpEkuWLEl2GSIivadrqKed60gG/Y1EpCW9IgiysrLYsmWLGro2uDtbtmwhKysr2aWISDfTK7qGSkpKKC8vp6KiItmldGtZWVmUlJQkuwwR6WYiDQIzOxf4BZAK3O/u32+23sL1fw/sBa5x98WH+jnp6emMGjWqEyoWEYmfyLqGzCwVmAOcB0wALjOzCc02Ow8YFz5uAHQXlIhIF4vyHMF0YIW7r3L3GuBxoPk4zDOBRzzwBlBgZoMjrElERJqJMgiGAmsT5svDZYe6jYiIRCjKcwQtDbzf/LKejmyDmd1A0HUEsNvMPjjMmgqBysN8bU+l7xwP+s7xcCTfeURrK6IMgnJgWMJ8CbD+MLbB3e8D7jvSgsysrLWxNnorfed40HeOh6i+c5RdQwuBcWY2yswygFnA/GbbzAeutsBJwA533xBhTSIi0kxkRwTuXmdmNwMvEFw+Os/d3zWz2eH6ucBzBJeOriC4fPTaqOoREZGWRXofgbs/R9DYJy6bmzDtwJeirKGZI+5e6oH0neNB3zkeIvnOPe73CEREpHP1irGGRETk8MUmCMzsXDP7wMxWmNmdya4namY2zMxeMrP3zexdM/tKsmvqCmaWamZ/NbP/TXYtXcXMCszsKTP7W/jf++Rk1xQlM7s1/Df9jpn92sx65UiKZjbPzDab2TsJy/qb2R/M7MPwuV9nfFYsgqCDw130NnXA/3P3Y4CTgC/F4DsDfAV4P9lFdLFfAM+7+3hgCr34+5vZUODLQKm7TyS4EGVWcquKzEPAuc2W3Qm86O7jgBfD+SMWiyCgY8Nd9CruvqFxAD9330XQOPTqu7bNrAT4NHB/smvpKmaWB8wAHgBw9xp3357UoqKXBmSbWRqQQwv3HvUG7r4A2Nps8Uzg4XD6YeAznfFZcQmCWA9lYWYjgeOA3v47mD8H/hFoSHIdXWk0UAE8GHaJ3W9mfZJdVFTcfR3wY+BjYAPBvUe/T25VXWpQ471W4XNRZ7xpXIKgQ0NZ9EZmlgs8DXzV3Xcmu56omNn5wGZ3X5TsWrpYGnA88Et3Pw7YQyd1F3RHYZ/4TGAUMAToY2ZXJreqni8uQdChoSx6GzNLJwiBx9z9mWTXE7FTgQvNbDVB19+ZZvZockvqEuVAubs3Hu09RRAMvdXZwEfuXuHutcAzwClJrqkrbWocoTl83twZbxqXIOjIcBe9SvijPw8A77v7T5NdT9Tc/S53L3H3kQT/ff/k7r1+T9HdNwJrzezocNFZwHtJLClqHwMnmVlO+G/8LHrxyfEWzAe+EE5/AfjvznjTXvFTle1pbbiLJJcVtVOBq4BlZrYkXPa18G5v6V1uAR4Ld3JW0YuHanH3N83sKWAxwZVxf6WX3mFsZr8GTgcKzawc+Cfg+8CTZnYdQShe0imfpTuLRUTiLS5dQyIi0goFgYhIzCkIRERiTkEgIhJzCgIRkZhTEEismVm9mS1JeHTaXblmNjJx5MgObN/HzP4QTr8SjqUjEjn9Q5O4q3L3qckuInQy8EY4jMIed69LdkESDzoiEGmBma02sx+Y2VvhY2y4fISZvWhmb4fPw8Plg8zst2a2NHw0DnuQamb/EY6f/3szy27hs8aEN/09ClwOLAKmhEconTKomEhbFAQSd9nNuoYuTVi3092nA/9GMLIp4fQj7j4ZeAz4l3D5vwAvu/sUgrF+Gu9cHwfMcfdjge3Axc0LcPeV4VHJIoIh0x8BrnP3qe7eKWPJiLRFdxZLrJnZbnfPbWH5auBMd18VDt630d0HmFklMNjda8PlG9y90MwqgBJ335fwHiOBP4Q/IoKZ3QGku/t3WqllobtPM7OngS+HQy6LRE5HBCKt81amW9umJfsSputp4bycmc0NTyqPC7uIzgWeNbNbD6FWkcOmIBBp3aUJz6+H06/R9NOIVwCvhNMvAjfC/t9Nzuvoh7j7bOCfgW8T/OLUs2G30M+OqHqRDtJVQxJ32Qmjs0Lw27+Nl5BmmtmbBDtMl4XLvgzMM7PbCX4ZrHGkz68A94WjQtYThMKGQ6jjNIJzA58AXj6cLyJyuHSOQKQF4TmCUnevTHYtIlFT15CISMzpiEBEJOZ0RCAiEnMKAhGRmFMQiIjEnIJARCTmFAQiIjGnIBARibn/D7Ds06j8w8R9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history.history['loss'], label='loss')\n",
    "plt.plot(history.history['val_loss'], label='val_loss')\n",
    "plt.ylim([0, max(plt.ylim())])\n",
    "plt.xlabel('Epoch #')\n",
    "plt.ylabel('CE/token')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "15036f8b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x22b9c5dafd0>"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAgCUlEQVR4nO3dfZQU9Z3v8fd3emaYAZQHGRBBM/gMCBN0IiZsBCVrMKsxiU+4rlHWh2uuGoP3XjVkjXjMyXFP4mb1avRiooYbXW4u6kaNiSuKevaqiYMaEY1KRMPEB0YQEGEeuvt7/6jqnp6e7pkGp6Znpj6vc+p01a9+Vf3rHvh9un7VXWXujoiIxFdFuRsgIiLlpSAQEYk5BYGISMwpCEREYk5BICISc5XlbsDuGjdunNfX15e7GSKxlUw7bR0pWpNpOpJpHHDCbx8W+RJi0e8mevF1nr8mv6JBhVk4gYWPXZc7y7LrKzrXW0mveGhYs2bNh+5eV2jdoAuC+vp6mpqayt0MkSEtlXaaP9rJ+k07stOfW4LH7a3JbL3hlRVUJyqCDjXsVY2wk7VsUbbTtbCidakLRvf6ZNZl63at05Fy2pIpdrWn2NWRIh0GRTqcSlGVMGoqE9RUJ6itCqaaqgpqqhLU5pQNCx9rqyuoTiRwHHdwd9IehFbaIe1BsqW9c7lQPXcnnS6+nZPZtmu9BdP35dSjJu/GX7KTmb1TbN2gCwIR6TutHSneavmE9S2dnf2fN+3grQ8/oT3Z2Z2OGzmMg+pGcHLDfhw8fiQHjx/JQXUjmTiqJttpl5O7055K09qepjUnHHZ1pGgNp13t6WxZW0ehOukuZVs+aQ+2C7fNzKfSnYcmmWDKHJlkgiu7HNapqLBsPcsesXTWA6ioILtN5766HtFsb+2I5P1TEIjEwNad7d0+2a9v2UHzR7vI/KbUDPYfM5yDx4/k2EPrOKhuRLbDHz28urwvoBdmxrDKBMMqE4yiKtLnSqW980hmAIRgX4hNEPx27Xtc/n9eorYqwfDMIV91MF/Tpayyy/ps/ex8JbXVFdRWVWa3z6yrSujcu5SPu/Puttagsw87+vWbdvBWyw4+3NGerVddWcGB40bQMHk035g1OfsJf8q4EdRUJcr4CgaHRMXQ6PxzxSYI6seNYNGcelrbU+xsT7GzI5Wd/7g1yabtbezsSAaHj+1Jdnak2N2rb1QlLBsqw6sruwRMRYWRuZyHh2OFwdhh5zyE58Ny1xP8B/fsdsGMF9wP3Z4jw3LHZXMOTbuMu+YcymbHeemshxUY0w0PW+lWP+hwaqoS1FQGYTmsqiIYjw3HYYPx2GB+WHZ8NhyjDesF47MVDKusiPzTl7vTlkzTnkrTnsyZwuW2vOX2ZJqOzLrwMZ32LuPDwTKkMuO9mXU59VJpz44hp3LGhVNhfQ/3FdTrnM+MNafc2byjnT+37GBneyr7ekbVVnHw+JEcf/j4bGd/cN1eTBpTOyQ7M9lzsQmCqRP3ZurEvUuun+kUWjvC4GhPZeeDMcMkuzLL4bSzo3M+u64jySftSdJOSZ1shRHOV2Q72vxOvNh+yOnUc7fzbsGSGy5dQ6a3sHLA0+Cku2yTOcGVG1Ltyc7x2tbwWya54867wwyGhcGSCYzMctdQSZCosG6deG5n3Z5MFezsO1LRXnfLDBI5Y8mJiu7zmW+zJHLnK3K+DZM7b5YdVx49vIozGvfPDuUcPH4k40ZWD5mhC4lWbIJgd5lZtmMZPbzcrRk60ukgYHNP4rV2BIHR2p4KHjvSneU5J/dak+nOE38dnfNtHWk2f9IeBE4yRSrlVFdWdE6J4HFUdRXVieDoIrc8v17B9Ymet8nMJyqsawff5SSiOmUZmBQE0q8qKiw4p1KtsWiRgUJnN0VEYk5BICIScxoaEhmo0mlItUGyDVLteY9tkGzvfEwne99fUZ/mJLlBZTVU1kJVDVSGU1Vt53zlsM6fB0vp3CHZCh27oGNn8Dhsb9hrQp8/lYJAZHe4Q9t22LUVWrfmPW4L/uMW6qiLduhFOvZU26fs3AcSC8OhJgiMymGdQVEVLmeDJH99D9skqsEqwikRPFZUFChL5JRVFCgrdducMEt1dHbO2cddBcp6WldCWb6/WQxfWtrnfyEFwWCT6gg6nNZtnZ1QbkfUujVn3bbgH1OiOpgy/3kyj4XKKodBYljwKa9LnUxZ7rrcspztKyoH9ifAdBratnXvzFsLlBWq4719BdaKvGd579fwEcXX9fq3yatfkaDzKjx7YE//Xp4Ogiu5CzpawyAMP8Vm51tz1u8Kgi+zvqMVWrdDclPhbXp9r/uTBYEA4KmeqxaSqA5CrGp43mMt1EzsXtat3nCoO7xvX1IoPkGw/nF4dEnOoWuBTxoFy3M/reQe9hYor6yBRFXP/6ncg865W+eT34kXWLdrK3R80vPrTFRDzWioGQW1o4O2ppLQsbX7J9NUe2dZqr3n/e4W6wyRikQQDImq4LEiARWZ+UpIVHbO506Jqs5tK3K2TVQVqZuz73Syl858Oz0Oh1RUBu9h7ejgcfg+MPagzuVijzWjgvd7oAfhYOEefPDpFh7hYzoJ6VT4w5acqUtZOJ9Ody/rVje/LPPoXcugeEddrBOvrA3+rQ9QA7dlfa16JNQdlvPpZBfs3JL3CSScPk2naBWFAybZ2vlJPt3LhaOG7R10KpnOZeyUzuXa0V3X5S7Xjg6ea086IffCwxZFhzDaex7WSLYF/3nSyeD1Zv7TpjLzeVOqI1ifbCtSP5VTngzCLbt9gfczE4iZTnrkBBh3WO+dee3o4D+vOvLys8z5h4F9naOhID5BcMDsYCpFpkMqeIibe5ibexjcVqA8Zz5RXaQTz5sftnd5PjlYOJxROaz/n/vTcg8/yYWBYhVBCKszFylJfIJgd1QkoHp4MMnAZxac3KtIDM4gEykz/Y5ARCTmFAQiIjGnIBARiTkFgYhIzCkIRERiTkEgIhJzCgIRkZhTEIiIxJyCQEQk5iINAjNbYGavm9l6M7u6wPpRZvaQmf3RzNaZ2aIo2yMiIt1FFgRmlgBuBU4EpgFnmdm0vGqXAK+6ewMwD7jRzHSFKRGRfhTlEcHRwHp3f8vd24EVwCl5dRzYy8wMGAlsAYbK3ThERAaFKINgErAxZ7k5LMt1CzAVeBdYC1zu3v1OFGZ2kZk1mVlTS0tLVO0VEYmlKIOg0DWA8+8G8mXgJWA/4LPALWa2d7eN3Je5e6O7N9bV1fV1O0VEYi3KIGgG9s9ZnkzwyT/XIuB+D6wHNgDR3ItNREQKijIIngcOMbMp4QnghcCDeXX+AswHMLMJwGHAWxG2SURE8kR2Yxp3T5rZpcCjQAK4093XmdnF4frbgeuBu81sLcFQ0lXu/mFUbRIRke4ivUOZuz8CPJJXdnvO/LvACVG2QUREeqZfFouIxJyCQEQk5hQEIiIxpyAQEYk5BYGISMwpCEREYk5BICIScwoCEZGYUxCIiMScgkBEJOYUBCIiMacgEBGJOQWBiEjMKQhERGJOQSAiEnMKAhGRmFMQiIjEnIJARCTmFAQiIjGnIBARiTkFgYhIzCkIRERiTkEgIhJzCgIRkZhTEIiIxJyCQEQk5hQEIiIxpyAQEYk5BYGISMwpCEREYk5BICIScwoCEZGYUxCIiMScgkBEJOYUBCIiMacgEBGJOQWBiEjMKQhERGIu0iAwswVm9rqZrTezq4vUmWdmL5nZOjN7Ksr2iIhId5VR7djMEsCtwN8CzcDzZvagu7+aU2c08FNggbv/xczGR9UeEREpLMojgqOB9e7+lru3AyuAU/Lq/D1wv7v/BcDdN0XYHhERKSDKIJgEbMxZbg7Lch0KjDGzJ81sjZl9s9COzOwiM2sys6aWlpaImisiEk9RBoEVKPO85UrgKODvgC8D15jZod02cl/m7o3u3lhXV9f3LRURibHIzhEQHAHsn7M8GXi3QJ0P3f0T4BMzexpoAN6IsF0iIpIjyiOC54FDzGyKmVUDC4EH8+r8GviimVWa2XBgNvBahG0SEZE8kR0RuHvSzC4FHgUSwJ3uvs7MLg7X3+7ur5nZ74CXgTTwM3d/Jao2iYhId+aeP2w/sDU2NnpTU1O5myEiMqiY2Rp3byy0Tr8sFhGJOQWBiEjMKQhERGJOQSAiEnMKAhGRmFMQiIjEnIJARCTmSv5BmZl9AajP3cbdl0fQJhER6UclBYGZ/W/gIOAlIBUWO6AgEBEZ5Eo9ImgEpvlg+xmyiIj0qtRzBK8A+0bZEBERKY9SjwjGAa+a2R+Atkyhu381klaJiEi/KTUIlkbZCBERKZ+SgsDdnzKzzwCHuPuq8N4BiWibJiIi/aGkcwRmdiGwEvhfYdEk4N8japOIiPSjUk8WXwLMAbYDuPubwPioGiUiIv2n1CBoc/f2zIKZVdL9RvQiIjIIlRoET5nZEqDWzP4W+L/AQ9E1S0RE+kupQXA10AKsBf4L8Ii7fy+yVomISL8p+euj7v594A4AM0uY2T3ufnZ0TRMRkf5Q6hHBAWb2XQAzqwbuB96MrFUiItJvSg2CRcCMMAweBp5096WRtUpERPpNj0NDZnZkzuJNBL8j+H8EJ4+PdPcXomyciIhEr7dzBDfmLX8ETAvLHTg+ikaJiEj/6TEI3P24/mqIiIiUR6mXmBhlZv9iZk3hdKOZjYq6cSIiEr1STxbfCXwMnBFO24G7omqUiIj0n1J/R3CQu5+as3ydmb0UQXtERKSflXpEsMvM/iazYGZzgF3RNElERPpTqUcEFwPLc84LfAScG02TRESkP5UaBNvdvcHM9gZw9+1mNiXCdomISD8pdWjoPggCwN23h2Uro2mSiIj0p95+WXw4MB0YZWbfyFm1N1ATZcNERKR/9DY0dBhwEjAaODmn/GPgwojaJCIi/ai3IBgO/Hdgmbs/2w/tERGRftZbEHyG4G5kVWb2OPBb4A/urttUiogMET2eLHb3G9z9eOArwB+BfwReMLN7zeybZjahPxopIiLRKenro+7+MfBAOGFm04ATgeXAlyNrnYiIRK7HIwIz+4ec+TmZeXd/FWhz9x5DwMwWmNnrZrbezK7uod7nzCxlZqftRttFRKQP9PY7gity5v9n3rp/7GlDM0sAtxIcOUwDzgqPJArV+2fg0V5bKyIifa63ILAi84WW8x0NrHf3t9y9HVgBnFKg3mUEP1jb1Mv+REQkAr0FgReZL7ScbxKwMWe5OSzLMrNJwNeB23vakZldlLkXQktLSy9PKyIiu6O3k8WHm9nLBJ/+DwrnCZcP7GXbQkcM+eHxr8BV7p4yK36A4e7LgGUAjY2N+uqqiEgf6i0IGoAJdP1kD8HvC97tZdtmYP+c5ckFtmkEVoQhMA74ipkl3f3fe9m3iIj0kd6Ghn5CcOXRd3InYGe4rifPA4eY2RQzqwYWAg/mVnD3Ke5e7+71BBex+68KARGR/tXbEUG9u7+cX+juTWZW39OG7p40s0sJvg2UAO5093VmdnG4vsfzAiIi0j96C4KerjBa29vO3f0R4JG8soIB4O7n9bY/ERHpe70NDT1vZt2uMmpm5wNrommSiIj0p96OCL4DPGBmZ9PZ8TcC1QRf+xQRkUGuxyBw9w+AL5jZccARYfFv3P2JyFsmIiL9otSLzq0GVkfcFhERKYNS71ksIiJDlIJARCTmFAQiIjGnIBARiTkFgYhIzCkIRERiTkEgIhJzCgIRkZhTEIiIxJyCQEQk5hQEIiIxpyAQEYk5BYGISMwpCEREYk5BICIScwoCEZGYUxCIiMScgkBEJOYUBCIiMacgEBGJOQWBiEjMKQhERGJOQSAiEnMKAhGRmFMQiIjEnIJARCTmFAQiIjGnIBARiTkFgYhIzCkIRERiTkEgIhJzCgIRkZhTEIiIxJyCQEQk5iINAjNbYGavm9l6M7u6wPqzzezlcHrGzBqibI+IiHQXWRCYWQK4FTgRmAacZWbT8qptAOa6+0zgemBZVO0REZHCojwiOBpY7+5vuXs7sAI4JbeCuz/j7h+Fi88BkyNsj4iIFBBlEEwCNuYsN4dlxZwP/LbQCjO7yMyazKyppaWlD5soIiJRBoEVKPOCFc2OIwiCqwqtd/dl7t7o7o11dXV92EQREamMcN/NwP45y5OBd/MrmdlM4GfAie6+OcL2iIhIAVEeETwPHGJmU8ysGlgIPJhbwcwOAO4HznH3NyJsi4iIFBHZEYG7J83sUuBRIAHc6e7rzOzicP3twPeBfYCfmhlA0t0bo2qTiIh0Z+4Fh+0HrMbGRm9qaip3M0REBhUzW1Psg7Z+WSwiEnMKAhGRmIvyW0P9pqOjg+bmZlpbW8vdFAFqamqYPHkyVVVV5W6KiJRgSARBc3Mze+21F/X19YQnnaVM3J3NmzfT3NzMlClTyt0cESnBkBgaam1tZZ999lEIDABmxj777KOjM5FBZEgEAaAQGED0txAZXIZMEIiIyJ5REIiIxJyCYJBJJpPlboKIDDFD4ltDua57aB2vvru9T/c5bb+9ufbk6b3W+9rXvsbGjRtpbW3l8ssv56KLLuJ3v/sdS5YsIZVKMW7cOB5//HF27NjBZZddRlNTE2bGtddey6mnnsrIkSPZsWMHACtXruThhx/m7rvv5rzzzmPs2LG8+OKLHHnkkZx55pl85zvfYdeuXdTW1nLXXXdx2GGHkUqluOqqq3j00UcxMy688EKmTZvGLbfcwgMPPADAY489xm233cb999/fp++RiAxeQy4IyunOO+9k7Nix7Nq1i8997nOccsopXHjhhTz99NNMmTKFLVu2AHD99dczatQo1q5dC8BHH33U024BeOONN1i1ahWJRILt27fz9NNPU1lZyapVq1iyZAn33Xcfy5YtY8OGDbz44otUVlayZcsWxowZwyWXXEJLSwt1dXXcddddLFq0KNL3QUQGlyEXBKV8co/KzTffnP3kvXHjRpYtW8axxx6b/T792LFjAVi1ahUrVqzIbjdmzJhe93366aeTSCQA2LZtG+eeey5vvvkmZkZHR0d2vxdffDGVlZVdnu+cc87hl7/8JYsWLeLZZ59l+fLlffSKRWQoGHJBUC5PPvkkq1at4tlnn2X48OHMmzePhoYGXn/99W513b3gVyxzy/K/hz9ixIjs/DXXXMNxxx3HAw88wNtvv828efN63O+iRYs4+eSTqamp4fTTT88GhYgI6GRxn9m2bRtjxoxh+PDh/OlPf+K5556jra2Np556ig0bNgBkh4ZOOOEEbrnlluy2maGhCRMm8Nprr5FOp7NHFsWea9Kk4K6fd999d7b8hBNO4Pbbb8+eUM4833777cd+++3HD37wA84777w+e80iMjQoCPrIggULSCaTzJw5k2uuuYZjjjmGuro6li1bxje+8Q0aGho488wzAfinf/onPvroI4444ggaGhpYvXo1ADfccAMnnXQSxx9/PBMnTiz6XFdeeSXf/e53mTNnDqlUKlt+wQUXcMABBzBz5kwaGhq49957s+vOPvts9t9/f6ZNmxbROyAig9WQuB/Ba6+9xtSpU8vUosHh0ksvZdasWZx//vn98nz6m4gMLD3dj0CDxTFw1FFHMWLECG688cZyN0VEBiAFQQysWbOm3E0QkQFM5whERGJOQSAiEnMKAhGRmFMQiIjEnIJARCTmFARlMHLkyHI3QUQka+h9ffS3V8P7a/t2n/vOgBNv6Nt9DgDJZFLXHRIRHRH0hauuuoqf/vSn2eWlS5dy3XXXMX/+fI488khmzJjBr3/965L2tWPHjqLbLV++PHv5iHPOOQeADz74gK9//es0NDTQ0NDAM888w9tvv80RRxyR3e7HP/4xS5cuBWDevHksWbKEuXPnctNNN/HQQw8xe/ZsZs2axZe+9CU++OCDbDsWLVrEjBkzmDlzJvfddx8///nPWbx4cXa/d9xxB1dcccUev28iMkC4+6CajjrqKM/36quvdivrTy+88IIfe+yx2eWpU6f6O++849u2bXN395aWFj/ooIM8nU67u/uIESOK7qujo6Pgdq+88oofeuih3tLS4u7umzdvdnf3M844w3/yk5+4u3symfStW7f6hg0bfPr06dl9/uhHP/Jrr73W3d3nzp3r3/rWt7LrtmzZkm3XHXfc4VdccYW7u1955ZV++eWXd6m3Y8cOP/DAA729vd3d3T//+c/7yy+/XPB1lPtvIiJdAU1epF/VuEAfmDVrFps2beLdd9+lpaWFMWPGMHHiRBYvXszTTz9NRUUFf/3rX/nggw/Yd999e9yXu7NkyZJu2z3xxBOcdtppjBs3Dui818ATTzyRvb9AIpFg1KhRvd7oJnPxO4Dm5mbOPPNM3nvvPdrb27P3Tih2z4Tjjz+ehx9+mKlTp9LR0cGMGTN2890SkYFGQdBHTjvtNFauXMn777/PwoULueeee2hpaWHNmjVUVVVRX1/f7R4DhRTbzovca6CQyspK0ul0drmnextcdtllXHHFFXz1q1/lySefzA4hFXu+Cy64gB/+8IccfvjhutOZyBChcwR9ZOHChaxYsYKVK1dy2mmnsW3bNsaPH09VVRWrV6/mnXfeKWk/xbabP38+v/rVr9i8eTPQea+B+fPnc9tttwGQSqXYvn07EyZMYNOmTWzevJm2tjYefvjhHp8vc2+DX/ziF9nyYvdMmD17Nhs3buTee+/lrLPOKvXtEZEBTEHQR6ZPn87HH3/MpEmTmDhxImeffTZNTU00NjZyzz33cPjhh5e0n2LbTZ8+ne9973vMnTuXhoaG7Enam266idWrVzNjxgyOOuoo1q1bR1VVFd///veZPXs2J510Uo/PvXTpUk4//XS++MUvZoedoPg9EwDOOOMM5syZU9ItNkVk4NP9CGS3nXTSSSxevJj58+cXraO/icjA0tP9CHREICXbunUrhx56KLW1tT2GgIgMLjpZXCZr167N/hYgY9iwYfz+978vU4t6N3r0aN54441yN0NE+tiQCYLd+VbNQDBjxgxeeumlcjcjEoNtuFEk7obE0FBNTQ2bN29WBzQAuDubN2+mpqam3E0RkRINiSOCyZMn09zcTEtLS7mbIgTBPHny5HI3Q0RKNCSCoKqqKvuLWBER2T2RDg2Z2QIze93M1pvZ1QXWm5ndHK5/2cyOjLI9IiLSXWRBYGYJ4FbgRGAacJaZTcurdiJwSDhdBNwWVXtERKSwKI8IjgbWu/tb7t4OrABOyatzCrA8vDjec8BoM5sYYZtERCRPlOcIJgEbc5abgdkl1JkEvJdbycwuIjhiANhhZq/vYZvGAR/u4baDlV5zPOg1x8Onec2fKbYiyiAo9KX+/O93llIHd18GLPvUDTJrKvYT66FKrzke9JrjIarXHOXQUDOwf87yZODdPagjIiIRijIIngcOMbMpZlYNLAQezKvzIPDN8NtDxwDb3P29/B2JiEh0IhsacvekmV0KPAokgDvdfZ2ZXRyuvx14BPgKsB7YCUR9p5NPPbw0COk1x4NeczxE8poH3WWoRUSkbw2Jaw2JiMieUxCIiMRcbIKgt8tdDDVmtr+ZrTaz18xsnZldXu429QczS5jZi2ZW/EbNQ4yZjTazlWb2p/Dv/flytylKZrY4/Df9ipn9m5kNyUvdmtmdZrbJzF7JKRtrZo+Z2ZvhY5/cLzYWQVDi5S6GmiTw39x9KnAMcEkMXjPA5cBr5W5EP7sJ+J27Hw40MIRfv5lNAr4NNLr7EQRfRFlY3lZF5m5gQV7Z1cDj7n4I8Hi4/KnFIggo7XIXQ4q7v+fuL4TzHxN0DpPK26pomdlk4O+An5W7Lf3FzPYGjgV+DuDu7e6+tayNil4lUGtmlcBwhuhvj9z9aWBLXvEpwC/C+V8AX+uL54pLEBS7lEUsmFk9MAsYuPfB7Bv/ClwJpMvcjv50INAC3BUOif3MzEaUu1FRcfe/Aj8G/kJwKZpt7v4f5W1Vv5qQ+a1V+Di+L3YalyAo6VIWQ5GZjQTuA77j7tvL3Z6omNlJwCZ3X1PutvSzSuBI4DZ3nwV8Qh8NFwxE4Zj4KcAUYD9ghJn9Q3lbNfjFJQhieSkLM6siCIF73P3+crcnYnOAr5rZ2wRDf8eb2S/L26R+0Qw0u3vmaG8lQTAMVV8CNrh7i7t3APcDXyhzm/rTB5krNIePm/pip3EJglIudzGkmJkRjBu/5u7/Uu72RM3dv+vuk929nuDv+4S7D/lPiu7+PrDRzA4Li+YDr5axSVH7C3CMmQ0P/43PZwifHC/gQeDccP5c4Nd9sdMhcavK3hS73EWZmxW1OcA5wFozeyksW+Luj5SvSRKRy4B7wg85bxH9pVrKxt1/b2YrgRcIvhn3IkP0UhNm9m/APGCcmTUD1wI3AL8ys/MJQvH0PnkuXWJCRCTe4jI0JCIiRSgIRERiTkEgIhJzCgIRkZhTEIiIxJyCQGLNzFJm9lLO1Ge/yjWz+twrR5ZQf4SZPRbO/2d4LR2RyOkfmsTdLnf/bLkbEfo88Fx4GYVP3D1Z7gZJPOiIQKQAM3vbzP7ZzP4QTgeH5Z8xs8fN7OXw8YCwfIKZPWBmfwynzGUPEmZ2R3j9/P8ws9oCz3VQ+KO/XwJ/D6wBGsIjlD65qJhITxQEEne1eUNDZ+as2+7uRwO3EFzZlHB+ubvPBO4Bbg7LbwaecvcGgmv9ZH65fghwq7tPB7YCp+Y3wN3/HB6VrCG4ZPpy4Hx3/6y798m1ZER6ol8WS6yZ2Q53H1mg/G3geHd/K7x43/vuvo+ZfQhMdPeOsPw9dx9nZi3AZHdvy9lHPfBYeBMRzOwqoMrdf1CkLc+7++fM7D7g2+Ell0UipyMCkeK8yHyxOoW05cynKHBezsxuD08qHxIOES0AfmNmi3ejrSJ7TEEgUtyZOY/PhvPP0HlrxLOB/wznHwe+Bdn7Ju9d6pO4+8XAdcD1BHec+k04LPSTT9V6kRLpW0MSd7U5V2eF4N6/ma+QDjOz3xN8YDorLPs2cKeZ/Q+CO4NlrvR5ObAsvCpkiiAU3tuNdswlODfwReCpPXkhIntK5whECgjPETS6+4flbotI1DQ0JCISczoiEBGJOR0RiIjEnIJARCTmFAQiIjGnIBARiTkFgYhIzP1/Z9H4P6OfebMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history.history['masked_acc'], label='accuracy')\n",
    "plt.plot(history.history['val_masked_acc'], label='val_accuracy')\n",
    "plt.ylim([0, max(plt.ylim())])\n",
    "plt.xlabel('Epoch #')\n",
    "plt.ylabel('CE/token')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "0886732b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'select current series from table where began new series began june '"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = model.translate(['What is the current series where the new series began in June 2011?']) \n",
    "result[0].numpy().decode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46e845ee",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}