lang_model_with_gt.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Language model with Good turing Smoothing</h1><br>\n",
    "\\begin{equation}\n",
    "c^*=(c+1) * \\frac{N_{c+1}}{N.N_c}\n",
    "\\end{equation}\n",
    "\n",
    "<p>$ c^*=$ smoothed count of N-grams that occur c times.<br>\n",
    "$N_c= N_c$ be the number of N-grams that occur c times<br>\n",
    "$N = $ Size of training data</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p> <code>Note:<code> If any changes you make in <code>language_model.ipynb<code>. \n",
    "In order to changes to reflect in this or any notebook that imported the file <code>language_model.ipynb<code>, \n",
    "you must restart the Kernel from the Kernel Tab Available above.<\\p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "import import_ipynb\n",
    "from language_model import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Wikipedia articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames=read_filename('./wikipedia/', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sents=create_tokens(filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_corpus, test_corpus=train_test_split(sents)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Obtaining probality distribution with Good Turing smoothing for Unigrams</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "fd_1gram=ngram_freq_dist(train_corpus, ngram=1)\n",
    "cpd_1gram_with_gt=nltk.SimpleGoodTuringProbDist(fd_1gram)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab=len(fd_1gram)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Obtaining conditional probality distribution with Good Turing smoothing for Bigrams</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dj/Desktop/Course_tutorial/tutorial/lib/python3.6/site-packages/nltk/probability.py:1422: UserWarning: SimpleGoodTuring did not find a proper best fit line for smoothing probabilities of occurrences. The probability estimates are likely to be unreliable.\n",
      "  'SimpleGoodTuring did not find a proper best fit '\n"
     ]
    }
   ],
   "source": [
    "cfd_2gram=ngram_freq_dist(train_corpus, ngram=2)\n",
    "cpd_2gram_with_gt = nltk.ConditionalProbDist(cfd_2gram, nltk.SimpleGoodTuringProbDist, bins=pow(vocab,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cpd_2gram_with_add1 = nltk.ConditionalProbDist(cfd_2gram, nltk.SimpleGoodTuringProbDist, bins=pow(vocab,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dict(cpd_2gram_with_add1['azerbaijan'].freqdist())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Obtaining conditional probality distribution with Good Turing smoothing for Trigrams</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "cfd_3gram=ngram_freq_dist(train_corpus, ngram=3)\n",
    "cpd_3gram_with_gt = nltk.ConditionalProbDist(cfd_3gram, nltk.SimpleGoodTuringProbDist, bins=pow(vocab,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#freq_3gram = nltk.FreqDist(ngrams(train_corpus,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cpd_3gram_with_gt = nltk.ConditionalProbDist(cfd_3gram, nltk.KneserNeyProbDist, bins=pow(vocab,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FreqDist({'a': 3, 'represented': 2, 'one': 1, 'an': 1, 'accredited': 1})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cpd_3gram_with_gt[('azerbaijan', 'is')].freqdist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3UAAAGDCAYAAACMWjkNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3de7xtdV0v/M9XQEEkfZCd4XVr3lJLVCTvF9IePWRqD16IVMw0TxfT6iiVJc9zsvA5lpfMY94OaKR4l0RPeUMQFAJFBZEwBVFQUSFAkev3/DHGwslir73n3qy51x6b9/v1mq817uM7fnOuueZn/cYYs7o7AAAATNNN1roAAAAAtpxQBwAAMGFCHQAAwIQJdQAAABMm1AEAAEyYUAcAADBhQh3AKquqQ6rqH7dw3YOq6tMbmX9MVf3Wlle39Syi1qr606p682puk21TVR1YVf+6gO16DQHbHaEOIElVnV1Vl1XVpVX1nao6rKpusdZ1rZaq+peqesnM+O2qqleY9jNrU+WmdfdfdfckQu2mrGbo3ZJ/JIyv+fXja/2g1ahjNXX3Ed39yzdkG1X1qKr65rLtrspraNz2MeOwL/0F1pRQB/ATT+juWyS5f5K9k7x0+QI1mOJ757FJHjEz/ogkX9nAtLO6+9ubs+Gq2vGGl8fUeN4Bth1T/GACsFDd/a0kH0lyn+TaHpWXV9XxSX6U5C5VdduqOqqqflBVX62q5y7bzM5VdWRVXVJVn6uq+y7NqKqDq+o/xnlfrqonL1u3qup1VfWfVfWVqvqllWqtqt+sqjOq6sKxN+5OKyx6bJKHzgTShyd5dZK9l007dmbbzx2P7Qfjsd52Zl5X1e9W1VlJzhqnPXas9z+r6nVJamb5u1bVp8Z536uqIzdyTM+sqnOq6vtV9edjj9JjxnnX9khV1Ueq6veWrfuFqvq1cfieVfXRsf4zq+qpM8sdVlV/X1VHj8/DiVX1sxup6WFVdUJVXVRV5y71bFXVLavqbVV1wVjzS5fas8ZTaavqlePz8/Wqevw47+Vje79u7B1+3cZqrqqbVtWpVfX74/gOVXV8Vf1FVT0uyZ8medq4rS+sdBybUlU3GY/hnKr67nhstxznrR+f9+dU1TeSfGKO52ufqvrM2G7nj6/rm87sr6vq+VV11rjM31dVzbbfOPzi8diWHldW1WHjvGePvwOXVNXXquq3x+m7Zvg9vu3MeretZb2aVfWrVXX6uP9jqurnZuadXVV/XFVfHF+7R1bVzlvavgAL090eHh4eN/pHkrOTPGYcvkOS05P893H8mCTfSHLvJDsm2SlD+Hl9kp2T7JXkgiT7jssfkuTKJPuPy/5xkq8n2Wmc/5Qkt83wj7WnJflhkj3HeQcluSrJi8Z1n5bkP5PsPlPLb43DT0zy1SQ/N9b10iQnrHB8N0tyWZL7jeOnJblLkuOXTXvmOLxvku9l6LW8WZK/S3LszPY6yUeT7J5klyR7JLlk5phfNB7HUq3vSPJn4zHvnORhK9R5rySXJnlYkpsmeeXYlo+Zadt/HIefmeT4ZeteNNa7a5Jzkzx7bJv7jcdzr3HZw5J8P8k+4/wjkrxzhZruNB7bAeOx3TrJXuO8tyX5YJLdkqxP8u9JnjPzXF6Z5LlJdkjyX5Ocl6SWP5fj+KZqvk+SC8fn+8+SfDbJDsvbZWZ7Byf50Gb+Hvzm+Jq6S5JbJHlfkreP89aPz/vbxlp3meP5ekCSB43Hsz7JGUleuOx19KEkt0pyxwy/R4+bab9Pb6DGO4zt+PhxfL8kP5vhnwiPzPCPl/uP8x6V5JvL1r+2rZLcPcPv32PH5/bF4/HfdOZ94aQMv6+7j/U/f63frzw8PDyWP/TUAfzEB6rqoiSfTvKpJH81M++w7j69u69K8jNJHprkJd394+4+NcmbM4SMJad093u6+8okf5shyDwoSbr73d19Xndf091HZujp2mdm3e8meXV3XznOPzPDB9flnp/kr7v7jLGuv0qyV22gt667L09yYpJHVNXuSW7Z3V9LctzMtHuNx50kByZ5a3d/blz3T5I8uKrWz2z2r7v7B919WZL/kuT0mWN+dZLZ0zivzBCObju22Uo3g9k/yT9396e7+4okf5Hhg/+GvH/Z8R6Y5H1jvb+S5Ozu/l/dfVV3fz7JezME6mvX7+6TxrY7IkM435BfT/Kx7n7H+Jx8v7tPraodkjw9yZ909yXdfXaSv0nyjJl1z+nuN3X31UkOT7JnktussJ+N1tzdpyX5yyQfyPCPgmeM292g7j60u39lpfkrODDJ33b317r70gzP+9PruqdaHtLdPxyf940+X919Snd/djyes5P8Q4bgNevQ7r6ou7+R5JNZ+XlIVe2S4fhf090fGfdxdHf/Rw8+leRfM/SCzuNpSY7u7o+Or9tXZgirD5lZ5rXj7+sPkvzzxuoDWCtCHcBPPKm7b9Xdd+ru3xk/tC45d2b4tkl+0N2XzEw7J8ntNrR8d1+T5Jvjekunq506nu51UYYemD1m1v1Wd88GmXOW1l3mTkleM7OdH2TorbjdBpZNfnJd3cMz9NAlQ4BdmnZud58zc4xLwxk/4H9/pWMcl5895l42/8VjbSeNp7r95go1Lt/Oj8b9Xs/Y/kdnCFbJ0JN2xDh8pyS/uNQ2Y/scmCGQL5kNnT/K0DO1IXdI8h8bmL5Hht6dc2amLX8dXLuP8Viykf3MU/Ph43If7u6zVtjODXGd530c3jHXDaIbe96v83xV1d2r6kNV9e2qujjDPx5mX+vJ/M9DkrwlyZnd/YqZfTy+qj47nrJ6UYZ/MCzfx0qWv86vGY9ng8/hHPUBrAmhDmA+syHrvCS7V9VuM9PumORbM+N3WBoYr7G6fZLzxl6lNyX5vSS37u5bZTjtsWbWvd3SdUUz2z5vAzWdm+S3xyC69Nilu09Y4RiOzRDeHpGhhy4Zwt1Dx2nHzix7XobwsHQMu2Y47XD2GGfb5Pxlx1yz49397e5+bnffNslvJ3l9Vd11AzWen6Gtlrazy7jflbwjyQFV9eAMvaGfHKefm+RTy9rmFt39XzeyrZWcm+H0vuW+l5/0QC5Z/jrYmOU9kPPU/PoMpyv+31X1sI1sa0td53nPcDxXJfnOCvva1PP1PzPckOdu3f1TGa79m31tz62qDs5wuuRzZqbdLENv5iuT3Gb8ffrwzD421S7LX+dLr9t5n0OAbYJQB7CZuvvcJCck+euq2rmqfiHDB83ZW8o/oKp+bTxt7YVJLs9wDdSuGT5oXpAMN3nIeEOWGT+d5AVVtVNVPSXDNVQf3kApb0jyJ1V173FbtxyXX8lnMly79BsZQ113XzjW8hu5bqh7R5JnV9Ve4wfnv0py4ngK3YYcneTeM8f8gsz0MFXVU6pq6cP/hWMbXLOB7bwnyROq6iHjDTUOycZDwIczfCj//5IcOfa0JEPwuXtVPWNsx52q6oGzN8HYDEckeUxVPbWqdqyqW1fVXuOpj+9K8vKq2m0M7H+Y674ONuY7Ga5dW7LRmqvqGRmuUTsoQ/seXj/52o3vJFlfN/zOrO9I8qKquvO47b/K0K5XrbD8pp6v3ZJcnOTSqrpnhusKN1sNN5h5QZInL+tBv2mGaygvSHLVuNzs1yB8J8mta7zZywa8K8l+VfVLVbVTkj/K8Lu60j9GALZJQh3Aljkgw40fzstwbdfLuvtjM/M/mOF6nQszXGP1a+P1WF/OcN3VZzJ84Pz5/ORUyCUnJrlbhp6glyfZv7uvdwpid78/ySuSvHM8te20JI9fqeDu/mGSUzJ8ED5tZtZxGYLksTPLfizJn2foBTk/Q0/V07OC7v5ehmu/Ds1w+t3dlh3XA5OcWFWXJjkqyR+M1/Qt387pSX4/yTvH/V6a4RrDy1fY7+UZbubxmCT/NDP9kgwf7p+e4Tn6doa2utlKx7CRY/tGhlP6/ijDKa6nJlm6m+nvZ7jRxtcynMr6T0neOuemX5Nk/xrujPnajdVcVXfMcJ3iM7v70u7+pyQnJ3nVuK13jz+/X1WfS679ku2PbObhvjXJ2zO8Fr6e5MfjMW7QHM/XH2e4JvGSDD3UK971dBOelmRdkjNm7mT5hrHNXpAhnF047uuomfq+kiGofm08pfU6pzF395kZ/qHxdxl+356Q4atNrtjCOgHWxNIduABgmzP2Fl2U4fS9r691PWyc5wtgbeipA2CbUlVPqKqbj9fxvTLJlzLcWp5tkOcLYO0JdQBsa56Y4fTD8zKcxvn0dlrJtszzBbDGnH4JAAAwYXrqAAAAJkyoAwAAmLAd17qAeeyxxx69fv36tS4DAABgTZxyyinf6+51G5o3iVC3fv36nHzyyWtdBgAAwJqoqnNWmuf0SwAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAnbca0LmLL1Bx+90flnH7rfVqoEAAC4sdJTBwAAMGFCHQAAwIQJdQAAABMm1AEAAEyYUAcAADBhQh0AAMCECXUAAAATJtQBAABMmFAHAAAwYUIdAADAhAl1AAAAEybUAQAATJhQBwAAMGFCHQAAwIQJdQAAABMm1AEAAEyYUAcAADBhQh0AAMCECXUAAAATJtQBAABMmFAHAAAwYUIdAADAhAl1AAAAEybUAQAATJhQBwAAMGFCHQAAwIQJdQAAABMm1AEAAEyYUAcAADBhQh0AAMCELTzUVdUOVfX5qvrQOH7nqjqxqr5aVUdW1U0XXQMAAMD2amv01P1BkjNmxl+R5FXdfdckFyZ5zlaoAQAAYLu00FBXVbdPsl+SN4/jlWTfJO8ZFzk8yZMWWQMAAMD2bNE9da9O8uIk14zjt05yUXdfNY5/M8ntFlwDAADAdmthoa6qfiXJd7v7lC1c/3lVdXJVnXzBBRescnUAAADbh0X21D00ya9W1dlJ3pnhtMvXJLlVVe04LnP7JN/a0Mrd/cbu3ru79163bt0CywQAAJiuhYW67v6T7r59d69P8vQkn+juA5N8Msn+42LPSvLBRdUAAACwvVuL76l7SZI/rKqvZrjG7i1rUAMAAMB2YcdNL3LDdfcxSY4Zh7+WZJ+tsV8AAIDt3Vr01AEAALBKhDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAnbca0LgCRZf/DRm1zm7EP32wqVAADAtOipAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmLCFhbqq2rmqTqqqL1TV6VX1/47T71xVJ1bVV6vqyKq66aJqAAAA2N4tsqfu8iT7dvd9k+yV5HFV9aAkr0jyqu6+a5ILkzxngTUAAABs1xYW6npw6Ti60/joJPsmec84/fAkT1pUDQAAANu7hV5TV1U7VNWpSb6b5KNJ/iPJRd191bjIN5PcbpE1AAAAbM8WGuq6++ru3ivJ7ZPsk+Se865bVc+rqpOr6uQLLrhgYTUCAABM2Va5+2V3X5Tkk0kenORWVbXjOOv2Sb61wjpv7O69u3vvdevWbY0yAQAAJmeRd79cV1W3God3SfLYJGdkCHf7j4s9K8kHF1UDAADA9m7HTS+yxfZMcnhV7ZAhPL6ruz9UVV9O8s6q+sskn0/ylgXWAAAAsF1bWKjr7i8mud8Gpn8tw/V1AAAA3EBb5Zo6AAAAFkOoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMLmCnVV9fOLLgQAAIDNN29P3eur6qSq+p2quuVCKwIAAGBuc4W67n54kgOT3CHJKVX1T1X12IVWBgAAwCbNfU1dd5+V5KVJXpLkkUleW1VfqapfW1RxAAAAbNy819T9QlW9KskZSfZN8oTu/rlx+FULrA8AAICN2HHO5f4uyZuT/Gl3X7Y0sbvPq6qXLqQyAAAANmneULdfksu6++okqaqbJNm5u3/U3W9fWHUAAABs1LzX1H0syS4z4zcfpwEAALCG5g11O3f3pUsj4/DNF1MSAAAA85o31P2wqu6/NFJVD0hy2UaWBwAAYCuY95q6FyZ5d1Wdl6SS/EySpy2sKgAAAOYyV6jr7n+rqnsmucc46czuvnJxZQEAADCPeXvqkuSBSdaP69y/qtLdb1tIVQAAAMxlrlBXVW9P8rNJTk1y9Ti5kwh1AAAAa2jenrq9k9yru3uRxQAAALB55r375WkZbo4CAADANmTenro9kny5qk5KcvnSxO7+1YVUBQAAwFzmDXWHLLIIAAAAtsy8X2nwqaq6U5K7dffHqurmSXZYbGkAAABsylzX1FXVc5O8J8k/jJNul+QDiyoKAACA+cx7o5TfTfLQJBcnSXefleSnF1UUAAAA85k31F3e3VcsjVTVjhm+pw4AAIA1NG+o+1RV/WmSXarqsUneneSfF1cWAAAA85g31B2c5IIkX0ry20k+nOSliyoKAACA+cx798trkrxpfAAAALCNmCvUVdXXs4Fr6Lr7LqteEQAAAHOb98vH954Z3jnJU5LsvvrlAAAAsDnmuqauu78/8/hWd786yX4Lrg0AAIBNmPf0y/vPjN4kQ8/dvL18AAAALMi8wexvZoavSnJ2kqeuejUAAABslnnvfvnoRRcCAADA5pv39Ms/3Nj87v7b1SkHAACAzbE5d798YJKjxvEnJDkpyVmLKAoAAID5zBvqbp/k/t19SZJU1SFJju7u31hUYQAAAGzaXF9pkOQ2Sa6YGb9inAYAAMAamren7m1JTqqq94/jT0py+GJKAgAAYF7z3v3y5VX1kSQPHyc9u7s/v7iyAAAAmMe8p18myc2TXNzdr0nyzaq684JqAgAAYE5zhbqqelmSlyT5k3HSTkn+cVFFAQAAMJ95e+qenORXk/wwSbr7vCS7LaooAAAA5jNvqLuiuztJJ0lV7bq4kgAAAJjXvKHuXVX1D0luVVXPTfKxJG9aXFkAAADMY967X76yqh6b5OIk90jyF9390YVWBgAAwCZtMtRV1Q5JPtbdj04iyAEAAGxDNnn6ZXdfneSaqrrlVqgHAACAzTDX6ZdJLk3ypar6aMY7YCZJd79gpRWq6g5J3pbkNhlusPLG7n5NVe2e5Mgk65OcneSp3X3hFlUPAABwIzdvqHvf+NgcVyX5o+7+XFXtluSUMRQelOTj3X1oVR2c5OAM34EHAADAZtpoqKuqO3b3N7r78M3dcHefn+T8cfiSqjojye2SPDHJo8bFDk9yTIQ6AACALbKpa+o+sDRQVe/d0p1U1fok90tyYpLbjIEvSb6d4fRMAAAAtsCmQl3NDN9lS3ZQVbdI8t4kL+zui2fnzX6h+QbWe15VnVxVJ19wwQVbsmsAAIDt3qZCXa8wPJeq2ilDoDuiu5euyftOVe05zt8zyXc3uOPuN3b33t2997p16zZ31wAAADcKmwp1962qi6vqkiS/MA5fXFWXVNXFG1uxqirJW5Kc0d1/OzPrqCTPGoefleSDW1o8AADAjd1Gb5TS3TvcgG0/NMkzMnwVwqnjtD9NcmiSd1XVc5Kck+SpN2AfAAAAN2rzfqXBZuvuT+e61+TN+qVF7RcAAODGZFOnXwIAALANE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDCdlzrAoDVt/7gozc6/+xD99tKlQAAsGh66gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACdtxrQsA2NatP/jojc4/+9D9tlIl07eptky0JwBsLj11AAAAEybUAQAATNjCQl1VvbWqvltVp81M272qPlpVZ40//69F7R8AAODGYJE9dYcledyyaQcn+Xh33y3Jx8dxAAAAttDCQl13H5vkB8smPzHJ4ePw4UmetKj9AwAA3Bhs7WvqbtPd54/D305ym628fwAAgO3Kmt0opbs7Sa80v6qeV1UnV9XJF1xwwVasDAAAYDq2dqj7TlXtmSTjz++utGB3v7G79+7uvdetW7fVCgQAAJiSrR3qjkryrHH4WUk+uJX3DwAAsF1Z5FcavCPJZ5Lco6q+WVXPSXJoksdW1VlJHjOOAwAAsIV2XNSGu/uAFWb90qL2CQAAcGOzZjdKAQAA4IYT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACdtxrQsAALbM+oOP3uj8sw/dbytVAsBa0lMHAAAwYUIdAADAhAl1AAAAEybUAQAATJhQBwAAMGFCHQAAwIQJdQAAABMm1AEAAEyYUAcAADBhQh0AAMCECXUAAAATJtQBAABM2I5rXQAAwLZg/cFHb3T+2Yfut5UqAdg8euoAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDBfaQAAwKra1NdDJL4iYnP4ug02RU8dAADAhAl1AAAAEybUAQAATJhQBwAAMGFCHQAAwIQJdQAAABPmKw0AAIAbhe316zb01AEAAEyYUAcAADBhQh0AAMCECXUAAAATJtQBAABMmFAHAAAwYUIdAADAhAl1AAAAEybUAQAATJhQBwAAMGFCHQAAwIQJdQAAABMm1AEAAEyYUAcAADBhQh0AAMCECXUAAAATJtQBAABMmFAHAAAwYUIdAADAhAl1AAAAEybUAQAATJhQBwAAMGFCHQAAwIStSairqsdV1ZlV9dWqOngtagAAANgebPVQV1U7JPn7JI9Pcq8kB1TVvbZ2HQAAANuDteip2yfJV7v7a919RZJ3JnniGtQBAAAweWsR6m6X5NyZ8W+O0wAAANhM1d1bd4dV+yd5XHf/1jj+jCS/2N2/t2y55yV53jh6jyRnbtVCt8weSb631kVsR7Tn6tGWq0t7ri7tuXq05erSnqtHW64u7bm6ptKed+rudRuasePWriTJt5LcYWb89uO06+juNyZ549YqajVU1cndvfda17G90J6rR1uuLu25urTn6tGWq0t7rh5tubq05+raHtpzLU6//Lckd6uqO1fVTZM8PclRa1AHAADA5G31nrruvqqqfi/JvyTZIclbu/v0rV0HAADA9mAtTr9Md384yYfXYt8LNqnTRSdAe64ebbm6tOfq0p6rR1uuLu25erTl6tKeq2vy7bnVb5QCAADA6lmLa+oAAABYJULdAlTVC6rqjKo6Yq1rmQLttRhVdcJa13BjVFWXrnUN25qqulVV/c44/Kiq+tBa1zQ1N7QNq+qgqrrtYqpbrHney6rqhVV1861Qy/qq+vWZ8b2r6rUL2M/ZVbXHam93W1NVx1TV9e44OL5eX7cWNW3L/F1nY4S6xfidJI/t7gPXupCJmLu9qmpNrgOdou5+yFrXAKNbZfg9Z8vd0DY8KMkkQ92c72UvTLJZoa6qdtiCctYnuTbUdffJ3f2CLdjOjd4Wtv+Nmr/rbIxQdwNV1R9W1Wnj44VV9YYkd0nykap60VrXt61b1l5/VFUfqKovVtVnq+oXxmUOqaq3V9XxSd6+pgVPyFKP0fhf/WOq6j1V9ZWqOqKqaq3r25aNr8NTqur0qnreOO3Sqnp5VX1hfH3eZpx+56r6TFV9qar+cm0r32YdmuRnq+rUJP8jyS029HqsqgdU1afGtv+XqtpzTavetszbhn9RVf82/k16Yw32T7J3kiOq6tSq2mUNj2Ozbeq9rKpekCGwfrKqPjku+8vj7+XnqurdVXWLcfrZVfWKqvpckqeM23tFVZ1UVf9eVQ8fl1tfVceN63+uqpY+TB+a5OFjO75otte0qnbfyN+wt477+tpY79KxXe+9Zgqq6r8tHUdVvaqqPjEO7zs+LweM74mnVdUrZta7tKr+pqq+kOTBy7b57PE5OCnJQ7fm8UzFzO/CnlV17Pg6PG3pdct86vqf3dfXcMbYm8bfxX+d2vtkkqS7PbbwkeQBSb6UZNckt0hyepL7JTk7yR5rXd9UHkvtleTvkrxsnLZvklPH4UOSnJJkl7WudUqPJJeOPx+V5D+T3D7DP3I+k+Rha13ftvxIsvv4c5ckpyW5dZJO8oRx+v+f5KXj8FFJnjkO/+5Su3tcpz3XJzltHN7g6zHJTklOSLJuXO5pGb7yZs3r3xYe87ThOG/3mXXePvOaPSbJ3mt9HFt47Jt8L5v9uzv+PTk2ya7j+EuS/MXMci+e2fYxSf5mHP4vST42Dt88yc7j8N2SnDxTw4dm1r92fBN/w05IcrOxtu8n2Wn2+Zp9r1l+PNviI8mDkrx7HD4uyUnj7/DLxsc3kqzLcJf1TyR50rhsJ3nqsrDL8OoAAAgCSURBVPbfO8meM+vcNMnxSV631se5rT1mfhf+KMmfjcM7JNltrWubyiMrf3a/Ksle4zLvSvIba13r5j701N0wD0vy/u7+YXdfmuR9Sfy3ZMs9LGNPXHd/Ismtq+qnxnlHdfdla1bZ9J3U3d/s7muSnJrhAyIre8H4n+TPJrlDhg91VyRZuo7plPykDR+a5B3jsJ7k+Wzo9XiPJPdJ8tGxN+qlGT68s2Er/U4/uqpOrKovZQgW916rAhdknveyByW5V5Ljx9fSs5LcaWb+kcuWf9/4c/b3eqckbxrb8d3j9jZlY3/Dju7uy7v7e0m+m+Q24/QNvddMwSlJHjAe3+UZAvbeGT4DXZTkmO6+oLuvSnJEkkeM612d5L0b2N4vzqxzRa7/HHFd/5bk2VV1SJKf7+5L1rieKVnps/vXu/vUcZnZ94LJcH0SU/HDtS5g4i6fGb46fvdXVFWPSvKYJA/u7h9V1TFJdk5yZY//wsv129B3w2yeDb0eK8np3f3gDa/CMtdrw6raOcnrM/TInTt+4Nt5LYpboHneyyrJR7v7gBW2sfzvydI2Z7f3oiTfSXLfDL2CP96iaq+/j2v3s5H3mm1ed19ZVV/PcK3mCUm+mOTRSe6aoZfxASus+uPuvnpr1Lg96+5jq+oRSfZLclhV/W13v22t65q45b+jkzv9Uk/dDXNckidV1c2ratckTx6nsWWOS3Jgcu0H6+9198VrWhE3RrdMcuH4IeueGf7rvzHHJ3n6OOzmSBt2SZLdNrHMmUnWVdWDk6Sqdqqq7a2X6YaYpw2XAsH3xmvI9t/M9ads9vg+m+ShVXXXJKmqXavq7pu5vVsmOX/sEXxGhlPclu9nuc39G7a57zXbmuOS/HGGU12PS/L8JJ/PcCrmI6tqjxpuhnJAkk9tYlsnjuvcuqp2SvKUxZU9fVV1pyTf6e43JXlzkvuvcUlTst1+dvff+huguz9XVYdleANLkjd39+fLPSi21CFJ3lpVX0zyowynzMDW9r+TPL+qzsgQND67ieX/IMk/VdVLknxw0cVNUXd/v6qOr6rTklyWoQdk+TJX1HBDj9dW1S0z/H16dYbrHW705mzDi6rqTRmuzfp2hlO0lhyW5A1VdVmGnqHt7XT2Nyb531V1Xnc/uqoOSvKOqrrZOP+lSf59M7b3+iTvrapnZnhPWOrd+2KSq8dTJg/LEGKWHJLN+xu2ue8125rjkvxZks909w+r6sdJjuvu86vq4CSfzNBrenR3b/S9cVznkAyncV6U4dRaVvaoJP+tqq5McmmSZ65tOdOxoc/uSS5cu4pWT/3kbCIAAACmxumXAAAAEybUAQAATJhQBwAAMGFCHQAAwIQJdQAAABMm1AGwzaiqq6vq1Ko6rareXVU338z1L93M5Q8bv0ph+fRHVdWHNmM776+qJ82Mn1lVL50Zf29V/drm1DZPnQCQCHUAbFsu6+69uvs+Sa7I8IXG16rBtvi36/gkD0mSqrp1hu81e/DM/AcnOWGeDVWV75AFYLNsi38YASAZvtz4rlW1fuz5eluGL9a+Q1UdUFVfGnv0XjG7UlW9qqpOr6qPV9W6cdpzq+rfquoLY6/ZbA/gY6rq5Kr696r6leVFVNWuVfXWqjqpqj5fVU/cQK0nZAx1489/TrJuDKF3zhBWv11VO1fV/xpr/3xVPXrcx0FVdVRVfSLJx8f1Xjce98eS/PRMPYdW1Zer6otV9cotbVwAth9CHQDbnLG36vFJvjROuluS13f3vZNcmeQVSfZNsleSB86c+rhrkpPH5T6V5GXj9Pd19wO7+75JzkjynJndrU+yT5L9kryhqnZeVs6fJflEd++T5NFJ/kdV7bpsmVOS3Keqbpoh1H0myZlJfm4cX+ql+90k3d0/n+SAJIfP7O/+Sfbv7kcmeXKSeyS5V5Jn5rq9gE9Ocu/u/oUkf7nRhgTgRkGoA2BbsktVnZrk5CTfSPKWcfo53f3ZcfiBSY7p7gu6+6okRyR5xDjvmiRHjsP/mORh4/B9quq4qvpSkgOT3Htmn+/q7mu6+6wkX0tyz2U1/XKSg8e6jkmyc5I7zi7Q3ZcnOT1DMHtQkhMzBLuHjI/jx0UfNtaV7v5KknOS3H2c99Hu/sE4/Igk7+juq7v7vCSfGKf/Z5IfJ3nLeI3ej67fhADc2DhvH4BtyWXdvdfshKpKhmvUtkSPPw9L8qTu/kJVHZTkURtYZqXxSvL/dPeZm9jX8RnC2G7dfWFVfTbJ7yW5X5J/mKPWTR5jd19VVfsk+aUk+4/b33eObQOwHdNTB8DUnJTkkVW1R1XtkOE0xk+N826SIewkya8n+fQ4vFuS86tqpww9dbOeUlU3qaqfTXKXDKdNzvqXJL9fY7qsqvutUNcJSX47yRfG8S9m6LW7Y4ZrAZPhOsEDx+3cfZy3obB4bJKnVdUOVbVnhtM+U1W3SHLL7v5wkhclue8KtQBwI6KnDoBJ6e7zq+rgJJ/M0It2dHd/cJz9wyT7jF8n8N0kTxun/3mGUyIvGH/uNrPJb2QIij+V5Pnd/eMxvy3570leneSL4503v57kejdUyRDq7pLkr8c6r6qq7yY5t7uvGZd5fZL/OZ4GelWSg7r78mX7S5L3Z+iB+/JY32fG6bsl+eB4HV4l+cONtRUANw7VvfwsEwAAAKbC6ZcAAAATJtQBAABMmFAHAAAwYUIdAADAhAl1AAAAEybUAQAATJhQBwAAMGFCHQAAwIT9H2+3AzcnJJwEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pws_2gram=probable_words('the world health organization', cpd_2gram_with_gt, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3UAAAGDCAYAAACMWjkNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deZhsZX3u/e8tYADBAdkxiMB2IBrUiLrlaEBFHF6VGDFBjSEqiZHkjWNijiFqoueNGnzjlIRoAmrAiLMiCHhywAEUkVGQSYIDiICAAgqKjL/zx3pairZ77+pNV1c/vb+f61pXr6nW+q1nV9Wuu561VqWqkCRJkiT16S7TLkCSJEmStP4MdZIkSZLUMUOdJEmSJHXMUCdJkiRJHTPUSZIkSVLHDHWSJEmS1DFDnSRNQZI3JfnQej523yRfWcvyLyX5k/WvbulMotYkr0vyvsXcptYuySFJ3jztOtZmXTUmqSQPWsqaJGmxGOokaUxJLkpyQ5Lrk1zRPiRuMe26FkuS/0ry1yPT27YPunPN+7XpVLluVfXWquoi1K7LYobe9fkioT3nV7fn+r5t3lq/VFgJZo63DYdMux5JWhdDnSQtzLOqagvgUcAa4A2zV8igx/fXE4AnjEw/AfjmHPMurKofLGTDSTa+8+VJ6yfJRtOuQZImqccPHZI0dVV1KfA54GHwix6VtyQ5EfgZ8IAk901yZJKrk3wryUtnbWbTJB9Lcl2SM5I8YmZBkv2TfLstOy/Jc2Y9NkkOTPLjJN9M8uT5ak3yx0nOT3JN643bYZ5VTwB2HQmkjwfeDayZNe+EkW2/tB3b1e1Y7zuyrJK8LMmFwIVt3lNbvT9OciCQkfUflOT4tuyHST62lmN6UZKLk/woyd+2HqWntGW/6JFK8rkkL5/12LOS/G4bf0iSY1v9FyR53sh6hyT51yRHt3+Hk5M8cC017Zbkq0muTXLJSM/WPZJ8MMlVreY3zLTnTK9Xkre3f5/vJnlGW/aW1t4Htt7hA9dWc5K7JjkzySva9EZJTkzyd0meDrwOeH7b1lnzHcfaJPkN4N+Ax7XtXDuy+F7ztdXa2nnW9p+U5OyR6WOTnDoy/eUke83U0l531yY5N8nvjKx3SJL3JjkmyU+BJ82xr/+Z5PIklyX54/VpD0laNqrKwcHBwWGMAbgIeEob3w44F/j7Nv0l4HvAQ4GNgU0Yws97gE2BnYGrgD3a+m8Cbgb2buv+FfBdYJO2/LnAfRm+fHs+8FNgm7ZsX+AW4C/aY58P/BjYaqSWP2njzwa+BfxGq+sNwFfnOb5fAW4AHtmmzwEeAJw4a96L2vgewA8Zei1/BfgX4ISR7RVwLLAVsBmwNXDdyDH/RTuOmVo/Ary+HfOmwG7z1LkTcD2wG3BX4O2tLZ8y0rYfauMvAk6c9dhrW713Ay4B/qi1zSPb8ezU1j0E+BGwS1t+GPDReWraoR3bC9qx3RvYuS37IHAEsCWwGvhv4CUj/5Y3Ay8FNgL+X+AyILP/Ldv0ump+GHBN+/d+PfA1YKPZ7TKyvf2Boxb4OtgX+MqsefO21bpqnrWdzYCft+fKJsAVwKWt7TZjeH7euy37FkNQvSvDc/E64MEj9fwY2JXbn0+HAG9uy5/etv2wVt+HGZ6vD5r2+4yDg4PD+gz21EnSwnym9U58BTgeeOvIskOq6tyqugX4NYYPlH9dVT+vqjOB9zGEjBmnV9Unq+pm4J0MHzwfC1BVn6iqy6rqtqr6GENP1y4jj70SeHdV3dyWXwDsOUe9fwb8Q1Wd3+p6K7DzXL11VXUjcDLwhCRbAfeoqu8AXx6Zt1M7boB9gA9U1RntsX/D0IOzemSz/1BVV1fVDcAzgXNHjvndwOhpnDczhKP7tjab77qtvYHPVtVXquom4O8YPpDP5fBZx7sP8OlW728DF1XVf1TVLVX1deBTDIH6F4+vqlNa2x3GEM7n8gfAcVX1kfZv8qOqOjPDaX+/D/xNVV1XVRcB7wBeOPLYi6vq4Kq6FTgU2Aa4zzz7WWvNVXUO8GbgMwxfFLywbXdOVXVAVf32fMsXaL62GqedZ+q5ATiV4TTfRwNnMXypsCvDa+PCqvpRG98COKCqbqqqLwBHMYTqGUdU1YntNfTzWbt6HvAfVXVOVf2UIfBKUrcMdZK0MHtV1T2raoeq+vP2IXTGJSPj9wWurqrrRuZdDGw71/pVdRvw/fa4mdMLz2ynll3L0KOw9chjL62q0SBz8cxjZ9kB+KeR7VzNcMrjtnOsC7dfV/d4hg/TMATYmXmXVNXFI8c4M05VXc/QWzPnMbb1R4+5Zi1/bavtlHY63XynxM3ezs/afn9Ja/+jGYIVDB/6D2vjOwD/Y6ZtWvvswxDIZ4yGzp8xBIm5bAd8e475Mz1OF4/Mm/08+MU+2rGwlv2MU/Ohbb1jqurCebYzCfO11Tg1jzoe2J3hOXc8Q2/lE9sw84XCfRmei7eNPG7e19cc7jtr+cXzrShJPfDCdUlaPKMh6zJgqyRbjgS77RlOJZux3cxIu8bqfsBlrVfpYODJwElVdWuSMxm5/gzYNklGgt32wJFz1HQJ8JaqOmyOZXM5gaF37yKGHjoYwt372rwTRta9jOED+8wx3I3h1LjRYxxtk8u54zFndLqGm6+8tC3bDTguyQlV9a1ZNV4OPHhkO5u1/c7nI8Abk5zA0Bv6xTb/EuD4qnrqWh47rku4Y0/qjB9yew/keW3e7OfB2szugRyn5vcw9Fr9P0l2G+nxnK83c6EWup2FtvPxDL2Z3wMOYDid9GDgRuBf2zqXAdsluctIsNue4dTWceq8w3OxPVaSumVPnSRNQFVdAnwV+Ickmyb5TeAlwOgt5R+d5Hcz3Bny1QwfWr/GcI1PMVyDR5I/ot2QZcSvAq9MskmS5zJcQ3XMHKX8G/A3SR7atnWPtv58TgLuCfwhLdRV1TWtlj/kjqHuI8AfJdk5ya8wnNp5cjvFcC5HAw8dOeZXMtJbk+S5Se7XJq9pbXDbL2+GTwLPSvJbSe7KcOpc5lhvxjEMoer/Az42EgKOAn49yQtbO26S5DHtZiALdRjwlCTPS7Jxknsn2bmd+vhx4C1JtmyB/S+54/Ngba5guK5xxlprTvJChtMW92Vo30Nz+89uXAGszp2/M+sVwP1a249joe38VYbQvgtwSlWdS+vt4/bn38kMvYGvbdvbHXgW8NExa/o4sG+SnZJsDrxxzMdJ0rJkqJOkyXkBw40xLmO4tuuNVXXcyPIjGG5ycg3DNVa/267HOo+hp+Ikhg/QD+f2UyFnnAzsyNAT9BZg73at0R1U1eHA24CPJvkJw41OnjFfwe36otMZbj5xzsiiLzMEyRNG1j0O+FuG66MuBx7I7ac5zrXtHzJcR3UAw+mSO846rscAJye5nqHX8VXtmr7Z2zkXeAXDB/jLGW6aciVDKJ5rvzcCnwaewnBDjJn51wFPazVfxnD64NsYbqKyIFX1PYZrBl/DcIrrmcDM3UxfwXCjm+8wnMr6YeADY276n4C9M9wZ85/XVnOS7RmuU3xRVV1fVR8GTgPe1bb1ifb3R0nOgF/8UPvnFni4X2C4SdAPkvxwXSsvtJ3bc/AMhusvb2qzT2K49vDKts5NDCHuGQyvgfcwHPc3xzmAqvocQ1t9geGGK18Y53GStFzN3F1LkqQutZ6oa4Edq+q7065HkqSlZk+dJKk7SZ6VZPN2Hd/bgbMZrvmTJGmDY6iTJPXo2Qyn8l3GcBrn75ennkiSNlCefilJkiRJHbOnTpIkSZI6ZqiTJEmSpI518ePjW2+9da1evXraZUiSJEnSVJx++uk/rKpVcy3rItStXr2a0047bdplSJIkSdJUJLl4vmWefilJkiRJHTPUSZIkSVLHDHWSJEmS1DFDnSRJkiR1zFAnSZIkSR0z1EmSJElSxwx1kiRJktQxQ50kSZIkdcxQJ0mSJEkdM9RJkiRJUscMdZIkSZLUMUOdJEmSJHXMUCdJkiRJHdt42gX0bPX+R691+UUH7LlElawMtqckSZK0cPbUSZIkSVLHDHWSJEmS1DFDnSRJkiR1zFAnSZIkSR0z1EmSJElSxwx1kiRJktQxQ50kSZIkdcxQJ0mSJEkdM9RJkiRJUscMdZIkSZLUMUOdJEmSJHXMUCdJkiRJHTPUSZIkSVLHDHWSJEmS1DFDnSRJkiR1zFAnSZIkSR0z1EmSJElSxwx1kiRJktQxQ50kSZIkdcxQJ0mSJEkdM9RJkiRJUscMdZIkSZLUMUOdJEmSJHXMUCdJkiRJHZtYqEuyaZJTkpyV5Nwk/6vNv3+Sk5N8K8nHktx1UjVIkiRJ0ko3yZ66G4E9quoRwM7A05M8Fngb8K6qehBwDfCSCdYgSZIkSSvaxEJdDa5vk5u0oYA9gE+2+YcCe02qBkmSJEla6SZ6TV2SjZKcCVwJHAt8G7i2qm5pq3wf2Haex+6X5LQkp1111VWTLFOSJEmSujXRUFdVt1bVzsD9gF2AhyzgsQdV1ZqqWrNq1aqJ1ShJkiRJPVuSu19W1bXAF4HHAfdMsnFbdD/g0qWoQZIkSZJWokne/XJVknu28c2ApwLnM4S7vdtqLwaOmFQNkiRJkrTSbbzuVdbbNsChSTZiCI8fr6qjkpwHfDTJm4GvA++fYA2SJEmStKJNLNRV1TeAR84x/zsM19dJkiRJku6kJbmmTpIkSZI0GYY6SZIkSeqYoU6SJEmSOmaokyRJkqSOGeokSZIkqWOGOkmSJEnqmKFOkiRJkjpmqJMkSZKkjhnqJEmSJKljhjpJkiRJ6pihTpIkSZI6ZqiTJEmSpI4Z6iRJkiSpY4Y6SZIkSeqYoU6SJEmSOmaokyRJkqSOGeokSZIkqWOGOkmSJEnqmKFOkiRJkjpmqJMkSZKkjhnqJEmSJKljhjpJkiRJ6pihTpIkSZI6ZqiTJEmSpI4Z6iRJkiSpY4Y6SZIkSeqYoU6SJEmSOmaokyRJkqSOGeokSZIkqWOGOkmSJEnqmKFOkiRJkjpmqJMkSZKkjhnqJEmSJKljhjpJkiRJ6pihTpIkSZI6ZqiTJEmSpI4Z6iRJkiSpY4Y6SZIkSerYxEJdku2SfDHJeUnOTfKqNv9NSS5NcmYbnjmpGiRJkiRppdt4gtu+BXhNVZ2RZEvg9CTHtmXvqqq3T3DfkiRJkrRBmFioq6rLgcvb+HVJzge2ndT+JEmSJGlDNMmeul9Ishp4JHAysCvw8iQvAk5j6M27Zo7H7AfsB7D99tsvRZnSirF6/6PXuc5FB+y5BJVIkiRp0iZ+o5QkWwCfAl5dVT8B3gs8ENiZoSfvHXM9rqoOqqo1VbVm1apVky5TkiRJkro00VCXZBOGQHdYVX0aoKquqKpbq+o24GBgl0nWIEmSJEkr2STvfhng/cD5VfXOkfnbjKz2HOCcSdUgSZIkSSvdJK+p2xV4IXB2kjPbvNcBL0iyM1DARcCfTrAGSZIkSVrRJnn3y68AmWPRMZPapyRJkiRtaCZ+oxRJkiRJ0uQY6iRJkiSpY4Y6SZIkSeqYoU6SJEmSOmaokyRJkqSOGeokSZIkqWOGOkmSJEnqmKFOkiRJkjpmqJMkSZKkjhnqJEmSJKljhjpJkiRJ6pihTpIkSZI6ZqiTJEmSpI4Z6iRJkiSpY4Y6SZIkSeqYoU6SJEmSOmaokyRJkqSOGeokSZIkqWOGOkmSJEnqmKFOkiRJkjpmqJMkSZKkjhnqJEmSJKljhjpJkiRJ6tjG0y5Akpa71fsfvdblFx2w5xJVIkmS9MvsqZMkSZKkjhnqJEmSJKljhjpJkiRJ6pihTpIkSZI6ZqiTJEmSpI4Z6iRJkiSpY4Y6SZIkSeqYoU6SJEmSOmaokyRJkqSOGeokSZIkqWOGOkmSJEnqmKFOkiRJkjpmqJMkSZKkjhnqJEmSJKljEwt1SbZL8sUk5yU5N8mr2vytkhyb5ML2916TqkGSJEmSVrpJ9tTdArymqnYCHgu8LMlOwP7A56tqR+DzbVqSJEmStB4mFuqq6vKqOqONXwecD2wLPBs4tK12KLDXpGqQJEmSpJVuSa6pS7IaeCRwMnCfqrq8LfoBcJ+lqEGSJEmSVqKJh7okWwCfAl5dVT8ZXVZVBdQ8j9svyWlJTrvqqqsmXaYkSZIkdWmsUJfk4euz8SSbMAS6w6rq0232FUm2acu3Aa6c67FVdVBVramqNatWrVqf3UuSJEnSijduT917kpyS5M+T3GOcByQJ8H7g/Kp658iiI4EXt/EXA0eMXa0kSZIk6Q7GCnVV9XhgH2A74PQkH07y1HU8bFfghcAeSc5swzOBA4CnJrkQeEqbliRJkiSth43HXbGqLkzyBuA04J+BR7beuNeNnFo5uv5XgMyzuSevT7GSJEmSpDsa95q630zyLoafJdgDeFZV/UYbf9cE65MkSZIkrcW4PXX/AryPoVfuhpmZVXVZ672TJEmSJE3BuKFuT+CGqroVIMldgE2r6mdV9Z8Tq06SJEmStFbj3v3yOGCzkenN2zxJkiRJ0hSNG+o2rarrZyba+OaTKUmSJEmSNK5xQ91PkzxqZiLJo4Eb1rK+JEmSJGkJjHtN3auBTyS5jOFnCn4NeP7EqpIkSZIkjWWsUFdVpyZ5CPDgNuuCqrp5cmVJkiRJksYx9o+PA48BVrfHPCoJVfXBiVQlSZIkSRrLWKEuyX8CDwTOBG5tswsw1EmSJEnSFI3bU7cG2KmqapLFSJIkSZIWZty7X57DcHMUSZIkSdIyMm5P3dbAeUlOAW6cmVlVvzORqiRJkiRJYxk31L1pkkVIkjYMq/c/ep3rXHTAnktQycqwrva0LSVpwzDuTxocn2QHYMeqOi7J5sBGky1NkiRJkrQuY11Tl+SlwCeBf2+ztgU+M6miJEmSJEnjGfdGKS8DdgV+AlBVFwK/OqmiJEmSJEnjGTfU3VhVN81MJNmY4XfqJEmSJElTNG6oOz7J64DNkjwV+ATw2cmVJUmSJEkax7ihbn/gKuBs4E+BY4A3TKooSZIkSdJ4xr375W3AwW2QJEmSJC0TY4W6JN9ljmvoquoBi16RJEmSJGls4/74+JqR8U2B5wJbLX45kiRJkqSFGOuauqr60chwaVW9G9hzwrVJkiRJktZh3NMvHzUyeReGnrtxe/kkSZIkSRMybjB7x8j4LcBFwPMWvRpJkiRJ0oKMe/fLJ026EEmSJEnSwo17+uVfrm15Vb1zccqRJEmSJC3EQu5++RjgyDb9LOAU4MJJFCVJkiRJGs+4oe5+wKOq6jqAJG8Cjq6qP5xUYZIkSZKkdRvrJw2A+wA3jUzf1OZJkiRJkqZo3J66DwKnJDm8Te8FHDqZkiRJkiRJ4xr37pdvSfI54PFt1h9V1dcnV5YkSdLSWr3/0WtdftEBey5RJZK0MOOefgmwOfCTqvon4PtJ7j+hmiRJkiRJYxor1CV5I/DXwN+0WZsAH5pUUZIkSZKk8YzbU/cc4HeAnwJU1WXAlpMqSpIkSZI0nnFD3U1VVUABJLnb5EqSJEmSJI1r3FD38ST/DtwzyUuB44CDJ1eWJEmSJGkcY4W6qno78EngU8CDgb+rqn9Z22OSfCDJlUnOGZn3piSXJjmzDc+8M8VLkiRJ0oZunT9pkGQj4LiqehJw7AK2fQhwIMNv3I16VwuJkiRJkqQ7aZ09dVV1K3BbknssZMNVdQJw9foWJkmSJElat7F+fBy4Hjg7ybG0O2ACVNUr12OfL0/yIuA04DVVdc16bEOSJEmSxPih7tNtuLPeC/w9w100/x54B/DHc62YZD9gP4Dtt99+EXYtSZKkpbB6/6PXuc5FB+y5BJVIG4a1hrok21fV96rq0MXYWVVdMbLtg4Gj1rLuQcBBAGvWrKnF2L8kSZIkrTTruqbuMzMjST51Z3eWZJuRyecA58y3riRJkiRp3dZ1+mVGxh+wkA0n+QiwO7B1ku8DbwR2T7Izw+mXFwF/upBtSpIkSZLuaF2hruYZX6eqesEcs9+/kG1IkiRJktZuXaHuEUl+wtBjt1kbp01XVd19otVJkiRJktZqraGuqjZaqkIkSZIkSQu3zh8flyRJkiQtX4Y6SZIkSerYuD8+LkmSJGkK1vVj7v6Qu+ypkyRJkqSOGeokSZIkqWOGOkmSJEnqmKFOkiRJkjpmqJMkSZKkjhnqJEmSJKljhjpJkiRJ6pihTpIkSZI65o+PS5IkSdpgrMQfc7enTpIkSZI6ZqiTJEmSpI4Z6iRJkiSpY4Y6SZIkSeqYoU6SJEmSOmaokyRJkqSOGeokSZIkqWOGOkmSJEnqmKFOkiRJkjpmqJMkSZKkjhnqJEmSJKljhjpJkiRJ6pihTpIkSZI6ZqiTJEmSpI4Z6iRJkiSpY4Y6SZIkSeqYoU6SJEmSOmaokyRJkqSOGeokSZIkqWOGOkmSJEnqmKFOkiRJkjpmqJMkSZKkjhnqJEmSJKljhjpJkiRJ6tjEQl2SDyS5Msk5I/O2SnJskgvb33tNav+SJEmStCGYZE/dIcDTZ83bH/h8Ve0IfL5NS5IkSZLW08RCXVWdAFw9a/azgUPb+KHAXpPavyRJkiRtCJb6mrr7VNXlbfwHwH2WeP+SJEmStKJM7UYpVVVAzbc8yX5JTkty2lVXXbWElUmSJElSP5Y61F2RZBuA9vfK+VasqoOqak1VrVm1atWSFShJkiRJPVnqUHck8OI2/mLgiCXevyRJkiStKJP8SYOPACcBD07y/SQvAQ4AnprkQuApbVqSJEmStJ42ntSGq+oF8yx68qT2KUmSJEkbmqndKEWSJEmSdOcZ6iRJkiSpY4Y6SZIkSeqYoU6SJEmSOmaokyRJkqSOGeokSZIkqWOGOkmSJEnqmKFOkiRJkjpmqJMkSZKkjhnqJEmSJKljhjpJkiRJ6pihTpIkSZI6ZqiTJEmSpI4Z6iRJkiSpY4Y6SZIkSeqYoU6SJEmSOmaokyRJkqSOGeokSZIkqWOGOkmSJEnqmKFOkiRJkjpmqJMkSZKkjhnqJEmSJKljhjpJkiRJ6pihTpIkSZI6ZqiTJEmSpI4Z6iRJkiSpY4Y6SZIkSeqYoU6SJEmSOmaokyRJkqSOGeokSZIkqWOGOkmSJEnqmKFOkiRJkjpmqJMkSZKkjhnqJEmSJKljhjpJkiRJ6pihTpIkSZI6ZqiTJEmSpI4Z6iRJkiSpYxtPY6dJLgKuA24FbqmqNdOoQ5IkSZJ6N5VQ1zypqn44xf1LkiRJUvc8/VKSJEmSOjatUFfA/0lyepL9plSDJEmSJHVvWqdf7lZVlyb5VeDYJN+sqhNGV2hhbz+A7bfffho1SpIkSdKyN5Weuqq6tP29Ejgc2GWOdQ6qqjVVtWbVqlVLXaIkSZIkdWHJQ12SuyXZcmYceBpwzlLXIUmSJEkrwTROv7wPcHiSmf1/uKr+9xTqkCRJkqTuLXmoq6rvAI9Y6v1KkiRJ0krkTxpIkiRJUscMdZIkSZLUMUOdJEmSJHXMUCdJkiRJHTPUSZIkSVLHDHWSJEmS1DFDnSRJkiR1zFAnSZIkSR0z1EmSJElSxwx1kiRJktQxQ50kSZIkdcxQJ0mSJEkdM9RJkiRJUscMdZIkSZLUMUOdJEmSJHXMUCdJkiRJHTPUSZIkSVLHDHWSJEmS1DFDnSRJkiR1zFAnSZIkSR0z1EmSJElSxwx1kiRJktQxQ50kSZIkdcxQJ0mSJEkdM9RJkiRJUscMdZIkSZLUMUOdJEmSJHXMUCdJkiRJHTPUSZIkSVLHDHWSJEmS1DFDnSRJkiR1zFAnSZIkSR0z1EmSJElSxwx1kiRJktQxQ50kSZIkdcxQJ0mSJEkdM9RJkiRJUscMdZIkSZLUsamEuiRPT3JBkm8l2X8aNUiSJEnSSrDkoS7JRsC/As8AdgJekGSnpa5DkiRJklaCafTU7QJ8q6q+U1U3AR8Fnj2FOiRJkiSpe9MIddsCl4xMf7/NkyRJkiQtUKpqaXeY7A08var+pE2/EPgfVfXyWevtB+zXJh8MXLCkha6frYEfTruIFcT2XDy25eKyPReX7bl4bMvFZXsuHttycdmei6uX9tyhqlbNtWDjpa4EuBTYbmT6fm3eHVTVQcBBS1XUYkhyWlWtmXYdK4XtuXhsy8Vley4u23Px2JaLy/ZcPLbl4rI9F9dKaM9pnH55KrBjkvsnuSvw+8CRU6hDkiRJkrq35D11VXVLkpcD/wVsBHygqs5d6jokSZIkaSWYxumXVNUxwDHT2PeEdXW6aAdsz8VjWy4u23Nx2Z6Lx7ZcXLbn4rEtF5ftubi6b88lv1GKJEmSJGnxTOOaOkmSJEnSIjHU3UlJvjrtGjZESa6fdg1LaVrHm2R1kj+Yxr6nKck9k/x5G989yVHTrmmlGG1bjae9Ds9ZhO3sm+TAxahpQ5PklUnOT3LYtGvpiZ+RxuP74uJLsl2SLyY5L8m5SV7V5m+V5NgkF7a/92rzH5LkpCQ3JvmrdW1nOTLU3UlV9VvTrkEalcFivbZXAxtcqAPuCfgf7GTYturRnwNPrap9pl1IT/yMNLYFvy8m2WhCtawUtwCvqaqdgMcCL0uyE7A/8Pmq2hH4fJsGuBp4JfD2Mbez7Bjq7qSZHpT2bf6XknwyyTeTHJYk065vOUvymSSnt28+9mvzrk/yliRnJflakvu0+fdv36CcneTN0618epJskeTzSc5obfHsNn91kguSfBA4B9guyUuS/HeSU5IcPPMNfZJVST6V5NQ27NrmPzHJmW34epItgQOAx7d5fzGt456CA4AHJjkT+Edgi7le20keneT49jz+ryTbTLXqPvyibZP8YxvOac/n50+7uGVs4/bcO789FzdP8nftNXxOkoNGnpdfSvK29tr/7ySPn72xJHu299Stl/5Qlrckf9na9Jwkr07yb8ADgM9tYO+Dd9rIZ6RtkpzQXvfnzPWc3MCNvi+empGzQ5IcmGTfNn5Re22fATx3vtd6koe2eWcm+UaSHadyVFNUVZdX1Rlt/DrgfGBb4NnAoW21Q4G92jpXVtWpwM1jbmf5qSqHOzEA17e/uwM/Zvgx9bsAJwG7Tbu+5TwAW7W/mzEEkXsDBTyrzf//gTe08SOBF7Xxl820+4YyjDzPNs23ydUAAAj1SURBVAbu3sa3Br4FhKFH7TbgsW3ZfYGLgK2ATYAvAwe2ZR+eeW4C2wPnt/HPAru28S3avnYHjpr28U+hvVcD57TxOV/brV2/Cqxq6z2f4Sdapl7/ch5mte3vAccy/LzNfYDvAdtMu8blNrQ2q5HX5weAv5p5D23z/nPkvfNLwDva+DOB49r4vsCBwHPae8K9pn1sy20AHg2cDdytvQ+eCzyyvZ9uPe36ehtG/u96DfD6Nr4RsOW0a1tOwxz/5xw1suxAYN82fhHw2pFl873W/wXYp43fFdhs2se4DNr3e8DdgWtH5md0us17E/BX69rOtI9prmEqP2mwgp1SVd8HaN/wrwa+MtWKlrdXJnlOG98O2BG4CZj5hup04KltfFeGD4AwfHh521IVucwEeGuSJzCEuG0ZPgwDXFxVX2vjuwDHV9XVAEk+Afx6W/YUYKeRjuS7J9kCOBF4Z4ZrRj5dVd+3s/kX5nptXws8DDi2tdNGwOXTKrBTuwEfqapbgSuSHA88huFLHN3RJVV1Yhv/EMNpQt9N8lpgc4YvcM5l+HIG4NPt7+kMz9cZewBrgKdV1U8mXXSHdgMOr6qfAiT5NGCv0p13KvCBJJsAn6mqM6ddUMc+Nmt6rtf6ScDrk9yP4f/zC5eotmWnfb75FPDqqvrJ6OeaqqokY/0MwOztTKTYO8nTLxfXjSPjtzKl3wHsQZLdGcLF46rqEcDXgU2Bm6t9HcIvt6G/vwH7AKuAR1fVzsAVDO0G8NMxt3EXhh69nduwbVVdX1UHAH/C0HN6YpKHLHbxHZvrtR3g3JF2fHhVPW065WkDMPv9r4D3AHtX1cOBg7n9vQBuf87Ofh/9NrAlt3/JI01cVZ0APAG4FDgkyYumXNJydgt3/Hy+6azls/+v/6XXelV9GPgd4AbgmCR7TKDOZa99ifAp4LCqmgm/V8xcKtH+Xrme21l2DHWalnsA11TVz1p4eOw61j8R+P02viFfqH4P4MqqujnJk4Ad5lnvVOCJSe6VZGNu7+UE+D/AK2Ymkuzc/j6wqs6uqre1xz8EuI7hA+CGZpzjvgBYleRxMLzpJ3noxCvr32jbfhl4fpKNkqxi+NB3ytQqW962n3muMdy8aOYskB+2b5D3HnM7FzO8H3zQ5+ucvgzs1a5ZvBu3n6qqOyHJDsAVVXUw8D7gUVMuabkZfV+8mOFsml9Jck/gyQvdWJIHAN+pqn8GjgB+c9Eq7US7xvj9DJeYvHNk0ZHAi9v4ixnaZ322s+zYk6Rp+d/AnyU5n+HD8dfWsf6rgA8n+WvW8QJc4Q4DPpvkbOA04JtzrVRVlyZ5K8MH5Kvbej9ui18J/GuSbzC8B5wA/Bnw6hYUb2M4jetzbfzWJGcBh1TVuyZ2ZMtIVf0oyYkZbiN/A0OP6Ox1bkqyN/DPSe7B0JbvZmg7zWNW234O+AZwFkPP02ur6gdTLXD5uoDhrmsfAM4D3gvci+F65B8wfBEzlqr6ZpJ9gE8keVZVfXsSBfeoqs5Icgi3f7nwvqr6uqei32m7A/8zyc3A9YA9dSPmeF/8OMNr+7sMZzIt1POAF7b2/gHw1kUrth+7Ai8Ezm6XTQC8juGmNB9P8hKGAP08gCS/xvC56u7AbUleDezEEIh/aTtVdcySHcmYcvuZbpJWkiRbVNX1rafucIabeBw+7bokSZK0uDz9Ulq53tS+VZr5tu8zU65HkiRJE2BPnSRJkiR1zJ46SZIkSeqYoU6SJEmSOmaokyRJkqSOGeokSctGkluTnJnknCSfSLL5Ah9//QLXP6T9NMXs+bsnOWoB2zk8yV4j0xckecPI9KeS/O5CahunTkmSwFAnSVpebqiqnavqYcBNDL+h+AsZLMf/u04Efgsgyb2BnwKPG1n+OOCr42yo/QyJJEljW47/MUqSBPBl4EFJVreerw8y/ETHdklekOTs1qP3ttEHJXlXknOTfD7JqjbvpUlOTXJW6zUb7QF8SpLTkvx3kt+eXUSSuyX5QJJTknw9ybPnqPWrtFDX/n4WWNVC6P0ZwuoPkmya5D9a7V9P8qS2j32THJnkC8Dn2+MObMd9HPCrI/UckOS8JN9I8vb1bVxJ0sphqJMkLTutt+oZwNlt1o7Ae6rqocDNwNuAPYCdgceMnPp4N+C0tt7xwBvb/E9X1WOq6hHA+cBLRna3GtgF2BP4tySbzirn9cAXqmoX4EnAPya526x1TgceluSuDKHuJOAC4Dfa9Ewv3cuAqqqHAy8ADh3Z36OAvavqicBzgAcDOwEv4o69gM8BHlpVvwm8ea0NKUnaIBjqJEnLyWZJzgROA74HvL/Nv7iqvtbGHwN8qaquqqpbgMOAJ7RltwEfa+MfAnZr4w9L8uUkZwP7AA8d2efHq+q2qroQ+A7wkFk1PQ3Yv9X1JWBTYPvRFarqRuBchmD2WOBkhmD3W204sa26W6uLqvomcDHw623ZsVV1dRt/AvCRqrq1qi4DvtDm/xj4OfD+do3ez365CSVJGxrP25ckLSc3VNXOozOSwHCN2vqo9vcQYK+qOivJvsDuc6wz33SA36uqC9axrxMZwtiWVXVNkq8BLwceCfz7GLWu8xir6pYkuwBPBvZu299jjG1LklYwe+okSb05BXhikq2TbMRwGuPxbdldGMIOwB8AX2njWwKXJ9mEoadu1HOT3CXJA4EHMJw2Oeq/gFekpcskj5ynrq8Cfwqc1aa/wdBrtz3DtYAwXCe4T9vOr7dlc4XFE4DnJ9koyTYMp32SZAvgHlV1DPAXwCPmqUWStAGxp06S1JWqujzJ/sAXGXrRjq6qI9rinwK7tJ8TuBJ4fpv/twynRF7V/m45ssnvMQTFuwN/VlU/b/ltxt8D7wa+0e68+V3gl26owhDqHgD8Q6vzliRXApdU1W1tnfcA722ngd4C7FtVN87aH8DhDD1w57X6TmrztwSOaNfhBfjLtbWVJGnDkKrZZ5lIkiRJknrh6ZeSJEmS1DFDnSRJkiR1zFAnSZIkSR0z1EmSJElSxwx1kiRJktQxQ50kSZIkdcxQJ0mSJEkdM9RJkiRJUsf+LzoIcn9EM2IAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pws_3gram=probable_words('the world', cpd_3gram_with_gt, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('in', 33),\n",
       " ('and', 33),\n",
       " ('largest', 25),\n",
       " ('the', 22),\n",
       " ('to', 18),\n",
       " ('bank', 18),\n",
       " ('of', 16),\n",
       " ('is', 15),\n",
       " ('turns', 14),\n",
       " ('2012', 13),\n",
       " ('at', 13),\n",
       " ('with', 13)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pws_3gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sent1='iran has an embassy in baku'\n",
    "test_sent2='iran embassy has an in baku'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.3195539760853673e-19\n"
     ]
    }
   ],
   "source": [
    "prob_1gram=find_sent_prob(test_sent2, cpd_1gram_with_gt)\n",
    "print(prob_1gram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.409292775550709e-41\n"
     ]
    }
   ],
   "source": [
    "prob_2gram=find_sent_prob(test_sent2, cpd_1gram_with_gt, cpd_2gram_with_gt, ngram=2)\n",
    "print(prob_2gram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.155916419280171e-08\n"
     ]
    }
   ],
   "source": [
    "prob_3gram=find_sent_prob(test_sent1, cpd_1gram_with_gt, cpd_2gram_with_gt, cpd_3gram_with_gt, ngram=3)\n",
    "print(prob_3gram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.052230149816279964"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy(cpd_1gram_with_gt, test_corpus, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.08028033838419461"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy(cpd_2gram_with_gt, test_corpus, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.03353764979254537"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy(cpd_3gram_with_gt, test_corpus, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0368664967503267"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(cpd_1gram_with_gt, test_corpus, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0572234557829687"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(cpd_2gram_with_gt, test_corpus, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0235188338803975"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(cpd_3gram_with_gt, test_corpus, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_wiki=generate_txt_bigram_model(cpd_2gram_with_gt, 'the', numwords=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'the second-largest european coal miners in april to portugal and south'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_wiki"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK Brown Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#nltk.corpus.brown.categories()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#text=nltk_brown_corpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_categories_text=brown_corpus_categories_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "sents=create_tokens(all_categories_text)\n",
    "train_corpus, test_corpus=train_test_split(sents)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Obtaining probality distribution with Good Turing smoothing for Unigrams</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "fd_1gram=ngram_freq_dist(train_corpus, ngram=1)\n",
    "cpd_1gram_with_gt=nltk.SimpleGoodTuringProbDist(fd_1gram)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab=len(fd_1gram)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Obtaining conditional probality distribution with Good Turing smoothing for Bigrams</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "cfd_2gram=ngram_freq_dist(train_corpus, ngram=2)\n",
    "cpd_2gram_with_gt = nltk.ConditionalProbDist(cfd_2gram, nltk.SimpleGoodTuringProbDist, bins=pow(vocab,2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Obtaining conditional probality distribution with Good Turing smoothing for Trigrams</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "cfd_3gram=ngram_freq_dist(train_corpus, ngram=3)\n",
    "cpd_3gram_with_gt = nltk.ConditionalProbDist(cfd_3gram, nltk.SimpleGoodTuringProbDist, bins=pow(vocab,3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3UAAAGDCAYAAACMWjkNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3de9xl93wv8M9XEhKRcshU3cagSlHXoXW/99BQ9ERxggZN6rQoraOhitc51aavOkWrSqgGTVN1rQqtqBIiEgkJiUjTEhJxiUsqcUvC9/yx1sjO43lm9sw8z7Nnzbzfr9d+zdprrb3Wd/3Wembvz/6ttXZ1dwAAAJimqy26AAAAAHacUAcAADBhQh0AAMCECXUAAAATJtQBAABMmFAHAAAwYUIdwCqrqhdV1d/u4GsPraoPb2X6B6rq13e8uvWzFrVW1fOq6rWruUx2TVV1SFW9dw2W6xgCdjtCHUCSqjqvqr5bVZdW1Veq6uiqutai61otVfUvVfV7M89vVFW9wrifWkyV29bdf9Tdkwi127KaoXdHvkgYj/lN47F+6GrUsZq6+5ju/sWdWUZV3a+qLliy3FU5hsZlf2Ac9qO/wEIJdQBXenh3XyvJnZNsTvL8pTPUYIr/d56Q5D4zz++T5DPLjDu3u7+8PQuuqr13vjymxn4H2HVM8YMJwJrq7i8meU+S2yU/6lF5cVWdmOQ7SW5eVTesqndW1Teq6j+q6rAli9m3qt5UVZdU1cer6g5bJlTVEVX1n+O0T1fVo5a8tqrqFVX1X1X1map64Eq1VtWTq+rsqvrm2Bt30xVmPSHJPWcC6b2TvCzJ5iXjTphZ9mHjtn1j3NYbzkzrqvqtqjo3ybnjuAeP9f5XVb0iSc3M/9NV9cFx2teq6k1b2aYnVtXnq+rrVfUHY4/Sg8ZpP+qRqqr3VNXTlrz2jKr6lXH41lV1/Fj/OVX1qzPzHV1Vf1lVx4374eSqusVWarpXVX2kqi6uqvO39GxV1bWr6g1VddFY8/O3tGeNp9JW1UvG/fO5qnroOO3FY3u/YuwdfsXWaq6qq1fV6VX19PH5XlV1YlW9oKoekuR5SR4zLuuMlbZjW6rqauM2fL6qvjpu27XHaZvG/f6UqvpCkvfPsb/uVlUnje32pfG4vvrM+rqqnlpV547z/GVV1Wz7jcPPGbdty+Pyqjp6nPak8W/gkqr6bFX9xjh+/wx/xzeced0Na0mvZlX9clWdNa7/A1X1szPTzquqZ1fVJ8dj901Vte+Oti/AmuluDw8Pjz3+keS8JA8ah2+S5Kwk/3d8/oEkX0hy2yR7J9knQ/h5ZZJ9k9wxyUVJHjDO/6Iklyc5eJz32Uk+l2Sfcfqjk9wwwxdrj0ny7SQ3GKcdmuSKJM8aX/uYJP+V5Loztfz6OPyIJP+R5GfHup6f5CMrbN81knw3yZ3G52cmuXmSE5eMe+I4/IAkX8vQa3mNJH+R5ISZ5XWS45NcN8l+SQ5McsnMNj9r3I4ttR6b5PfHbd43yb1WqPM2SS5Ncq8kV0/ykrEtHzTTtn87Dj8xyYlLXnvxWO/+Sc5P8qSxbe40bs9txnmPTvL1JHcbpx+T5O9XqOmm47Y9bty26yW54zjtDUn+MckBSTYl+fckT5nZl5cnOSzJXkn+V5ILk9TSfTk+31bNt0vyzXF//36SjybZa2m7zCzviCTv2s6/gyePx9TNk1wryduSvHGctmnc728Ya91vjv11lyS/MG7PpiRnJ3nmkuPoXUmuk2Rjhr+jh8y034eXqfEmYzs+dHx+UJJbZPgS4b4Zvni58zjtfkkuWPL6H7VVkp/J8Pf34HHfPmfc/qvP/L9wSoa/1+uO9T910f9feXh4eCx96KkDuNI7quriJB9O8sEkfzQz7ejuPqu7r0jyU0numeT3uvt73X16ktdmCBlbnNbdb+nuy5P8WYYg8wtJ0t1v7u4Lu/uH3f2mDD1dd5t57VeTvKy7Lx+nn5Phg+tST03yx9199ljXHyW5Yy3TW9fd309ycpL7VNV1k1y7uz+b5EMz424zbneSHJLkdd398fG1z01y96raNLPYP+7ub3T3d5P8UpKzZrb5ZUlmT+O8PEM4uuHYZivdDObgJP/U3R/u7suSvCDDB//lvH3J9h6S5G1jvQ9Lcl53/013X9Hdn0jy1gyB+kev7+5TxrY7JkM4X87/TPK+7j523Cdf7+7Tq2qvJI9N8tzuvqS7z0vy/5I8Yea1n+/u13T3D5K8PskNklx/hfVstebuPjPJHyZ5R4YvCp4wLndZ3X1kdz9spekrOCTJn3X3Z7v70gz7/bF11VMtX9Td3x73+1b3V3ef1t0fHbfnvCSvzhC8Zh3Z3Rd39xeS/FtW3g+pqv0ybP/Lu/s94zqO6+7/7MEHk7w3Qy/oPB6T5LjuPn48bl+SIazeY2aePx//Xr+R5J+2Vh/Aogh1AFd6ZHdfp7tv2t2/OX5o3eL8meEbJvlGd18yM+7zSW603Pzd/cMkF4yv23K62unj6V4XZ+iBOXDmtV/s7tkg8/ktr13ipklePrOcb2TorbjRMvMmV15Xd+8MPXTJEGC3jDu/uz8/s41bhjN+wP/6Sts4zj+7zb1k+nPG2k4ZT3V78go1Ll3Od8b1/pix/Y/LEKySoSftmHH4pkl+fkvbjO1zSIZAvsVs6PxOhp6p5dwkyX8uM/7ADL07n58Zt/Q4+NE6xm3JVtYzT82vH+d7d3efu8JydsZV9vs4vHeuGkS3tt+vsr+q6meq6l1V9eWq+laGLx5mj/Vk/v2QJH+d5Jzu/pOZdTy0qj46nrJ6cYYvGJauYyVLj/Mfjtuz7D6coz6AhRDqAOYzG7IuTHLdqjpgZtzGJF+ceX6TLQPjNVY3TnLh2Kv0miRPS3K97r5OhtMea+a1N9pyXdHMsi9cpqbzk/zGGES3PPbr7o+ssA0nZAhv98nQQ5cM4e6e47gTZua9MEN42LIN+2c47XB2G2fb5EtLtrlmn3f3l7v7sO6+YZLfSPLKqvrpZWr8Uoa22rKc/cb1ruTYJI+rqrtn6A39t3H8+Uk+uKRtrtXd/2sry1rJ+RlO71vqa7myB3KLpcfB1iztgZyn5ldmOF3xv1fVvbayrB11lf2eYXuuSPKVFda1rf31VxluyHPL7v6JDNf+zR7bc6uqIzKcLvmUmXHXyNCb+ZIk1x//nt49s45ttcvS43zLcTvvPgTYJQh1ANupu89P8pEkf1xV+1bV7TN80Jy9pfxdqupXxtPWnpnk+xmugdo/wwfNi5LhJg8Zb8gy4yeTPKOq9qmqR2e4hurdy5TyqiTPrarbjsu69jj/Sk7KcO3S4zOGuu7+5ljL43PVUHdskidV1R3HD85/lOTk8RS65RyX5LYz2/yMzPQwVdWjq2rLh/9vjm3ww2WW85YkD6+qe4w31HhRth4C3p3hQ/n/SfKmsaclGYLPz1TVE8Z23Keq7jp7E4ztcEySB1XVr1bV3lV1vaq643jq4z8keXFVHTAG9t/JVY+DrflKhmvXtthqzVX1hAzXqB2aoX1fX1f+7MZXkmyqnb8z67FJnlVVNxuX/UcZ2vWKFebf1v46IMm3klxaVbfOcF3hdqvhBjPPSPKoJT3oV89wDeVFSa4Y55v9GYSvJLlejTd7WcY/JDmoqh5YVfsk+d0Mf6srfTECsEsS6gB2zOMy3PjhwgzXdr2wu983M/0fM1yv880M11j9yng91qczXHd1UoYPnD+XK0+F3OLkJLfM0BP04iQHd/ePnYLY3W9P8idJ/n48te3MJA9dqeDu/naS0zJ8ED5zZtKHMgTJE2bmfV+SP8jQC/KlDD1Vj80KuvtrGa79OjLD6Xe3XLJdd01yclVdmuSdSX57vKZv6XLOSvL0JH8/rvfSDNcYfn+F9X4/w808HpTk72bGX5Lhw/1jM+yjL2doq2ustA1b2bYvZDil73cznOJ6epItdzN9eoYbbXw2w6msf5fkdXMu+uVJDq7hzph/vrWaq2pjhusUn9jdl3b33yU5NclLx2W9efz361X18eRHP7L9nu3c3NcleWOGY+FzSb43buOy5thfz85wTeIlGXqoV7zr6TY8JsmGJGfP3MnyVWObPSNDOPvmuK53ztT3mQxB9bPjKa1XOY25u8/J8IXGX2T4e3t4hp82uWwH6wRYiC134AKAXc7YW3RxhtP3Prfoetg6+wtgMfTUAbBLqaqHV9U1x+v4XpLkUxluLc8uyP4CWDyhDoBdzSMynH54YYbTOB/bTivZldlfAAvm9EsAAIAJ01MHAAAwYUIdAADAhO296ALmceCBB/amTZsWXQYAAMBCnHbaaV/r7g3LTZtEqNu0aVNOPfXURZcBAACwEFX1+ZWmOf0SAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwvZedAFTtumI47Y6/bwjD1qnSgAAgD2VnjoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwtYs1FXV66rqq1V15jLTfrequqoOXKv1AwAA7AnWsqfu6CQPWTqyqm6S5BeTfGEN1w0AALBHWLNQ190nJPnGMpNemuQ5SXqt1g0AALCnWNdr6qrqEUm+2N1nrOd6AQAAdld7r9eKquqaSZ6X4dTLeeY/PMnhSbJx48Y1rAwAAGC61rOn7hZJbpbkjKo6L8mNk3y8qn5quZm7+6ju3tzdmzds2LCOZQIAAEzHuvXUdfenkvzkludjsNvc3V9brxoAAAB2N2v5kwbHJjkpya2q6oKqesparQsAAGBPtWY9dd39uG1M37RW6wYAANhTrOvdLwEAAFhdQh0AAMCECXUAAAATJtQBAABMmFAHAAAwYUIdAADAhAl1AAAAEybUAQAATJhQBwAAMGFCHQAAwIQJdQAAABMm1AEAAEyYUAcAADBhQh0AAMCECXUAAAATJtQBAABMmFAHAAAwYUIdAADAhAl1AAAAEybUAQAATJhQBwAAMGFCHQAAwIQJdQAAABMm1AEAAEyYUAcAADBhQh0AAMCECXUAAAATJtQBAABMmFAHAAAwYUIdAADAhAl1AAAAEybUAQAATJhQBwAAMGFrFuqq6nVV9dWqOnNm3J9W1Weq6pNV9faqus5arR8AAGBPsJY9dUcneciScccnuV133z7Jvyd57hquHwAAYLe3ZqGuu09I8o0l497b3VeMTz+a5MZrtX4AAIA9wd4LXPeTk7xppYlVdXiSw5Nk48aN61UTC7TpiOO2Ov28Iw9ap0oAAGA6FnKjlKr6/SRXJDlmpXm6+6ju3tzdmzds2LB+xQEAAEzIuvfUVdWhSR6W5IHd3eu9fgAAgN3Juoa6qnpIkuckuW93f2c91w0AALA7WsufNDg2yUlJblVVF1TVU5K8IskBSY6vqtOr6lVrtX4AAIA9wZr11HX345YZ/ddrtT4AAIA90UJulAIAAMDqEOoAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMLWLNRV1euq6qtVdebMuOtW1fFVde74739bq/UDAADsCdayp+7oJA9ZMu6IJP/a3bdM8q/jcwAAAHbQmoW67j4hyTeWjH5EktePw69P8si1Wj8AAMCeYL2vqbt+d39pHP5ykuuv8/oBAAB2Kwu7UUp3d5JeaXpVHV5Vp1bVqRdddNE6VgYAADAd6x3qvlJVN0iS8d+vrjRjdx/V3Zu7e/OGDRvWrUAAAIApWe9Q984kvzYO/1qSf1zn9QMAAOxW1vInDY5NclKSW1XVBVX1lCRHJnlwVZ2b5EHjcwAAAHbQ3mu14O5+3AqTHrhW6wQAANjTLOxGKQAAAOw8oQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJ23uemarq57r7U2tdDLA6Nh1x3DbnOe/Ig9ahEgAA1tq8PXWvrKpTquo3q+raa1oRAAAAc5sr1HX3vZMckuQmSU6rqr+rqgevaWUAAABs09zX1HX3uUmen+T3ktw3yZ9X1Weq6lfWqjgAAAC2bq5QV1W3r6qXJjk7yQOSPLy7f3Ycfuka1gcAAMBWzHWjlCR/keS1SZ7X3d/dMrK7L6yq569JZQAAAGzTvKHuoCTf7e4fJElVXS3Jvt39ne5+45pVBwAAwFbNe03d+5LsN/P8muM4AAAAFmjeULdvd1+65ck4fM21KQkAAIB5zRvqvl1Vd97ypKrukuS7W5kfAACAdTDvNXXPTPLmqrowSSX5qSSPWbOqAAAAmMtcoa67P1ZVt05yq3HUOd19+dqVBQAAwDzm7alLkrsm2TS+5s5Vle5+w5pUBQAAwFzmCnVV9cYkt0hyepIfjKM7iVAHAACwQPP21G1Ocpvu7rUsBgAAgO0z790vz8xwcxQAAAB2IfP21B2Y5NNVdUqS728Z2d2/vCZVAQAAMJd5Q92L1rIIAAAAdsxcp1929weTnJdkn3H4Y0k+vqMrrapnVdVZVXVmVR1bVfvu6LIAAAD2ZHOFuqo6LMlbkrx6HHWjJO/YkRVW1Y2SPCPJ5u6+XZK9kjx2R5YFAACwp5v3Rim/leSeSb6VJN19bpKf3In17p1kv6raO8k1k1y4E8sCAADYY817Td33u/uyqkqSjGFsh37eoLu/WFUvSfKFJN9N8t7ufu/S+arq8CSHJ8nGjRt3ZFUAq2LTEcdtdfp5Rx60TpUAAPy4eXvqPlhVz8vQu/bgJG9O8k87ssKq+m9JHpHkZklumGT/qnr80vm6+6ju3tzdmzds2LAjqwIAANjtzRvqjkhyUZJPJfmNJO9O8vwdXOeDknyuuy/q7suTvC3JPXZwWQAAAHu0uU6/7O4fJnnN+NhZX0jyC1V1zQynXz4wyamrsFwAAIA9zlyhrqo+l2Wuoevum2/vCrv75Kp6S4afRLgiySeSHLW9ywEAAGD+G6VsnhneN8mjk1x3R1fa3S9M8sIdfT0AAACDeX98/Oszjy9298uSuN0bAADAgs17+uWdZ55eLUPP3by9fAAAAKyReYPZ/5sZviLJeUl+ddWrAQAAYLvMe/fL+691IQAAAGy/eU+//J2tTe/uP1udcgAAANge23P3y7smeef4/OFJTkly7loUBQAAwHzmDXU3TnLn7r4kSarqRUmO6+7Hr1VhAAAAbNtcP2mQ5PpJLpt5ftk4DgAAgAWat6fuDUlOqaq3j88fmeT1a1MSAAAA85r37pcvrqr3JLn3OOpJ3f2JtSsLAACAecx7+mWSXDPJt7r75UkuqKqbrVFNAAAAzGmuUFdVL0zye0meO47aJ8nfrlVRAAAAzGfenrpHJfnlJN9Oku6+MMkBa1UUAAAA85k31F3W3Z2kk6Sq9l+7kgAAAJjXvKHuH6rq1UmuU1WHJXlfktesXVkAAADMY967X76kqh6c5FtJbpXkBd19/JpWBgAAwDZtM9RV1V5J3tfd908iyAEAAOxCtnn6ZXf/IMkPq+ra61APAAAA22Gu0y+TXJrkU1V1fMY7YCZJdz9jTaoCAABgLvOGureNDwAAAHYhWw11VbWxu7/Q3a9fr4IAAACY37auqXvHloGqeusa1wIAAMB22laoq5nhm69lIQAAAGy/bYW6XmEYAACAXcC2bpRyh6r6VoYeu/3G4YzPu7t/Yk2rAwAAYKu2Guq6e6/1KgQAAIDtt80fHwcAAGDXJdQBAABMmFAHAAAwYUIdAADAhAl1AAAAEybUAQAATNhCQl1VXaeq3lJVn6mqs6vq7ouoAwAAYOq29ePja+XlSf65uw+uqqsnueaC6gAAAJi0dQ91VXXtJPdJcmiSdPdlSS5b7zoAAAB2B4s4/fJmSS5K8jdV9Ymqem1V7b+AOgAAACZvEaFu7yR3TvJX3X2nJN9OcsTSmarq8Ko6tapOveiii9a7RgAAgElYRKi7IMkF3X3y+PwtGULeVXT3Ud29ubs3b9iwYV0LBAAAmIp1D3Xd/eUk51fVrcZRD0zy6fWuAwAAYHewqLtfPj3JMeOdLz+b5EkLqgMAAGDSFhLquvv0JJsXsW4AAIDdyUJ+fBwAAIDVIdQBAABMmFAHAAAwYUIdAADAhAl1AAAAEybUAQAATJhQBwAAMGFCHQAAwIQJdQAAABMm1AEAAEyYUAcAADBhQh0AAMCECXUAAAATJtQBAABMmFAHAAAwYXsvugAA9hybjjhum/Ocd+RB61DJ7mFb7aktAfYMeuoAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJmxhoa6q9qqqT1TVuxZVAwAAwNQtsqfut5OcvcD1AwAATN5CQl1V3TjJQUleu4j1AwAA7C72XtB6X5bkOUkOWGmGqjo8yeFJsnHjxnUqCwDYU2064ritTj/vyIPWqRKA7bPuPXVV9bAkX+3u07Y2X3cf1d2bu3vzhg0b1qk6AACAaVnE6Zf3TPLLVXVekr9P8oCq+tsF1AEAADB56x7quvu53X3j7t6U5LFJ3t/dj1/vOgAAAHYHfqcOAABgwhZ1o5QkSXd/IMkHFlkDAADAlOmpAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAnbe9EFAACwe9l0xHHbnOe8Iw9ah0p2D9tqT22JnjoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwtY91FXVTarq36rq01V1VlX99nrXAAAAsLvYewHrvCLJ73b3x6vqgCSnVdXx3f3pBdQCAAAwaeveU9fdX+ruj4/DlyQ5O8mN1rsOAACA3cEieup+pKo2JblTkpOXmXZ4ksOTZOPGjetaFwAAsHvadMRxW51+3pEHrVMlq2dhN0qpqmsleWuSZ3b3t5ZO7+6juntzd2/esGHD+hcIAAAwAQsJdVW1T4ZAd0x3v20RNQAAAOwOFnH3y0ry10nO7u4/W+/1AwAA7E4W0VN3zyRPSPKAqjp9fPzSAuoAAACYvHW/UUp3fzhJrfd6AQAAdkcLu1EKAAAAO0+oAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJE+oAAAAmTKgDAACYMKEOAABgwoQ6AACACRPqAAAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJW0ioq6qHVNU5VfUfVXXEImoAAADYHax7qKuqvZL8ZZKHJrlNksdV1W3Wuw4AAIDdwSJ66u6W5D+6+7PdfVmSv0/yiAXUAQAAMHmLCHU3SnL+zPMLxnEAAABsp+ru9V1h1cFJHtLdvz4+f0KSn+/upy2Z7/Akh49Pb5XknHUtdMccmORriy5iN6I9V4+2XF3ac3Vpz9WjLVeX9lw92nJ1ac/VNZX2vGl3b1huwt7rXUmSLya5yczzG4/jrqK7j0py1HoVtRqq6tTu3rzoOnYX2nP1aMvVpT1Xl/ZcPdpydWnP1aMtV5f2XF27Q3su4vTLjyW5ZVXdrKqunuSxSd65gDoAAAAmb9176rr7iqp6WpJ/SbJXktd191nrXQcAAMDuYBGnX6a7353k3YtY9xqb1OmiE6A9V4+2XF3ac3Vpz9WjLVeX9lw92nJ1ac/VNfn2XPcbpQAAALB6FnFNHQAAAKtEqFsDVfWMqjq7qo5ZdC1TUlUfWXQNU1VV16mq3xyH71dV71p0TbuDqrp0hfFPraonbuV19sEqWWkfsLyqendVXWccvnT8d1NVnbnYynZt3rd3zs6+B1XVoVV1w7Wpbpr83a6N3bldhbq18ZtJHtzdhyy6kCnp7nssuoYJu06G44510N2v6u43LLoOWKq7f6m7L150HRM09/t2VS3kfgS7uJ19Dzo0iVAHO0Go20lV9TtVdeb4eGZVvSrJzZO8p6qetej6pmTmW+X7VdUHquotVfWZqjqmqmrR9e3ijkxyi6o6PcmfJrnWcu1XVXepqg9W1WlV9S9VdYOFVr1gVfW/q+oZ4/BLq+r94/ADtnxjX1UvrqozquqjVXX9cdyLqurZ4/BPV9X7xnk+XlW3GBe/7D7YE1XVO8Zj7qyqOnwcd+kKbXuzqjqpqj5VVX+42Mp3Pds6ZqvqvKo6cLFVTsuS9+3fHY/XT47H5e3HeV5UVW+sqhOTvHGhBe+a5n0PekFVfWz8zHRUDQ5OsjnJMVV1elXtt8Dt2NXsVVWvGf/vfG9V7VdVh41teEZVvbWqrpkkVfXosV3PqKoTFl34Lm65dr1FVf3z+F71oaq69aKL3F5C3U6oqrskeVKSn0/yC0kOS/LqJBcmuX93v3SB5U3dnZI8M8ltMrzZ3nOx5ezyjkjyn919xyT/O8u0X1Xtk+Qvkhzc3XdJ8rokL0+g+E4AAAfoSURBVF5QvbuKDyW59zi8OcMHkX3GcSck2T/JR7v7DuPzw5ZZxjFJ/nKc5x5JvjSOdwxf6cnjMbc5yTOq6npZuW1fnuSvuvvncmVbcqVtHbNsp+5+asb37SSbknyiu2+f5HlJZnvkb5PkQd39uHUvcte3zfegcb5XdPddu/t2SfZL8rDufkuSU5Mc0t137O7vrn/5u6xbZnh/uW2Si5P8jyRvG9vwDknOTvKUcd4XJPnv4/hfXki107Fcux6V5Onje9Wzk7xygfXtEKFu59wrydu7+9vdfWmSt+XKN1t2zindfUF3/zDJ6RneaJnfcu13qyS3S3L8+G3q85PceHEl7hJOS3KXqvqJJN9PclKGD8r3zvDh+bIk75qZd9Psi6vqgCQ36u63J0l3f6+7vzNOdgxf6RlVdUaSjya5SYY31JXa9p5Jjh2H9Yj8uG0ds+yce2U87rr7/UmuN7Z1krxT4JjbSv//3b+qTq6qTyV5QJLbLqrAifhcd58+Dm/5f/J2Y0/Sp5Ickivb8MQkR1fVYRl+B5qVLdeu90jy5vHz0auTTO5MJueFs6v6/szwD+JY3V7LtV8lOau7776YknY93X15VX0uw/UcH0nyyQzf1v90hm9AL+8rf/dle49Dx3CG06mTPCjJ3bv7O1X1gST7Zutt67d2VjDHMcva+faiC5iQH/v/r6r2zdD7sbm7z6+qF2X4v4CVLW3H/ZIcneSR3X1GVR2a5H7J0ONcVT+f5KAkp1XVXbr76+tb7mQsbdfrJ7l47GmeLD11O+dDSR5ZVdesqv2TPCq+KWUxLklywDbmOSfJhqq6e5JU1T5V5VvS4W/22RlOXftQkqdmOP1qm8Giuy9JckFVPTJJquoaW65v4EeuneSbY6C7dYZT1bfmxCSPHYfdbGp5O3zMsk0fynjcjV9IfK27v7XQiqZhnvegLQHua1V1rSQHb+frGRyQ5Evjadc/+j+yqm7R3Sd39wuSXJThrAjm860kn6uqRyfJeK3nHRZc03YT6nZCd388wzcmpyQ5Oclru/sTCy2KPdL4bdyJNdym909XmOeyDG+ifzKeCnd6htMN9nQfynCaxUnd/ZUk38v2fTnzhAynF34yQ8/JT61+iZP2zxm+pT87w80UPrqN+X87yW+NpxbdaK2Lm6idPWZZ2YsynN76yQzH668ttpxpmPM96OIkr0lyZpJ/SfKxmclHJ3mVG6XM5Q8yfOY8MclnZsb/6XiDqTMzvBedsYjiJuyQJE8ZPx+dleQRC65nu5Uv9gAAAKZLTx0AAMCECXUAAAATJtQBAABMmFAHAAAwYUIdAADAhAl1AOwyquoH423Nz6yqN2/v7/5V1aXbOf/RVXXwMuPvV1Xv2o7lvH3L7xWOz8+pqufPPH9rVf3K9tQ2T50AkAh1AOxavtvdd+zu2yW5LMMPa//I+KOwu+J714kZf/exqq6X5NtJ7j4z/e4Zfjtqm6pq71WvDoDd2q74xggAyfCD2j9dVZvGnq83ZPjh4ptU1eO2/NBuVf3J7Iuq6qVVdVZV/WtVbRjHHVZVH6uqM8Zes9kewAdV1alV9e9V9bClRVTV/lX1uqo6pao+UVXL/SjtRzKGuvHff0qyYQyhN8sQVr9cVftW1d+MtX+iqu4/ruPQqnpnVb0/yb+Or3vFuN3vS/KTM/UcWVWfrqpPVtVLdrRxAdh9CHUA7HLG3qqHJvnUOOqWSV7Z3bdNcnmSP0nygCR3THLXmVMf909y6jjfB5O8cBz/tu6+a3ffIcnZSZ4ys7pNSe6W5KAkr6qqfZeU8/tJ3t/dd0ty/yR/WlX7L5nntCS3q6qrZwh1JyU5J8nPjs+39NL9VpLu7p9L8rgkr59Z352THNzd903yqCS3SnKbJE/MVXsBH5Xktt19+yR/uNWGBGCPINQBsCvZr6pOT3Jqki8k+etx/Oe7+6Pj8F2TfKC7L+ruK5Ick+Q+47QfJnnTOPy3Se41Dt+uqj5UVZ9KckiS286s8x+6+4fdfW6Szya59ZKafjHJEWNdH0iyb5KNszN09/eTnJUhmP1CkpMzBLt7jI8Tx1nvNdaV7v5Mks8n+Zlx2vHd/Y1x+D5Jju3uH3T3hUneP47/ryTfS/LX4zV63/nxJgRgT+O8fQB2Jd/t7jvOjqiqZLhGbUf0+O/RSR7Z3WdU1aFJ7rfMPCs9ryT/o7vP2ca6TswQxg7o7m9W1UeTPC3JnZK8eo5at7mN3X1FVd0tyQOTHDwu/wFzLBuA3ZieOgCm5pQk962qA6tqrwynMX5wnHa1DGEnSf5nkg+Pwwck+VJV7ZOhp27Wo6vqalV1iyQ3z3Da5Kx/SfL0GtNlVd1phbo+kuQ3kpwxPv9khl67jRmuBUyG6wQPGZfzM+O05cLiCUkeU1V7VdUNMpz2maq6VpJrd/e7kzwryR1WqAWAPYieOgAmpbu/VFVHJPm3DL1ox3X3P46Tv53kbuPPCXw1yWPG8X+Q4ZTIi8Z/D5hZ5BcyBMWfSPLU7v7emN+2+L9JXpbkk+OdNz+X5MduqJIh1N08yR+PdV5RVV9Ncn53/3Cc55VJ/mo8DfSKJId29/eXrC9J3p6hB+7TY30njeMPSPKP43V4leR3ttZWAOwZqnvpWSYAAABMhdMvAQAAJkyoAwAAmDChDgAAYMKEOgAAgAkT6gAAACZMqAMAAJgwoQ4AAGDChDoAAIAJ+/8n14qrwduyIAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pws_2gram=probable_words('the world health organization', cpd_2gram_with_gt, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA24AAAGDCAYAAABTFEMSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deZxldXkn/s8joCASHaVjFGnbfY2itsQFjaLmpyHGZTBqUH86RrK4m8yIS5SZaIIzjktijFFj0Ih73CKaRKIIIkJAdpBoFEQxCq6ABASf+eOchktZ3V0NdbtOV7/fr9d91blnfc63zr1Vn/s959zq7gAAADBd11vpAgAAANg0wQ0AAGDiBDcAAICJE9wAAAAmTnADAACYOMENAABg4gQ3gDmrqoOr6t3XctmnV9XnNzH9yKr6nWtf3dYzj1qr6qVV9fblXCebVlWHVtWrtsJ21lbVxVW1wxLmXVdVXVU7zrsugJUiuAEsoqrOqapLx38cvzP+s3qjla5ruVTVP1XVi2ee7zH+47vYuF9amSo3r7v/tLu3ieC6OcsZbK/NhwXjMb9uPNafPo7b5AcH89Td3+juG3X3ldd1XYu1x9jeDxmnHXxdtwEwb4IbwMY9urtvlOTeSdYnefnCGWqwLb6XHpXkwTPPH5zky4uM+0p3/8eWrFivBwAsv23xnw2Araq7v5XkU0nunlz1Sf2rq+qYJD9JctuqumVVfbyqvl9VX62qZy1Yzc5V9f6quqiqvlRV99wwoaoOqqp/H6edWVWPW7BsVdWbqupHVfXlqnrYxmqtqv9WVWdV1Q/GXrVbb2TWo5I8cCZ0PijJG5KsXzDuqJl1P2vct++P+3rLmWldVc+uqq8k+co47hFjvT+qqjclqZn5b19VnxunXVhV79/EPj2tqs6tqu9V1R+PPUMPH6dd1ZNSVZ+qqucsWPaUqnr8OHznqvr0WP/ZVfVbM/MdWlV/WVWHj7+H46rqdpuoaZ+q+kJV/bCqzpvpobpxVb2rqi4Ya375hvbc0HtVVa8dfz9fr6pHjdNePbb3m8Ze3jdtquaqun5VnVxVzx2f71BVx1TVK6rqkUlemuSJ47pO2dh+bEpV3SXJW5Lcf1zPD2cm/5eNtdWm2nmRbRxZVX8y1n5RVf1zVe0+TrvG6Y9VdZuqOmqc74jx97WwV/GAqvrGeEy9bFxuWdoDYMV1t4eHh4fHgkeSc5I8fBzeM8kZSf5kfH5kkm8kuVuSHZPslCHgvDnJzkn2SnJBkn3H+Q9O8tMk+4/z/lGSryfZaZz+hCS3zPBh2hOTXJLkFuO0pye5IskLx2WfmORHSW46U8vvjMOPSfLVJHcZ63p5ki9sZP9ukOTSJPcan5+e5LZJjlkw7mnj8L5JLszQ+3iDJH+R5KiZ9XWSTye5aZJdkuye5KKZfX7huB8ban1vkpeN+7xzkn02Uuddk1ycZJ8k10/y2rEtHz7Ttu8eh5+W5JgFy/5wrHfXJOclecbYNvca9+eu47yHJvlekr3H6Ycled9Garr1uG9PHvftZkn2Gqe9K8nHkuyWZF2Sf0vyzJnf5U+TPCvJDkl+P8n5SWrh73J8vrma757kB+Pv+2VJvphkh4XtMrO+g5J8YgtfB09P8vkF4zbaVpureZH1H5nk35PccTxujkxyyDhtXYbjasfx+bHj7//64/Hw45nf/YZ53zau555JLktyl421h4eHh8e29tDjBrBxHx17GT6f5HNJ/nRm2qHdfUZ3X5Hkl5I8MMmLu/s/u/vkJG/PECQ2OLG7P9TdP03yugxh5X5J0t0f7O7zu/tn3f3+DD1We88s+90kb+jun47Tz06y3yL1/l6SP+vus8a6/jTJXov1unX3ZUmOS/Lgqrppkht399eSHD0z7q7jfifJAUne0d1fGpd9SYaemHUzq/2z7v5+d1+a5NeTnDGzz29IMnvK5U8zBKBbjm22seuo9k/yD939+e6+PMkrMvyDvpiPLNjfA5J8eKz3N5Kc091/291XdPdJSf4+Q2i+avnuPn5su8MyBPDF/HaSI7r7vePv5HvdfXINN9F4UpKXdPdF3X1Okv+b5Kkzy57b3W/r4bqtdya5RZKbb2Q7m6y5u09P8qokH83wYcBTexPXg3X3Id39GxubvoU21lZLaeeF/ra7/208bj6QRdq9qtYmuW+SV3T35ePx8vFF1vU/u/vS7j4lySkZAhzAqiC4AWzcY7v7Jt196+7+g/Efyw3Omxm+ZZLvd/dFM+POTbLHYvN398+SfHNcbsOpgCePp939MENPyu4zy36ru2fDyrkbll3g1kneOLOe72c4PXGPReZNrr7O7UEZetqSIaRuGHded587s48bhtPdF2fodVl0H8f5Z/e5F0z/H2Ntx1fVGVX13zZS48L1/GTc7s8Z2//wDOEpGXrEDhuHb53kVza0zdg+B2QI3RvMBsufJNnYzWj2zNBLtNDuGXrgzp0Zt/A4uGob475kE9tZSs3vHOf7ZHd/ZSPrmYeNtdVSal7qumZteI39ZGbceYvMt9TfIcA2xwXkANfObJA6P8lNq2q3mfC2Nsm3ZubZc8PAeM3TrZKcP/YOvS3Jw5Ic291XVtXJmbkeLMkeVVUz4W1tFu9tOC/Jq7v7sEWmLeaoDL1052ToaUuGAPf2cdxRM/Oen+Gf8g37sGuGUwRn93G2Tb6da+5zzT7v4YYnzxqn7ZPkiKo6qru/uqDGbye508x6dhm3uzHvTfLKqjoqQ6/mZ8fx5yX5XHc/YhPLLtV5uWaP6AYX5uqexDPHcQuPg01Z2JO4lJrfnOQTSf6/qtpnpudyY72SW2pL17Oc7Tzr2xleYzecCW97bmqBBZarPQBWjB43gOuou89L8oUkf1ZVO1fVPZI8M8nsjRPuU1WPH2+08IIM1998McM1QZ3hmrhU1TMy3gRlxi8meV5V7VRVT8hwTdMnFynlLUleUlV3G9d143H+jTk2yU2SPCVjcOvuH4y1PCXXDG7vTfKMqtqrqm6Q4TTM48bTARdzeJK7zezz8zLT61JVT6iqW41PfzC2wc8WWc+Hkjy6qh5QVdfPcK1SLTLfBp/MEJz+V5L3j72byRBu7lhVTx3bcaequu94A44tdViSh1fVb1XVjlV1s6raazxN8QNJXl1Vu42h/EW55nGwKd/JcJ3hBpusuaqemuQ+Ga5De16Sd9bVX1nxnSTr6rrf8fQ7SW41tv1SLGc7X2Xs+T0hycHjjVnun+TRW7CK5WoPgBXjDQxgeTw5ww0Szs9wrdUru/uImekfy3BjkR9kuObp8eP1UWdmuA7q2Az/XP5yrj5tcYPjktwhQ4/Oq5Ps390/d7pgd38kyWuSvK+qfpzh5iKP2ljB3X1JkhMz3Ozh9JlJR2cIi0fNzHtEkj/OcL3St5PcLlefkrjYui/McF3TIRlObbzDgv26b5LjquriDL2Hzx+vsVu4njOSPDfJ+8btXpzhmr/LNrLdy5J8OMnDk7xnZvxFSX5trPn8DKfUvSbDjUu2SHd/I8M1fH+Y4XTUk3P1tVTPzXBzma9lOO30PUnescRVvzHJ/jXccfLPN1XzeM3XGzLcPObi7n5PhmDz+nFdHxx/fq+qvpRc9WXln9rC3f1Mhhvz/EdVXbi5mZeznRdxQJL7ZzieXpXk/dnIcbCIn2sPgG3NhjtZAcDkjT1KP0xyh+7++krXw8qp4Sskvtzdr1zpWgC2Bj1uAExaVT26qm44Xlf32iSnZbgGj+3IeMrl7arqeuN3sz0mwx01AbYLghsAU/eYDKfdnZ/hlMsntdNFtke/lOF73i5O8udJfn/8ugGA7YJTJQEAACZOjxsAAMDECW4AAAATN6kv4N5999173bp1K10GAADAijjxxBMv7O41C8dPKritW7cuJ5xwwkqXAQAAsCKq6tzFxjtVEgAAYOIENwAAgIkT3AAAACZOcAMAAJg4wQ0AAGDiBDcAAICJE9wAAAAmTnADAACYOMENAABg4gQ3AACAiZtrcKuqm1TVh6rqy1V1VlXdf57bAwAAWI12nPP635jkH7t7/6q6fpIbznl7AAAAq87cgltV3TjJg5M8PUm6+/Ikl89rewAAAKvVPHvcbpPkgiR/W1X3THJikud39yWzM1XVgUkOTJK1a9fOsZxrb91Bh29y+jmH7LeVKgEAALZH87zGbcck907yV919rySXJDlo4Uzd/dbuXt/d69esWTPHcgAAALZN8wxu30zyze4+bnz+oQxBDgAAgC0wt+DW3f+R5LyqutM46mFJzpzX9gAAAFared9V8rlJDhvvKPm1JM+Y8/YAAABWnbkGt+4+Ocn6eW4DAABgtZvrF3ADAABw3QluAAAAEye4AQAATJzgBgAAMHGCGwAAwMQJbgAAABMnuAEAAEyc4AYAADBxghsAAMDECW4AAAATJ7gBAABMnOAGAAAwcYIbAADAxAluAAAAEye4AQAATJzgBgAAMHGCGwAAwMQJbgAAABMnuAEAAEyc4AYAADBxghsAAMDECW4AAAATJ7gBAABMnOAGAAAwcYIbAADAxAluAAAAEye4AQAATJzgBgAAMHGCGwAAwMQJbgAAABMnuAEAAEyc4AYAADBxghsAAMDECW4AAAATJ7gBAABMnOAGAAAwcYIbAADAxAluAAAAEye4AQAATJzgBgAAMHGCGwAAwMQJbgAAABMnuAEAAEzcjvNceVWdk+SiJFcmuaK7189zewAAAKvRXIPb6KHdfeFW2A4AAMCq5FRJAACAiZt3cOsk/1xVJ1bVgYvNUFUHVtUJVXXCBRdcMOdyAAAAtj3zDm77dPe9kzwqybOr6sELZ+jut3b3+u5ev2bNmjmXAwAAsO2Za3Dr7m+NP7+b5CNJ9p7n9gAAAFajuQW3qtq1qnbbMJzk15KcPq/tAQAArFbzvKvkzZN8pKo2bOc93f2Pc9weAADAqjS34NbdX0tyz3mtHwAAYHvh6wAAAAAmTnADAACYOMENAABg4gQ3AACAiRPcAAAAJk5wAwAAmDjBDQAAYOIENwAAgIkT3AAAACZOcAMAAJg4wQ0AAGDiBDcAAICJE9wAAAAmTnADAACYOMENAABg4gQ3AACAiRPcAAAAJk5wAwAAmDjBDQAAYOIENwAAgIkT3AAAACZOcAMAAJg4wQ0AAGDiBDcAAICJE9wAAAAmTnADAACYOMENAABg4gQ3AACAiRPcAAAAJk5wAwAAmDjBDQAAYOIENwAAgIkT3AAAACZOcAMAAJg4wQ0AAGDiBDcAAICJE9wAAAAmTnADAACYOMENAABg4gQ3AACAiRPcAAAAJk5wAwAAmLi5B7eq2qGqTqqqT8x7WwAAAKvR1uhxe36Ss7bCdgAAAFaluQa3qrpVkv2SvH2e2wEAAFjN5t3j9oYk/yPJz+a8HQAAgFVrx3mtuKp+I8l3u/vEqnrIJuY7MMmBSbJ27dp5lcNErDvo8M3Oc84h+22FSlaHzbWntgQAWB3m2eP2wCS/WVXnJHlfkn2r6t0LZ+rut3b3+u5ev2bNmjmWAwAAsG2aW3Dr7pd09626e12SJyX5THc/ZV7bAwAAWK18jxsAAMDEze0at1ndfWSSI7fGtgAAAFYbPW4AAAATJ7gBAABMnOAGAAAwcYIbAADAxAluAAAAEye4AQAATJzgBgAAMHGCGwAAwMQJbgAAABMnuAEAAEyc4AYAADBxghsAAMDECW4AAAATJ7gBAABMnOAGAAAwcUsKblX1y/MuBAAAgMUttcftzVV1fFX9QVXdeK4VAQAAcA1LCm7d/aAkByTZM8mJVfWeqnrEXCsDAAAgyRZc49bdX0ny8iQvTvKrSf68qr5cVY+fV3EAAAAs/Rq3e1TV65OclWTfJI/u7ruMw6+fY30AAADbvR2XON9fJHl7kpd296UbRnb3+VX18rlUBgAAQJKlB7f9klza3VcmSVVdL8nO3f2T7v67uVUHAADAkq9xOyLJLjPPbziOAwAAYM6WGtx27u6LNzwZh284n5IAAACYtdTgdklV3XvDk6q6T5JLNzE/AAAAy2Sp17i9IMkHq+r8JJXkl5I8cW5VAQAAcJUlBbfu/tequnOSO42jzu7un86vLAAAADZYao9bktw3ybpxmXtXVbr7XXOpCgAAgKssKbhV1d8luV2Sk5NcOY7uJIIbAADAnC21x219krt2d8+zGAAAAH7eUu8qeXqGG5IAAACwlS21x233JGdW1fFJLtswsrt/cy5VAQAAcJWlBreD51kEAAAAG7fUrwP4XFXdOskduvuIqrphkh3mWxoAAADJEq9xq6pnJflQkr8eR+2R5KPzKgoAAICrLfXmJM9O8sAkP06S7v5Kkl+cV1EAAABcbanB7bLuvnzDk6raMcP3uAEAADBnSw1un6uqlybZpaoekeSDSf5hfmUBAACwwVKD20FJLkhyWpLfTfLJJC+fV1EAAABcbal3lfxZkreNDwAAALaiJQW3qvp6Frmmrbtvu+wVAQAAcA1L/QLu9TPDOyd5QpKbLn85AAAALLSka9y6+3szj2919xuS7Dfn2gAAAMjST5W898zT62XogdvkslW1c5KjktxgnPdD3f3Ka1knAADAdmupp0r+35nhK5Kck+S3NrPMZUn27e6Lq2qnJJ+vqk919xe3vEwAAIDt11LvKvnQLV1xd3eSi8enO40PX9oNAACwhZZ6quSLNjW9u1+3keV2SHJiktsn+cvuPm6ReQ5McmCSrF27dinlACy7dQcdvtl5zjnEpb1Ltbn21JYAsGWW+gXc65P8fpI9xsfvJbl3kt3Gx6K6+8ru3ivJrZLsXVV3X2Set3b3+u5ev2bNmi2tHwAAYNVb6jVut0py7+6+KEmq6uAkh3f3U5aycHf/sKo+m+SRSU6/NoUCAABsr5ba43bzJJfPPL98HLdRVbWmqm4yDu+S5BFJvnxtigQAANieLbXH7V1Jjq+qj4zPH5vknZtZ5hZJ3jle53a9JB/o7k9cuzIBAAC2X0u9q+Srq+pTSR40jnpGd5+0mWVOTXKv61gfAADAdm+pp0omyQ2T/Li735jkm1V1mznVBAAAwIwlBbeqemWSFyd5yThqpyTvnldRAAAAXG2pPW6PS/KbSS5Jku4+P5v4GgAAAACWz1KD2+Xd3Uk6Sapq1/mVBAAAwKylBrcPVNVfJ7lJVT0ryRFJ3ja/sgAAANhgqXeVfG1VPSLJj5PcKckruvvTc60MAACAJEsIbuP3sB3R3Q9NIqwBAABsZZs9VbK7r0zys6q68VaoBwAAgAWWdKpkkouTnFZVn854Z8kk6e7nzaUqAAAArrLU4Pbh8QEAAMBWtsngVlVru/sb3f3OrVUQAAAA17S5a9w+umGgqv5+zrUAAACwiM0Ft5oZvu08CwEAAGBxmwtuvZFhAAAAtpLN3ZzknlX14ww9b7uMwxmfd3f/wlyrAwAAYNPBrbt32FqFAAAAsLjNfgE3AAAAK0twAwAAmDjBDQAAYOIENwAAgIkT3AAAACZOcAMAAJg4wQ0AAGDiBDcAAICJE9wAAAAmTnADAACYOMENAABg4gQ3AACAiRPcAAAAJk5wAwAAmDjBDQAAYOIENwAAgIkT3AAAACZOcAMAAJg4wQ0AAGDiBDcAAICJE9wAAAAmTnADAACYOMENAABg4gQ3AACAiRPcAAAAJk5wAwAAmDjBDQAAYOLmFtyqas+q+mxVnVlVZ1TV8+e1LQAAgNVsxzmu+4okf9jdX6qq3ZKcWFWf7u4z57hNAACAVWduPW7d/e3u/tI4fFGSs5LsMa/tAQAArFbz7HG7SlWtS3KvJMctMu3AJAcmydq1a7dGOQCwTVl30OGbnH7OIfttpUq2fZtry0R7AtM095uTVNWNkvx9khd0948XTu/ut3b3+u5ev2bNmnmXAwAAsM2Za3Crqp0yhLbDuvvD89wWAADAajXPu0pWkr9JclZ3v25e2wEAAFjt5tnj9sAkT02yb1WdPD5+fY7bAwAAWJXmdnOS7v58kprX+gEAALYXc785CQAAANeN4AYAADBxghsAAMDECW4AAAATJ7gBAABMnOAGAAAwcYIbAADAxAluAAAAEye4AQAATJzgBgAAMHGCGwAAwMQJbgAAABMnuAEAAEyc4AYAADBxghsAAMDECW4AAAATJ7gBAABMnOAGAAAwcYIbAADAxAluAAAAEye4AQAATJzgBgAAMHGCGwAAwMQJbgAAABMnuAEAAEyc4AYAADBxghsAAMDECW4AAAATJ7gBAABMnOAGAAAwcYIbAADAxAluAAAAEye4AQAATJzgBgAAMHGCGwAAwMQJbgAAABMnuAEAAEyc4AYAADBxghsAAMDECW4AAAATJ7gBAABMnOAGAAAwcYIbAADAxM0tuFXVO6rqu1V1+ry2AQAAsD2YZ4/boUkeOcf1AwAAbBfmFty6+6gk35/X+gEAALYXO650AVV1YJIDk2Tt2rUrXA0AAEu17qDDNzn9nEP220qVrA7ac/lsri2Tba89V/zmJN391u5e393r16xZs9LlAAAATM6KBzcAAAA2TXADAACYuHl+HcB7kxyb5E5V9c2qeua8tgUAALCaze3mJN395HmtGwAAYHviVEkAAICJE9wAAAAmTnADAACYOMENAABg4gQ3AACAiRPcAAAAJk5wAwAAmDjBDQAAYOIENwAAgIkT3AAAACZOcAMAAJg4wQ0AAGDiBDcAAICJE9wAAAAmTnADAACYOMENAABg4gQ3AACAiRPcAAAAJk5wAwAAmDjBDQAAYOIENwAAgIkT3AAAACZOcAMAAJg4wQ0AAGDiBDcAAICJE9wAAAAmTnADAACYOMENAABg4gQ3AACAiRPcAAAAJk5wAwAAmDjBDQAAYOIENwAAgIkT3AAAACZOcAMAAJg4wQ0AAGDiBDcAAICJE9wAAAAmTnADAACYOMENAABg4gQ3AACAiRPcAAAAJm6uwa2qHllVZ1fVV6vqoHluCwAAYLWaW3Crqh2S/GWSRyW5a5InV9Vd57U9AACA1WqePW57J/lqd3+tuy9P8r4kj5nj9gAAAFaleQa3PZKcN/P8m+M4AAAAtkB193xWXLV/kkd29++Mz5+a5Fe6+zkL5jswyYHj0zslOXsuBS2v3ZNcuNJFrBLacnlpz+WlPZePtlxe2nP5aMvlpT2Xl/ZcPttSW966u9csHLnjHDf4rSR7zjy/1TjuGrr7rUneOsc6ll1VndDd61e6jtVAWy4v7bm8tOfy0ZbLS3suH225vLTn8tKey2c1tOU8T5X81yR3qKrbVNX1kzwpycfnuD0AAIBVaW49bt19RVU9J8k/JdkhyTu6+4x5bQ8AAGC1muepkunuTyb55Dy3sUK2qVM7J05bLi/tuby05/LRlstLey4fbbm8tOfy0p7LZ5tvy7ndnAQAAIDlMc9r3AAAAFgG221wq6p1VXX6Fsx/56o6uapOqqrbzbM2rlZVF690DduiqnpeVZ1VVYetdC1TVlU3qao/GIcfUlWfWOmagPnZ0r/9bB1VdWRVbdN3+1tOVfWFla5hW1ZVB1fVH817mZWw3Qa3a+GxST7U3ffq7n/f3Mw10L6slD9I8ojuPmClC5m4m2RoKwC2QFXN9T4J27PufsBK17A92ZaO5e09WOxYVYeNPRMfqqobVtV9qupzVXViVf1TVd2iqn49yQuS/H5VfTZJqupFVXX6+HjBOG5dVZ1dVe9KcnqSPavq16rq2Kr6UlV9sKputHK7uzKq6qNje54xfuF6quriqnp1VZ1SVV+sqpuP428zttdpVfWqla1827DwWKyqtyS5bZJPVdULV7q+iTskye2q6uQk/yfJjcb3gi+P7w2VJIu9L6xo1RNTVf+9qp43Dr++qj4zDu87tuNfVdUJ43vA/5xZ7pCqOrOqTq2q165U/duSxd5P2WI7VNXbxjb856rapapuV1X/OLbt0VV155Uucgqq6mnj6/OUqvq7qjq0qt5SVccl+d9VddPxmDx1/Ft+j3G508YzGqqqvldVTxvHv6uqHjG2+fvG/78+kmSXldzPqanxbKPxf9Cjajjj6/SqetBK1zZFC4/TBdMWfW0vPJYXLPOsqvpUVU3vuOzu7fKRZF2STvLA8fk7kvz3JF9IsmYc98QMX2OQJAcn+aNx+D5JTkuya5IbJTkjyb3Gdf4syf3G+XZPclSSXcfnL07yipXe9xVo65uOP3fJEGhvNrb9o8fx/zvJy8fhjyd52jj87CQXr3T9U35s4lg8J8nuK13f1B/ja/b0cfghSX6U5FYZPtQ6Nsk+SXba2PuCx1XteL8kHxyHj05y/Nhur0zyuzPvATskOTLJPcb3gbNz9U2ybrLS+7EtPBZ7P13pmralx/iavyLJXuPzDyR5SpJ/SXKHcdyvJPnMSte60o8kd0vybxv+liS5aZJDk3wiyQ7juL9I8spxeN8kJ4/Db0myX5K7Z/he37eN478y/r16Ua7+/+oe4+9k/Urv81QeG/73SfKHSV42Du+QZLeVrm1qj40cpwfn6v/ZF31tL3IsH5zkj5I8J8nHktxgpfdtscc20zU4J+d19zHj8LuTvDTDm8ynxw/ad0jy7UWW2yfJR7r7kiSpqg8neVCG0HFud39xnO9+Se6a5JhxfdfP8M/g9uZ5VfW4cXjPJHdIcnmGF0ySnJjkEePwA5P813H475K8ZmsVuY3a2LHItXN8d38zScZeuHVJfpilvS9sz05Mcp+q+oUklyX5UpL1GY7F5yX5rbF3aMckt8jwvnhmkv9M8jc1XFvo+sKlWez99HsrWM+26OvdffI4fGKG1/kDknxwfI0nyQ1WoK6p2TfDBzIXJkl3f39snw9295XjPPtk/Jvd3Z+pqpuN7wNHJ3lwknOT/FWSA6tqjyQ/6O5LqurBSf58XO7Uqjp1a+7YNuRfk7yjqnZK8tGZ45arbew4zXiW26Ze27PHcpI8Lcl5SR7b3T+dd+HXxvYe3BZ+F8JFSc7o7vtfh3VeMjNcST7d3U++DuvbplXVQ5I8PMn9u/snVXVkkp2T/LTHjziSXJlrHou+o4KVctnM8IbjsnLd3xdWte7+aVV9PcnTM/ROnprkoUlun+TSDJ9i3re7f1BVhybZubuvqKq9kzwsyf4ZPsoSOwcAAAaOSURBVOXcdwXK32Zs4v2ULbPwdX7zJD/s7r1WqJ5tzSWbnyVHZThrZm2SlyV5XIbX+dFzrGvV6e6jxpC7X5JDq+p13f2ula5rG3K9bPq1vfBYPi3JXhnOvPn6PAu7trb3a9zWVtWGf8Z+O8kXk6zZMK6qdqqquy2y3NFJHlvDNXG7ZnhDWuzN6ItJHlhVtx/Xt2tV3XHZ92LabpzhE7afjOcV328z8x+T5EnjsBtrbN5Sj0UWd1GS3TYzz9lZ2vvC9u7oDAHtqHH495KclOQXMvxx/FEN17I+Krnqk9Abd/cnk7wwyT1XouhtzJa+n7I0P07y9ap6QnLVzcUcj8lnkjyhqm6WJFV100XmOTrj3+rxg4ULu/vH3X1ehstF7tDdX0vy+Vz9/pDx52+Py909w+mSLFBVt07yne5+W5K3J7n3Cpc0RRs9Trt7S1/bJ2U4vf/jVXXLOdZ8rW3vwe3sJM+uqrOS/JcM52rvn+Q1VXVKkpMzdLFeQ3d/KcO5sccnOS7J27v7pEXmuyDDJ9DvHU8DODbJ9nbB8z9muAnMWRluBPHFzcz//Ay/k9OS7DHv4rZ1Sz0WWVx3fy/DqcynZ7g5yWLzXJ4lvC+QozOcBnlsd38nw2mQR3f3KRn+GH45yXsyfDiTDIH5E+N74+czXPPCpm3p+ylLd0CSZ46v8TOSPGaF61lx3X1Gklcn+dzYLq9bZLaDM5wmfWqGY/L/n5l2XIZrj5Lh/WGPDK/1ZDh98kbjsfy/Mpyyys97SJJTquqkDNdXv3Fly5meJRynW/Ta7u4NHzIcXlW7z6Hk66SuPlsNAACAKdree9wAAAAmT3ADAACYOMENAABg4gQ3AACAiRPcAAAAJk5wA2Crq6orq+rkqjq9qj5YVTfcwuUv3sL5D62q/RcZ/5Cq+sQWrOcjVfXYmednV9XLZ57/fVU9fktqW0qdACC4AbASLu3uvbr77kkuz/CF3VcZvyh1in+jjsn4PX7jF75ekuT+M9Pvn+QLS1lRVe247NUBsGpN8Y8iANuXo5PcvqrWjT1Y70pyepI9q+rJVXXa2DP3mtmFqur1VXVGVf1LVa0Zxz2rqv61qk4Ze79me/IeXlUnVNW/VdVvLCyiqnatqndU1fFVdVJVLfZFrV/I1V/A/oAk/5BkzRg0b5MhkP5HVe1cVX871n5SVT103MbTq+rjVfWZJP8yLvemcb+PSPKLM/UcUlVnVtWpVfXaa9u4AKwOghsAK2bsdXpUktPGUXdI8ubuvluSnyZ5TZJ9k+yV5L4zpynumuSEcb7PJXnlOP7D3X3f7r5nkrOSPHNmc+uS7J1kvyRvqaqdF5TzsiSf6e69kzw0yf+pql0XzHNikrtX1fUzBLdjk5yd5C7j8w29bc9O0t39y0menOSdM9u7d5L9u/tXkzwuyZ2S3DXJ03LN3rzHJblbd98jyas22ZAArHqCGwArYZeqOjnJCUm+keRvxvHndvcXx+H7Jjmyuy/o7iuSHJbkweO0nyV5/zj87iT7jMN3r6qjq+q0JAckudvMNj/Q3T/r7q8k+VqSOy+o6deSHDTWdWSSnZOsnZ2huy9LckaG8HW/JMdlCG8PGB/HjLPuM9aV7v5yknOT3HGc9unu/v44/OAk7+3uK7v7/CSfGcf/KMl/Jvmb8Zq5n/x8EwKwPXF+PQAr4dLu3mt2RFUlwzVj10aPPw9N8tjuPqWqnp7kIYvMs7HnleS/dvfZm9nWMRkC127d/YOq+mKS5yS5V5K/XkKtm93H7r6iqvZO8rAk+4/r33cJ6wZgldLjBsBUHZ/kV6tq96raIcMph58bp10vQ6BJkt9O8vlxeLck366qnTL0uM16QlVdr6pul+S2GU5xnPVPSZ5bY4KsqnttpK4vJPndJKeMz0/N0Pu2NsO1eclw3d4B43ruOE5bLBAeleSJVbVDVd0iwymaqaobJblxd38yyQuT3HMjtQCwndDjBsAkdfe3q+qgJJ/N0Bt2eHd/bJx8SZK9x1vxfzfJE8fxf5zh9MULxp+7zazyGxnC4C8k+b3u/s8xo23wJ0nekOTU8Y6WX0/yczcxyRDcbpvkz8Y6r6iq7yY5r7t/Ns7z5iR/NZ6yeUWSp3f3ZQu2lyQfydCTduZY37Hj+N2SfGy8Lq6SvGhTbQXA6lfdC88UAQAAYEqcKgkAADBxghsAAMDECW4AAAATJ7gBAABMnOAGAAAwcYIbAADAxAluAAAAEye4AQAATNz/AznTaeCF0SIiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pws_3gram=probable_words('the night', cpd_3gram_with_gt, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sent1='iran has an embassy in baku'\n",
    "test_sent2='iran embassy has an in baku'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.313144094162263e-24\n"
     ]
    }
   ],
   "source": [
    "prob_1gram=find_sent_prob(test_sent2, cpd_1gram_with_gt, ngram=1)\n",
    "print(prob_1gram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.581601190573749e-41\n"
     ]
    }
   ],
   "source": [
    "prob_2gram=find_sent_prob(test_sent2,cpd_1gram_with_gt, cpd_2gram_with_gt, ngram=2)\n",
    "print(prob_2gram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8.07727283762361e-72\n"
     ]
    }
   ],
   "source": [
    "prob_3gram=find_sent_prob(test_sent1,cpd_1gram_with_gt, cpd_2gram_with_gt, cpd_3gram_with_gt, ngram=3)\n",
    "print(prob_3gram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.04721527743679202"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy(cpd_1gram_with_gt, test_corpus, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.08709011402122996"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy(cpd_2gram_with_gt, test_corpus, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.03988889009336946"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy(cpd_3gram_with_gt, test_corpus, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0332685594289752"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(cpd_1gram_with_gt, test_corpus, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.062225533451682"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(cpd_2gram_with_gt, test_corpus, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0280346489856098"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(cpd_3gram_with_gt, test_corpus, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_brown=generate_txt_bigram_model(cpd_2gram_with_gt, 'the', numwords=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'the spiritual character nikolai cherkasov the spreading her father forgive me'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_brown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"he certainly did n't own a producing oil well or ill has\""
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_txt_trigram_model(cpd_3gram_with_gt, 'he','certainly', numwords=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cpd_3gram_with_gt.conditions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}