FAA_PyTorch_Tuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "96741ef2-6770-428a-ba3f-3c10490e076d",
   "metadata": {},
   "source": [
    "# this code is the final version of PyTorch NNs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3c377aa-3648-40dd-ac54-b6e532ed781f",
   "metadata": {
    "id": "d3c377aa-3648-40dd-ac54-b6e532ed781f"
   },
   "source": [
    "# Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7f9773d3-4ad2-4989-b8c3-8609dbf1476e",
   "metadata": {
    "id": "7f9773d3-4ad2-4989-b8c3-8609dbf1476e"
   },
   "outputs": [],
   "source": [
    "# load libaries\n",
    "import pandas as pd                  # for dataframe\n",
    "import numpy as np                   # for dealing with numpy\n",
    "import matplotlib.pyplot as plt      # for plots\n",
    "import seaborn as sns                # for statistical data visualization\n",
    "\n",
    "# a magic function that renders the figure in a notebook \n",
    "%matplotlib Inline\n",
    "\n",
    "# allow to display all columns of dataframe\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ee11871-afd8-4f6f-8fe2-92a2fb716913",
   "metadata": {
    "id": "5ee11871-afd8-4f6f-8fe2-92a2fb716913"
   },
   "source": [
    "# Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "60c6355c-3114-4b71-b5af-52067c7f22cb",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "60c6355c-3114-4b71-b5af-52067c7f22cb",
    "outputId": "a6fc174a-dc05-4a17-a681-7c9814431a32"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ENROLID</th>\n",
       "      <th>AGE</th>\n",
       "      <th>SEX</th>\n",
       "      <th>I</th>\n",
       "      <th>II</th>\n",
       "      <th>III</th>\n",
       "      <th>IV</th>\n",
       "      <th>V</th>\n",
       "      <th>VI</th>\n",
       "      <th>VII</th>\n",
       "      <th>VIII</th>\n",
       "      <th>X</th>\n",
       "      <th>XI</th>\n",
       "      <th>XII</th>\n",
       "      <th>XIII</th>\n",
       "      <th>XIV</th>\n",
       "      <th>XV</th>\n",
       "      <th>XVI</th>\n",
       "      <th>XVII</th>\n",
       "      <th>XVIII</th>\n",
       "      <th>XIX</th>\n",
       "      <th>XX</th>\n",
       "      <th>XXI</th>\n",
       "      <th>XXII</th>\n",
       "      <th>LABEL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>571103</td>\n",
       "      <td>57</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13452502</td>\n",
       "      <td>54</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14357306</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14828304</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15194602</td>\n",
       "      <td>59</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    ENROLID  AGE  SEX  I  II  III  IV  V  VI  VII  VIII  X  XI  XII  XIII  \\\n",
       "0    571103   57    1  1   0    0   0  0   1    0     0  5   2    0     1   \n",
       "1  13452502   54    1  3   0    0   0  1   0    0     0  0   0    2     0   \n",
       "2  14357306   27    1  0   0    0   0  2   0    0     0  0   0    0     0   \n",
       "3  14828304   16    2  0   0    0   0  0   3    0     0  0   0    0     1   \n",
       "4  15194602   59    2  0   1    0   0  0   0    0     0  1   0    0     0   \n",
       "\n",
       "   XIV  XV  XVI  XVII  XVIII  XIX  XX  XXI  XXII  LABEL  \n",
       "0    4   0    0     0      9    2   0    1     0      1  \n",
       "1    5   0    0     0      2    1   0    1     0      0  \n",
       "2    0   0    0     0      0    0   0    0     0      1  \n",
       "3    2   0    0     1      3    2   0    2     0      0  \n",
       "4    0   0    0     0      1    0   0    7     0      0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import data\n",
    "df = pd.read_csv('df19_20.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ba330f5-7ed2-4326-aedc-aca45b0b50fe",
   "metadata": {
    "id": "7ba330f5-7ed2-4326-aedc-aca45b0b50fe"
   },
   "source": [
    "# Import Sklearn Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e2f31b74-ea43-4925-976d-7341ef62832c",
   "metadata": {
    "id": "e2f31b74-ea43-4925-976d-7341ef62832c"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split   # even though this is usually used for train-test split, I am using it to sample the data for now to have stratified sampling\n",
    "from sklearn.preprocessing import MinMaxScaler         # to normalize data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e69b91a-e324-4652-a8bf-8c9cf4bdebb2",
   "metadata": {
    "id": "5e69b91a-e324-4652-a8bf-8c9cf4bdebb2"
   },
   "source": [
    "# Split Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e5d033f5-722b-4177-94b2-d7e14ee90aa1",
   "metadata": {
    "id": "e5d033f5-722b-4177-94b2-d7e14ee90aa1"
   },
   "outputs": [],
   "source": [
    "# Set Input Variables and Target Variable\n",
    "X = df.drop(columns = ['ENROLID', 'LABEL'], axis = 1)\n",
    "y = df['LABEL']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ee02eb74-32ad-4204-b979-c455c6c46285",
   "metadata": {
    "id": "ee02eb74-32ad-4204-b979-c455c6c46285"
   },
   "outputs": [],
   "source": [
    "# use train_test_split function to split data into training and testing data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=1, stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7ba3c741-85ee-4443-aa82-cd2bdce41ac4",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 633
    },
    "id": "7ba3c741-85ee-4443-aa82-cd2bdce41ac4",
    "outputId": "3f6f96ea-11bb-498c-a792-642c3881c5d3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Class Distribution')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmMAAAGoCAYAAAAKD+KXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAdYUlEQVR4nO3df7CmZXkf8O9VVolWsQirwV3IoqAtMJUMG0omNWOHtKL5AWk1WWKVJKSrDM7EaptAkok2lY4mTcjYBDKkEMAalfgjkERSjSYxTlFcDYpoiKuirGxgFaIYA3Xx6h/vc9KXw9k9y57F++zx85l55n3e677v573f/WPnO/dzP++p7g4AAGP8o9ETAAD4ViaMAQAMJIwBAAwkjAEADCSMAQAMJIwBAAwkjAGPqKp6dVX9r9HzmFdV11fVOQfoWs+qqlvn3t9WVd93IK49Xe+Wqnr2gboesPoIY8CKVdWPVdW2qvpqVe2cws6/HDSXrqq/m+bypap6T1X96Hyf7n5ud1+1j9c6bm99uvsvuvsZK5339HlXVtVrFl3/xO7+swNxfWB1EsaAFamqVyT59ST/LcmTkxyT5JIkZw6c1jO7+3FJnpHkyiS/UVWvOtAfUlXrDvQ1gW89whiw36rqCUl+Kcn53f327v677v56d/9Bd//nPYz5var6m6r6clW9r6pOnGt7XlV9oqruraovVNV/mupHVtUfVtXfVtXdVfUXVbXs/1/d/cXufkOS85JcWFVHTNf7s6r6qen8uKr682k+X6yqt0z1902X+ei0yvajVfXsqtpRVT9bVX+T5HcWaos++rum73FPVf1OVX3bdM0fr6r3L/r36GkOW5O8MMnPTJ/3B1P7P9z2rKpDq+rXq+qO6fj1qjp0aluY2yur6q5phfInlvs3AsYTxoCV+O4k35bkHQ9jzPVJjk/ypCQfSfLGubbLk7ykux+f5KQk753qr0yyI8n6zFbffi7Jw/lbbtcmWZfk1CXa/muSdyU5PMnGJP8jSbr7e6f2Z3b347r7LdP7b0/yxCTfkWTrHj7vhUmek+RpSZ6e5BeWm2B3X5bZv8UvT5/3g0t0+/kkpyU5Ockzp+8zf+1vT/KEJBuSnJvkN6vq8OU+GxhLGANW4ogkX+zu3fs6oLuv6O57u/v+JK9O8sxphS1Jvp7khKo6rLvv6e6PzNWPSvId08rbX/TD+MO63f31JF/MLEQt9vXMgtVTuvu+7n7/En3mfSPJq7r7/u7++z30+Y3uvr27705yUZKz93Wuy3hhkl/q7ru6e1eS/5LkRXPtX5/av97d70zy1cxu1QKrmDAGrMSXkhy5r3unquqQqnptVX26qr6S5Lap6cjp9d8leV6Sz023Dr97qv9Kku1J3lVVn6mqCx7OJKvqUZmtqt29RPPPJKkkN05PLv7kMpfb1d33LdPn9rnzzyV5yj5Pdu+eMl1vT9f+0qJg/LUkjztAnw08QoQxYCVuSHJfkrP2sf+PZbax//syu522aapXknT3h7r7zMxuYf5+kmum+r3d/crufmqSH0zyiqo6/WHM88wku5PcuLihu/+mu/9Ddz8lyUuSXLLME5T7siJ39Nz5MUnumM7/LsljFxqq6tsf5rXvyGwVb6lrAwcpYQzYb9395SS/mNnepLOq6rFV9aiqem5V/fISQx6f5P7MVtQem9kTmEmSqnp0Vb2wqp4w3Vb8SpIHprYfmDa511z9geXmV1VPrKoXJvnNJK/r7i8t0ecFVbVxentPZoFo4dp3JnnqPvxTLHZ+VW2sqidmtr9tYb/ZR5OcWFUnT5v6X71o3HKf96Ykv1BV66vqyMz+7VfVb7gBD58wBqxId/9akldktpF8V2a36F6W2crWYldndmvtC0k+keQDi9pflOS26RbmS5P8+6l+fJI/yWwP1A1JLlnmt7c+WlVfzezW5k8l+Y/d/Yt76PtdST449b8uyU9392entlcnuWp6ivNH9vJ5i/1uZg8FfGY6XpMk3f3XmT19+idJPpVk8f60yzPbM/e3VfX7S1z3NUm2JflYkpszewDiNUv0Aw4i9TD2wAIAcIBZGQMAGEgYAwAYSBgDABhIGAMAGEgYAwAYaJ9+NXs1OvLII3vTpk2jpwEAsKwPf/jDX+zu9Uu1HbRhbNOmTdm2bdvoaQAALKuqPrenNrcpAQAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGWjd6AhxYmy74o9FT4CBx22u/f/QUAIiVMQCAoYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBlg1jVXV0Vf1pVX2yqm6pqp+e6k+sqndX1aem18PnxlxYVdur6taqes5c/ZSqunlqe31V1VQ/tKreMtU/WFWbHoHvCgCw6uzLytjuJK/s7n+W5LQk51fVCUkuSPKe7j4+yXum95natiQ5MckZSS6pqkOma12aZGuS46fjjKl+bpJ7uvu4JBcned0B+G4AAKvesmGsu3d290em83uTfDLJhiRnJrlq6nZVkrOm8zOTvLm77+/uzybZnuTUqjoqyWHdfUN3d5KrF41ZuNZbk5y+sGoGALCWPaw9Y9Ptw+9M8sEkT+7unckssCV50tRtQ5Lb54btmGobpvPF9QeN6e7dSb6c5IiHMzcAgIPRPoexqnpckrcleXl3f2VvXZeo9V7qexuzeA5bq2pbVW3btWvXclMGAFj19imMVdWjMgtib+zut0/lO6dbj5le75rqO5IcPTd8Y5I7pvrGJeoPGlNV65I8Icndi+fR3Zd19+bu3rx+/fp9mToAwKq2L09TVpLLk3yyu39trum6JOdM5+ckuXauvmV6QvLYzDbq3zjdyry3qk6brvniRWMWrvX8JO+d9pUBAKxp6/ahz/ckeVGSm6vqpqn2c0lem+Saqjo3yeeTvCBJuvuWqromyScyexLz/O5+YBp3XpIrkzwmyfXTkczC3huqantmK2JbVva1AAAODsuGse5+f5be05Ukp+9hzEVJLlqivi3JSUvU78sU5gAAvpX4BX4AgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgZYNY1V1RVXdVVUfn6u9papumo7bquqmqb6pqv5+ru235sacUlU3V9X2qnp9VdVUP3S63vaq+mBVbTrwXxMAYHXal5WxK5OcMV/o7h/t7pO7++Qkb0vy9rnmTy+0dfdL5+qXJtma5PjpWLjmuUnu6e7jklyc5HX780UAAA5Gy4ax7n5fkruXaptWt34kyZv2do2qOirJYd19Q3d3kquTnDU1n5nkqun8rUlOX1g1AwBY61a6Z+xZSe7s7k/N1Y6tqr+sqj+vqmdNtQ1Jdsz12THVFtpuT5Lu3p3ky0mOWOrDqmprVW2rqm27du1a4dQBAMZbaRg7Ow9eFduZ5Jju/s4kr0jyu1V1WJKlVrp6et1b24OL3Zd19+bu3rx+/foVTBsAYHVYt78Dq2pdkn+b5JSFWnffn+T+6fzDVfXpJE/PbCVs49zwjUnumM53JDk6yY7pmk/IHm6LAgCsNStZGfu+JH/V3f9w+7Gq1lfVIdP5UzPbqP+Z7t6Z5N6qOm3aD/biJNdOw65Lcs50/vwk7532lQEArHn78tMWb0pyQ5JnVNWOqjp3atqSh27c/94kH6uqj2a2Gf+l3b2wynVekv+ZZHuSTye5fqpfnuSIqtqe2a3NC1bwfQAADirL3qbs7rP3UP/xJWpvy+ynLpbqvy3JSUvU70vyguXmAQCwFvkFfgCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgYQxAICBhDEAgIGEMQCAgZYNY1V1RVXdVVUfn6u9uqq+UFU3Tcfz5tourKrtVXVrVT1nrn5KVd08tb2+qmqqH1pVb5nqH6yqTQf4OwIArFr7sjJ2ZZIzlqhf3N0nT8c7k6SqTkiyJcmJ05hLquqQqf+lSbYmOX46Fq55bpJ7uvu4JBcned1+fhcAgIPOsmGsu9+X5O59vN6ZSd7c3fd392eTbE9yalUdleSw7r6huzvJ1UnOmhtz1XT+1iSnL6yaAQCsdSvZM/ayqvrYdBvz8Km2Icntc312TLUN0/ni+oPGdPfuJF9OcsQK5gUAcNDY3zB2aZKnJTk5yc4kvzrVl1rR6r3U9zbmIapqa1Vtq6ptu3btelgTBgBYjfYrjHX3nd39QHd/I8lvJzl1atqR5Oi5rhuT3DHVNy5Rf9CYqlqX5AnZw23R7r6suzd39+b169fvz9QBAFaV/Qpj0x6wBT+cZOFJy+uSbJmekDw2s436N3b3ziT3VtVp036wFye5dm7MOdP585O8d9pXBgCw5q1brkNVvSnJs5McWVU7krwqybOr6uTMbifeluQlSdLdt1TVNUk+kWR3kvO7+4HpUudl9mTmY5JcPx1JcnmSN1TV9sxWxLYcgO8FAHBQWDaMdffZS5Qv30v/i5JctER9W5KTlqjfl+QFy80DAGAt8gv8AAADCWMAAAMJYwAAAwljAAADCWMAAAMJYwAAAwljAAADCWMAAAMJYwAAAwljAAADCWMAAAMJYwAAAwljAAADCWMAAAMJYwAAAwljAAADCWMAAAMJYwAAAwljAAADCWMAAAMJYwAAAwljAAADCWMAAAMJYwAAAwljAAADCWMAAAMJYwAAAwljAAADCWMAAAMJYwAAAwljAAADCWMAAAMJYwAAAwljAAADCWMAAAMtG8aq6oqququqPj5X+5Wq+quq+lhVvaOq/slU31RVf19VN03Hb82NOaWqbq6q7VX1+qqqqX5oVb1lqn+wqjYd+K8JALA67cvK2JVJzlhUe3eSk7r7nyf56yQXzrV9urtPno6XztUvTbI1yfHTsXDNc5Pc093HJbk4yese9rcAADhILRvGuvt9Se5eVHtXd++e3n4gyca9XaOqjkpyWHff0N2d5OokZ03NZya5ajp/a5LTF1bNAADWugOxZ+wnk1w/9/7YqvrLqvrzqnrWVNuQZMdcnx1TbaHt9iSZAt6XkxxxAOYFALDqrVvJ4Kr6+SS7k7xxKu1Mckx3f6mqTkny+1V1YpKlVrp64TJ7aVv8eVszu9WZY445ZiVTBwBYFfZ7ZayqzknyA0leON16THff391fms4/nOTTSZ6e2UrY/K3MjUnumM53JDl6uua6JE/IotuiC7r7su7e3N2b169fv79TBwBYNfYrjFXVGUl+NskPdffX5urrq+qQ6fypmW3U/0x370xyb1WdNu0He3GSa6dh1yU5Zzp/fpL3LoQ7AIC1btnblFX1piTPTnJkVe1I8qrMnp48NMm7p732H5ienPzeJL9UVbuTPJDkpd29sMp1XmZPZj4msz1mC/vMLk/yhqrantmK2JYD8s0AAA4Cy4ax7j57ifLle+j7tiRv20PbtiQnLVG/L8kLlpsHAMBa5Bf4AQAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGEsYAAAYSxgAABhLGAAAGEsYAAAZaNoxV1RVVdVdVfXyu9sSqendVfWp6PXyu7cKq2l5Vt1bVc+bqp1TVzVPb66uqpvqhVfWWqf7Bqtp0gL8jAMCqtS8rY1cmOWNR7YIk7+nu45O8Z3qfqjohyZYkJ05jLqmqQ6YxlybZmuT46Vi45rlJ7unu45JcnOR1+/tlAAAONsuGse5+X5K7F5XPTHLVdH5VkrPm6m/u7vu7+7NJtic5taqOSnJYd9/Q3Z3k6kVjFq711iSnL6yaAQCsdfu7Z+zJ3b0zSabXJ031DUlun+u3Y6ptmM4X1x80prt3J/lykiOW+tCq2lpV26pq265du/Zz6gAAq8eB3sC/1IpW76W+tzEPLXZf1t2bu3vz+vXr93OKAACrx/6GsTunW4+ZXu+a6juSHD3Xb2OSO6b6xiXqDxpTVeuSPCEPvS0KALAm7W8Yuy7JOdP5OUmunatvmZ6QPDazjfo3Trcy762q06b9YC9eNGbhWs9P8t5pXxkAwJq3brkOVfWmJM9OcmRV7UjyqiSvTXJNVZ2b5PNJXpAk3X1LVV2T5BNJdic5v7sfmC51XmZPZj4myfXTkSSXJ3lDVW3PbEVsywH5ZgAAB4Flw1h3n72HptP30P+iJBctUd+W5KQl6vdlCnMAAN9q/AI/AMBAwhgAwEDCGADAQMIYAMBAwhgAwEDCGADAQMIYAMBAwhgAwEDCGADAQMIYAMBAwhgAwEDCGADAQMIYAMBAwhgAwEDCGADAQMIYAMBAwhgAwEDCGADAQMIYAMBAwhgAwEDCGADAQMIYAMBAwhgAwEDCGADAQMIYAMBAwhgAwEDCGADAQMIYAMBAwhgAwEDCGADAQMIYAMBAwhgAwEDCGADAQMIYAMBAwhgAwED7Hcaq6hlVddPc8ZWqenlVvbqqvjBXf97cmAurantV3VpVz5mrn1JVN09tr6+qWukXAwA4GOx3GOvuW7v75O4+OckpSb6W5B1T88ULbd39ziSpqhOSbElyYpIzklxSVYdM/S9NsjXJ8dNxxv7OCwDgYHKgblOenuTT3f25vfQ5M8mbu/v+7v5sku1JTq2qo5Ic1t03dHcnuTrJWQdoXgAAq9qBCmNbkrxp7v3LqupjVXVFVR0+1TYkuX2uz46ptmE6X1x/iKraWlXbqmrbrl27DtDUAQDGWXEYq6pHJ/mhJL83lS5N8rQkJyfZmeRXF7ouMbz3Un9osfuy7t7c3ZvXr1+/kmkDAKwKB2Jl7LlJPtLddyZJd9/Z3Q909zeS/HaSU6d+O5IcPTduY5I7pvrGJeoAAGvegQhjZ2fuFuW0B2zBDyf5+HR+XZItVXVoVR2b2Ub9G7t7Z5J7q+q06SnKFye59gDMCwBg1Vu3ksFV9dgk/zrJS+bKv1xVJ2d2q/G2hbbuvqWqrknyiSS7k5zf3Q9MY85LcmWSxyS5fjoAANa8FYWx7v5akiMW1V60l/4XJbloifq2JCetZC4AAAcjv8APADCQMAYAMJAwBgAwkDAGADCQMAYAMJAwBgAwkDAGADCQMAYAMJAwBgAwkDAGADCQMAYAMJAwBgAwkDAGADCQMAYAMJAwBgAwkDAGADCQMAYAMJAwBgAwkDAGADCQMAYAMJAwBgAwkDAGADCQMAYAMJAwBgAw0LrREwBg9dt0wR+NngIHidte+/2jp3DQsTIGADCQMAYAMJAwBgAwkDAGADCQMAYAMJAwBgAwkDAGADCQMAYAMJAwBgAw0IrCWFXdVlU3V9VNVbVtqj2xqt5dVZ+aXg+f639hVW2vqlur6jlz9VOm62yvqtdXVa1kXgAAB4sDsTL2r7r75O7ePL2/IMl7uvv4JO+Z3qeqTkiyJcmJSc5IcklVHTKNuTTJ1iTHT8cZB2BeAACr3iNxm/LMJFdN51clOWuu/ubuvr+7P5tke5JTq+qoJId19w3d3UmunhsDALCmrTSMdZJ3VdWHq2rrVHtyd+9Mkun1SVN9Q5Lb58bumGobpvPFdQCANW/dCsd/T3ffUVVPSvLuqvqrvfRdah9Y76X+0AvMAt/WJDnmmGMe7lwBAFadFa2Mdfcd0+tdSd6R5NQkd063HjO93jV135Hk6LnhG5PcMdU3LlFf6vMu6+7N3b15/fr1K5k6AMCqsN9hrKr+cVU9fuE8yb9J8vEk1yU5Z+p2TpJrp/PrkmypqkOr6tjMNurfON3KvLeqTpueonzx3BgAgDVtJbcpn5zkHdOvUKxL8rvd/cdV9aEk11TVuUk+n+QFSdLdt1TVNUk+kWR3kvO7+4HpWucluTLJY5JcPx0AAGvefoex7v5MkmcuUf9SktP3MOaiJBctUd+W5KT9nQsAwMHKL/ADAAwkjAEADCSMAQAMJIwBAAwkjAEADCSMAQAMJIwBAAwkjAEADCSMAQAMJIwBAAwkjAEADCSMAQAMJIwBAAwkjAEADCSMAQAMJIwBAAwkjAEADCSMAQAMJIwBAAwkjAEADCSMAQAMJIwBAAwkjAEADCSMAQAMJIwBAAwkjAEADCSMAQAMJIwBAAwkjAEADCSMAQAMJIwBAAwkjAEADCSMAQAMJIwBAAwkjAEADLTfYayqjq6qP62qT1bVLVX101P91VX1haq6aTqeNzfmwqraXlW3VtVz5uqnVNXNU9vrq6pW9rUAAA4O61YwdneSV3b3R6rq8Uk+XFXvntou7u7/Pt+5qk5IsiXJiUmekuRPqurp3f1AkkuTbE3ygSTvTHJGkutXMDcAgIPCfq+MdffO7v7IdH5vkk8m2bCXIWcmeXN339/dn02yPcmpVXVUksO6+4bu7iRXJzlrf+cFAHAwOSB7xqpqU5LvTPLBqfSyqvpYVV1RVYdPtQ1Jbp8btmOqbZjOF9cBANa8FYexqnpckrcleXl3fyWzW45PS3Jykp1JfnWh6xLDey/1pT5ra1Vtq6ptu3btWunUAQCGW1EYq6pHZRbE3tjdb0+S7r6zux/o7m8k+e0kp07ddyQ5em74xiR3TPWNS9Qforsv6+7N3b15/fr1K5k6AMCqsJKnKSvJ5Uk+2d2/Nlc/aq7bDyf5+HR+XZItVXVoVR2b5PgkN3b3ziT3VtVp0zVfnOTa/Z0XAMDBZCVPU35PkhclubmqbppqP5fk7Ko6ObNbjbcleUmSdPctVXVNkk9k9iTm+dOTlElyXpIrkzwms6coPUkJAHxL2O8w1t3vz9L7vd65lzEXJbloifq2JCft71wAAA5WfoEfAGAgYQwAYCBhDABgIGEMAGAgYQwAYCBhDABgIGEMAGAgYQwAYCBhDABgIGEMAGAgYQwAYCBhDABgIGEMAGAgYQwAYCBhDABgIGEMAGAgYQwAYCBhDABgIGEMAGAgYQwAYCBhDABgIGEMAGAgYQwAYCBhDABgIGEMAGAgYQwAYCBhDABgIGEMAGAgYQwAYCBhDABgIGEMAGAgYQwAYCBhDABgIGEMAGAgYQwAYKBVE8aq6oyqurWqtlfVBaPnAwDwzbAqwlhVHZLkN5M8N8kJSc6uqhPGzgoA4JG3KsJYklOTbO/uz3T3/03y5iRnDp4TAMAjbt3oCUw2JLl97v2OJP9icaeq2ppk6/T2q1V16zdhbqwNRyb54uhJrCb1utEzgDXB/y2L+L9lj75jTw2rJYzVErV+SKH7siSXPfLTYa2pqm3dvXn0PIC1xf8tHAir5TbljiRHz73fmOSOQXMBAPimWS1h7ENJjq+qY6vq0Um2JLlu8JwAAB5xq+I2ZXfvrqqXJfnfSQ5JckV33zJ4Wqwtbm8DjwT/t7Bi1f2QrVkAAHyTrJbblAAA35KEMQCAgYQxAICBVsUGfjiQquqfZvYXHDZk9nt1dyS5rrs/OXRiALAEK2OsKVX1s5n9Oa1KcmNmP5tSSd7kD9ADj5Sq+onRc+Dg5WlK1pSq+uskJ3b31xfVH53klu4+fszMgLWsqj7f3ceMngcHJ7cpWWu+keQpST63qH7U1AawX6rqY3tqSvLkb+ZcWFuEMdaalyd5T1V9Kv//j88fk+S4JC8bNSlgTXhykuckuWdRvZL8n2/+dFgrhDHWlO7+46p6epJTM9vAX5n97dMPdfcDQycHHOz+MMnjuvumxQ1V9Wff9NmwZtgzBgAwkKcpAQAGEsYAAAYSxgAABhLGAAAGEsYAAAb6f+CWMb0FHm+EAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# label frequency\n",
    "plt.figure(figsize=(10, 7))\n",
    "ax = y_train.value_counts().plot.bar()\n",
    "ax.set_title('Class Distribution')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "873a720d-ce90-4abc-a813-4658b9189611",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    20717\n",
       "1     5813\n",
       "Name: LABEL, dtype: int64"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2ba98e0-b795-46ae-95d3-a8077183a5ab",
   "metadata": {
    "id": "f2ba98e0-b795-46ae-95d3-a8077183a5ab"
   },
   "source": [
    "## Normalize data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "76138624-6b3d-409f-934b-273655a59195",
   "metadata": {
    "id": "76138624-6b3d-409f-934b-273655a59195"
   },
   "outputs": [],
   "source": [
    "# normalize data\n",
    "scaler = MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f896580-4dfe-40c5-93e2-fd90f38bc1ee",
   "metadata": {
    "id": "9f896580-4dfe-40c5-93e2-fd90f38bc1ee"
   },
   "source": [
    "# Import SMOTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "11b3bfd4-3c0a-4098-9263-48c50809889d",
   "metadata": {
    "id": "11b3bfd4-3c0a-4098-9263-48c50809889d"
   },
   "outputs": [],
   "source": [
    "# https://imbalanced-learn.org/stable/references/generated/imblearn.over_sampling.SMOTE.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a6ae9077-6cb2-41fa-88b1-1d83035d3efd",
   "metadata": {
    "id": "a6ae9077-6cb2-41fa-88b1-1d83035d3efd"
   },
   "outputs": [],
   "source": [
    "# install first if using it first-time\n",
    "# import sys\n",
    "# !{sys.executable} -m pip install imblearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2fa601c1-ad80-480f-9922-da22c33fe5f1",
   "metadata": {
    "id": "2fa601c1-ad80-480f-9922-da22c33fe5f1"
   },
   "outputs": [],
   "source": [
    "# import SMOTE module from imblearn library\n",
    "# pip install imblearn (if you don't have imblearn in your system)\n",
    "from imblearn.over_sampling import SMOTE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97824df6-2634-42aa-a118-0469615fb18b",
   "metadata": {
    "id": "97824df6-2634-42aa-a118-0469615fb18b"
   },
   "source": [
    "# SMOTE to oversample the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "024c58aa-425b-43c8-ad41-87488c317122",
   "metadata": {
    "id": "024c58aa-425b-43c8-ad41-87488c317122"
   },
   "outputs": [],
   "source": [
    "sm = SMOTE(random_state = 1, sampling_strategy='auto', k_neighbors=10)\n",
    "X_train_sm, y_train_sm = sm.fit_resample(X_train, y_train.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "395ac094-31e2-4e4b-a65b-53a1ba9e5950",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "395ac094-31e2-4e4b-a65b-53a1ba9e5950",
    "outputId": "8f14715f-d822-4e1f-9afc-a9d23afef871"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.859375  , 1.        , 0.        , ..., 0.        , 0.125     ,\n",
       "        0.        ],\n",
       "       [0.59375   , 0.        , 0.        , ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       [0.03125   , 0.        , 0.11111111, ..., 0.        , 0.        ,\n",
       "        0.        ],\n",
       "       ...,\n",
       "       [0.79166964, 1.        , 0.        , ..., 0.        , 0.08333333,\n",
       "        0.        ],\n",
       "       [0.73417135, 0.        , 0.        , ..., 0.        , 0.01055244,\n",
       "        0.        ],\n",
       "       [0.9376154 , 0.        , 0.        , ..., 0.        , 0.08333333,\n",
       "        0.        ]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8eb66856-396a-4010-bd83-a1b18726ea08",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8eb66856-396a-4010-bd83-a1b18726ea08",
    "outputId": "216894d3-56d3-46b4-8326-f4a610dcd936"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 0, ..., 1, 1, 1], dtype=int64)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6df8b799-3a41-41e4-8a6d-ce1783806097",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 504
    },
    "id": "6df8b799-3a41-41e4-8a6d-ce1783806097",
    "outputId": "81018d09-507d-400c-a348-ce7acb43ea94"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([<matplotlib.axis.XTick at 0x287ae6e2ca0>,\n",
       "  <matplotlib.axis.XTick at 0x287ae6e2c70>],\n",
       " [Text(0, 0, ''), Text(0, 0, '')])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEICAYAAABfz4NwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAXd0lEQVR4nO3df7DddZ3f8eerRFmshgK5KibRoESnwNQ4ZFN2rA6dbEu0uxtsZQ1rJdvNNsrgjFbbXXB3VmpNR9wqDl1hJxY2wSqQFV2yrbQi7IrORPDCIuGHrJcfmmsiuQiL8Qcpie/+cT5358vNyb03917uTeD5mDlzv+f9+Xy+53MySV73+/l+z/mmqpAk6R/M9QQkSYcHA0GSBBgIkqTGQJAkAQaCJKkxECRJgIGgI0CSi5P8z7meR1eSG5OsnaF9vSnJA53njyT51ZnYd9vfvUnOnKn96bnLQNBhIclvJRlM8pMku9p/uP9sjuZSSX7a5vKjJDcneUe3T1W9pao2T3JfJ4/Xp6q+XlWvm+682+ttSvLRMfs/tar+eib2r+c2A0FzLskHgE8B/xV4GfBK4HJg9RxO6/VV9WLgdcAm4E+SfHimXyTJvJnepzRVBoLmVJJjgY8AF1TVF6vqp1X1dFX9ZVX9p4OM+fMkP0zyZJJbk5zaaXtrkvuS7EnygyT/sdUXJPlfSf4uyeNJvp5kwr//VfVYVX0WOB+4KMkJbX9/neR32/bJSb7W5vNYkuta/da2m2+3o413JDkzyXCS30/yQ+DPRmtjXvqX2/t4IsmfJfmlts/fTvKNMX8e1eawHngn8Hvt9f6ytf/9ElSSo5N8KsnO9vhUkqNb2+jcPphkdztS+3cT/RnpucNA0Fz7FeCXgC8dwpgbgaXAS4E7gc912q4E3l1VLwFOA25p9Q8Cw8AAvaOQDwGH8r0tNwDzgBV92v4L8BXgOGAR8N8BqurNrf31VfXiqrquPX85cDzwKmD9QV7vncBZwGuA1wJ/ONEEq2ojvT+Lj7fX+/U+3f4AOANYBry+vZ/uvl8OHAssBNYBn05y3ESvrecGA0Fz7QTgsaraN9kBVXVVVe2pqr3AxcDr25EGwNPAKUnmV9UTVXVnp34i8Kp2BPL1OoQv8qqqp4HH6P1HPtbT9P5zf0VVPVVV3+jTp+sXwIeram9V/fwgff6kqnZU1ePABuDcyc51Au8EPlJVu6tqBPjPwLs67U+39qer6svAT+gtm+l5wEDQXPsRsGCya+lJjkrysSQPJvkx8EhrWtB+/hvgrcD32jLOr7T6HwNDwFeSPJTkwkOZZJIX0Du6eLxP8+8BAW5vV/T8zgS7G6mqpybos6Oz/T3gFZOe7Phe0fZ3sH3/aEw4/wx48Qy9tg5zBoLm2jbgKeDsSfb/LXonm3+V3tLGklYPQFV9q6pW01tO+gtgS6vvqaoPVtWrgV8HPpBk5SHMczWwD7h9bENV/bCq/n1VvQJ4N3D5BFcWTebIZHFn+5XAzrb9U+BFow1JXn6I+95J72im3771PGcgaE5V1ZPAH9Fbqz47yYuSvCDJW5J8vM+QlwB76R1ZvIjelUkAJHlhkncmObYt8fwY2N/afq2deE2nvn+i+SU5Psk7gU8Dl1TVj/r0OSfJovb0CXr/KY/u+1Hg1ZP4oxjrgiSLkhxP73zH6PmHbwOnJlnWTjRfPGbcRK93DfCHSQaSLKD3Z39YfcZDc8dA0Jyrqk8CH6B3cnOE3nLJe+n9hj/W1fSWOX4A3Ad8c0z7u4BH2nLSe4B/2+pLga/SWxPfBlw+wbX5307yE3rLTL8L/Ieq+qOD9P1l4LbWfyvwvqp6uLVdDGxuVzf95jivN9bn6Z2ofqg9PgpQVX9L76qsrwLfBcaer7iS3jmUv0vyF332+1FgELgb2E7vpPxH+/TT81C8QY4kCTxCkCQ1BoIkCTAQJEmNgSBJAnofxT8iLViwoJYsWTLX05CkI8odd9zxWFUN9Gs7YgNhyZIlDA4OzvU0JOmIkuR7B2tzyUiSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEHMGfVJ6OJRf+77megvp45GP/aq6noDH8t3J4erb+rXiEIEkCDARJUmMgSJKASQRCksVJ/irJ/UnuTfK+Vj8+yU1Jvtt+HtcZc1GSoSQPJDmrUz89yfbWdlmStPrRSa5r9duSLHkW3qskaRyTOULYB3ywqv4xcAZwQZJTgAuBm6tqKXBze05rWwOcCqwCLk9yVNvXFcB6YGl7rGr1dcATVXUycClwyQy8N0nSIZgwEKpqV1Xd2bb3APcDC4HVwObWbTNwdtteDVxbVXur6mFgCFiR5ERgflVtq6oCrh4zZnRfXwBWjh49SJJmxyGdQ2hLOW8AbgNeVlW7oBcawEtbt4XAjs6w4VZb2LbH1p8xpqr2AU8CJ/R5/fVJBpMMjoyMHMrUJUkTmHQgJHkxcD3w/qr68Xhd+9RqnPp4Y55ZqNpYVcuravnAQN87wEmSpmhSgZDkBfTC4HNV9cVWfrQtA9F+7m71YWBxZ/giYGerL+pTf8aYJPOAY4HHD/XNSJKmbjJXGQW4Eri/qj7ZadoKrG3ba4EbOvU17cqhk+idPL69LSvtSXJG2+d5Y8aM7uvtwC3tPIMkaZZM5qsr3gi8C9ie5K5W+xDwMWBLknXA94FzAKrq3iRbgPvoXaF0QVXtb+POBzYBxwA3tgf0AuezSYboHRmsmd7bkiQdqgkDoaq+Qf81foCVBxmzAdjQpz4InNan/hQtUCRJc8NPKkuSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSM5lbaF6VZHeSezq165Lc1R6PjN5JLcmSJD/vtP1pZ8zpSbYnGUpyWbuNJu1Wm9e1+m1Jlsz825QkTWQyRwibgFXdQlW9o6qWVdUy4Hrgi53mB0fbquo9nfoVwHp691he2tnnOuCJqjoZuBS4ZCpvRJI0PRMGQlXdSu8+xwdov+X/JnDNePtIciIwv6q2VVUBVwNnt+bVwOa2/QVg5ejRgyRp9kz3HMKbgEer6rud2klJ/ibJ15K8qdUWAsOdPsOtNtq2A6Cq9gFPAif0e7Ek65MMJhkcGRmZ5tQlSV3TDYRzeebRwS7glVX1BuADwOeTzAf6/cZf7ed4bc8sVm2squVVtXxgYGAa05YkjTVvqgOTzAP+NXD6aK2q9gJ72/YdSR4EXkvviGBRZ/giYGfbHgYWA8Ntn8dykCUqSdKzZzpHCL8KfKeq/n4pKMlAkqPa9qvpnTx+qKp2AXuSnNHOD5wH3NCGbQXWtu23A7e08wySpFk0mctOrwG2Aa9LMpxkXWtaw4Enk98M3J3k2/ROEL+nqkZ/2z8f+B/AEPAgcGOrXwmckGSI3jLThdN4P5KkKZpwyaiqzj1I/bf71K6ndxlqv/6DwGl96k8B50w0D0nSs8tPKkuSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSM5k7pl2VZHeSezq1i5P8IMld7fHWTttFSYaSPJDkrE799CTbW9tl7VaaJDk6yXWtfluSJTP8HiVJkzCZI4RNwKo+9Uurall7fBkgySn0bq15ahtz+eg9loErgPX07rO8tLPPdcATVXUycClwyRTfiyRpGiYMhKq6FXh8on7NauDaqtpbVQ/Tu3/yiiQnAvOraltVFXA1cHZnzOa2/QVg5ejRgyRp9kznHMJ7k9zdlpSOa7WFwI5On+FWW9i2x9afMaaq9gFPAif0e8Ek65MMJhkcGRmZxtQlSWNNNRCuAF4DLAN2AZ9o9X6/2dc49fHGHFis2lhVy6tq+cDAwCFNWJI0vikFQlU9WlX7q+oXwGeAFa1pGFjc6boI2Nnqi/rUnzEmyTzgWCa/RCVJmiFTCoR2TmDU24DRK5C2AmvalUMn0Tt5fHtV7QL2JDmjnR84D7ihM2Zt2347cEs7zyBJmkXzJuqQ5BrgTGBBkmHgw8CZSZbRW9p5BHg3QFXdm2QLcB+wD7igqva3XZ1P74qlY4Ab2wPgSuCzSYboHRmsmYH3JUk6RBMGQlWd26d85Tj9NwAb+tQHgdP61J8CzploHpKkZ5efVJYkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkZsJASHJVkt1J7unU/jjJd5LcneRLSf5Rqy9J8vMkd7XHn3bGnJ5ke5KhJJe1W2nSbrd5XavflmTJzL9NSdJEJnOEsAlYNaZ2E3BaVf0T4G+BizptD1bVsvZ4T6d+BbCe3n2Wl3b2uQ54oqpOBi4FLjnkdyFJmrYJA6GqbqV3r+Nu7StVta89/SawaLx9JDkRmF9V26qqgKuBs1vzamBz2/4CsHL06EGSNHtm4hzC7wA3dp6flORvknwtyZtabSEw3Okz3GqjbTsAWsg8CZzQ74WSrE8ymGRwZGRkBqYuSRo1rUBI8gfAPuBzrbQLeGVVvQH4APD5JPOBfr/x1+huxml7ZrFqY1Utr6rlAwMD05m6JGmMeVMdmGQt8GvAyrYMRFXtBfa27TuSPAi8lt4RQXdZaRGws20PA4uB4STzgGMZs0QlSXr2TekIIckq4PeB36iqn3XqA0mOatuvpnfy+KGq2gXsSXJGOz9wHnBDG7YVWNu23w7cMhowkqTZM+ERQpJrgDOBBUmGgQ/Tu6roaOCmdv73m+2KojcDH0myD9gPvKeqRn/bP5/eFUvH0DvnMHre4Urgs0mG6B0ZrJmRdyZJOiQTBkJVndunfOVB+l4PXH+QtkHgtD71p4BzJpqHJOnZ5SeVJUmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKmZMBCSXJVkd5J7OrXjk9yU5Lvt53GdtouSDCV5IMlZnfrpSba3tsvarTRJcnSS61r9tiRLZvg9SpImYTJHCJuAVWNqFwI3V9VS4Ob2nCSn0LsF5qltzOWj91gGrgDW07vP8tLOPtcBT1TVycClwCVTfTOSpKmbMBCq6lZ69zruWg1sbtubgbM79Wuram9VPQwMASuSnAjMr6ptVVXA1WPGjO7rC8DK0aMHSdLsmeo5hJdV1S6A9vOlrb4Q2NHpN9xqC9v22PozxlTVPuBJ4IR+L5pkfZLBJIMjIyNTnLokqZ+ZPqnc7zf7Gqc+3pgDi1Ubq2p5VS0fGBiY4hQlSf1MNRAebctAtJ+7W30YWNzptwjY2eqL+tSfMSbJPOBYDlyikiQ9y6YaCFuBtW17LXBDp76mXTl0Er2Tx7e3ZaU9Sc5o5wfOGzNmdF9vB25p5xkkSbNo3kQdklwDnAksSDIMfBj4GLAlyTrg+8A5AFV1b5ItwH3APuCCqtrfdnU+vSuWjgFubA+AK4HPJhmid2SwZkbemSTpkEwYCFV17kGaVh6k/wZgQ5/6IHBan/pTtECRJM0dP6ksSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSc2UAyHJ65Lc1Xn8OMn7k1yc5Aed+ls7Yy5KMpTkgSRndeqnJ9ne2i5rt9mUJM2iKQdCVT1QVcuqahlwOvAz4Eut+dLRtqr6MkCSU+jdHvNUYBVweZKjWv8rgPX07sG8tLVLkmbRTC0ZrQQerKrvjdNnNXBtVe2tqoeBIWBFkhOB+VW1raoKuBo4e4bmJUmapJkKhDXANZ3n701yd5KrkhzXaguBHZ0+w622sG2PrR8gyfokg0kGR0ZGZmjqkiSYgUBI8kLgN4A/b6UrgNcAy4BdwCdGu/YZXuPUDyxWbayq5VW1fGBgYDrTliSNMRNHCG8B7qyqRwGq6tGq2l9VvwA+A6xo/YaBxZ1xi4Cdrb6oT12SNItmIhDOpbNc1M4JjHobcE/b3gqsSXJ0kpPonTy+vap2AXuSnNGuLjoPuGEG5iVJOgTzpjM4yYuAfwG8u1P+eJJl9JZ9Hhltq6p7k2wB7gP2ARdU1f425nxgE3AMcGN7SJJm0bQCoap+BpwwpvaucfpvADb0qQ8Cp01nLpKk6fGTypIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUTCsQkjySZHuSu5IMttrxSW5K8t3287hO/4uSDCV5IMlZnfrpbT9DSS5rt9KUJM2imThC+OdVtayqlrfnFwI3V9VS4Ob2nCSnAGuAU4FVwOVJjmpjrgDW07vP8tLWLkmaRc/GktFqYHPb3gyc3alfW1V7q+phYAhYkeREYH5VbauqAq7ujJEkzZLpBkIBX0lyR5L1rfayqtoF0H6+tNUXAjs6Y4dbbWHbHls/QJL1SQaTDI6MjExz6pKkrnnTHP/GqtqZ5KXATUm+M07ffucFapz6gcWqjcBGgOXLl/ftI0mammkdIVTVzvZzN/AlYAXwaFsGov3c3boPA4s7wxcBO1t9UZ+6JGkWTTkQkvzDJC8Z3Qb+JXAPsBVY27qtBW5o21uBNUmOTnISvZPHt7dlpT1JzmhXF53XGSNJmiXTWTJ6GfCldoXoPODzVfV/knwL2JJkHfB94ByAqro3yRbgPmAfcEFV7W/7Oh/YBBwD3NgekqRZNOVAqKqHgNf3qf8IWHmQMRuADX3qg8BpU52LJGn6/KSyJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDXTuYXm4iR/leT+JPcmeV+rX5zkB0nuao+3dsZclGQoyQNJzurUT0+yvbVd1m6lKUmaRdO5heY+4INVdWe7t/IdSW5qbZdW1X/rdk5yCrAGOBV4BfDVJK9tt9G8AlgPfBP4MrAKb6MpSbNqykcIVbWrqu5s23uA+4GF4wxZDVxbVXur6mFgCFiR5ERgflVtq6oCrgbOnuq8JElTMyPnEJIsAd4A3NZK701yd5KrkhzXaguBHZ1hw622sG2Prfd7nfVJBpMMjoyMzMTUJUnNtAMhyYuB64H3V9WP6S3/vAZYBuwCPjHatc/wGqd+YLFqY1Utr6rlAwMD0526JKljWoGQ5AX0wuBzVfVFgKp6tKr2V9UvgM8AK1r3YWBxZ/giYGerL+pTlyTNoulcZRTgSuD+qvpkp35ip9vbgHva9lZgTZKjk5wELAVur6pdwJ4kZ7R9ngfcMNV5SZKmZjpXGb0ReBewPcldrfYh4Nwky+gt+zwCvBugqu5NsgW4j94VShe0K4wAzgc2AcfQu7rIK4wkaZZNORCq6hv0X///8jhjNgAb+tQHgdOmOhdJ0vT5SWVJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJag6bQEiyKskDSYaSXDjX85Gk55vDIhCSHAV8GngLcAq923CeMrezkqTnl8MiEIAVwFBVPVRV/w+4Flg9x3OSpOeVKd9TeYYtBHZ0ng8D/3RspyTrgfXt6U+SPDALczvcLQAem+tJzIRcMtcz0HOc/1Z6XnWwhsMlENKnVgcUqjYCG5/96Rw5kgxW1fK5nod0uPPfysQOlyWjYWBx5/kiYOcczUWSnpcOl0D4FrA0yUlJXgisAbbO8Zwk6XnlsFgyqqp9Sd4L/F/gKOCqqrp3jqd1pHAJTZoc/61MIFUHLNVLkp6HDpclI0nSHDMQJEmAgXDE8qs+pMlJclWS3Unumeu5HO4MhCOQX/UhHZJNwKq5nsSRwEA4MvlVH9IkVdWtwONzPY8jgYFwZOr3VR8L52gukp4jDIQj06S+6kOSDoWBcGTyqz4kzTgD4cjkV31ImnEGwhGoqvYBo1/1cT+wxa/6kPpLcg2wDXhdkuEk6+Z6Tocrv7pCkgR4hCBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSp+f+OpybSpaWWcgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#search unique labels\n",
    "lebels = list(np.unique(y_train_sm))\n",
    "\n",
    "#count occurrences of each unique label\n",
    "lebel_count = list(np.unique(y_train_sm, return_counts=True)[1])\n",
    "\n",
    "# label frequency\n",
    "plt.bar(lebels, lebel_count, width = 0.5)\n",
    "plt.title('Class Distribution')\n",
    "\n",
    "import math\n",
    "new_list = range(math.floor(min(lebels)), math.ceil(max(lebels))+1)\n",
    "plt.xticks(new_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f207e1f-6819-468c-91c1-51ab087da600",
   "metadata": {
    "id": "6f207e1f-6819-468c-91c1-51ab087da600"
   },
   "source": [
    "# PyTorch Neural Network"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6078f206-32a7-4cfc-bc16-6782922bdf90",
   "metadata": {
    "id": "6078f206-32a7-4cfc-bc16-6782922bdf90"
   },
   "source": [
    "https://www.learnpytorch.io/02_pytorch_classification/#1-make-classification-data-and-get-it-ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "3d664130-ff2b-4be7-8598-dc7dea0f3217",
   "metadata": {
    "id": "3d664130-ff2b-4be7-8598-dc7dea0f3217"
   },
   "outputs": [],
   "source": [
    "# install PyTorch first if using it first-time\n",
    "# uncomment below to install -- or may be better to install on the terminal\n",
    "# import sys\n",
    "# !{sys.executable} -m pip uninstall torch --yes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19c619d5-33c3-4977-96d5-4fb50faaacbc",
   "metadata": {
    "id": "19c619d5-33c3-4977-96d5-4fb50faaacbc"
   },
   "outputs": [],
   "source": [
    "# !{sys.executable} -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 --yes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f15b12fb-9774-41f2-bd38-f9486981d849",
   "metadata": {
    "id": "f15b12fb-9774-41f2-bd38-f9486981d849"
   },
   "outputs": [],
   "source": [
    "# import PyTorch\n",
    "import torch\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "baea4d96-53d7-451c-a266-5ad2e2a26fe6",
   "metadata": {
    "id": "baea4d96-53d7-451c-a266-5ad2e2a26fe6"
   },
   "outputs": [],
   "source": [
    "# convert np.arrays to tensors\n",
    "\n",
    "# training data\n",
    "X_train_sm = torch.from_numpy(X_train_sm).type(torch.float)\n",
    "y_train_sm = torch.from_numpy(y_train_sm).type(torch.float)\n",
    "\n",
    "# testing data\n",
    "X_test = torch.from_numpy(X_test).type(torch.float)\n",
    "# since y_test is still a pd.series, need to convert to np.array\n",
    "y_test = torch.from_numpy(y_test.to_numpy()).type(torch.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "83192359-6975-4597-a9f7-5b56fbf9eb7d",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "83192359-6975-4597-a9f7-5b56fbf9eb7d",
    "outputId": "46d3e8f1-0439-41b0-83f8-9bd71a7097f0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([41434, 23])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_sm.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "26c245b8-07d2-4e84-86a9-2d20c82477c5",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "26c245b8-07d2-4e84-86a9-2d20c82477c5",
    "outputId": "cc773c7f-3777-4b05-d291-f9d4d793695e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.8594, 1.0000, 0.0000,  ..., 0.0000, 0.1250, 0.0000],\n",
       "        [0.5938, 0.0000, 0.0000,  ..., 0.0000, 0.0000, 0.0000],\n",
       "        [0.0312, 0.0000, 0.1111,  ..., 0.0000, 0.0000, 0.0000],\n",
       "        ...,\n",
       "        [0.7917, 1.0000, 0.0000,  ..., 0.0000, 0.0833, 0.0000],\n",
       "        [0.7342, 0.0000, 0.0000,  ..., 0.0000, 0.0106, 0.0000],\n",
       "        [0.9376, 0.0000, 0.0000,  ..., 0.0000, 0.0833, 0.0000]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "73373a69-6ee8-4d95-90d7-acc9274adcf9",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "73373a69-6ee8-4d95-90d7-acc9274adcf9",
    "outputId": "0913f55a-5f8b-42fc-f845-de243cd2faa3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1., 0., 0.,  ..., 1., 1., 1.])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "17b27234-92ac-4c56-9c69-3e18d24baef3",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "17b27234-92ac-4c56-9c69-3e18d24baef3",
    "outputId": "624b7b79-d9a5-4686-f422-14a90e6abd1a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cuda'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# setting up device agnostic code (so our model can run on CPU or GPU if it's available)\n",
    "# Make device agnostic code\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b241d22e-c03d-4ca7-b259-8bc99b2d6779",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "b241d22e-c03d-4ca7-b259-8bc99b2d6779",
    "outputId": "62509a0f-4894-462b-89e3-b916d6a7bf42"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check for GPU\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b9b2363d-f712-474e-955e-800e99ccf146",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 287
    },
    "id": "b9b2363d-f712-474e-955e-800e99ccf146",
    "outputId": "81efb7df-273b-44ec-9032-412cdbc3ef36"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.], device='cuda:0')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.zeros(1).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "59793caf-05e7-4991-a2e0-90b7cf4e02d0",
   "metadata": {
    "id": "59793caf-05e7-4991-a2e0-90b7cf4e02d0",
    "outputId": "45f68103-9394-459b-ee9a-ce6510727af8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thu Apr 20 18:17:46 2023       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 527.56       Driver Version: 527.56       CUDA Version: 12.0     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name            TCC/WDDM | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  NVIDIA GeForce ... WDDM  | 00000000:01:00.0 Off |                  N/A |\n",
      "| N/A   53C    P2    25W /  80W |    510MiB /  6144MiB |      5%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|    0   N/A  N/A     15872    C+G   ...y\\ShellExperienceHost.exe    N/A      |\n",
      "|    0   N/A  N/A     16176      C   ...asri\\anaconda3\\python.exe    N/A      |\n",
      "|    0   N/A  N/A     21380    C+G   ...lPanel\\SystemSettings.exe    N/A      |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89d82f89-2f5c-4951-b6d6-ca6273591bb8",
   "metadata": {
    "id": "89d82f89-2f5c-4951-b6d6-ca6273591bb8"
   },
   "source": [
    "# hyperparameter tuning\n",
    "https://machinelearningmastery.com/how-to-grid-search-hyperparameters-for-pytorch-models/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2c5a038a-4832-4d4f-9215-ffb3bfd803e0",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2c5a038a-4832-4d4f-9215-ffb3bfd803e0",
    "outputId": "c9ae7442-b1e8-496d-a8da-04475275c864"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting skorch\n",
      "  Downloading skorch-0.12.1-py3-none-any.whl (193 kB)\n",
      "Requirement already satisfied: scipy>=1.1.0 in c:\\users\\nasri\\anaconda3\\lib\\site-packages (from skorch) (1.7.3)\n",
      "Requirement already satisfied: tabulate>=0.7.7 in c:\\users\\nasri\\anaconda3\\lib\\site-packages (from skorch) (0.8.9)\n",
      "Requirement already satisfied: tqdm>=4.14.0 in c:\\users\\nasri\\anaconda3\\lib\\site-packages (from skorch) (4.64.0)\n",
      "Requirement already satisfied: numpy>=1.13.3 in c:\\users\\nasri\\anaconda3\\lib\\site-packages (from skorch) (1.22.4)\n",
      "Requirement already satisfied: scikit-learn>=0.22.0 in c:\\users\\nasri\\anaconda3\\lib\\site-packages (from skorch) (1.0.2)\n",
      "Requirement already satisfied: joblib>=0.11 in c:\\users\\nasri\\anaconda3\\lib\\site-packages (from scikit-learn>=0.22.0->skorch) (1.2.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\nasri\\anaconda3\\lib\\site-packages (from scikit-learn>=0.22.0->skorch) (2.2.0)\n",
      "Requirement already satisfied: colorama in c:\\users\\nasri\\anaconda3\\lib\\site-packages (from tqdm>=4.14.0->skorch) (0.4.4)\n",
      "Installing collected packages: skorch\n",
      "Successfully installed skorch-0.12.1\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "!{sys.executable} -m pip install skorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d3b355c-4fa1-46fc-bc92-838a880a4bc7",
   "metadata": {
    "id": "7d3b355c-4fa1-46fc-bc92-838a880a4bc7",
    "outputId": "a7ca8f78-2971-48f3-99be-dd1ed3d76b56"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([41434, 23])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_sm.squeeze().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc78da0b-5ecc-4749-b2f1-6dd7a15f277a",
   "metadata": {
    "id": "fc78da0b-5ecc-4749-b2f1-6dd7a15f277a",
    "outputId": "043ed12d-9ded-412e-f470-5e5101346443"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([41434, 1])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_sm.unsqueeze(1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "wlOG5bXWkv4D",
   "metadata": {
    "id": "wlOG5bXWkv4D"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from skorch import NeuralNetClassifier\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1315651f-4b0d-4bcb-92ba-3ba77088822b",
   "metadata": {},
   "source": [
    "#### computation time for below gridsearch: 6166 sec\n",
    "'batch_size': [10, 20, 50, 100],\n",
    "    'max_epochs': [100, 500, 1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be5942a6-ffe1-426e-9d14-54d10eefb266",
   "metadata": {
    "id": "be5942a6-ffe1-426e-9d14-54d10eefb266",
    "outputId": "cfa0dc20-765b-43e6-b341-b39479443604"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.601077 using {'batch_size': 100, 'max_epochs': 1000}\n",
      "0.500893 (0.000515) with: {'batch_size': 10, 'max_epochs': 100}\n",
      "0.502317 (0.002066) with: {'batch_size': 10, 'max_epochs': 500}\n",
      "0.503331 (0.002138) with: {'batch_size': 10, 'max_epochs': 1000}\n",
      "0.500362 (0.000452) with: {'batch_size': 20, 'max_epochs': 100}\n",
      "0.502100 (0.001224) with: {'batch_size': 20, 'max_epochs': 500}\n",
      "0.503548 (0.001693) with: {'batch_size': 20, 'max_epochs': 1000}\n",
      "0.502076 (0.001143) with: {'batch_size': 50, 'max_epochs': 100}\n",
      "0.509533 (0.004289) with: {'batch_size': 50, 'max_epochs': 500}\n",
      "0.509582 (0.005209) with: {'batch_size': 50, 'max_epochs': 1000}\n",
      "0.592581 (0.042323) with: {'batch_size': 100, 'max_epochs': 100}\n",
      "0.586452 (0.067047) with: {'batch_size': 100, 'max_epochs': 500}\n",
      "0.601077 (0.052402) with: {'batch_size': 100, 'max_epochs': 1000}\n"
     ]
    }
   ],
   "source": [
    "# PyTorch classifier  \n",
    "class PimaClassifier(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.layer = nn.Linear(23, 100)\n",
    "        self.act = nn.ReLU()\n",
    "        self.output = nn.Linear(100, 1)\n",
    "        self.prob = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.act(self.layer(x))\n",
    "        x = self.prob(self.output(x))\n",
    "        return x\n",
    "\n",
    "# create model with skorch\n",
    "model = NeuralNetClassifier(\n",
    "    PimaClassifier,\n",
    "    criterion=nn.BCELoss,\n",
    "    optimizer=optim.Adam,\n",
    "    verbose=False\n",
    ")\n",
    "\n",
    "# define the grid search parameters\n",
    "param_grid = {\n",
    "    'batch_size': [10, 20, 50, 100],\n",
    "    'max_epochs': [100, 500, 1000]\n",
    "}\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=5)\n",
    "grid_result = grid.fit(X_train_sm, y_train_sm.unsqueeze(1))\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b37fd09-c33d-4a71-b99b-a3f0264594ae",
   "metadata": {
    "id": "0b37fd09-c33d-4a71-b99b-a3f0264594ae"
   },
   "outputs": [],
   "source": [
    "# computation time: 6166 sec"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5a702bc-5084-4a35-bcb2-3b957279a924",
   "metadata": {},
   "source": [
    "#### computation time for below gridsearch: 23410 sec\n",
    "'optimizer': [optim.SGD, optim.RMSprop, optim.Adagrad, optim.Adadelta,\n",
    "                  optim.Adam, optim.Adamax, optim.NAdam],"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "L6oHiLYqk9E5",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "L6oHiLYqk9E5",
    "outputId": "5c723dc3-efa4-43cc-b351-ad1c0a4aea68"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.709683 using {'optimizer': <class 'torch.optim.adagrad.Adagrad'>}\n",
      "0.504706 (0.002230) with: {'optimizer': <class 'torch.optim.sgd.SGD'>}\n",
      "0.506565 (0.003953) with: {'optimizer': <class 'torch.optim.rmsprop.RMSprop'>}\n",
      "0.709683 (0.005781) with: {'optimizer': <class 'torch.optim.adagrad.Adagrad'>}\n",
      "0.707366 (0.004902) with: {'optimizer': <class 'torch.optim.adadelta.Adadelta'>}\n",
      "0.589107 (0.064675) with: {'optimizer': <class 'torch.optim.adam.Adam'>}\n",
      "0.539774 (0.015812) with: {'optimizer': <class 'torch.optim.adamax.Adamax'>}\n",
      "0.614544 (0.043026) with: {'optimizer': <class 'torch.optim.nadam.NAdam'>}\n"
     ]
    }
   ],
   "source": [
    "# tune optimizer\n",
    "class PimaClassifier(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.layer = nn.Linear(23, 100)\n",
    "        self.act = nn.ReLU()\n",
    "        self.output = nn.Linear(100, 1)\n",
    "        self.prob = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.act(self.layer(x))\n",
    "        x = self.prob(self.output(x))\n",
    "        return x\n",
    "\n",
    "# create model with skorch\n",
    "model = NeuralNetClassifier(\n",
    "    PimaClassifier,\n",
    "    criterion=nn.BCELoss,\n",
    "    max_epochs=1000,\n",
    "    batch_size=100,\n",
    "    verbose=False\n",
    ")\n",
    "\n",
    "# define the grid search parameters\n",
    "param_grid = {\n",
    "    'optimizer': [optim.SGD, optim.RMSprop, optim.Adagrad, optim.Adadelta,\n",
    "                  optim.Adam, optim.Adamax, optim.NAdam],\n",
    "}\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=5)\n",
    "grid_result = grid.fit(X_train_sm, y_train_sm.unsqueeze(1))\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "FyLOLdyHmWKp",
   "metadata": {
    "id": "FyLOLdyHmWKp"
   },
   "outputs": [],
   "source": [
    "# 23410 sec"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5443514-10d6-41f9-a850-110fe286d735",
   "metadata": {},
   "source": [
    "#### computation time for below gridsearch: 6033 sec\n",
    "'optimizer__lr': [0.01, 0.1, 0.5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "EYcQ4l7QzbNj",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EYcQ4l7QzbNj",
    "outputId": "3b36d194-2d37-49f0-e131-a2ba03fb70ad"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.709562 using {'optimizer__lr': 0.01}\n",
      "0.709562 (0.005343) with: {'optimizer__lr': 0.01}\n",
      "0.626973 (0.026114) with: {'optimizer__lr': 0.1}\n",
      "0.541294 (0.018587) with: {'optimizer__lr': 0.5}\n"
     ]
    }
   ],
   "source": [
    "# Tune learning rate 6033 sec\n",
    "class PimaClassifier(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.layer = nn.Linear(23, 100)\n",
    "        self.act = nn.ReLU()\n",
    "        self.output = nn.Linear(100, 1)\n",
    "        self.prob = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.act(self.layer(x))\n",
    "        x = self.prob(self.output(x))\n",
    "        return x\n",
    "\n",
    "# create model with skorch\n",
    "model = NeuralNetClassifier(\n",
    "    PimaClassifier,\n",
    "    criterion=nn.BCELoss,\n",
    "    optimizer=optim.Adagrad,\n",
    "    max_epochs=1000,\n",
    "    batch_size=100,\n",
    "    verbose=False\n",
    ")\n",
    "\n",
    "# define the grid search parameters\n",
    "param_grid = {\n",
    "    'optimizer__lr': [0.01, 0.1, 0.5],\n",
    "}\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=5)\n",
    "grid_result = grid.fit(X_train_sm, y_train_sm.unsqueeze(1))\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c2c442b-692e-4a80-b80e-70e33a7ff44e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a756ef72-5d27-44b9-aab0-4099e0e9707a",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EYcQ4l7QzbNj",
    "outputId": "3b36d194-2d37-49f0-e131-a2ba03fb70ad"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.709562 using {'optimizer__lr': 0.01}\n",
      "0.709562 (0.005343) with: {'optimizer__lr': 0.01}\n",
      "0.626973 (0.026114) with: {'optimizer__lr': 0.1}\n",
      "0.541294 (0.018587) with: {'optimizer__lr': 0.5}\n"
     ]
    }
   ],
   "source": [
    "# Tune learning rate -- 6033 sec\n",
    "class PimaClassifier(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.layer = nn.Linear(23, 100)\n",
    "        self.act = nn.ReLU()\n",
    "        self.output = nn.Linear(100, 1)\n",
    "        self.prob = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.act(self.layer(x))\n",
    "        x = self.prob(self.output(x))\n",
    "        return x\n",
    "\n",
    "# create model with skorch\n",
    "model = NeuralNetClassifier(\n",
    "    PimaClassifier,\n",
    "    criterion=nn.BCELoss,\n",
    "    optimizer=optim.Adagrad,\n",
    "    max_epochs=1000,\n",
    "    batch_size=100,\n",
    "    verbose=False\n",
    ")\n",
    "\n",
    "# define the grid search parameters\n",
    "param_grid = {\n",
    "    'optimizer__lr': [0.01, 0.1, 0.5],\n",
    "}\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=5)\n",
    "grid_result = grid.fit(X_train_sm, y_train_sm.unsqueeze(1))\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "081a5f29-b55e-47b1-98d1-fdfb4b4b3e38",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.init as init\n",
    "import torch.optim as optim\n",
    "from skorch import NeuralNetClassifier\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "7043bc64-4e58-40ef-aa28-161241c7f1b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.709321 using {'module__n_neurons': 500}\n",
      "0.708235 (0.006461) with: {'module__n_neurons': 20}\n",
      "0.709273 (0.006934) with: {'module__n_neurons': 100}\n",
      "0.709321 (0.005270) with: {'module__n_neurons': 500}\n"
     ]
    }
   ],
   "source": [
    "# tune number of neurons in the hidden layer -- 4563 sec\n",
    "class PimaClassifier(nn.Module):\n",
    "    def __init__(self, n_neurons=50):\n",
    "        super().__init__()\n",
    "        self.layer = nn.Linear(23, n_neurons)\n",
    "        self.act = nn.ReLU()\n",
    "        self.dropout = nn.Dropout(0.1)\n",
    "        self.output = nn.Linear(n_neurons, 1)\n",
    "        self.prob = nn.Sigmoid()\n",
    "        self.weight_constraint = 2.0\n",
    "        # manually init weights\n",
    "        init.kaiming_uniform_(self.layer.weight)\n",
    "        init.kaiming_uniform_(self.output.weight)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # maxnorm weight before actual forward pass\n",
    "        with torch.no_grad():\n",
    "            norm = self.layer.weight.norm(2, dim=0, keepdim=True).clamp(min=self.weight_constraint / 2)\n",
    "            desired = torch.clamp(norm, max=self.weight_constraint)\n",
    "            self.layer.weight *= (desired / norm)\n",
    "        # actual forward pass\n",
    "        x = self.act(self.layer(x))\n",
    "        x = self.dropout(x)\n",
    "        x = self.prob(self.output(x))\n",
    "        return x\n",
    "\n",
    "# create model with skorch\n",
    "model = NeuralNetClassifier(\n",
    "    PimaClassifier,\n",
    "    criterion=nn.BCELoss,\n",
    "    optimizer=optim.Adagrad,\n",
    "    max_epochs=1000,\n",
    "    batch_size=100,\n",
    "    verbose=False\n",
    ")\n",
    "\n",
    "# define the grid search parameters\n",
    "param_grid = {\n",
    "    'module__n_neurons': [20, 100, 500]\n",
    "}\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=5)\n",
    "grid_result = grid.fit(X_train_sm, y_train_sm.unsqueeze(1))\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "286d1a93-9986-4aa0-9b5f-cf220deffed5",
   "metadata": {},
   "outputs": [],
   "source": [
    "NN_pred_proba = grid_result.predict_proba(X_test)[:, 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21c79d3c-5a94-42ac-bf21-5d475be97779",
   "metadata": {},
   "source": [
    "#### Best model\n",
    "model = NeuralNetClassifier(\n",
    "    PimaClassifier,\n",
    "    criterion=nn.BCELoss,\n",
    "    optimizer=optim.Adagrad,\n",
    "    max_epochs=1000,\n",
    "    batch_size=100,\n",
    "    verbose=False\n",
    ")\n",
    "\n",
    "+\n",
    "\n",
    "using {'module__n_neurons': 500}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "2cb7b98b-78bc-4aed-9b65-e4356d5f1edc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving the model\n",
    "NN_clf = grid_result\n",
    "import pickle\n",
    "filename = 'NNs_model.sav'\n",
    "pickle.dump(NN_clf, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "932f535b-0bcb-42bc-8474-c7b43c7f1867",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix, classification_report, roc_auc_score, roc_curve, plot_confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "7a3bebd3-08a3-4f90-9b19-84f52d9dbbb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA8p0lEQVR4nO3dd3wUdfrA8c+THtKAhBoIJaF3CCAi0kQBQawgYNcfxwl6emfB07Oc2D3sDbuegicqogiKCNJEOgihSCf0JBBCevn+/pglhhiSDWR3srvP+/XKi52Z7848Y+I+O9/vzPMVYwxKKaV8l5/dASillLKXJgKllPJxmgiUUsrHaSJQSikfp4lAKaV8XIDdAVRWTEyMadq0qd1hKKWUR1m9enWKMaZOWds8LhE0bdqUVatW2R2GUkp5FBHZc6Zt2jWklFI+ThOBUkr5OE0ESinl4zxujKAs+fn5JCcnk5OTY3coqpoKCQmhUaNGBAYG2h2KUtWOVySC5ORkIiIiaNq0KSJidziqmjHGkJqaSnJyMs2aNbM7HKWqHZd1DYnIeyJyREQ2nmG7iMjLIrJdRDaISNezPVZOTg7R0dGaBFSZRITo6Gi9YlTqDFw5RvABMLic7UOAFo6fccAb53IwTQKqPPr3odSZuaxryBizSESaltNkBPCRsepgLxeRmiLSwBhz0FUxKaWUp0g5mcuh9Bx2HD1JRlY2afu20blLdy5sWeYzYefEzruGYoF9JZaTHev+RETGicgqEVl19OhRtwR3Nr766itEhC1bthSvW7hwIcOGDTut3U033cSMGTMAa6B70qRJtGjRgvbt29OjRw/mzJnzp33369ePVq1a0alTJ7p37866deuKt6Wnp3PDDTcQHx9PfHw8N9xwA+np6cXbt23bxtChQ0lISKBNmzaMHDmSw4cPn9O5pqWlMWjQIFq0aMGgQYM4duzYn9ps3bqVzp07F/9ERkby4osvFm9/5ZVXaNWqFe3ateO+++477b179+4lPDyc559/vnhdXl4e48aNo2XLlrRu3ZovvvgCgNzcXEaNGkVCQgI9e/Zk9+7d53RuSrlTTn4hh9Jz2HY4gxfmbWPCp2to/sBsEif/yLBXljD1s5l0mXslI5Nu54d1O10Sg52DxWVdq5c5S44xZiowFSAxMbHazqQzbdo0LrjgAqZPn86jjz7q1Hv+9a9/cfDgQTZu3EhwcDCHDx/m559/LrPtJ598QmJiIu+//z733nsv8+bNA+DWW2+lffv2fPTRRwA88sgj3HbbbXz++efk5ORw6aWXMmXKFIYPHw7AggULOHr0KPXq1Tvrc3366acZOHAgkyZN4umnn+bpp5/mmWeeOa1Nq1atihNWYWEhsbGxXHHFFcUxfP3112zYsIHg4GCOHDly2nvvvvtuhgwZctq6J554grp167Jt2zaKiopIS0sD4N1336VWrVps376d6dOnc//99/PZZ5+d9bkpVRWMMWTmFXLweDa7U7NIOnCCAH9hy6EM8goK2bj/BEdP5pJXUHTa+5rHhNGuYRQNwoR7gr+kxfb3KQqtTd4lLzC5U0+XxGpnIkgGGpdYbgQcsCmWc3by5EmWLl3KggULuOyyy5xKBFlZWbz99tvs2rWL4OBgAOrVq8fIkSPLfV+vXr147rnnANi+fTurV68+7YPv4YcfJiEhgR07dvDzzz/Tq1ev4iQA0L9//7M4w9N9/fXXLFy4EIAbb7yRfv36/SkRlDR//nzi4+Np0qQJAG+88QaTJk0qPu+6desWt505cybNmzcnLCzstH289957xVdbfn5+xMTEFMdy6r/31VdfzcSJEzHG6LiAciljDAfTc9idmsmJ7AJSTuaSnVfIydwC3vx5B7mlPuBL69uyDrG1QmlTP4JaYUE0rlWD+LrhdG5c02rw8ZXw+3zofB3+l0wmNLSWy87FzkQwC5goItOBnkB6VYwPPPbNJpIOnDjn4Epq2zCSR4a3K7fNzJkzGTx4MC1btqR27dqsWbOGrl3LvxFq+/btxMXFERkZWal45s6dy+WXXw5AUlISnTt3xt/fv3i7v78/nTt3ZtOmTWzcuJFu3bpVuM+MjAz69OlT5rZPP/2Utm3bnrbu8OHDNGjQAIAGDRr86Rt9adOnT2f06NHFy9u2bWPx4sU8+OCDhISE8Pzzz9O9e3cyMzN55plnmDdv3mndQsePHwesK6iFCxcSHx/Pq6++Sr169di/fz+NG1vfKQICAoiKiiI1NbU4UShVEWMMu1IyOXwil5STuZzMLaCgyFBUZCgoMuw4epLkY9kArN1zjFphQaRl5nEyt6DM/cXWDCXAXxiZ2Jggfz+6N6tNZEgADWuGEhzgd+YvKbkZkJ8DgSFwwd1w/kSIH+Cq0y7mskQgItOAfkCMiCQDjwCBAMaYN4HvgKHAdiALuNlVsbjDtGnTuOuuuwC49tprmTZtGl27dj3jL/xsvq2OHTuWzMxMCgsLWbNmDcAZv/lW9htxRETEaeMOVSkvL49Zs2bx1FNPFa8rKCjg2LFjLF++nJUrVzJy5Eh27tzJI488wt133014ePhp+ygoKCA5OZnevXszZcoUpkyZwj333MPHH39MWfNu69WAKi23oJADx62++GOZeWw9nEFaZh67UzJZn5xe8Q6A4AA/OjWuSXZeIf1b1aFFvQiax4QRVSOQmPBgagT5ExzgT1DAWQy/bv8RvrkLOo6EgQ9Ds7K/mLmCK+8aGl3BdgNMqOrjVvTN3RVSU1P56aef2LhxIyJCYWEhIsKzzz5LdHT0nwZS09LSiImJISEhgb1795KRkUFERESFx/nkk0/o1KkTkyZNYsKECXz55Ze0a9eOtWvXUlRUhJ+f9cdXVFTE+vXradOmDUeOHDnjmENJlb0iqFevHgcPHqRBgwYcPHjwtK6d0ubMmUPXrl1PG5No1KgRV155JSJCjx498PPzIyUlhV9//ZUZM2Zw3333cfz4cfz8/AgJCWHChAnUqFGjeIzhmmuu4d133y3e1759+2jUqBEFBQWkp6dTu3btCs9ZeadjmXlMW7mXQ+k5rNt3nMMncigsMqRm5lHyO0NooD91IoKJrRnKjb2aEBTgR/vYKJrFhBETHkyAvxDg54e/nxDgJ4QG+uPn54IvGFlp8P2DsP5TiGkJLS6p+mNUwCueLLbbjBkzuOGGG3jrrbeK1/Xt25clS5bQo0cPDhw4wObNm2nTpg179uxh/fr1dO7cmRo1anDrrbdy55138tZbbxEUFMTBgweZP38+1113XZnHCgwMZPLkycTHxxfvs0uXLkyePJmHH34YgMmTJ9O1a1cSEhKIjY3lqaeeYvbs2Vx66aWA1bUUGxtLhw4divdb2SuCyy67jA8//JBJkybx4YcfMmLEiDO2nTZt2mndQgCXX345P/30E/369WPbtm3k5eURExPD4sWLi9s8+uijhIeHM3HiRACGDx/OwoULGTBgAPPnzy9OTqdi6dWrFzNmzGDAgAF6ReAjjDGs2XuclbvT2Lg/nYPpOaze88cXr+iwIPz9hNhaNbgmsTENo0KoHRZMs5gwWtYLJ8Df5nJrOxfCF/8H2WnQ5x648F6rW8jdjDEe9dOtWzdTWlJS0p/WuVPfvn3NnDlzTlv30ksvmfHjxxtjjFmyZInp2bOn6dSpk0lMTDQ//PBDcbvc3Fxz7733mvj4eNOuXTvTo0cPM3fu3DKPsXLlyuLl559/3txyyy3GGGPS0tLM2LFjTXx8vGnevLkZO3asOXbsWHHbzZs3m0suucQkJCSYNm3amFGjRplDhw6d0zmnpKSYAQMGmISEBDNgwACTmppqjDFm//79ZsiQIcXtMjMzTe3atc3x48dPe39ubq4ZO3asadeunenSpYuZP3/+n47xyCOPmOeee654effu3aZPnz6mQ4cOZsCAAWbPnj3GGGOys7PN1VdfbeLj40337t3Njh07yozZ7r8Tde5ST+aar9ftN2Pe/sW0fPA70+T+b0/7aTbpW3PNm8vMzLXJJr+g0O5wK3ZoozFvDzTmwHqXHwpYZc7wuSqmjP7V6iwxMdGUnpjm1DdjpcqjfyfVnzGGPalZ/LIzlXV7jxMa5M/B9GySj2WTkVPA3rSs4rbtGkZSLzKEptFhXNqxPu0aRhES6F/O3qsBY2Ddp3BwPQx99o91briCFZHVxpjEsrZp15BSyha5BYVsPZTBkRO5/H7kJBuSjzN/8xHyCk+/7TI6LIjMvAJ6NY/m6m6NaFU/gibRNWhdv3J329nu2G5rMHjnAog7H/KzITDULUmgIpoIlFJuUVhk+P1IBtNX7ONQeg5zNx06bXuT6Bq0bhBB0+gwRveIo1GtUBrXrmFTtFWoqBBWvA3zHwPxg0v/A91uAb/qMx2M1yQCow8QqXJ4WheoN9h/PJsfNh0iK6+Q3w9nMHPd6c+LXtKuHo1r1WBA67rUDg/yvG/4zspKhQVPQpPeMOwFqNm44ve4mVckgpCQEFJTU7UUtSqTccxHEBJiw90YPsIYw/7j2fyYdJg5Gw+RfCyb/cezi7eHBvozuF19mtUJY1jHBrRtEOnd/68W5sOG/0Gn0RBeF/7yM9RqWi26gcriFYmgUaNGJCcnU50L0il7nZqhTJ277LxCdqacZOfRTI5n55N04ATf/XaQ9Oz84jY1gvwZmdiIAa3r0Ss+msiQAO/+4C/pwFr4eiIc3ggR9SDhIqhdvSdE8opEEBgYqDNPKeUCKSdz+S05neTj2azde4wv1+z/U5uIkAC6xNWibkQw7RtGcmW3RkSG+OCUoPnZsPBpWPYKhNWBUZ9YScADeEUiUEqdu1O3bm45dILlO9P4fNU+MvMK/9SuXcNIxvSMIzIkkCbRNWjfMMo1T9x6muljYMdP0PUGGPQ4hNa0OyKnaSJQygcdz8pjx9GT5OQXsf94NhuSj/PJr3tPK8FwYcs6tGsYSVztGnRsFEWtGkHUjQi2/2nc6iTnBPgHWU8D9/kH9P4bNO9nd1SVpolAKS+3OyWTPWlZpGTk8sWaZJbtSP1Tm/DgALrG1SI6LIjRPeNIqBPuHbduutK2H+Dbu60icRc9Ak0vsDuis6aJQCkvdCwzj/u+2MC8pLJnorusU0O6N61Fy3oRxEQE0yw6TLt3nJWZCt8/ABs+gzqtodVQuyM6Z5oIlPIie1IzGf7KEk7k/FEn/9KODbi6WyMiggNoEh1GnYhgGyP0cDt+sorE5RyHvvdb3UEBnv/fUxOBUh4qPSufuZsOsnH/CTJy8tl2+CRJB/+YlOnVMV0Y1rGhjRF6ofD6EJ0Aw6ZAPfeXvHcVTQRKeQhjDLkFRazec4wp87adVm4ZICY8iCHt63NFl1gualNPu3qqgjGw5iM4tMEqDVGvLdwyt9o+GHa2NBEoVU0ZY9iXls20lXv5/fBJFv1+9LSJzrvE1eTqbo0Y2r4BNWsE+s4DW+6Stgu+uRN2LYKmfapVkbiqpolAqWoiPSufLYdOsCcti3cW7yQtM4+Uk3nF2y9pV4+OjWrSJLoGvZpHEx3u+X3T1VJRIfz6Jsx/HPwCYNiL0PXGalUkrqppIlDKRumOEg2PfbOJLYcyitf7+wlRoYHcdVEL2jSIZJB29bhPViosfAaa94VLp0BUrN0RuZwmAqXc7EROPvM2Heal+b+fNtFKREgAdwxI4Pz4GNo0iMRfP/jdpyDPuh2081irSNz4xVAzziu7gcqiiUApN9iXlsWUedtYvjOVg+k5p22b2D+Bwe3r0z42yqbofNz+1VaRuCNJENkQEgZCrSZ2R+VWmgiUcqGsvAIe/3YzX65JJtcx0HtRm3p0bBTF8E4NaRYTZnOEPiwvCxY8Actft24LHT3dSgI+SBOBUlWsoLCIL9fu54Olu0+7r3/WxN50iI3Su3uqi+mjYedC6HYTDPo3hPjuFZkmAqWqQE5+IZNnJ/Hf5XtPW98kugZ3DmjBlV1jNQFUBznp4B9sFYm78D7ryeBmF9odle00EShVSYVFhr1pWWx1lGs+fCKHORv/mH+3eZ0wLm5bn1svaKblHKqTrXOtInGdRsFFj0LT3nZHVG1oIlCqAoVFhnlJh/gh6TBbDmac1t1zSqfGNRncrj7j+zbXb/7VTWYKzLkfNs6Auu2gzXC7I6p2NBEoVYbdKZks35nK9JX7SDp4oviJ3ka1Qq37+tvWo33DSFrXj6Rx7VD98K+uts+HL//Pmjeg3z/hgrshIMjuqKodTQRKlbJqdxpXv/nLaetu7xfP4Pb16diopj1BqbMT2RBiWllF4uq2sTuaaksTgfJ5J3MLmJd0iJW7jzEv6TBHM3IBuKV3M/42sAVRNXxw/l1PVVQEaz60isQNe8H68L9ljt1RVXuaCJTP+mnLYe74dO1p8/K2rBdOx9goHhjamoS6ETZGpyotdQd88zfYvfj0InGqQpoIlE/Jzivkh6RD3P/FBnLyrX7/BlEhXN+rCWN6xFGzhvYfe5yiQuuhsJ+eAP9AGP6yNYG8jts4zaWJQEQGAy8B/sA7xpinS22PAv4LxDlied4Y874rY1K+Jc9Rv/+9pbtYtTuNY1n5xdvqRQbz/k09aNsw0sYI1TnLSoVFz0F8f2vOgEidjKeyXJYIRMQfeA0YBCQDK0VkljEmqUSzCUCSMWa4iNQBtorIJ8aYvDJ2qZRTjDEs3Z7K+0t3sWbvseIP/4jgAIa0r8/58dFc3iWWiBDt+/dYBbmwfhp0ucFRJG4JRDXWq4Cz5Morgh7AdmPMTgARmQ6MAEomAgNEiHXvXTiQBhSU3pFSzjDG8M+vNjJtxR9P99aPDOHBoQkM69SABlHaX+wVkldZReKObrY+/BMGWpVC1VlzZSKIBfaVWE4GepZq8yowCzgARACjjDFFpdogIuOAcQBxcfoLV6c7mpHL1EU7eHvxruJ1o3vEcUvvprSopwO+XiMv0xoHWP661f0z5nOfLRJX1VyZCMq6RjOlli8B1gEDgHhgnogsNsac9uimMWYqMBUgMTGx9D6Uj1q/7zjvLtnFrPUHAGgaXYOY8GDev7m7dvt4o+ljrCJxibdaJSJCdGynqrgyESQDjUssN8L65l/SzcDTxhgDbBeRXUBrYIUL41IerqCwiBmrk3nsmySy861bP2/p3YyHh7e1OTJV5bKPQ0CwdRto3/utQnFaI6jKuTIRrARaiEgzYD9wLTCmVJu9wEBgsYjUA1oBO10Yk/JQ+9KyWLjtKF+v3c+qPccA67bPhRP6US8yxObolEts+Q5m/x06joJBj0GT8+2OyGu5LBEYYwpEZCLwPdbto+8ZYzaJyHjH9jeBx4EPROQ3rK6k+40xKa6KSXme1xZs562fd3Aix7qHQMRKAH1axPDEFR0I9PfeCcV91smjMOc+2PQl1GsPbUfYHZHXc+lzBMaY74DvSq17s8TrA8DFroxBeaaTuQX0e24BKSetO4lHJTYmsWktLu8Sqx/+3uz3H+HL26yB4f4PwQV3WQ+JKZfSJ4tVtbJxfzp3fbaO7UdOFq+bPu48zmsebWNUym2iYq1S0Zf+B+q2tjsan6GJQFUb85IO838frQKsmb0eH9GeC1vWsTkq5VJFRbD6PTj0Gwx/ySoSd/Nsu6PyOZoIlK0OHM9m2oq9zFidzMH0HAAm9I/n3kv026DXS9kOs+6AvcugeX/Iz7GmkFRup4lA2WJfWhbPzN3CtxsOFq9rHhPGf0Z2oktcLRsjUy5XWAC/vAILnrI++Ee8Dp3HaHkIG2kiUG51+EQOEz9dw8rdx4rXvTamK0M71NdZvnxFdhoseRFaDLLGAiLq2x2Rz9NEoNzmzZ938PScLcXLj13Wjht6NdEE4AsKcmHdJ9D1JqtI3F+XQlQju6NSDpoIlEsVFhneX7qLVxds57ijCuj7N3enf6u6Nkem3GbfCqtIXMpWqNXMKhetSaBa0USgXKb0FUC7hpG8f3N36kbogKBPyD0JP02GX9+0Pviv+8JKAqra0USgqlxmbgGjpv7Cxv1W7cDhnRpy3yWtaFy7hs2RKbeaPgZ2/Qw9xsHAhyFYK8FWV5oIVJUyxtDl8XnkFRRRI8ifb++4gOZ1wu0OS7lL9jEICLGKxPV7wPpp0svuqFQFnH5WX0TCXBmI8nwrdqXR6qG55BUU0at5NJseu0STgC9JmgWv9YSFT1nLTXppEvAQFSYCETlfRJKAzY7lTiLyussjUx4jPTuf1xZsZ+Rbv5BXWMTwTg355LaeejeQr8g4DJ9dD/+73rojqP1VdkekKsmZrqEXsCaQmQVgjFkvIhe6NCrlMbYfyeCiKYuKl8f3jWfSEH0q2Gf8Pg++uA3ys61xgPPv1CJxHsipMQJjzL5S3+4KXROO8iSz1h/gzmlrARjdozH/HNpGZwbzNVGNoUFHGPofqNPS7mjUWXImEewTkfMBIyJBwJ04uomUbyosMtz64UoWbj0KwEvXdmZE51ibo1JuUVQEK9+Bw7/BZa9YFUJv/MbuqNQ5ciYRjAdewpqMPhn4AbjdlUGp6mvzwRMMeWlx8fKafw2idliQjREpt0n53XowbN9yiB+oReK8iDOJoJUxZmzJFSLSG1jqmpBUdbUh+TiXvWr92i9qU5e3rk/E308HhL1eYT4sexkWPmPdFnr5G9BptBaJ8yLOJIJXgK5OrFNeyhjDB8t289g3SQA8OrwtN/VuZnNUym2yj8PSl6HVYBjyHETUszsiVcXOmAhEpBdwPlBHRP5eYlMk1hzEykeMfns5y3em4e8nPHNVR67upnVivF5+Dqz9GBJvhfA68Ndl1uxhyiuVd0UQBIQ72pR8NvwEcLUrg1LVQ/KxLO75fD3Ld6YRFODH2n8NIixYH0b3ent+gVkTIXU7RCc4isRpEvBmZ/y/2hjzM/CziHxgjNnjxphUNTB340HG/3cNAJd2aMCUUZ0IDtALQa+WmwE/PgYr34aacXD9V1okzkc48/UuS0SeA9oBxbcIGGMGuCwqZZvcgkLGfbSan7dZt4Y+f00n7QryFdPHwK7F0POvMOAhCNbyIL7CmUTwCfAZMAzrVtIbgaOuDErZ55IXFrE7NQuAXx4YQIOoUJsjUi6VlWYViQuqAf0fggECjXvYHZVyM2eKzkUbY94F8o0xPxtjbgHOc3FcygYvzNvG7tQsggL82PXUUE0C3m7TTHitxx9F4uJ6ahLwUc5cEeQ7/j0oIpcCBwDtK/Ay/5q5kY+XW0NBC+/ppwXjvFnGIZj9D9jyLTToDB1H2h2RspkziWCyiEQB/8B6fiASuMuVQSn3KSoydHrsBzJyC4gMCeCXBwbqnUHebNv38OX/WXMIX/QY9JoI/vr79nUV/gUYY751vEwH+kPxk8XKwyUfy+Km91eSkVtAvchglt4/gAB/p6eoUJ6oVlNo2BWGPg8xCXZHo6qJ8h4o8wdGYtUYmmuM2Sgiw4B/AqFAF/eEqFxh/b7jjHjNKhdRNyKYhff01yTgjYoKYcVUOLwRRrwGdVrBDTPtjkpVM+VdEbwLNAZWAC+LyB6gFzDJGDPTDbEpF1m95xhXvbEMgPsHt+av/eJtjki5xJEtMOsOSF4BLS7WInHqjMpLBIlAR2NMkYiEAClAgjHmkHtCU66wcX96cRJ4bUxXLu3YwOaIVJUryIOlL8GiZyEoHK58Gzpco0Xi1BmV1xeQZ4wpAjDG5ADbKpsERGSwiGwVke0iMukMbfqJyDoR2SQiP1dm/6py0rPzGfbKEgCevrKDJgFvlZMOy1+D1sNgwgrrriBNAqoc5V0RtBaRDY7XAsQ7lgUwxpiO5e3YMcbwGjAIax6DlSIyyxiTVKJNTeB1YLAxZq+I1D37U1Hl2ZWSSf/nFwIwukcc1/aIszcgVbXys2HNx9D9NkeRuF8gUhO9ck55iaDNOe67B7DdGLMTQESmAyOApBJtxgBfGmP2AhhjjpzjMVUZkg6cYNTUXwC4vV889w3WOYW9yu6l1lhA2g5rusjm/TQJqEopr+jcuRaaiwX2lVhOBnqWatMSCBSRhVgVTl8yxnxUekciMg4YBxAXp99kK2ProQyGvmzNKDZlZCeu7KrPAnqNnBPw46Ow6l2o2QRu+NpKAkpVkiufJCmrU9KUcfxuwECsW1J/EZHlxphtp73JmKnAVIDExMTS+1Dl+Pe3mwBrcnlNAl5m+hjYvQTOmwADHoSgMLsjUh7KlYkgGev201MaYZWnKN0mxRiTCWSKyCKgE7ANdc5env87S7enct15cUy+vIPd4aiqkJlqTRcZVAMGPgwINO5ud1TKwzn1BJGIhIpIq0rueyXQQkSaiUgQcC0wq1Sbr4E+IhIgIjWwuo42V/I4qgw/bDrElHnbiAkP4rHL2tsdjjpXxsBvM+C17rDwSWtd4x6aBFSVqDARiMhwYB0w17HcWURKf6D/iTGmAJgIfI/14f4/Y8wmERkvIuMdbTY79rsB68G1d4wxG8/yXJTDyt1pjPt4NQDPXt1RJ5j3dCcOWN1AX9xqjQV0Gm13RMrLiDHld7mLyGpgALDQGNPFsW5DRbePukpiYqJZtWqVHYf2CHkFRbR8aA4An9zWk94JMTZHpM7J1rlWkbjCfGsc4LzbwU9nilOVJyKrjTGJZW1zZoygwBiTrmWJPcPdn60DYGiH+poEvEHt5lYX0JBnIVpLgSjXcGaMYKOIjAH8RaSFiLwCLHNxXOosrNl7jNm/HaRrXE1eHd3V7nDU2SgqhF9eg6/+ai3XaQnXfaFJQLmUM4ngDqz5inOBT7HKUd/lwpjUWXjr5x1c+bqVnx+7rD1+Oi7geY5shncvhu//CVmpVpE4pdzAma6hVsaYB4EHXR2MOjtPzE7i7cW7AKuGUIdGUTZHpCqlIA+WvACLnoOQSLjqXWh/ldYHUm7jTCKYIiINgM+B6caYTS6OSVXCzqMni5OADg57qJx0+PVNaHc5DH4awvR3qNyrwq4hY0x/oB9wFJgqIr+JyEOuDkxVzBjDoBcWAXD3RS01CXiSvCxY/oY1JhBeB27/Ba56R5OAsoVTD5QZYw4ZY14GxmM9U/CwK4NSFSsoLOLG91dSWGTo16oOf7uohd0hKWftWgRv9IK5k2C3VQeKiPr2xqR8WoVdQyLSBhgFXA2kAtOxJrJXNrrkxUXsOJpJ2waRvH+TPl3qEXLSYd7DsPoDqNUMbvwWmvWxOyqlnBojeB+YBlxsjCldK0jZ4PNV+9hxNBOAWRN7o894eIjpY2HPUjj/Tuj3gFUvSKlqoMJEYIw5zx2BKOcUFBZx7wxrvqD5/+irE85Xd5kpEFjDUSTuEfDzg9hudkel1GnOmAhE5H/GmJEi8hunl492aoYyVfWMMZz31E8AjO0ZR3ydcJsjUmd0qkjcnPugy1i4eLIWiFPVVnlXBH9z/DvMHYGoin22ch8pJ3OpHRbE5Mu1omi1lb4fZv8dts2F2EToPNbuiJQqV3kzlB10vLzdGHN/yW0i8gxw/5/fpVxlb2oWk778DYBv77hAxwWqqy3fwZfjwBTCJU9Bz79okThV7TnTwTyojHVDqjoQdWYFhUVc+NwCAMb3jadhzVCbI1JnFJ0AcefBX5dBL60UqjxDeWMEfwVuB5qLyIYSmyKApa4OTP3h1ENjfVvWYdIQnXi+WiksgOWvw+FNcOVbjiJxM+yOSqlKKW+M4FNgDvAUMKnE+gxjTJpLo1LF/rdyH7tSrFtF37mxzFLiyi6HNsKsiXBgLbS61CoSFxhid1RKVVp5icAYY3aLyITSG0SktiYD10vLzOO+L6yLsXUPDyJQbxWtHgpyYfF/rJ/QWnDNB9D2ci0SpzxWRVcEw4DVWLePlvwrN0BzF8algIH/WQjArRc0o2aNIHuDUX/IzYCV70D7q2HwU1Cjtt0RKXVOyrtraJjj32buC0cBFBUZxr7zK8ey8unetBYPXdrG7pBUXqZVGqLneKsw3O3LIbyu3VEpVSWcmby+t4iEOV5fJyJTRCTO9aH5rn9/m8QvO1OJCQ/i0/87T28VtdvOhfB6L2vCmN1LrHWaBJQXcabT+Q0gS0Q6AfcBe4CPXRqVDzuelccHy3YTHRbEskkDdVzATtnH4euJ8NEI8AuAm76D5n3tjkqpKufs5PVGREYALxlj3hWRG10dmC8qKCxi/H9XA/D0VR0JCtAkYKvProM9y6D3XdBvEgTq8xvKOzmTCDJE5AHgeqCPiPgDga4Nyzf9Z942lu9M48ousQxqW8/ucHzTySMQFGb9XPSo9UBYwy52R6WUSznzlXMU1sT1txhjDgGxwHMujcoHpWfn88bCHQD60JgdjIH10+G1HrDgSWtdo0RNAsonODNV5SHgEyBKRIYBOcaYj1wemQ/JyS+k02M/APDfW3tSN1IfSnKr4/vgk2vgq79AdAvoeoPdESnlVs7MUDYS6wpgIdazBK+IyL3GGH2Ovoo84CgmV7NGIBe00Dlr3WrLbEeROANDnoXut2l9IOVznBkjeBDobow5AiAidYAfAU0EVWTzwRMArP1XWfX9lEsYYz0JHNMSml5gJYFaTeyOSilbODNG4HcqCTikOvk+5YR9aVlsOZTByMRG+ryAOxQWwJIXrKsAgJgWMOYzTQLKpzlzRTBXRL7HmrcYrMHj71wXkm+55YOVAAzt0MDmSHzAod/g6wlwcD20HqZF4pRycGbO4ntF5ErgAqwxgqnGmK9cHpkPePHHbfx+5CRXdImlXyt9UtVl8nNg0XOw9EUIrQ0jP4K2I+yOSqlqo7z5CFoAzwPxwG/APcaY/e4KzNvN3XiQF3/8HYAH9HZR18o7Cavfhw4j4ZIntEicUqWU19f/HvAtcBVWBdJXKrtzERksIltFZLuITCqnXXcRKRSRqyt7DE90+EQOEz5dC8DMCb31dlFXyD0JS1+GokKrSNyEFXDFG5oElCpDeV1DEcaYtx2vt4rImsrs2PEE8mtYU10mAytFZJYxJqmMds8A31dm/57qaEYuPZ+cD8DLo7vQuXFNewPyRtvnwzd3Qfo+aNgZml1oJQOlVJnKSwQhItKFP+YhCC25bIypKDH0ALYbY3YCiMh0YASQVKrdHcAXQPdKxu6RHv1mEwBXdInlsk4NbY7Gy2SlwQ8PwbpPrAfDbplrzR+slCpXeYngIDClxPKhEssGGFDBvmOBfSWWk4GeJRuISCxwhWNfZ0wEIjIOGAcQF+e5FbAzcvKZveEg9SNDeGFUZ7vD8T6fXQd7l0Off8CF9+kdQUo5qbyJafqf477LuinelFp+EbjfGFNY3j30xpipwFSAxMTE0vvwCIVFhvMcXUI3925qbzDeJOMwBIdbReIGPQ7+gdCgo91RKeVRnHmO4GwlA41LLDcCDpRqkwhMdySBGGCoiBQYY2a6MC63KywyXPPmMjLzCqkfGcJf+sbbHZLnMwbWfWpNFtPlOutuoEbd7I5KKY/kykSwEmghIs2A/cC1wJiSDUpOgykiHwDfelsSALjs1SVsOnCCuhHBLLrvXC+0FMf2wLd3wY6fIK4XdLvJ7oiU8mguSwTGmAIRmYh1N5A/8J4xZpOIjHdsf9NVx65OZq0/wKYDJwgO8OPXfw7UMhLnavM38OVfrDpBQ5+HxFvBTyueKHUunKk+KsBYoLkx5t+O+YrrG2NWVPReY8x3lCpHcaYEYIy5yamIPYgxhsnfWjdJLby3nyaBc3GqSFydNtC8Hwx5Gmp67o0DSlUnznyVeh3oBYx2LGdgPR+gKvD56mSOZOQyukdjGkTpNIdnpTAfFj0PX9xmLcckwOhPNQkoVYWcSQQ9jTETgBwAY8wxIMilUXmBwiLDfTM2APDoZe1sjsZDHVgHb/eHnx4HUwgFuXZHpJRXcmaMIN/x9K+B4vkIilwalRcY8tIiAIZ3akhwgE50Uin52fDzM1aJiLAYGPUJtBlmd1RKeS1nEsHLwFdAXRF5ArgaeMilUXm4r9ftZ9vhk9SNCGbKyE52h+N58rJgzcfQeTRcPBlCa9kdkVJezZky1J+IyGpgINZDYpcbYza7PDIPVVRkeGHeNgDm3nUhgf56R4tTcjNg5btw/h0QFm0ViQuLtjsqpXyCM3cNxQFZwDcl1xlj9royME913xcb2J2axdiecdQO06EUp/z+o/VcQHoyxHaDZn00CSjlRs50Dc3GGh8QIARoBmwFdAS0lGOZecxYnQzAQ5e2tTkaD5CVZj0ZvH4axLSCW3+Axj3sjkopn+NM11CHkssi0hX4i8si8mD3fL4egFt6NyM0SAeIK/TZdbDvV6tA3IX3QECw3REp5ZMq/WSxMWaNiPhEyejK2HzwBPO3HAHg4eF6NXBGGYcgKNwqFHfx4+AfBPU7VPw+pZTLODNG8PcSi35AV+CoyyLyQMYYhry0GIAnr9APtTIZA2v/C98/aBWJG/ykNR6glLKdM1cEESVeF2CNGXzhmnA808Rp1rSTV3VtxJie+sTrn6TtsgaDdy6EJr0h8Ra7I1JKlVBuInA8SBZujLnXTfF4nOy8QmZvOAjAf/SZgT9LmgVf/QXEHy6dAt1u1iJxSlUzZ0wEIhLgqCDa1Z0BeZpxH68C4B+DWtocSTVzqkhcvXaQMBAGPw1RjeyOSilVhvKuCFZgjQesE5FZwOdA5qmNxpgvXRxbtff1uv0s/j2FptE1uGNgC7vDqR4K8mDpS3B0M1z1LkTHw6j/2h2VUqoczowR1AZSseYVPvU8gQF8PhHM32zdJfTOjYk2R1JN7F8Ds+6Awxuh/VVQmKe3hCrlAcpLBHUddwxt5I8EcIpHzhtclQ6l5zBr/QEa1w4loW5ExW/wZvnZsOBJ+OVVCK8H106D1kPtjkop5aTyEoE/EI5zk9D7nJfm/w7ApR0a2hxJNZCXZc0f3OV6GPRvCK1pd0RKqUooLxEcNMb8222ReJAjGTlMW2GVWrp7kI+ODeScgJXvQO+/WXWBJq6EGrXtjkopdRbKSwQ6r+IZPPXdFgAeurSNb841sO17+PZuyDgIjbpbReI0CSjlscq7oXug26LwIHtTs/hq7X6ax4RxW5/mdofjXpkp1pSRn46E4Ei4dZ6VBJRSHu2MVwTGmDR3BuIpJs+2JqP/ly/WE/rsekheCf0egAv+DgFaZlspb1DponO+LD0rnx+SDlOrRiD9W9W1Oxz3OHHA+vYfHG7VB/IPhno+mASV8mL6rH8ljP/vagD+ObSNzZG4gTGw+gN4rad1ayhAwy6aBJTyQnpF4KT8wiJ+2ZlKkL8f1yQ2tjsc10rbCbPuhN2LoWkf6HGb3REppVxIE4GTnphtTdN8VTcvr5ezaSZ8NR78A2H4S9D1RqtmkFLKa2kicELKyVw+WLabFnXDefKK9naH4xqnisTV7wAtL4ZLnoKoWLujUkq5gY4ROOH577cCcMfAFoi3fTsuyIOFT8OMm61kEB0PIz/SJKCUD9FE4IRfd1l30g7v2MDmSKpY8mqY2hcWPgV+AVaROKWUz9GuoQokHTjBrpRMusTV9J6rgbwsWPAELH8dwuvD6M+g1WC7o1JK2UQTQQVe+ckqLvfE5V40F3FBDmz4H3S7CS56DEIi7Y5IKWUjl3YNichgEdkqIttFZFIZ28eKyAbHzzIRqXZzPS75PYWo0EDaNvTwD8ucdFj0HBQWWHWBJq6AYS9oElBKuS4ROOY7fg0YArQFRotI6aeRdgF9jTEdgceBqa6K52zsPHqSjNwCLm5bz+5Qzs3WOX88GLb3F2tdaC17Y1JKVRuuvCLoAWw3xuw0xuQB04ERJRsYY5YZY445FpcD1eom/Z+2WDOQXd+ric2RnKXMFJhxC0y7FkJrw23ztUicUupPXDlGEAvsK7GcDPQsp/2twJyyNojIOGAcQFxcXFXFV6FXF2wntmYoHWKj3HbMKnWqSFz/B6H3XVokTilVJlcmAqdnNhOR/liJ4IKythtjpuLoNkpMTHTL7GjfbzrE8ax8Lmlb37PuFkrfDyFRjiJxT1lzBtf1gdpISqmz5squoWSgZFGeRsCB0o1EpCPwDjDCGJPqwngqZdZ6K9Q7BibYHImTiopg1XuOsYAnrHUNO2sSUEpVyJVXBCuBFiLSDNgPXAuMKdlAROKAL4HrjTHbXBhLpWzcn87sDQdpVS+CRrVq2B1OxVJ3WEXi9iyBZn2hxzi7I1JKeRCXJQJjTIGITAS+B/yB94wxm0RkvGP7m8DDQDTwuqP7pcAYk+iqmJw17JUlADxymQeUXN70laNIXDBc9ip0uU6LxCmlKsWlD5QZY74Dviu17s0Sr28DqlWN46QDJwAY0Lou58fH2BxNOYqLxHWEVkPhkich0stKYCil3EJrDZUy9OXFAPxzaGubIzmDglz46Qn4/MY/isRd874mAaXUWdNEUMKmA+nFrxPqRtgYyRnsWwlvXQiLnoWAUC0Sp5SqElprqIQPl+0G4Kvbz7c3kNLyMuGnybD8DYiMhbEzoMUgu6NSSnkJTQQlzFx7gFo1AukSV83KLxTkwsYvoPttcNEjEFwNr1aUUh5LE4HDiZx88gqL6Ne0jt2hWLKPw4qpcMHfrSJxE1ZAaE27o1JKeSFNBA4bk63xgf6t69ocCbD5W5j9D8g8Ck16Q9PemgSUUi6jicDhpfnWvAN9W9p4RXDyCHx3LyTNhHodYMx0aNjFvniUUj5BEwGQV1BUPB1lw5qh9gXyvxtg/2oY8JBVJM4/0L5YlFI+QxMBsPj3owD85cLm7j/48X1Wt09wBAx5xnpCuG41fYZBKeWVfP45AmMMf/1kDSGBftze340F5oqKYMXb8Pp51oQxAA06aRJQSrmdz18RHDqRQ15BEdef14SoUDd1xaT8DrPusGYLa94feo53z3GVUqoMPp8Ifth0GIAhHeq754Abv7SKxAWGwIjXofMYLRKnlLKVTycCYwyPf5tE7bAgejWPdvXBrA/8hp2hzXCrSFyEh8+FrJTyCj49RvDE7M0UFBkuSIhx3Sxk+Tkw/9/wv+utZFC7OVz9riYBpVS14dOJYL5jcvrHL2/vmgPs/RXe6gOL/wNBEVokTilVLfls19CS31PYlZLJbRc0q/pB4tyT1lXAiqkQ1Qiu+wISLqraYyilVBXx2UTwwbJdANx5UYuq33lhHiR9DT3+DwY+rEXilFLVms8mgh83H6Fx7VAiQ6roaiArDX59Cy681yoSN3EFhERVzb6VUsqFfDIRHD6RA0Bik9pVs8Okr2H2PZCVCs0utIrEaRJQSnkIn0wEi39PAWBgm3OsNJpxCL67BzZ/Y80dfN0X0KBjFUSolFLu45OJYKWjwNxFbc7xFs7Pb4L9a+CiR6HXHeDvk/85lVIezic/uTY65iYOCfSv/JuP74XQWo4icc9CYCjEuGDAWSml3MQnnyM4lplHg6iQyr2pqMgaDH7tPPjpCWtdg46aBJRSHs/nrgiWbU/hQHoOEytTafToNqtI3L7l1vMAvW53XYBKKeVmPpcIZq7bD8ANvZo494bfZsDMv0JQGFzxFnQcpUXilFJexecSwfebDhMS6EfdyAq6hoqKwM8PYrtC28vhkicgvBrMZ6yUUlXMp8YI9qZmkZ6dT49m5VQazc+GeY+cXiTuqrc1CSilvJZPJYJ1yccBGNMjruwGe5bBmxfA0hetO4MK890Wm1JK2cWnuoYWOKqNDmhd6tt9bgb8+CisfAdqNoHrZ0J8f7fHp5RSdvCpRLA3LQsRCAoodSFUmA9bZsN5t8OAh6yBYaWU8hE+lQhW7zlGvchgayErDZa/AX3vdxSJW6lVQpVSPsmlYwQiMlhEtorIdhGZVMZ2EZGXHds3iEhXV8WSV1AEQIeGUbDpK3itByyZAskrrAaaBJRSPspliUBE/IHXgCFAW2C0iLQt1WwI0MLxMw54w1XxbD2UQV2O8Vj2k1aNoMhYGLcQmpzvqkMqpZRHcGXXUA9guzFmJ4CITAdGAEkl2owAPjLGGGC5iNQUkQbGmINVHcyRjBxeC3qJ+il7YNC/4bwJWiROKaVwbSKIBfaVWE4GejrRJhY4LRGIyDisKwbi4s5w62cFokIDmRN3D80HtSe6Sbuz2odSSnkjVyaCsuowmLNogzFmKjAVIDEx8U/bnZHYtDaJt406m7cqpZRXc+VgcTLQuMRyI+DAWbRRSinlQq5MBCuBFiLSTESCgGuBWaXazAJucNw9dB6Q7orxAaWUUmfmsq4hY0yBiEwEvgf8gfeMMZtEZLxj+5vAd8BQYDuQBdzsqniUUkqVzaW3zRhjvsP6sC+57s0Srw0wwZUxKKWUKp9PFZ1TSin1Z5oIlFLKx2kiUEopH6eJQCmlfJxY47WeQ0SOAnvO8u0xQEoVhuMJ9Jx9g56zbziXc25ijKlT1gaPSwTnQkRWGWMS7Y7DnfScfYOes29w1Tlr15BSSvk4TQRKKeXjfC0RTLU7ABvoOfsGPWff4JJz9qkxAqWUUn/ma1cESimlStFEoJRSPs4rE4GIDBaRrSKyXUQmlbFdRORlx/YNItLVjjirkhPnPNZxrhtEZJmIdLIjzqpU0TmXaNddRApF5Gp3xucKzpyziPQTkXUisklEfnZ3jFXNib/tKBH5RkTWO87Zo6sYi8h7InJERDaeYXvVf34ZY7zqB6vk9Q6gORAErAfalmozFJiDNUPaecCvdsfthnM+H6jleD3EF865RLufsKrgXm133G74PdfEmhc8zrFc1+643XDO/wSecbyuA6QBQXbHfg7nfCHQFdh4hu1V/vnljVcEPYDtxpidxpg8YDowolSbEcBHxrIcqCkiDdwdaBWq8JyNMcuMMccci8uxZoPzZM78ngHuAL4AjrgzOBdx5pzHAF8aY/YCGGM8/bydOWcDRIiIAOFYiaDAvWFWHWPMIqxzOJMq//zyxkQQC+wrsZzsWFfZNp6ksudzK9Y3Ck9W4TmLSCxwBfAm3sGZ33NLoJaILBSR1SJyg9uicw1nzvlVoA3WNLe/AX8zxhS5JzxbVPnnl0snprGJlLGu9D2yzrTxJE6fj4j0x0oEF7g0Itdz5pxfBO43xhRaXxY9njPnHAB0AwYCocAvIrLcGLPN1cG5iDPnfAmwDhgAxAPzRGSxMeaEi2OzS5V/fnljIkgGGpdYboT1TaGybTyJU+cjIh2Bd4AhxphUN8XmKs6ccyIw3ZEEYoChIlJgjJnplgirnrN/2ynGmEwgU0QWAZ0AT00EzpzzzcDTxupA3y4iu4DWwAr3hOh2Vf755Y1dQyuBFiLSTESCgGuBWaXazAJucIy+nwekG2MOujvQKlThOYtIHPAlcL0HfzssqcJzNsY0M8Y0NcY0BWYAt3twEgDn/ra/BvqISICI1AB6ApvdHGdVcuac92JdASEi9YBWwE63RuleVf755XVXBMaYAhGZCHyPdcfBe8aYTSIy3rH9Taw7SIYC24EsrG8UHsvJc34YiAZed3xDLjAeXLnRyXP2Ks6cszFms4jMBTYARcA7xpgyb0P0BE7+nh8HPhCR37C6Te43xnhseWoRmQb0A2JEJBl4BAgE131+aYkJpZTycd7YNaSUUqoSNBEopZSP00SglFI+ThOBUkr5OE0ESinl4zQRqGrJUS10XYmfpuW0PVkFx/tARHY5jrVGRHqdxT7eEZG2jtf/LLVt2bnG6NjPqf8uGx0VN2tW0L6ziAytimMr76W3j6pqSUROGmPCq7ptOfv4APjWGDNDRC4GnjfGdDyH/Z1zTBXtV0Q+BLYZY54op/1NQKIxZmJVx6K8h14RKI8gIuEiMt/xbf03EflTpVERaSAii0p8Y+7jWH+xiPzieO/nIlLRB/QiIMHx3r879rVRRO5yrAsTkdmO+vcbRWSUY/1CEUkUkaeBUEccnzi2nXT8+1nJb+iOK5GrRMRfRJ4TkZVi1Zj/ixP/WX7BUWxMRHqINc/EWse/rRxP4v4bGOWIZZQj9vccx1lb1n9H5YPsrr2tP/pT1g9QiFVIbB3wFdZT8JGObTFYT1WeuqI96fj3H8CDjtf+QISj7SIgzLH+fuDhMo73AY75CoBrgF+xirf9BoRhlTfeBHQBrgLeLvHeKMe/C7G+fRfHVKLNqRivAD50vA7CqiIZCowDHnKsDwZWAc3KiPNkifP7HBjsWI4EAhyvLwK+cLy+CXi1xPufBK5zvK6JVYMozO7ft/7Y++N1JSaU18g2xnQ+tSAigcCTInIhVumEWKAecKjEe1YC7znazjTGrBORvkBbYKmjtEYQ1jfpsjwnIg8BR7EqtA4EvjJWATdE5EugDzAXeF5EnsHqTlpcifOaA7wsIsHAYGCRMSbb0R3VUf6YRS0KaAHsKvX+UBFZBzQFVgPzSrT/UERaYFWiDDzD8S8GLhORexzLIUAcnl2PSJ0jTQTKU4zFmn2qmzEmX0R2Y32IFTPGLHIkikuBj0XkOeAYMM8YM9qJY9xrjJlxakFELiqrkTFmm4h0w6r38pSI/GCM+bczJ2GMyRGRhVilk0cB004dDrjDGPN9BbvINsZ0FpEo4FtgAvAyVr2dBcaYKxwD6wvP8H4BrjLGbHUmXuUbdIxAeYoo4IgjCfQHmpRuICJNHG3eBt7Fmu5vOdBbRE71+dcQkZZOHnMRcLnjPWFY3TqLRaQhkGWM+S/wvOM4peU7rkzKMh2rUFgfrGJqOP7966n3iEhLxzHLZIxJB+4E7nG8JwrY79h8U4mmGVhdZKd8D9whjssjEelypmMo36GJQHmKT4BEEVmFdXWwpYw2/YB1IrIWqx//JWPMUawPxmkisgErMbR25oDGmDVYYwcrsMYM3jHGrAU6ACscXTQPApPLePtUYMOpweJSfsCal/ZHY02/CNY8EUnAGrEmLX+LCq7YHbGsxyrN/CzW1clSrPGDUxYAbU8NFmNdOQQ6YtvoWFY+Tm8fVUopH6dXBEop5eM0ESillI/TRKCUUj5OE4FSSvk4TQRKKeXjNBEopZSP00SglFI+7v8BWEsLFZX5SfMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr, tpr, thresholds = roc_curve(y_test, NN_pred_proba)\n",
    "\n",
    "# Calculate the AUC ROC score for SVM\n",
    "auc_roc = roc_auc_score(y_test, NN_pred_proba)\n",
    "\n",
    "# Plot the ROC curve\n",
    "plt.plot(fpr, tpr, label=f'AUC ROC = {auc_roc:.5f}')\n",
    "plt.plot([0, 1], [0, 1], linestyle='--')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "4859d4e1-435a-4f05-bf46-22c67627ab6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\nasri\\anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:87: FutureWarning: Function plot_confusion_matrix is deprecated; Function `plot_confusion_matrix` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: ConfusionMatrixDisplay.from_predictions or ConfusionMatrixDisplay.from_estimator.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'NNs')"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUgAAAEWCAYAAAAEkA60AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAg+0lEQVR4nO3de7xVdZ3/8df7HJDL4X5HLuGFNLTyQnhLE20CtRKbmDBLa5whGUxrqhlr+uWY0cOxmikrbzWOmBmZl8QEzAglS1RUFEVRFJUjCB5Q7pdz9vn8/tjrwPaw12Ef2PtcNu/n47EeZ+3v+q61vvsc/fC9rPX9KiIwM7PdVbR2AczM2ioHSDOzFA6QZmYpHCDNzFI4QJqZpXCANDNL4QBpZpbCAdL2iqRXJa2WVJWT9k+SHkz2Q9JiSRU5x78n6eaWL63Z3nGAtH3RAbi0ieMHApNaqCxmRecAafviB8DXJfVKOX41cIWkDo0PSOos6VZJayW9I+lxSQNLWViz5nKAtH2xEHgQ+HrK8buADcAX8hy7AOgJDAP6AhcBW4teQrN94ABp++o7wJcl9c9zLID/B3xHUqdGx2rJBsZDIyITEU9ExIYSl9WsWRwgbZ9ExLPAH4DLUo7PAl4HJjc69CvgfmCGpJWSrpbUsaSFNWsmB0grhsuBfwaGpBz/NvAfQNeGhIiojYgrImIUcCLwceD8UhfUrDkcIG2fRcQy4LfAJSnHHwQWk+13BEDSWEnvl1RJtp+yFsiUvrRmhXOAtGL5LlDVxPFvA31yPg8C7iAbHJ8HHgJuLVnpzPaCPGGumVl+rkGamaVwgDQzS+EAaWaWwgHSzCzFbu/ItnX9+lTGiGF+nrg9Wbwu30s21pbtqK6uiYi9/sONG1sVa9cV9tTWE89svz8ixu/tvUqp3QXIEcM68tj9w1q7GNYMI2+d0tpFsGZ65Rtfe21fzq9Zl+HR+4cWlLfj4Jf77cu9SqndBUgzaw+CTNS3diH2mQOkmRVdAPW0/2esHSDNrCTqcQ3SzGw3QVDrJraZ2e4CyLiJbWaWn/sgzczyCCBTBhPhOECaWUm0/x5IB0gzK4Eg3AdpZpZPBNS2//joAGlmpSAyqLULsc8cIM2s6AKodw3SzCw/1yDNzPLIPijuAGlmtpsAaqP9z8ftAGlmRReITBksWOAAaWYlUR9uYpuZ7cZ9kGZmqUTGfZBmZrvLzijuAGlmtpsIsSMqW7sY+8wB0sxKot59kGZmu8sO0riJbWaWhwdpzMzy8iCNmVkTMn5Q3Mxsd4GojfYfXtr/NzCzNqdcBmna/zcwszYnEJkobCuEpFclLZa0SNLCJK2PpAckvZT87J2T/5uSlklaKmlcTvqxyXWWSbpGUpMFcIA0s5Kop6KgrRnGRsRRETE6+XwZMDciRgJzk89IGgVMAo4AxgPXSmp4av06YDIwMtnGN3VDB0gzK7oIyERFQds+OBuYnuxPBybkpM+IiO0RsRxYBoyRNBjoERGPREQAt+Sck5f7IM2s6LKDNAW/ativodmcuDEibtztkvBHSQHckBwfGBGrACJilaQBSd4hwIKcc6uTtNpkv3F6KgdIMyuJZgzS1OQ0m9OcFBErkyD4gKQXmsibr18xmkhP5QBpZkUXqKgT5kbEyuTnGkl3A2OA1ZIGJ7XHwcCaJHs1MCzn9KHAyiR9aJ70VO6DNLOSyFBR0LYnkqokdW/YBz4GPAvMBC5Isl0A3JPszwQmSeok6SCygzGPJc3xjZKOT0avz885Jy/XIM2s6LLrYhet/jUQuDt5IqcDcFtEzJH0OHC7pAuB14GJABHxnKTbgSVAHTA1IjLJtaYANwNdgNnJlsoB0sxKQEVbciEiXgE+mCd9LXB6yjnTgGl50hcCRxZ6bwdIMyu67LKvnjDXzGw3ESpmE7vVOECaWUl4Pkgzszyy80F6ujMzszw8o7iZWV7Zx3xcgzQz200z38VusxwgzawkvCaNmVke2enO3MQ2M8vLfZBmZnlkZ/NxE9vMbDfZVw0dIG0Pzh8zii7dMlRUQGWH4GdzXmTal95D9cudAdi8oZKqHhmu+9NS6mrhf74+nGWLu5CpEx+duI5JX85OcffSM1344VeGs31bBWNO28CUK9+g6eWGrLkOqKjjto/dwwGV9XRQPXNeP5hrnvkQ44e/zCUfWMghPd/m72d/imfXDdh5zpeOeJKJh75AJsSVj3+Yh1dlpyE8os9b/NcJ8+jcoY6H3hjOlQtPIv98reXKNcg9kjQe+AlQCfwyIq5qdFzJ8TOBLcAXIuLJUpapNVz9u2X07JvZ+fk/bnht5/4NVxxIVffssfn39qJ2u7jhz0vZtkVMPvV9nDrhHQYN28E1lw3l0qtX8L5jt/Dtzx3Mwnnd+dBpG1v8u5SzHfWVnP+nT7KlriMdlGHGuHuYv3I4L73Th6nzx3HlcQ+9K/+hPddx1oiXOfPezzCgy2amf/QP/N3MSdRHBVeMmc+3Hz2FRTUD+eXYWZxy4ArmrxzeSt+sdZTDmzQlC/HJKmI/B84ARgHnJquN5TqDXauLTSa74th+IwLmz+zF2AlvAyDBti0VZOpgx7YKOhxQT9duGdau7sCWjZWMGr0FCT766XX8bU7PVi59ORJb6joC0KGing4V9UTAyxt6s3xDr91ynz70Ve579RB21FdSvbkHr23swQf6rqF/l81061jLoppBgPj98vfyd8OWt+xXaWUNo9jFWva1tZSyBjkGWJbM5YakGWRXG1uSk+ds4JZkhbEFkno1TKFewnK1LAXfOvcQEJz1+bWc+bm1Ow89+2gVvfvXMeTgHQCc/PF3eOT+npx71JFs2youumIlPXpnePHpLvQbXLvzvH4H1lLzZscW/yr7gwrV8/sz7mR49/X8+sUjeXrtwNS8A7tuZlHNruNvbunGoK6bqauv4M0tVbvSN3djYJfNJS13W+QmdtOGACtyPlcDxxWQZwjwrgApaTLZGibDh7SvbtP/uecl+g6q452aDlw26RCGHbqN9x+f/Z9l3u97c2pSewRY+lQVFZXBbU89y6b1HfjahEM5+uSNRJ5lhdr2v7vtV31U8MlZE+necTvXfuR+RvZcx0vr++TNm3cFqFDevuHYz/5ixV6TprWUMsQXsoJYQauMRcSNETE6Ikb379u+Xl/qO6gOgF796jhp/HpeeKorAJk6+Ousnnzkk+/szDvv7l6MHruRDh2z+Ud9aDMvPt2VfoNrqVm1q8ZYs7IjfQfVYqWzsbYTj64+kFMOfD01z5tbqhjcddPOz4O6bmL11q68uaWKQV131RgHVW1izdauJS1vWxNAXVQUtLVlpSxd2spizc3Tbm3bUsGWTRU79594qDsjDt8GwJN/6c6wQ7fT/8Bdga7/kFoWPdyNiGz+F56sYtih2+g7sI6u3ep5/omuRMCf7ujDCePWt8p3Kmd9Om2le8ftAHSqrOPEwdW8sqF3av651SM4a8TLHFCRYWjVBkZ0X88zawfw1tYqNtd15Kh+q4FgwkEv8qcVI1rmS7Qh9VFR0NaWlbK9+jgwMllV7A1gEvDZRnlmAhcn/ZPHAevLqf/x7bc6cMWFBwHZGuPYc97hQ2OzI88P3fPu5jXAJ79Yw4++OpzJYw+DEB/7zFoOHpUNqF++agU//MpwdmyrYPTYDR7BLoH+XbZw9Yl/pkJBhYLZrx3CvDfew98NW853Rj9Mn85b+cXY2Tz/dl/+8c8fZ9n6Psx+7WBmf+K31IX4z8dP3vk//OWPnsx/nTiPzpUZHlo5jIf2sxFsojya2Ip8HVzFurh0JvBjso/53BQR0yRdBBAR1yeP+fwMGE/2MZ8vJovqpBr9wc7x2P3DmspibczIW6e0dhGsmV75xteeiIjRe3t+78MHxGk3fbqgvHeddN0+3auUSjriERGzgFmN0q7P2Q9gainLYGatoxxqkO1rSNjM2gVPmGtmliIQdfVtewCmEA6QZlYS5fCqoQOkmRVfuIltZpaX+yDNzJrgAGlmlkcgMh6kMTPLz4M0ZmZ5hAdpzMzShQOkmVk+5TFZRfvvRTWzNilCBW2FklQp6SlJf0g+95H0gKSXkp+9c/J+U9IySUsljctJP1bS4uTYNcmEOakcIM2s6CIgU6+Ctma4FHg+5/NlwNyIGAnMTT6TrH01CTiC7Exh1yZrZEF23avJ7FoLa3xTN3SANLOSqEcFbYWQNBQ4C/hlTvLZwPRkfzowISd9RkRsj4jlwDJgjKTBQI+IeCSZSeyWnHPych+kmRVd0KxBmn6ScueBvTEibmyU58fAvwHdc9IGNkywHRGrJDUsWD4EWJCTr2Gtq9pkv3F6KgdIMyuBZg3S1DQ1Ya6kjwNrIuIJSacWdPPdRRPpqRwgzawkirhYwUnAJ5MVCjoDPSTdCqxuWCY6aT6vSfKnrXVVnew3Tk/lPkgzK4lijWJHxDcjYmhEjCA7+PLniPgc2TWtLkiyXQDck+zPBCZJ6pSsiTUSeCxpjm+UdHwyen1+zjl5uQZpZkWXHcUuef3rKuB2SRcCrwMTs/eO5yTdDiwB6oCpEZFJzpkC3Ax0AWYnWyoHSDMriVKsBxgRDwIPJvtrgdNT8k0DpuVJXwgcWej9HCDNrCT8qqGZWR5B896SaascIM2sJErQwm5xDpBmVnwB0bzXCNskB0gzKwk3sc3MUpRiFLulpQZIST+liW6EiLikJCUys3avme9it1lN1SAXNnHMzCxdAOUcICNieu5nSVURsbn0RTKzclAOTew9vgsk6QRJS0gmqpT0QUnXlrxkZtaOiagvbGvLCnlZ8sfAOGAtQEQ8DZxSwjKZWTmIArc2rKBR7IhY0WjphkxaXjMzovwHaRqskHQiEJIOAC7h3etCmJntro3XDgtRSBP7ImAq2anJ3wCOSj6bmTVBBW5t1x5rkBFRA5zXAmUxs3JS39oF2HeFjGIfLOleSW9JWiPpHkkHt0ThzKydangOspCtDSukiX0bcDswGDgQ+B3wm1IWyszav4jCtraskACpiPhVRNQl262URfermZVUOT/mI6lPsjtP0mXADLJf5zPAfS1QNjNrz9p487kQTQ3SPMG715L9Us6xAK4sVaHMrP1TG68dFqKpd7EPasmCmFkZCUEbf42wEAW9SSPpSGAU2UW7AYiIW0pVKDMrA+Vcg2wg6XLgVLIBchZwBvAw4ABpZunKIEAWMor9abJrz74ZEV8EPgh0KmmpzKz9K+dR7BxbI6JeUp2kHsAawA+Km1m6cp8wN8dCSb2AX5Ad2d4EPFbKQplZ+1fWo9gNIuJfkt3rJc0BekTEM6Utlpm1e+UcICUd09SxiHiyNEUys3JQ7jXIHzVxLIDTilyWgrz4TFfGHXhUa9za9tKgc8pgWpf9zCvFuEg590FGxNiWLIiZlZF2MEJdiIIeFDczazYHSDOz/FQGPSsOkGZWGmVQgyxkRnFJ+pyk7ySfh0saU/qimVl7pSh82+O1pM6SHpP0tKTnJF2RpPeR9ICkl5KfvXPO+aakZZKWShqXk36spMXJsWvUaLnWxgp51fBa4ATg3OTzRuDnBZxnZvuz4i25sB04LSI+SHbRwPGSjgcuA+ZGxEhgbvIZSaOAScARwHjgWkmVybWuAyYDI5NtfFM3LiRAHhcRU4FtABHxNnBAId/KzPZjRXoXO7I2JR87JlsAZwPTk/TpwIRk/2xgRkRsj4jlwDJgjKTBZF90eSQiguyEOw3n5FVIgKxNom8ASOpPWaxXZmal1Iwmdj9JC3O2ybtdS6qUtIjsXBAPRMSjwMCIWAWQ/ByQZB8CrMg5vTpJG5LsN05PVcggzTXA3cAASdPIzu7z7QLOM7P9VTRrFLsmIkY3ebmIDHBUMi/E3ckctWnytdujifRUhbyL/WtJT5Cd8kzAhIh4fk/nmdl+rgSj2BHxjqQHyfYdrpY0OCJWJc3nNUm2amBYzmlDgZVJ+tA86akKGcUeDmwB7gVmApuTNDOzdEXqg5TUP6k5IqkL8FHgBbLx6IIk2wXAPcn+TGCSpE6SDiI7GPNY0gzfKOn4ZPT6/Jxz8iqkiX0fu6qnnYGDgKVkR4jMzPIq4mQVg4HpyVhIBXB7RPxB0iPA7ZIuBF4HJgJExHOSbgeWAHXA1KSJDjAFuBnoAsxOtlSFNLHfn/s5meXnSynZzcyKKple8eg86WvJdv3lO2caMC1P+kKgqf7Ld2n2mzQR8aSkDzX3PDPbz5TBmzSFLNr1rzkfK4BjgLdKViIza/+aN4rdZhVSg+yes19Htk/yztIUx8zKRrnXIJNO0W4R8Y0WKo+ZlQFR5jOKS+oQEXVNLb1gZpaqnAMk2ZULjwEWSZoJ/A7Y3HAwIu4qcdnMrL0qcKaetq6QPsg+wFqya9A0PA8ZgAOkmaUr80GaAckI9rPs/h5jGfzbYGalVO41yEqgG3vxgreZWTlEiaYC5KqI+G6LlcTMysd+sKph+1/U1sxaTbk3sfO+42hmVpByDpARsa4lC2Jm5WV/edXQzKx59oM+SDOzvSLKYxDDAdLMSsM1SDOz/Mp9FNvMbO85QJqZ5bEfTZhrZtZ8rkGameXnPkgzszQOkGZm+bkGaWaWT1D2E+aame2Vsl+0y8xsnzhAmpnlp2j/EdIB0syKz7P5mJmlcx+kmVkKv2poZpbGNUgzszzCTWwzs3RlECArWrsAZlZ+Gh4UL2Tb47WkYZLmSXpe0nOSLk3S+0h6QNJLyc/eOed8U9IySUsljctJP1bS4uTYNZKaXBnCAdLMSkL1UdBWgDrgaxHxPuB4YKqkUcBlwNyIGAnMTT6THJsEHAGMB66VVJlc6zpgMjAy2cY3dWMHSDMrvmjGtqdLRayKiCeT/Y3A88AQ4GxgepJtOjAh2T8bmBER2yNiObAMGCNpMNAjIh6JiABuyTknL/dBtqDpjy5h66ZK6ushUye+fMZ7OXjUVr58VTVdqupZXX0A/zV1OFs2VXLYUVu49AcrgGxz5Vc/GsTf5vRs3S+wH7jscw9y4pGv8/bGLlwwbSIAhw6t4euTHuaAjhkyGfHfv/0wz782gA6VGb5x7l84bPhbRIif3HEii146EIDTj13G58c9BYia9V258ubTWL+5cyt+s5bXjMd8+klamPP5xoi4Me81pRHA0cCjwMCIWAXZICppQJJtCLAg57TqJK022W+cnqpkAVLSTcDHgTURcWSe4wJ+ApwJbAG+0PCvRDn7t4mHsGHdrl/7V364gl9890AWL+jGxyat5dNT1nDLDwbz6tLOXDz+vdRnRJ8BtVz3pxdZ8EAP6jPlsJhm2zV7wWHc9dCR/Mf583amTZnwKP836xgeXTKc4494nSkTHuWSn3yCT5z0AgBf+P5EenXbyg+nzuafrz6HCgWXTvwbn7/yH1i/uTNTJizgUx95lv+bNbq1vlbrKHyQpiYi9vjLkdQNuBP4SkRsaKL7MN+BaCI9VSmb2DfTdPv+DHb1A0wm2zew3xl6yHYWL6gC4Kn53fnwWesB2L61Ymcw7NipnjJ4rbVdeHrZYDZs7vTuxBBVnWsBqOq8g5r1XQEYMehtnliarYC8s6kLm7YewOHD3wKy/yd27lQLBF0711KzvqqlvkKbUaxBGgBJHckGx19HxF1J8uqk2Uzyc02SXg0Myzl9KLAySR+aJz1VyQJkRMwH1jWR5WzglshaAPRq+LJlK8T3f/MKP5vzImectxaA15Z25oRxGwA4+ePr6X9g7c7shx29mRvnvcANf36Ra/59qGuPreSaO07gX85ZwB3f+zVTP7WAG2aOAWDZG3358AdepbKinsF9N/DeYTUM6L2JTH0FP/rth5n+rTv4/fdvZcTgt7nvb4e18rdoYQFEFLbtQdLa/F/g+Yj475xDM4ELkv0LgHty0idJ6iTpILKVsMeS5vhGSccn1zw/55y8WrMPcgiwIudzQ3/AqsYZJU0mW8ukM11bpHCl8NWzD2Xd6o707FvLVTNeYcWyTvz3vw5jypVvcN5XV/PIH3tQt2NXEFz6VBWTxx7OsEO38Y2fvM7j87pTu93jai1twilL+OmdJ/DQooMZe8zLXHbefL7607OY9chhjBj0Nr/497t5c103nl0+kEx9BZUV9Uw4eQn/eNXfs7KmO1/5h7/yuXGLuGXOMa39VVpUEV81PAn4PLBY0qIk7VvAVcDtki4EXgcmAkTEc5JuB5aQHQGfGhGZ5LwpZFu3XYDZyZaqNQNkwf0BSYftjQA91KfdNjbXre4IwPq1HfnrnJ4cfvQW7rh+AN869xAAhhy8neNO37DbeSuWdWbblgpGHLaNl55pv/9AtFfjj3uRn/zuRADmPXkw//7Z+QBk6iv46Z0n7sx37dfuoXpNT0YOrQFgZU2P5JxDOO9ji1q20K2smBPmRsTD5I8XAKennDMNmJYnfSGw25hImtasjqT1E5SlTl0ydKnK7Nw/9iMbefWFzvTsm21SS8FnL13NH37VF4CBw7ZTUZn9L2zAkB0MPWQ7q6sPaJ3C7+dq1ldx1Mhsw+bYw1ZS/Vb2aYJOHevofED27zf68Goy9eLVN3vz1voqRgx6m17dtu489tqbvVql7K2m0OZ1G+9cb80a5EzgYkkzgOOA9Q1D9uWod/86Lv/fVwGo7BDMu7s3Cx/swYQL3+ITX8jWOP46uyd/nNEHgCPHbOYzFy+nrk7U14uffmvou0a/rTQu/+Jcjh65kp7dtnHn937NTfcdy9W3ncKln/4blRX17Kir5OrbTgagd/et/OjiWdSHqHmniu9NHwvA2vVV/N+sY/npV+8lk6ngzXXd+P6vTm3Fb9U6yuFdbEWJIrik3wCnAv2A1cDlQEeAiLg+6ST9GdmR7i3AF5Pqb5N6qE8cp7y1amujtpxzXGsXwZrpb3d944lCHr1J073X0Dj6lEsLyvuXe/9tn+5VSiWrkkTEuXs4HsDUUt3fzFpXOdQg3WYzs+ILINP+I6QDpJmVhGuQZmZp2vgIdSEcIM2sJFyDNDPLx8u+mpnlJ0AepDEzy0/ugzQzy8NNbDOzNG3/PetCOECaWUl4FNvMLI1rkGZmeYRHsc3M0rX/+OgAaWal4cd8zMzSOECameURQPEW7Wo1DpBmVnQi3MQ2M0tV3/6rkA6QZlZ8bmKbmaVzE9vMLI0DpJlZPp6swswsP69qaGaWzn2QZmZpHCDNzPIIoN4B0swsDw/SmJmlc4A0M8sjgEz7f5WmorULYGblKCDqC9v2QNJNktZIejYnrY+kByS9lPzsnXPsm5KWSVoqaVxO+rGSFifHrpGkPd3bAdLMSiOisG3PbgbGN0q7DJgbESOBuclnJI0CJgFHJOdcK6kyOec6YDIwMtkaX3M3DpBmVnwNo9iFbHu6VMR8YF2j5LOB6cn+dGBCTvqMiNgeEcuBZcAYSYOBHhHxSEQEcEvOOancB2lmpVH4IE0/SQtzPt8YETfu4ZyBEbEqe5tYJWlAkj4EWJCTrzpJq032G6c3yQHSzEqj8ABZExGji3TXfP2K0UR6kxwgzaz4IiCTKeUdVksanNQeBwNrkvRqYFhOvqHAyiR9aJ70JrkP0sxKo3iDNPnMBC5I9i8A7slJnySpk6SDyA7GPJY0xzdKOj4ZvT4/55xUrkGaWWkU6UFxSb8BTiXbV1kNXA5cBdwu6ULgdWBi9pbxnKTbgSVAHTA1IhqqslPIjoh3AWYnW5McIM2sBAoboS7oShHnphw6PSX/NGBanvSFwJHNubcDpJkVX0AU8BB4W+cAaWalUQavGjpAmlnxRXjZVzOzVJ7Nx8wsv3AN0swsH0+Ya2aWn5dcMDPLL4Ao7auGLcIB0syKL6KgyXDbOgdIMyuJcBPbzCxFGdQgFe1spEnSW8BrrV2OEukH1LR2Iaxg5fz3ek9E9N/bkyXNIfv7KURNROxx+YPW0O4CZDmTtLCIE4daifnvVf48H6SZWQoHSDOzFA6QbcueFiqytsV/rzLnPkgzsxSuQZqZpXCANDNL4QDZwiSNl7RU0jJJl+U5LknXJMefkXRMa5TTsiTdJGmNpGdTjvvvVcYcIFuQpErg58AZwCjgXEmjGmU7g+xSlSOBycB1LVpIa+xmoKmHmP33KmMOkC1rDLAsIl6JiB3ADODsRnnOBm6JrAVAr2RhdGsFETEfWNdEFv+9ypgDZMsaAqzI+VydpDU3j7Ud/nuVMQfIlqU8aY2fsyokj7Ud/nuVMQfIllUNDMv5PBRYuRd5rO3w36uMOUC2rMeBkZIOknQAMAmY2SjPTOD8ZHT0eGB9RKxq6YJawfz3KmOeD7IFRUSdpIuB+4FK4KaIeE7SRcnx64FZwJnAMmAL8MXWKq+BpN8ApwL9JFUDlwMdwX+v/YFfNTQzS+EmtplZCgdIM7MUDpBmZikcIM3MUjhAmpmlcIAsQ5IykhZJelbS7yR13Ydr3Szp08n+L/NMrpGb91RJJ+7FPV6VtNsKeGnpjfJsaua9/lPS15tbRts/OUCWp60RcVREHAnsAC7KPZjMKtRsEfFPEbGkiSynAs0OkGZtlQNk+fsLcGhSu5sn6TZgsaRKST+Q9Hgyj+GXYOf8hj+TtETSfcCAhgtJelDS6GR/vKQnJT0taa6kEWQD8VeT2uvJkvpLujO5x+OSTkrO7Svpj5KeknQD+d9nfhdJv5f0hKTnJE1udOxHSVnmSuqfpB0iaU5yzl8kHV6U36btV/wmTRmT1IHsfIVzkqQxwJERsTwJMusj4kOSOgF/lfRH4GjgMOD9wEBgCXBTo+v2B34BnJJcq09ErJN0PbApIn6Y5LsN+J+IeFjScLJvEL2P7NsoD0fEdyWdRXYexT35x+QeXYDHJd0ZEWuBKuDJiPiapO8k176Y7IJaF0XES5KOA64FTtuLX6Ptxxwgy1MXSYuS/b8A/0u26ftYRCxP0j8GfKChfxHoSXbS11OA30REBlgp6c95rn88ML/hWhGRNl/iR4FR0s4KYg9J3ZN7fCo59z5JbxfwnS6RdE6yPywp61qgHvhtkn4rcJekbsn3/V3OvTsVcA+zd3GALE9bI+Ko3IQkUGzOTQK+HBH3N8p3JnuerksF5IFsF84JEbE1T1kKfsdV0qlkg+0JEbFF0oNA55Tskdz3nca/A7Pmch/k/ut+YIqkjgCS3iupCpgPTEr6KAcDY/Oc+wjwEUkHJef2SdI3At1z8v2RbHOXJN9Rye584Lwk7Qyg9x7K2hN4OwmOh5OtwTaoABpqwZ8l23TfACyXNDG5hyR9cA/3MNuNA+T+65dk+xefVHZBqhvItijuBl4CFpNdX+WhxidGxFtk+w3vkvQ0u5q49wLnNAzSAJcAo5NBoCXsGk2/AjhF0pNkm/qv76Gsc4AOkp4BrgQW5BzbDBwh6QmyfYzfTdLPAy5Myvccuy9tYbZHns3HzCyFa5BmZikcIM3MUjhAmpmlcIA0M0vhAGlmlsIB0swshQOkmVmK/w+rNKPNNu+fwAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figsize = (20, 14)\n",
    "plot_confusion_matrix(NN_clf, X_test, y_test)\n",
    "plt.title(\"NNs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "71eda390-a174-4c4c-b25d-26ce3115eefa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# PRC\n",
    "from sklearn.metrics import auc, precision_recall_curve\n",
    "precision, recall, thresholds = precision_recall_curve(y_test, NN_pred_proba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "4fb958af-c063-48dd-a4b5-3963a9ffec93",
   "metadata": {},
   "outputs": [],
   "source": [
    "# AUPRC\n",
    "auprc = auc(recall, precision)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "ed26e273-208a-4695-8615-eee5ea106439",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAqyElEQVR4nO3deXxU1d3H8c8vCwRkFQIKAQPKKgjSiCuKG2WxVbStoC116eODila0KorVqqhUrVqr1Yeqj9YF9LGoKIi74gayGPYtIktABEEWgQBJfs8fM4QskzBA7kyS+b5fr7yYe+65d36XwPzm3HPuOebuiIhI4kqKdwAiIhJfSgQiIglOiUBEJMEpEYiIJDglAhGRBJcS7wD2V9OmTT0zMzPeYYiIVCszZ878wd3TI+2rdokgMzOTGTNmxDsMEZFqxcxWlLdPt4ZERBKcEoGISIJTIhARSXDVro9ARKK3e/ducnNzycvLi3coEiNpaWlkZGSQmpoa9TFKBCI1WG5uLvXr1yczMxMzi3c4EjB3Z8OGDeTm5tKmTZuojwvs1pCZPWNm68xsXjn7zcweNbMcM5tjZj2CikUkUeXl5dGkSRMlgQRhZjRp0mS/W4BB9hE8C/StYH8/oF345wrgiQBjEUlYSgKJ5UB+34ElAnefAmysoMq5wL89ZCrQyMwODyqexWu38tC7i3lvwfdBvYWISLUUz1FDLYFVxbZzw2VlmNkVZjbDzGasX7/+gN4sZ91PPPphDre+NveAjheRA/faa69hZixatKio7OOPP+acc84pUe+SSy7h1VdfBaB379506NCBbt26cfLJJ7N48eIy5ccddxzZ2dlFx7/99ttkZWXRqVMnOnbsyJ/+9KeDjn3mzJl07dqVo446imuvvZaK1nBZuXIl9erV48EHHwRg69atdO/eveinadOmXHfddQA8+eSTdO3ale7du3PKKaewYMGCovPcfPPNdOnShS5duvDyyy8Xlbs7I0eOpH379nTq1IlHH330oK8P4psIIrVfIv4Nu/sYd89y96z09IhPSO/TgGMOZ3DP1gd0rIgcnLFjx3LKKacwbty4/TruxRdfZPbs2fz+97/nxhtvLFN+1VVXFZXPmzePYcOG8cILL7Bw4ULmzZtH27ZtDzr2K6+8kjFjxrB06VKWLl3K5MmTy607fPhw+vXrV7Rdv359srOzi36OOOIIzj//fAAuuugi5s6dS3Z2NjfddBPXX389ABMnTmTWrFlkZ2czbdo0HnjgAbZs2QLAs88+y6pVq1i0aBELFy5k0KBBB319EN9EkAu0KradAayJUywiEpCffvqJzz//nKeffnq/E8Eep556Kjk5OWXKTzzxRFavXg3A/fffz8iRI+nYsSMAKSkpXHXVVQceOPDdd9+xZcsWTjzxRMyMIUOG8Prrr0es+/rrr9O2bVuOPvroiPuXLl3KunXr6NWrFwANGjQo2rdt27aie/sLFizgtNNOIyUlhUMOOYRu3boVJZ8nnniC22+/naSk0Ed3s2bNDur69ojn8NEJwDAzGwccD2x29+/iGI9IjXbnm/NZsGZLpZ6zc4sG3PGLyB98e7z++uv07duX9u3bc+ihhzJr1ix69Ni/QYJvvvkmXbt2LVM+efJkzjvvPCDUIrjhhhv2ea6PPvqI4cOHlymvW7cuX3zxRYmy1atXk5GRUbSdkZFRlHiK27ZtG3/961957733im4LlTZ27FguvPDCEp25jz/+OA899BC7du3iww8/BKBbt27ceeedXH/99Wzfvp2PPvqIzp07A/DNN9/w8ssv89prr5Gens6jjz5Ku3bt9nnN+xJYIjCzsUBvoKmZ5QJ3AKkA7v4kMAnoD+QA24FLg4pFROJn7NixRffFBw0axNixY+nRo0e5o1uKl1988cXUqVOHzMxM/vGPf5Qo37ZtGwUFBcyaNWu/4jn99NNL9CtUJFJ/QKS477jjDoYPH069evXKPde4ceN4/vnnS5RdffXVXH311bz00kuMGjWK5557jj59+jB9+nROOukk0tPTOfHEE0lJCX1U79y5k7S0NGbMmMH48eO57LLL+PTTT6O6looElgjcffA+9jtwdVDvLyIl7eubexA2bNjAhx9+yLx58zAzCgoKMDPuv/9+mjRpwo8//lii/saNG2natGnR9osvvkhWVlaZ87744ot069aNESNGcPXVVzN+/HiOPvpoZs6cSbdu3SqMaX9aBBkZGeTm5hZt5+bm0qJFizLHTps2jVdffZWbbrqJTZs2kZSURFpaGsOGDQNg9uzZ5Ofn87Of/SxiTIMGDeLKK68s2h45ciQjR44EQn0Je771Z2RkcMEFFwAwcOBALr20cr4/a64hEQnMq6++ypAhQ1ixYgXLly9n1apVtGnThs8++4x27dqxZs0aFi5cCMCKFSuYPXs23bt3j+rcqampjBo1iqlTp7Jw4UJuvPFG7r33XpYsWQJAYWEhDz30UJnj9rQISv+UTgIAhx9+OPXr12fq1Km4O//+978599xzy9T79NNPWb58OcuXL+e6667j1ltvLUoCEGoVDR5c8rvx0qVLi15PnDix6MO+oKCADRs2ADBnzhzmzJlDnz59ADjvvPOKbiF98skntG/fPqq/q33RFBMiEpixY8cyYsSIEmUXXHABL730Er169eKFF17g0ksvJS8vj9TUVJ566ikaNmwY9fnr1KnDDTfcwIMPPsjTTz/NI488wuDBg9m+fTtmxoABAw76Gp544gkuueQSduzYQb9+/YpGBU2YMIEZM2Zw11137fMcr7zyCpMmTSpR9thjj/H++++TmppK48aNee6554DQ/FDFO5RfeOGFoltDI0aM4OKLL+bhhx+mXr16PPXUUwd9fQBW0ZjYqigrK8sPdGGaW8bP5f2F3zN95FmVHJVI1bRw4UI6deoU7zAkxiL93s1spruXvc+Gbg2JiCQ8JQIRkQSnRCBSw1W3279ycA7k961EIFKDpaWlsWHDBiWDBLFnPYK0tLT9Ok6jhkRqsD3j4A90skapfvasULY/lAhEarDU1NT9WqlKEpNuDYmIJDglAhGRBKdEICKS4JQIREQSnBKBiEiCUyIQEUlwSgQiIgku0ERgZn3NbLGZ5ZjZiAj7G5vZa2Y2x8y+MrMuQcYjIiJlBZYIzCwZeBzoB3QGBptZ51LVbgWy3f0YYAjw96DiERGRyIJsEfQEctx9mbvvAsYBpZf26Qx8AODui4BMM2seYEwiIlJKkImgJbCq2HZuuKy42cD5AGbWEzgC2L9JMkRE5KAEmQgsQlnpKRBHA43NLBu4BvgayC9zIrMrzGyGmc3Q5FkiIpUryEnncoFWxbYzgDXFK7j7FuBSADMz4NvwD6XqjQHGQGipyoDiFRFJSEG2CKYD7cysjZnVAgYBE4pXMLNG4X0AfwCmhJODiIjESGAtAnfPN7NhwDtAMvCMu883s6Hh/U8CnYB/m1kBsAC4PKh4REQkskDXI3D3ScCkUmVPFnv9JdAuyBhERKRiCfVk8QcLv2f91p0UFqqbQURkj4RKBOu27gRgV0FhnCMREak6EioR7JFkkUa2iogkpoRMBCIispcSgYhIglMiEBFJcAmVCIaf1T7eIYiIVDkJlQhSkkOdxKt+3B7nSEREqo6ESgTuoecHfvXEF3GORESk6kioRLArP/T8wI/bd8c5EhGRqiOhEsHOcCKolZxQly0iUqGE+kTckwhqpyTUZYuIVCihPhF35hcAsHVnPuvD002IiCS6xEoEu/fOMXTps1/FMRIRkaojsRJB/t5E8N2mvDhGIiJSdSRUIsjbXVD0ul5aoEsxiIhUGwmVCIq3CA6ppUQgIgIBJwIz62tmi80sx8xGRNjf0MzeNLPZZjbfzC4NMp49ncWgFoGIyB6BJQIzSwYeB/oBnYHBZta5VLWrgQXu3g3oDfyt2GL2la54i6BebSUCEREItkXQE8hx92XuvgsYB5xbqo4D9c3MgHrARiA/qICKjxpSIhARCQkyEbQEVhXbzg2XFfcY0AlYA8wF/ujuZdaRNLMrzGyGmc1Yv379AQe0u9gSlYfUTuG1r3O5/pXsAz6fiEhNEGQiiLQeZOlV438OZAMtgO7AY2bWoMxB7mPcPcvds9LT0w84oOJrFddPS2H4y7MZP2v1AZ9PRKQmCDIR5AKtim1nEPrmX9ylwHgPyQG+BToGFVB+wd48VLdWclBvIyJSrQSZCKYD7cysTbgDeBAwoVSdlcCZAGbWHOgALAsqoOK3hoq/FhFJZIH1mLp7vpkNA94BkoFn3H2+mQ0N738SuBt41szmErqVdLO7/xBUTMU//B//6Jug3kZEpFoJdOiMu08CJpUqe7LY6zVAnyBjKO6sTs0Z/7X6BEREikuoJ4tHX3BMvEMQEalyEioR1NI6BCIiZeiTUUQkwSVcInj5ihPK3Td12QYu+tdUjSgSkYSScPMsZGUeWmK7TmroeYIXp61g5GvzAFi/dSctGtWJeWwiIvGQcC2C0nbsLmD7rvyiJAAw7dsNcYxIRCS2Ej4RAPR5eEqJ7eyVm+ITiIhIHCgRALk/7iixrdtCIpJIlAgiyGhcN94hiIjEjBJBBFe/NIuvvt0Y7zBERGJCiaAYKzZx9oPvLo5fICIiMZRwiSDSIgl7vHPdqUWvG6Slsn1XYIuliYhUGQmXCJKSjOcu61mm/JozjsKLLZvz/sLv6Xz7OzGMTEQkPhIuEQA0q1+7TFmvdums25oXh2hEROIr4Z4sBuh0eAPe/mMvWjSqQ7c73wUgo3Ed6qcl5F+HiCS4hP3k63R4yaWRmzdIIznJuKVfR5KTjFETF8YpMhGR2Ar01pCZ9TWzxWaWY2YjIuy/0cyywz/zzKzAzA6NdK6gPHJhd/p3PYzkpFA38n+fdiR/6NWWszo1i2UYIiJxE1iLwMySgceBswktZD/dzCa4+4I9ddz9AeCBcP1fAMPdPaYD+M87tiXnHduyTHl6/doR+xJERGqaIFsEPYEcd1/m7ruAccC5FdQfDIwNMB4REYkgyETQElhVbDs3XFaGmdUF+gL/KWf/FWY2w8xmrF+/vtIDjWT1pjzWbd3Jui15XP9yNn94bnpM3ldEJNaC7CyO9OyWRygD+AXweXm3hdx9DDAGICsrq7xzVKopS0IJp+e9H8Ti7URE4ibIFkEu0KrYdgawppy6g6gGt4Xmr9kc7xBERCpdkIlgOtDOzNqYWS1CH/YTSlcys4bAacAbAcay3x676NgyZTt2FcQhEhGRYEWVCMzsZDN7z8yWmNkyM/vWzJZVdIy75wPDgHeAhcAr7j7fzIaa2dBiVQcC77r7tgO9iCA0OWTviKEerRsB8KsnvyxTb27uZtreMpGfdmpeIhGpnsx937fczWwRMByYCRR9LXb3mK/pmJWV5TNmzIjJez3+UQ7bd+VzfJsmDHnmKwC+va8/Vmya0swREwG44tS2NG+QxhvZq5kw7JSYxCciEi0zm+nuWZH2RdtZvNnd367EmKqFq08/qkzZTzvzqZ+WSt7uAtZv3VlUPmbK3gZSzrqtNKxTi6b1apVIGiIiVVG0ieAjM3sAGA8Uffq5+6xAoqqCjm7RgPlrtvDKjFwuPSmTTrdPprzG1FkPhdZAzmxSl49vPD2GUYqI7L9oE8Hx4T+LNyscOKNyw6m6jkyvx/w1W7j7rQXc/daCEvuuOeMo/vFhTpljlm/YHqvwREQOWFSJwN0T/mtt2/RDIpa/8t8n0rPNoaSlJtO7QzoDHv0sxpGJiBycaEcNNTSzh/Y83WtmfwsP+0wYpfsLfv2zDL69rz892xxatP/oFg3pXGpW0yXfb41ZjCIiByLa5wieAbYCvwn/bAH+N6igqqLU5CQuPr41XVs2ZNK1vXjg190idgRP+mMvlo8eULTd5+EpsQxTRGS/RTt8NNvdu++rLBZiOXz0YOTtLqDjnycD8N+nteWWfp3iHJGIJLKKho9G2yLYYWZFg+PN7GRgR2UEV1OlpSYXvf6fT5bxrykVPn8nIhI30SaCK4HHzWy5ma0AHgOG7uOYhFd8FbR7JmnFMxGpmqIdNZQNdDOzBuHtLUEGVVO8/cdebNq+i+53vQfAuq15NKufFueoRERKqjARmNlv3f0FM7u+VDkA7v5QgLHVCI3q1ip6feH/TOWjP/WOXzAiIhHs69bQnsHz9cv5kShcfkobANo3rxfnSEREyopq1FBVUl1GDZW2Z3I6oMTwUhGRWDjoUUNmdr+ZNTCzVDP7wMx+MLPfVm6YiePcxz9nx64CMkdMJHPERPJ2a50DEYmfaEcN9Ql3EJ9DaOWx9sCNgUVVA3018kyOyQg9jD171SY63T65aF/puYtERGIp2kSQGv6zPzC2vLWFpXzN6qcx5MTMiPtenLaS3QWFsQ1IRCQs2kTwZnhxmizgAzNLB/L2dZCZ9TWzxWaWY2YjyqnT28yyzWy+mX0SfejVz69+lsFnN5/ONWeE5i2a85c+Rfsefm8J81ZvZld+yYSQX1Co1c9EJFBRdxabWWNgi7sXmFldoIG7r62gfjKwBDib0O2k6cBgd19QrE4j4Augr7uvNLNm7r6uojiqa2dxecbPyuX6V2YXbfdq15TnLw/N+r1q43Z63f9RmWOev7wnvdqlxyxGEan+KuosrjARmNkZ7v6hmZ0fab+7j6/g2BOBv7j7z8Pbt4SPua9YnauAFu5+W1RXQs1LBIWFTttbJ+33cUvv6UdqcrQNOhFJdAczaui08J+/iPBzzj6ObQmsKradGy4rrj3Q2Mw+NrOZZjYk0onM7Io9U2CvX79+H29bvSQlGe8NP7Xc/Xf+8mju+EVn0uvX5vxj9/71tRuZcCuHikhAAnuOwMx+Dfzc3f8Q3v4d0NPdrylW5zFC/Q5nAnWAL4EB7r6kvPPWtBZBcXNzN/OLxz7jpCObsHHbLv4+6Fg6HFbyub3Pc37g4qemlTl29h19aFgntUy5iAhUznME94bv5+/Zbmxmo/ZxWC7Qqth2BrAmQp3J7r7N3X8ApgDdoompJuqa0ZDlowfw0n+dwOTrTi2TBABOPqop9wzsUqa8253vxiJEEamBor3J3M/dN+3ZcPcfCQ0lrch0oJ2ZtTGzWsAgYEKpOm8AvcwsJdwBfTygaTr34eLjj+CrW8+kS8sGDDpub64tLKxeT4mLSNUQbSJINrPaezbMrA5Qu4L6uHs+MAx4h9CH+yvuPt/MhprZ0HCdhcBkYA7wFfCUu8/b/8tIPM0apPHWNb0YfcExRWVtb53Euq37HNUrIlJCtCuU3QT8ktDylA5cBkxw9/uDDa+smtxHcKBWbtjOqQ+UHWY67ooTaFgnlY6H1cfM2Jq3m0KHgY9/zpW9j+TXWa0inE1EaqIDHj5a6iR9gbMAA95193cqL8ToKRFEtnZzHifc98F+H3d8m0O57/yunPG3Tzih7aG89IcTSEoquxaziFRvlZUIjgDaufv74fv5ye6+tRLjjIoSQfk2bd/Fr578klv6deTy5w7876hBWgof3NCb9PoV3v0TkWrkoBOBmf0XcAVwqLsfaWbtgCfd/czKDXXflAiiV1joOLBh206a1U9j6fdbOapZPcyMzdt3c9VLM/k8ZwMAg3u2ZuxXK0scP/uOPny8eB3dWzXiiCaHRHgHEakuKiMRZAM9gWnufmy4bK67d63MQKOhRFC51m3NI71e7aJV5xat3ULfRz4tU+/jP/WmTq1kmjfQUpsi1dFBP0cA7HT3XcVOmEKo01iquWb104qSAEDHwxrw8IVlH+Xo/eDHHH/vB2SOmMgr01dpqKpIDRJti+B+YBMwBLgGuApY4O4jA40uArUIYmtr3m66/qX8h9Ue+k03zu+REcOIRORAVMatIQP+APQhNGroHUJj/mP+tVCJIL7+/eVybn9jfpny9Pq1Wb91J1//+WxWb9pB3u4CsjIPjUOEIhLJQSUCM0sC5rh72XkN4kCJoGrYviufv769iOe+XFFhvVeHnqiEIFIFVEaL4EXgFndfuc/KAVMiqFq+35LHmCnL6NWuKZf87/Ry6902oBNHptfj9I7NYhidiOxRGYngQ+A4QtNAbNtT7u6/rKwgo6VEUD1sydtN34ensGZz2SkvlozqR60UraUgEkuVkQhOi1Tu7jFfWlKJoHqZtmwDD767mLVb8li1cUdR+XvDT6Vd8/q4e4lRSyISjINZoSwNGAocBcwFng5PJhc3SgTV1878AjrcNrlM+S39OjKwR0ua1dczCiJBOZhE8DKwG/gU6AescPc/BhJllJQIqr8Ot73NzvzCiPvaNj2EnfmFPPHbHnRp0VDzHolUkoNJBEVPD4cfIvvK3XsEE2Z0lAhqlt8/8xWfLCl/+dHZt/ehYV2tvCZysCpKBCn7OHb3nhfunq97uVLZnrusZ9HriXO+I3vVjyxau5VPl/4AQLe7Sj7Mln372TSqWyumMYrUdPtqERSwd5SQEVpXeHv4tbt7g8AjLEUtgsRQWOi0vXVShXV+d8IR3H1elXi8RaTKq5RpqKsKJYLEk19QyKYdu8ka9X6ZfYc1SGPqrTGfBFek2qmMSecO9I37mtliM8sxsxER9vc2s81mlh3+uT3IeKR6SklOomm92iwfPYDlowewZFQ//j6oOwBrt+SROWIiy9b/FN8gRaqxffURHDAzSwYeB84GcoHpZjbB3ReUqvqpu58TVBxS89RKSeLc7i1ZsWE7D723BIAz/lb2kZapt5zJYQ01JFVkX4JsEfQEctx9WXgK63HAuQG+nySYa89sx/LRA8rdf8J9H/Dzh6fEMCKR6imwFgHQElhVbDsXOD5CvRPNbDawBviTu5eZ2tLMriC0QhqtW7cOIFSpzkong135hbS/7W0AFn+/lcwRE0lOMnLu6QdQ5knmwkLHrGy5SKIIMhFE+l9Vumd6FnCEu/9kZv2B14F2ZQ5yHwOMgVBncSXHKTVMrZQklo8ewMoN2zn1gY8AKCh02tyydxRS7ZSkMg+1/bJbC+4Z2IVN23fTslEdPcwmCSPIRJALtCq2nUHoW38Rd99S7PUkM/unmTV19x8CjEsSROsmdVk+egDrtuTR894PSuyL9GTzhNlrmDC7xD9Rnrkkiw6HNeCwBmkkKzFIDRVkIpgOtDOzNsBqYBBwUfEKZnYY8L27u5n1JNRnsSHAmCQBNWuQVuL20aqN22l8SC3SUpJISQ51k02et5ahL8ykQVoKW/L2Tqd12bMlhyqfdGQTXrj8eLUWpEYJ9DmC8O2eR4Bk4Bl3v8fMhgK4+5NmNgy4EsgHdgDXu/sXFZ1TzxFI0PJ2F5CanMSwl2Yxb83mErOmFjfw2JYMP6s9dWsn07Re7RhHKbJ/9ECZSCVYtXE7ve7/qMI6o87rwq+zMqidkhyjqESio0QgUokKC53ZuZu4b9IiftqZz4LvtlRY/42rT6ZN+iHUr52ikUkSN0oEIgHbtH0XT3z8Df8zZVlU9f/3kuO0bKfElBKBSIztWXntxv+bzda8fCbPX1th/S4tG/DQb7rTvnn9GEUoiUaJQKQKeX7qCv78+rxy9z86+Fh+2a1FDCOSRKBEIFJFuTvzVm/h+leyWbqu/Inz2jevR7tm9Xnowm7qiJYDokQgUk38Z2YuN/zfbA5rkMbaLXn7rN+heX0uPK4VFx3fmrRUJQgpnxKBSDX20858ctb9xMX/msq2XQUV1h11Xhd+e8IRMYpMqhMlApEaxt1Zv3Un7y9cx62vzS233s+Pbs7d53WhWX1Nx53olAhEEsC81Zu5/LnpfL9lZ8T9vdo15fnLI00ALIlAiUAkAS1b/xN/e28JE+d8V2bf5ae04c/ndI5DVBIvSgQiCczdmbVyExc8EXkar9+e0JrFa7fy0G+60+rQujGOTmJFiUBEiny6dD2/e/qrcvff+cujGXLiEZoOo4ZRIhCRiGat/JE6qck89lFOmVtIp7ZP57qz2tE9o5Gm3a4BlAhEJCrvL/ieP/y77P+vgce25Oa+HTmsoUYfVVdKBCKyXzZt38X4Wau5660FEff/a0gWZ3ZsppZCNaJEICIHbN7qzfzfjFU89+WKiPtv6deR/+rVVkmhiotbIjCzvsDfCa1Q9pS7jy6n3nHAVOBCd3+1onMqEYjEj7vz5pzvuHbs12X2XXdWO371swwyGmvkUVUUl0RgZsnAEuBsQgvZTwcGu/uCCPXeA/IILWepRCBSDbg7M1b8yK+f/LLMvjqpyVx4XCtG9OuoOZCqiIoSQVKA79sTyHH3Ze6+CxgHnBuh3jXAf4B1AcYiIpXMzDgu81CW3dufZy89ju6tGhXt27G7gGe/WE7HP08mc8REduxjjiSJr5QAz90SWFVsOxco8Xy7mbUEBgJnAMeVdyIzuwK4AqB169aVHqiIHLikJKN3h2b07rB3xbU1m3Zwz6SFRUNSO90+mf5dD+OGPh04Mr1evEKVcgTZIojUc1T6PtQjwM3uXuHXBXcf4+5Z7p6Vnp5eWfGJSEBaNKrD4xf1YOk9/YrKJs1dy5l/+4TMERP5Zn35ay9I7AXZIsgFWhXbzgDWlKqTBYwLP8HYFOhvZvnu/nqAcYlIjKQmJ7F89ABmr9rEuOkrGftV6CbBmX/7hDqpybwx7GQtz1kFBNlZnEKos/hMYDWhzuKL3H1+OfWfBd5SZ7FIzVVQ6Pz5jXm8NG1lUdm7w0+lXbN6mtIiYHHpLHb3fGAY8A6wEHjF3eeb2VAzGxrU+4pI1ZWcZNw7sCvf3tefjMZ1AOjz8BTa3DKJzBETmbJkfZwjTEx6oExE4ubZz7/lrrcWUFjqY+jnRzdn+Nnt6XhYg/gEVgPpyWIRqfLemrOGYS+VfFDtz+d05vJT2sQpopqlokQQZGexiEjUzjmmBecc04LN23dz4ZgvWbR2K3e/tYC83QVcffpR8Q6vRlOLQESqpN89PY1Pl/5Qouz960/lqGYaZXQg4vVksYjIAXv+8uOZMOxkurZsWFR21kNTyBwxkUfeX0J+QWEco6tZ1CIQkSpv2858bnt9Hq99vbpE+RMX96Bf18PjFFX1os5iEakxPlmynt8/s3epzUMPqcXUW86kVopucFREt4ZEpMY4rX06y0cP4OELuwGwcdsu2t/2tia2OwhKBCJSLQ08NoOcYnMZdbo9NNPp5Hlr4xhV9aREICLVVkpyEjn39KNH60ZFZUNfmMnJoz+MX1DVkPoIRKTGeO6L5dwxoeR0Zved35XBPTV9vfoIRCQh/P6kTMZfdVKJslvGz6XNLRNZuzkvTlFVfUoEIlKj9GjdmOWjB7B89ADu/OXRALjDCfd9wM58dShHokQgIjXW70/K5Nv7+hdtd7htsh5Ei0CJQERqNDMrsVLaUSPfJnPERApKT3mawJQIRKTGS01OYvGoviWmqzjy1tAaCE98/A0Lv9sSx+jiT6OGRCSh5BcU0ufhKSz7YVuZfUvv6Udqcs38fhy3KSbMrC/wdyAZeMrdR5fafy5wN1AI5APXuftnFZ1TiUBEKsu2nfnc9eYCXp6xqkT507/P4sxOzeMUVTDikgjMLJnQmsVnE1rIfjow2N0XFKtTD9jm7m5mxxBazrJjRedVIhCRypZfUMjN/5nLf2bllihfdm9/kpJqxlrK8XqOoCeQ4+7L3H0XMA44t3gFd//J92aiQ4DqdZ9KRGqElOQk/vabbnxzb3/+eXGPovK2t07imc++jWNksRFkImgJFG9v5YbLSjCzgWa2CJgIXBbpRGZ2hZnNMLMZ69drcWsRCUZyktG/6+EsGbV3lNFdby0gc8RERr21oIIjq7cgE0Gk9lSZb/zu/lr4dtB5hPoLyh7kPsbds9w9Kz09vXKjFBEppVZKEstHD+CZS/beSXnqs2/JHDGRL3J+qHHPIgS5ZnEu0KrYdgawprzK7j7FzI40s6bu/kN59UREYuWMjs1ZPnoA89dsZsCjoXEsFz01rWj/e8NPpV3z6r90ZpAtgulAOzNrY2a1gEHAhOIVzOwoM7Pw6x5ALWBDgDGJiOy3o1s0ZMmoftzavyN9Ou8dTXT2w1O4ZuzXcYyscgQ9fLQ/8Aih4aPPuPs9ZjYUwN2fNLObgSHAbmAHcKOGj4pIdXDy6A9ZvWkHALVTkpj557OpVzvImywHR0tViogEYPaqTZz7+OdF260PrcuUm06PY0Tl0zTUIiIB6NaqEYvu7suR6YcAsHLjdjJHTGTasup1h1stAhGRSvDDTzvJGvV+0XbDOql8evPpNEhLjWNUe6lFICISsKb1arN89ABuP6czAJt37OaYv7xL97veZeO2XXGOrmJKBCIileiyU9qQc08/encIPfO0aftuetz9HpkjJjK1it4y0q0hEZGAFBY6E2av4bqXs0uUvznsFLpmNIx8UEB0a0hEJA6Skozzjm3J8tEDePuPvYrKf/HYZ/xryrIqs3SmEoGISAx0OrwBy0cPIOuIxgDcM2khHW6bzG+fmsb6rTvjGptuDYmIxNiitVsY8vRXrCuWABrVTSX79j6BvaduDYmIVCEdD2vAVyPPIvv2s7nmjKOAUKdy+9veZsWGsiunBU2JQEQkThrVrcUNfTqw6O6+HNGkLrvyCzntgY9ZtXF7TONQIhARibO01GQ+ufF0Bh0XmrC51/0fUVAYu9v2SgQiIlXEqPO6FL3u+8gUlv8Qm9tESgQiIlVESnIS397Xn/5dD2Ppup/o/eDH/PPjnMDfV4lARKQKMTP+efHPuDbciXz/5MVkjpjIhNnlrut10JQIRESqoOv7dOCta04p2r527Nfc9OrsQN6r6q6iICKS4Lq0bMjy0QNYvHYrD767mF7tglmzXYlARKSK63BYff41JOKzYJUi0FtDZtbXzBabWY6ZjYiw/2IzmxP++cLMugUZj4iIlBVYIjCzZOBxoB/QGRhsZp1LVfsWOM3djwHuBsYEFY+IiEQWZIugJ5Dj7svcfRcwDji3eAV3/8LdfwxvTgUyAoxHREQiCDIRtARWFdvODZeV53Lg7Ug7zOwKM5thZjPWr19fiSGKiEiQicAilEV8ZtrMTieUCG6OtN/dx7h7lrtnpacH02suIpKoghw1lAu0KradAZR5IsLMjgGeAvq5e9Vcx01EpAYLskUwHWhnZm3MrBYwCJhQvIKZtQbGA79z9yUBxiIiIuUIrEXg7vlmNgx4B0gGnnH3+WY2NLz/SeB2oAnwTzMDyC9v4QQREQlGtVuhzMzWAysO8PCmwA+VGE51oGtODLrmxHAw13yEu0fsZK12ieBgmNmMRGtx6JoTg645MQR1zZp0TkQkwSkRiIgkuERLBIk4hYWuOTHomhNDINecUH0EIiJSVqK1CEREpBQlAhGRBFcjE0EU6yCYmT0a3j/HzHrEI87KlIhrP+zrmovVO87MCszsV7GMLwjRXLOZ9TazbDObb2afxDrGyhbFv+2GZvammc0OX/Ol8YizspjZM2a2zszmlbO/8j+/3L1G/RB6ivkboC1QC5gNdC5Vpz+hmU4NOAGYFu+4Y3DNJwGNw6/7JcI1F6v3ITAJ+FW8447B77kRsABoHd5uFu+4Y3DNtwJ/Db9OBzYCteId+0Fc86lAD2BeOfsr/fOrJrYI9rkOQnj73x4yFWhkZofHOtBKlIhrP0Tzewa4BvgPsC6WwQUkmmu+CBjv7isB3L26X3c01+xAfQvNU1OPUCLIj22YlcfdpxC6hvJU+udXTUwE0ayDsL9rJVR1lbb2QzWyz2s2s5bAQODJGMYVpGh+z+2Bxmb2sZnNNLMhMYsuGNFc82NAJ0KzG88F/ujuhbEJLy4q/fOrJi5eH806CFGvlVBNHMjaD6cEGlHwornmR4Cb3b0gPKlhdRfNNacAPwPOBOoAX5rZVK++s/tGc80/B7KBM4AjgffM7FN33xJwbPFS6Z9fNTERRLMOQlRrJVQjibj2QzTXnAWMCyeBpkB/M8t399djEmHli/bf9g/uvg3YZmZTgG5AdU0E0VzzpcBoD91AzzGzb4GOwFexCTHmKv3zqybeGtrnOgjh7SHh3vcTgM3u/l2sA61Eibj2wz6v2d3buHumu2cCrwJXVeMkANH9234D6GVmKWZWFzgeWBjjOCtTNNe8klALCDNrDnQAlsU0ytiq9M+vGtci8OjWQZhEqOc9B9hO6BtFtRXlNdeotR+ivOYaJZprdveFZjYZmAMUAk+5e8RhiNVBlL/nu4FnzWwuodsmN7t7tZ2e2szGAr2BpmaWC9wBpEJwn1+aYkJEJMHVxFtDIiKyH5QIREQSnBKBiEiCUyIQEUlwSgQiIglOiUAkgvBspdlmNi88s2WjSj7/cjNrGn79U2WeW2R/KRGIRLbD3bu7exdCE4BdHe+ARIKiRCCyb18SntTLzI40s8nhCd0+NbOO4fLmZvZaeE782WZ2Urj89XDd+WZ2RRyvQaRcNe7JYpHKZGbJhKYveDpcNAYY6u5Lzex44J+EJjt7FPjE3QeGj6kXrn+Zu280szrAdDP7Tw2Y50lqGCUCkcjqmFk2kAnMJDSjZT1CC/z8X7HZTGuH/zwDGALg7gXA5nD5tWY2MPy6FdAOUCKQKkWJQCSyHe7e3cwaAm8R6iN4Ftjk7t2jOYGZ9QbOAk509+1m9jGQFkSwIgdDfQQiFXD3zcC1wJ+AHcC3ZvZrKFo7ds/azx8AV4bLk82sAdAQ+DGcBDoSWlZQpMpRIhDZB3f/mtBauYOAi4HLzWw2MJ+9yyb+ETg9PAPmTOBoYDKQYmZzCM2QOTXWsYtEQ7OPiogkOLUIREQSnBKBiEiCUyIQEUlwSgQiIglOiUBEJMEpEYiIJDglAhGRBPf/8AYASsVOZiAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the PRC\n",
    "plt.plot(recall, precision, label=f'AUPRC = {auprc:.5f}')\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "bb75dc0b-f929-4960-b7c6-29aa318a43ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# another way to account for when denominator is zero\n",
    "numerator = 2 * recall * precision\n",
    "denom = recall + precision\n",
    "f1_scores = np.divide(numerator, denom, out=np.zeros_like(denom), where=(denom!=0))\n",
    "max_f1 = np.max(f1_scores)\n",
    "max_f1_thresh = thresholds[np.argmax(f1_scores)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "a9497f0c-6060-4e01-a552-44924e732301",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best threshold:  0.61284286\n",
      "Best F1-Score:  0.51973265436028\n"
     ]
    }
   ],
   "source": [
    "print('Best threshold: ', max_f1_thresh)\n",
    "print('Best F1-Score: ', max_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a713026-edd8-460d-8220-059c863e0137",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}