task4-pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hackingforfuture Task 4 - Predictive Quality Data Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "some details on this hackathon... To be filled during the presentation phase."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Libs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import feature_engine as fe\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "from feature_engine.categorical_encoders import OneHotCategoricalEncoder\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "np.random.seed(48)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading file\n",
    "The file in our case is transformed into csv-file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time</th>\n",
       "      <th>Date</th>\n",
       "      <th>asm_Force #D</th>\n",
       "      <th>asm_Length #E</th>\n",
       "      <th>asm_Height #F</th>\n",
       "      <th>asm_Press-fit #G</th>\n",
       "      <th>asm_force #H</th>\n",
       "      <th>asm_Length #I</th>\n",
       "      <th>asm_Depth #J</th>\n",
       "      <th>asm_depth #K</th>\n",
       "      <th>...</th>\n",
       "      <th>fin_Tracking, Zone3 #BU</th>\n",
       "      <th>fin_Tracking, Zone4 #BV</th>\n",
       "      <th>fin_Tracking, Zone1, G #BW</th>\n",
       "      <th>fin_Tracking, Zone2, G #BX</th>\n",
       "      <th>fin_Tracking, Zone3, G #BY</th>\n",
       "      <th>fin_Tracking, Zone4, G #BZ</th>\n",
       "      <th>fin_Shape #CA</th>\n",
       "      <th>fin_pressure #CB</th>\n",
       "      <th>fin_flow #CC</th>\n",
       "      <th>fin_temperature #CD</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>02:02:09</td>\n",
       "      <td>03.01.18</td>\n",
       "      <td>0.149</td>\n",
       "      <td>102.52</td>\n",
       "      <td>0.215</td>\n",
       "      <td>12.747</td>\n",
       "      <td>1.094</td>\n",
       "      <td>135.43</td>\n",
       "      <td>1.8541</td>\n",
       "      <td>1.8517</td>\n",
       "      <td>...</td>\n",
       "      <td>96000.0</td>\n",
       "      <td>139700.0</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.08</td>\n",
       "      <td>V-shape</td>\n",
       "      <td>2.915</td>\n",
       "      <td>0.828</td>\n",
       "      <td>88.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>02:02:41</td>\n",
       "      <td>03.01.18</td>\n",
       "      <td>0.163</td>\n",
       "      <td>102.61</td>\n",
       "      <td>0.191</td>\n",
       "      <td>12.731</td>\n",
       "      <td>1.116</td>\n",
       "      <td>135.48</td>\n",
       "      <td>1.8583</td>\n",
       "      <td>1.8556</td>\n",
       "      <td>...</td>\n",
       "      <td>97800.0</td>\n",
       "      <td>138100.0</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.12</td>\n",
       "      <td>V-shape</td>\n",
       "      <td>2.933</td>\n",
       "      <td>1.068</td>\n",
       "      <td>89.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02:05:06</td>\n",
       "      <td>03.01.18</td>\n",
       "      <td>0.160</td>\n",
       "      <td>102.56</td>\n",
       "      <td>0.192</td>\n",
       "      <td>12.734</td>\n",
       "      <td>1.033</td>\n",
       "      <td>135.48</td>\n",
       "      <td>1.8533</td>\n",
       "      <td>1.8487</td>\n",
       "      <td>...</td>\n",
       "      <td>112100.0</td>\n",
       "      <td>140000.0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.15</td>\n",
       "      <td>V-shape</td>\n",
       "      <td>2.916</td>\n",
       "      <td>0.828</td>\n",
       "      <td>87.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>02:05:31</td>\n",
       "      <td>03.01.18</td>\n",
       "      <td>0.158</td>\n",
       "      <td>102.47</td>\n",
       "      <td>0.195</td>\n",
       "      <td>12.726</td>\n",
       "      <td>1.079</td>\n",
       "      <td>135.48</td>\n",
       "      <td>1.8578</td>\n",
       "      <td>1.8546</td>\n",
       "      <td>...</td>\n",
       "      <td>96000.0</td>\n",
       "      <td>140000.0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.18</td>\n",
       "      <td>Ascending</td>\n",
       "      <td>2.925</td>\n",
       "      <td>1.068</td>\n",
       "      <td>88.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>02:08:00</td>\n",
       "      <td>03.01.18</td>\n",
       "      <td>0.154</td>\n",
       "      <td>102.61</td>\n",
       "      <td>0.207</td>\n",
       "      <td>12.752</td>\n",
       "      <td>1.061</td>\n",
       "      <td>135.48</td>\n",
       "      <td>1.8594</td>\n",
       "      <td>1.8543</td>\n",
       "      <td>...</td>\n",
       "      <td>100500.0</td>\n",
       "      <td>137700.0</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.23</td>\n",
       "      <td>Ascending</td>\n",
       "      <td>2.904</td>\n",
       "      <td>0.816</td>\n",
       "      <td>87.29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 78 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Time      Date  asm_Force #D  asm_Length #E  asm_Height #F  \\\n",
       "0  02:02:09  03.01.18         0.149         102.52          0.215   \n",
       "1  02:02:41  03.01.18         0.163         102.61          0.191   \n",
       "2  02:05:06  03.01.18         0.160         102.56          0.192   \n",
       "3  02:05:31  03.01.18         0.158         102.47          0.195   \n",
       "4  02:08:00  03.01.18         0.154         102.61          0.207   \n",
       "\n",
       "   asm_Press-fit #G  asm_force #H  asm_Length #I  asm_Depth #J  asm_depth #K  \\\n",
       "0            12.747         1.094         135.43        1.8541        1.8517   \n",
       "1            12.731         1.116         135.48        1.8583        1.8556   \n",
       "2            12.734         1.033         135.48        1.8533        1.8487   \n",
       "3            12.726         1.079         135.48        1.8578        1.8546   \n",
       "4            12.752         1.061         135.48        1.8594        1.8543   \n",
       "\n",
       "   ...  fin_Tracking, Zone3 #BU  fin_Tracking, Zone4 #BV  \\\n",
       "0  ...                  96000.0                 139700.0   \n",
       "1  ...                  97800.0                 138100.0   \n",
       "2  ...                 112100.0                 140000.0   \n",
       "3  ...                  96000.0                 140000.0   \n",
       "4  ...                 100500.0                 137700.0   \n",
       "\n",
       "   fin_Tracking, Zone1, G #BW  fin_Tracking, Zone2, G #BX  \\\n",
       "0                        0.14                        0.12   \n",
       "1                        0.16                        0.14   \n",
       "2                        0.04                        0.03   \n",
       "3                        0.25                        0.20   \n",
       "4                        0.14                        0.13   \n",
       "\n",
       "   fin_Tracking, Zone3, G #BY  fin_Tracking, Zone4, G #BZ  fin_Shape #CA  \\\n",
       "0                        0.07                        0.08        V-shape   \n",
       "1                        0.06                        0.12        V-shape   \n",
       "2                        0.02                        0.15        V-shape   \n",
       "3                        0.07                        0.18      Ascending   \n",
       "4                        0.10                        0.23      Ascending   \n",
       "\n",
       "   fin_pressure #CB  fin_flow #CC  fin_temperature #CD  \n",
       "0             2.915         0.828                88.15  \n",
       "1             2.933         1.068                89.14  \n",
       "2             2.916         0.828                87.30  \n",
       "3             2.925         1.068                88.68  \n",
       "4             2.904         0.816                87.29  \n",
       "\n",
       "[5 rows x 78 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filename = 'csvdata.csv'\n",
    "df = pd.read_csv(filename)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Distinguish columns\n",
    "Due to my laziness to join data table, I created one universal dataframe with prepended column names: `asm_`, `int_`, and `fin_` which stand for assembly data, initial inspection data, and final inspection data respectively."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Ti',\n",
       " 'Da',\n",
       " 'asm_Force',\n",
       " 'asm_Length',\n",
       " 'asm_Height',\n",
       " 'asm_Press-fit',\n",
       " 'asm_force',\n",
       " 'asm_Length',\n",
       " 'asm_Depth',\n",
       " 'asm_depth',\n",
       " 'asm_Depth',\n",
       " 'asm_parallelism',\n",
       " 'asm_force',\n",
       " 'asm_Length',\n",
       " 'asm_force',\n",
       " 'asm_Length',\n",
       " 'asm_torque 1',\n",
       " 'asm_angle 1',\n",
       " 'asm_torque 2',\n",
       " 'asm_angle 2',\n",
       " 'asm_torque 3',\n",
       " 'asm_angle 3',\n",
       " 'asm_torque 1',\n",
       " 'asm_torque 2',\n",
       " 'asm_torque 3',\n",
       " 'asm_angle 1',\n",
       " 'asm_angle 2',\n",
       " 'asm_angle 3',\n",
       " 'asm_torque 1',\n",
       " 'asm_torque 2',\n",
       " 'asm_torque 3',\n",
       " 'asm_torque 4',\n",
       " 'asm_angle 1',\n",
       " 'asm_angle 2',\n",
       " 'asm_angle 3',\n",
       " 'asm_angle 4',\n",
       " 'asm_temperature',\n",
       " 'asm_force',\n",
       " 'asm_Length',\n",
       " 'asm_force',\n",
       " 'asm_Temperature',\n",
       " 'asm_angle',\n",
       " 'asm_torque',\n",
       " 'asm_angle',\n",
       " 'asm_torque',\n",
       " 'asm_clearance',\n",
       " 'int_weight',\n",
       " 'int_Angle position ',\n",
       " 'int_weight',\n",
       " 'int_Angle position',\n",
       " 'int_Tracking, Zone1',\n",
       " 'int_Tracking, Zone2',\n",
       " 'int_Tracking, Zone3',\n",
       " 'int_Tracking, Zone4',\n",
       " 'int_Tracking, Zone1, G',\n",
       " 'int_Tracking, Zone2, G',\n",
       " 'int_Tracking, Zone3, G',\n",
       " 'int_Tracking, Zone4, G',\n",
       " 'int_Shape',\n",
       " 'int_pressure',\n",
       " 'int_flow',\n",
       " 'int_temperature',\n",
       " 'fin_weight',\n",
       " 'fin_Angle',\n",
       " 'fin_weight',\n",
       " 'fin_Angle',\n",
       " 'fin_Tracking, Zoe1',\n",
       " 'fin_Tracking, Zone2',\n",
       " 'fin_Tracking, Zone3',\n",
       " 'fin_Tracking, Zone4',\n",
       " 'fin_Tracking, Zone1, G',\n",
       " 'fin_Tracking, Zone2, G',\n",
       " 'fin_Tracking, Zone3, G',\n",
       " 'fin_Tracking, Zone4, G',\n",
       " 'fin_Shape',\n",
       " 'fin_pressure',\n",
       " 'fin_flow',\n",
       " 'fin_temperature']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols = df.columns.tolist()\n",
    "# remove # from column names\n",
    "for idc, c in enumerate(cols):\n",
    "    for idx, char in enumerate(c):\n",
    "        if char == '#':\n",
    "            break\n",
    "    cols[idc] = c[:idx-1]\n",
    "cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Assembly data columns: 44\n",
      "Initial inspection data columns: 16\n",
      "Final inspection data columns: 16\n"
     ]
    }
   ],
   "source": [
    "def redefine_cols(cols):\n",
    "    cols_asm = [c for c in cols if 'asm_' == c[:4]]\n",
    "    cols_int = [c for c in cols if 'int_' == c[:4]]\n",
    "    cols_fin = [c for c in cols if 'fin_' == c[:4]]\n",
    "    return cols_asm, cols_int, cols_fin\n",
    "cols_asm, cols_int, cols_fin = redefine_cols(cols)\n",
    "print(\"Assembly data columns: {}\".format(len(cols_asm)))\n",
    "print(\"Initial inspection data columns: {}\".format(len(cols_int)))\n",
    "print(\"Final inspection data columns: {}\".format(len(cols_fin)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns = cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploratory Data Analysis\n",
    "This part involves EDA process. The first step is to find Nan."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ti                        0.000000\n",
       "Da                        0.000000\n",
       "asm_Force                 0.000000\n",
       "asm_Length                0.000000\n",
       "asm_Height                0.000000\n",
       "asm_Press-fit             0.000000\n",
       "asm_force                 0.000000\n",
       "asm_Length                0.000000\n",
       "asm_Depth                 0.000000\n",
       "asm_depth                 0.000000\n",
       "asm_Depth                 0.000000\n",
       "asm_parallelism           0.000000\n",
       "asm_force                 0.000000\n",
       "asm_Length                0.000000\n",
       "asm_force                 0.000000\n",
       "asm_Length                0.000000\n",
       "asm_torque 1              0.000000\n",
       "asm_angle 1               0.000000\n",
       "asm_torque 2              0.000000\n",
       "asm_angle 2               0.000000\n",
       "asm_torque 3              0.000000\n",
       "asm_angle 3               0.000000\n",
       "asm_torque 1              0.000000\n",
       "asm_torque 2              0.000000\n",
       "asm_torque 3              0.000000\n",
       "asm_angle 1               0.000000\n",
       "asm_angle 2               0.000000\n",
       "asm_angle 3               0.000000\n",
       "asm_torque 1              0.000000\n",
       "asm_torque 2              0.000000\n",
       "                            ...   \n",
       "int_weight                0.000000\n",
       "int_Angle position        0.000000\n",
       "int_Tracking, Zone1       0.000000\n",
       "int_Tracking, Zone2       0.000000\n",
       "int_Tracking, Zone3       0.000000\n",
       "int_Tracking, Zone4       0.000000\n",
       "int_Tracking, Zone1, G    0.000000\n",
       "int_Tracking, Zone2, G    0.000000\n",
       "int_Tracking, Zone3, G    0.000000\n",
       "int_Tracking, Zone4, G    0.000000\n",
       "int_Shape                 0.000000\n",
       "int_pressure              0.000000\n",
       "int_flow                  0.000000\n",
       "int_temperature           0.000000\n",
       "fin_weight                0.042976\n",
       "fin_Angle                 0.042976\n",
       "fin_weight                0.042976\n",
       "fin_Angle                 0.042976\n",
       "fin_Tracking, Zoe1        0.042976\n",
       "fin_Tracking, Zone2       0.042976\n",
       "fin_Tracking, Zone3       0.042976\n",
       "fin_Tracking, Zone4       0.042976\n",
       "fin_Tracking, Zone1, G    0.042976\n",
       "fin_Tracking, Zone2, G    0.042976\n",
       "fin_Tracking, Zone3, G    0.042976\n",
       "fin_Tracking, Zone4, G    0.042976\n",
       "fin_Shape                 0.042976\n",
       "fin_pressure              0.042976\n",
       "fin_flow                  0.042976\n",
       "fin_temperature           0.042976\n",
       "Length: 78, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Find Nan\n",
    "df.isnull().mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are around 5% of Nan in this data set. From the readme file, this is due to quality stuffs or whatever. The Nan are only in the final inspection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 16963 entries, 0 to 16962\n",
      "Data columns (total 78 columns):\n",
      "Ti                        16963 non-null object\n",
      "Da                        16963 non-null object\n",
      "asm_Force                 16963 non-null float64\n",
      "asm_Length                16963 non-null float64\n",
      "asm_Height                16963 non-null float64\n",
      "asm_Press-fit             16963 non-null float64\n",
      "asm_force                 16963 non-null float64\n",
      "asm_Length                16963 non-null float64\n",
      "asm_Depth                 16963 non-null float64\n",
      "asm_depth                 16963 non-null float64\n",
      "asm_Depth                 16963 non-null float64\n",
      "asm_parallelism           16963 non-null float64\n",
      "asm_force                 16963 non-null float64\n",
      "asm_Length                16963 non-null float64\n",
      "asm_force                 16963 non-null float64\n",
      "asm_Length                16963 non-null float64\n",
      "asm_torque 1              16963 non-null float64\n",
      "asm_angle 1               16963 non-null float64\n",
      "asm_torque 2              16963 non-null float64\n",
      "asm_angle 2               16963 non-null float64\n",
      "asm_torque 3              16963 non-null float64\n",
      "asm_angle 3               16963 non-null float64\n",
      "asm_torque 1              16963 non-null float64\n",
      "asm_torque 2              16963 non-null float64\n",
      "asm_torque 3              16963 non-null float64\n",
      "asm_angle 1               16963 non-null float64\n",
      "asm_angle 2               16963 non-null float64\n",
      "asm_angle 3               16963 non-null float64\n",
      "asm_torque 1              16963 non-null float64\n",
      "asm_torque 2              16963 non-null float64\n",
      "asm_torque 3              16963 non-null float64\n",
      "asm_torque 4              16963 non-null float64\n",
      "asm_angle 1               16963 non-null float64\n",
      "asm_angle 2               16963 non-null float64\n",
      "asm_angle 3               16963 non-null float64\n",
      "asm_angle 4               16963 non-null float64\n",
      "asm_temperature           16963 non-null float64\n",
      "asm_force                 16963 non-null float64\n",
      "asm_Length                16963 non-null float64\n",
      "asm_force                 16963 non-null float64\n",
      "asm_Temperature           16963 non-null float64\n",
      "asm_angle                 16963 non-null float64\n",
      "asm_torque                16963 non-null float64\n",
      "asm_angle                 16963 non-null float64\n",
      "asm_torque                16963 non-null float64\n",
      "asm_clearance             16963 non-null float64\n",
      "int_weight                16963 non-null float64\n",
      "int_Angle position        16963 non-null float64\n",
      "int_weight                16963 non-null float64\n",
      "int_Angle position        16963 non-null float64\n",
      "int_Tracking, Zone1       16963 non-null int64\n",
      "int_Tracking, Zone2       16963 non-null int64\n",
      "int_Tracking, Zone3       16963 non-null int64\n",
      "int_Tracking, Zone4       16963 non-null int64\n",
      "int_Tracking, Zone1, G    16963 non-null float64\n",
      "int_Tracking, Zone2, G    16963 non-null float64\n",
      "int_Tracking, Zone3, G    16963 non-null float64\n",
      "int_Tracking, Zone4, G    16963 non-null float64\n",
      "int_Shape                 16963 non-null object\n",
      "int_pressure              16963 non-null float64\n",
      "int_flow                  16963 non-null float64\n",
      "int_temperature           16963 non-null float64\n",
      "fin_weight                16234 non-null float64\n",
      "fin_Angle                 16234 non-null float64\n",
      "fin_weight                16234 non-null float64\n",
      "fin_Angle                 16234 non-null float64\n",
      "fin_Tracking, Zoe1        16234 non-null float64\n",
      "fin_Tracking, Zone2       16234 non-null float64\n",
      "fin_Tracking, Zone3       16234 non-null float64\n",
      "fin_Tracking, Zone4       16234 non-null float64\n",
      "fin_Tracking, Zone1, G    16234 non-null float64\n",
      "fin_Tracking, Zone2, G    16234 non-null float64\n",
      "fin_Tracking, Zone3, G    16234 non-null float64\n",
      "fin_Tracking, Zone4, G    16234 non-null float64\n",
      "fin_Shape                 16234 non-null object\n",
      "fin_pressure              16234 non-null float64\n",
      "fin_flow                  16234 non-null float64\n",
      "fin_temperature           16234 non-null float64\n",
      "dtypes: float64(70), int64(4), object(4)\n",
      "memory usage: 10.1+ MB\n"
     ]
    }
   ],
   "source": [
    "# Check data type\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Most of the data are number variables. There are `Da`, `Ti`, `int_Shape`, and `fin_Shape` which are categorical variables. These variables shall be encoded afterwards."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# After this point will be data processing. The initial dataframe shall be backed up first.\n",
    "df_bkup = df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Nan processing\n",
    "This section dedicates for Nan processing. The quick-dirty process is to remove the Nan out. If there is time left, data imputation can be done."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ti                        0.0\n",
       "Da                        0.0\n",
       "asm_Force                 0.0\n",
       "asm_Length                0.0\n",
       "asm_Height                0.0\n",
       "asm_Press-fit             0.0\n",
       "asm_force                 0.0\n",
       "asm_Length                0.0\n",
       "asm_Depth                 0.0\n",
       "asm_depth                 0.0\n",
       "asm_Depth                 0.0\n",
       "asm_parallelism           0.0\n",
       "asm_force                 0.0\n",
       "asm_Length                0.0\n",
       "asm_force                 0.0\n",
       "asm_Length                0.0\n",
       "asm_torque 1              0.0\n",
       "asm_angle 1               0.0\n",
       "asm_torque 2              0.0\n",
       "asm_angle 2               0.0\n",
       "asm_torque 3              0.0\n",
       "asm_angle 3               0.0\n",
       "asm_torque 1              0.0\n",
       "asm_torque 2              0.0\n",
       "asm_torque 3              0.0\n",
       "asm_angle 1               0.0\n",
       "asm_angle 2               0.0\n",
       "asm_angle 3               0.0\n",
       "asm_torque 1              0.0\n",
       "asm_torque 2              0.0\n",
       "                         ... \n",
       "int_weight                0.0\n",
       "int_Angle position        0.0\n",
       "int_Tracking, Zone1       0.0\n",
       "int_Tracking, Zone2       0.0\n",
       "int_Tracking, Zone3       0.0\n",
       "int_Tracking, Zone4       0.0\n",
       "int_Tracking, Zone1, G    0.0\n",
       "int_Tracking, Zone2, G    0.0\n",
       "int_Tracking, Zone3, G    0.0\n",
       "int_Tracking, Zone4, G    0.0\n",
       "int_Shape                 0.0\n",
       "int_pressure              0.0\n",
       "int_flow                  0.0\n",
       "int_temperature           0.0\n",
       "fin_weight                0.0\n",
       "fin_Angle                 0.0\n",
       "fin_weight                0.0\n",
       "fin_Angle                 0.0\n",
       "fin_Tracking, Zoe1        0.0\n",
       "fin_Tracking, Zone2       0.0\n",
       "fin_Tracking, Zone3       0.0\n",
       "fin_Tracking, Zone4       0.0\n",
       "fin_Tracking, Zone1, G    0.0\n",
       "fin_Tracking, Zone2, G    0.0\n",
       "fin_Tracking, Zone3, G    0.0\n",
       "fin_Tracking, Zone4, G    0.0\n",
       "fin_Shape                 0.0\n",
       "fin_pressure              0.0\n",
       "fin_flow                  0.0\n",
       "fin_temperature           0.0\n",
       "Length: 78, dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove nan\n",
    "df = df.dropna()\n",
    "df.isnull().mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Start Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:3940: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  errors=errors)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>int_Shape</th>\n",
       "      <th>fin_Shape</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ascending</td>\n",
       "      <td>V-shape</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>V-shape</td>\n",
       "      <td>V-shape</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>V-shape</td>\n",
       "      <td>V-shape</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>V-shape</td>\n",
       "      <td>Ascending</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Descending</td>\n",
       "      <td>Ascending</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    int_Shape  fin_Shape\n",
       "0   Ascending    V-shape\n",
       "1     V-shape    V-shape\n",
       "2     V-shape    V-shape\n",
       "3     V-shape  Ascending\n",
       "4  Descending  Ascending"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_col = 'fin_temperature'\n",
    "\n",
    "if 'Da' in df.columns:\n",
    "    df.drop(columns=['Da', 'Ti'], inplace=True)\n",
    "features_numerical = df.select_dtypes(include=['int64', 'float64']).columns.tolist()\n",
    "features_categorical = df.select_dtypes(include=['object']).columns.tolist()\n",
    "\n",
    "if label_col in features_categorical:\n",
    "    features_categorical.remove(label_col)\n",
    "if label_col in features_numerical:\n",
    "    features_numerical.remove(label_col)\n",
    "df[features_categorical].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asm_Force</th>\n",
       "      <th>asm_Length</th>\n",
       "      <th>asm_Length</th>\n",
       "      <th>asm_Length</th>\n",
       "      <th>asm_Length</th>\n",
       "      <th>asm_Length</th>\n",
       "      <th>asm_Height</th>\n",
       "      <th>asm_Press-fit</th>\n",
       "      <th>asm_force</th>\n",
       "      <th>asm_force</th>\n",
       "      <th>...</th>\n",
       "      <th>fin_Tracking, Zoe1</th>\n",
       "      <th>fin_Tracking, Zone2</th>\n",
       "      <th>fin_Tracking, Zone3</th>\n",
       "      <th>fin_Tracking, Zone4</th>\n",
       "      <th>fin_Tracking, Zone1, G</th>\n",
       "      <th>fin_Tracking, Zone2, G</th>\n",
       "      <th>fin_Tracking, Zone3, G</th>\n",
       "      <th>fin_Tracking, Zone4, G</th>\n",
       "      <th>fin_pressure</th>\n",
       "      <th>fin_flow</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.149</td>\n",
       "      <td>102.52</td>\n",
       "      <td>135.43</td>\n",
       "      <td>89.25</td>\n",
       "      <td>149.25</td>\n",
       "      <td>120.00</td>\n",
       "      <td>0.215</td>\n",
       "      <td>12.747</td>\n",
       "      <td>1.094</td>\n",
       "      <td>0.00747</td>\n",
       "      <td>...</td>\n",
       "      <td>54700.0</td>\n",
       "      <td>66200.0</td>\n",
       "      <td>96000.0</td>\n",
       "      <td>139700.0</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.08</td>\n",
       "      <td>2.915</td>\n",
       "      <td>0.828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.163</td>\n",
       "      <td>102.61</td>\n",
       "      <td>135.48</td>\n",
       "      <td>89.27</td>\n",
       "      <td>149.28</td>\n",
       "      <td>120.00</td>\n",
       "      <td>0.191</td>\n",
       "      <td>12.731</td>\n",
       "      <td>1.116</td>\n",
       "      <td>0.00692</td>\n",
       "      <td>...</td>\n",
       "      <td>52100.0</td>\n",
       "      <td>66700.0</td>\n",
       "      <td>97800.0</td>\n",
       "      <td>138100.0</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.12</td>\n",
       "      <td>2.933</td>\n",
       "      <td>1.068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.160</td>\n",
       "      <td>102.56</td>\n",
       "      <td>135.48</td>\n",
       "      <td>89.28</td>\n",
       "      <td>149.27</td>\n",
       "      <td>120.05</td>\n",
       "      <td>0.192</td>\n",
       "      <td>12.734</td>\n",
       "      <td>1.033</td>\n",
       "      <td>0.00687</td>\n",
       "      <td>...</td>\n",
       "      <td>50900.0</td>\n",
       "      <td>65000.0</td>\n",
       "      <td>112100.0</td>\n",
       "      <td>140000.0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.15</td>\n",
       "      <td>2.916</td>\n",
       "      <td>0.828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.158</td>\n",
       "      <td>102.47</td>\n",
       "      <td>135.48</td>\n",
       "      <td>89.28</td>\n",
       "      <td>149.28</td>\n",
       "      <td>120.05</td>\n",
       "      <td>0.195</td>\n",
       "      <td>12.726</td>\n",
       "      <td>1.079</td>\n",
       "      <td>0.00788</td>\n",
       "      <td>...</td>\n",
       "      <td>51400.0</td>\n",
       "      <td>65000.0</td>\n",
       "      <td>96000.0</td>\n",
       "      <td>140000.0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.18</td>\n",
       "      <td>2.925</td>\n",
       "      <td>1.068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.154</td>\n",
       "      <td>102.61</td>\n",
       "      <td>135.48</td>\n",
       "      <td>89.28</td>\n",
       "      <td>149.28</td>\n",
       "      <td>119.96</td>\n",
       "      <td>0.207</td>\n",
       "      <td>12.752</td>\n",
       "      <td>1.061</td>\n",
       "      <td>0.00688</td>\n",
       "      <td>...</td>\n",
       "      <td>55100.0</td>\n",
       "      <td>67400.0</td>\n",
       "      <td>100500.0</td>\n",
       "      <td>137700.0</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.23</td>\n",
       "      <td>2.904</td>\n",
       "      <td>0.816</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 161 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   asm_Force  asm_Length  asm_Length  asm_Length  asm_Length  asm_Length  \\\n",
       "0      0.149      102.52      135.43       89.25      149.25      120.00   \n",
       "1      0.163      102.61      135.48       89.27      149.28      120.00   \n",
       "2      0.160      102.56      135.48       89.28      149.27      120.05   \n",
       "3      0.158      102.47      135.48       89.28      149.28      120.05   \n",
       "4      0.154      102.61      135.48       89.28      149.28      119.96   \n",
       "\n",
       "   asm_Height  asm_Press-fit  asm_force  asm_force  ...  fin_Tracking, Zoe1  \\\n",
       "0       0.215         12.747      1.094    0.00747  ...             54700.0   \n",
       "1       0.191         12.731      1.116    0.00692  ...             52100.0   \n",
       "2       0.192         12.734      1.033    0.00687  ...             50900.0   \n",
       "3       0.195         12.726      1.079    0.00788  ...             51400.0   \n",
       "4       0.207         12.752      1.061    0.00688  ...             55100.0   \n",
       "\n",
       "   fin_Tracking, Zone2  fin_Tracking, Zone3  fin_Tracking, Zone4  \\\n",
       "0              66200.0              96000.0             139700.0   \n",
       "1              66700.0              97800.0             138100.0   \n",
       "2              65000.0             112100.0             140000.0   \n",
       "3              65000.0              96000.0             140000.0   \n",
       "4              67400.0             100500.0             137700.0   \n",
       "\n",
       "   fin_Tracking, Zone1, G  fin_Tracking, Zone2, G  fin_Tracking, Zone3, G  \\\n",
       "0                    0.14                    0.12                    0.07   \n",
       "1                    0.16                    0.14                    0.06   \n",
       "2                    0.04                    0.03                    0.02   \n",
       "3                    0.25                    0.20                    0.07   \n",
       "4                    0.14                    0.13                    0.10   \n",
       "\n",
       "   fin_Tracking, Zone4, G  fin_pressure  fin_flow  \n",
       "0                    0.08         2.915     0.828  \n",
       "1                    0.12         2.933     1.068  \n",
       "2                    0.15         2.916     0.828  \n",
       "3                    0.18         2.925     1.068  \n",
       "4                    0.23         2.904     0.816  \n",
       "\n",
       "[5 rows x 161 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[features_numerical].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train-test split\n",
    "Do `train_test_split` here to avoid information leakage. But first of all the label columns should be identified."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((16234, 76), (11364, 76), (4870, 76))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This train_test_split is done under an assumption that the sequence does not matter.\n",
    "# There will be another analysis that consider the sequence of data.\n",
    "\n",
    "nrows = df.shape[0]\n",
    "test_size = int(0.3*nrows)\n",
    "\n",
    "train = df.head(nrows-test_size)\n",
    "test = df.tail(test_size)\n",
    "\n",
    "df.shape, train.shape, test.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "def transform_label(data, label_col, ok_std=1.5):\n",
    "    label_mean = data[label_col].mean()\n",
    "    label_std = data[label_col].std()\n",
    "    data['label'] = (np.abs(data[label_col] - label_mean) < ok_std*label_std)*1\n",
    "    return data\n",
    "train = transform_label(train, label_col, ok_std=1.5)\n",
    "test = transform_label(test, label_col, ok_std=1.5)\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select features\n",
    "def redefine_cols(cols):\n",
    "    cols_asm = [c for c in cols if 'asm_' == c[:4]]\n",
    "    cols_int = [c for c in cols if 'int_' == c[:4]]\n",
    "    cols_fin = [c for c in cols if 'fin_' == c[:4]]\n",
    "    return cols_asm, cols_int, cols_fin\n",
    "cols_asm, cols_int, cols_fin = redefine_cols(train.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = train.drop(columns=[label_col]+['label'])\n",
    "X_train = X_train[cols_asm]\n",
    "X_test = test.drop(columns=[label_col]+['label'])\n",
    "X_test = X_test[cols_asm]\n",
    "y_train = train['label']\n",
    "y_test = test['label'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20328,)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Number of observations per bin')"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAELCAYAAAA2mZrgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGPZJREFUeJzt3XuUXWWZ5/HvkxgIgSRgRehAwGRovNCIXCKKMAgqmMgMCJ1GQFyKdIdpYWBEGJNpTANqd2wGRFyoHds0jJimEUXTEpqIExBFyAWiJgGaAKFThIGQhhAuQQjP/HFOHU+KStVOUftscur7WatW7fv5FavIU+9+3/3uyEwkSQIYUnUASdIbh0VBktRgUZAkNVgUJEkNFgVJUoNFQZLUYFGQJDVYFCRJDRYFSVLDm6oOsLXGjBmT48ePrzqGJG1TlixZ8lRmvqWv47a5ojB+/HgWL15cdQxJ2qZExKNFjvP2kSSpwaIgSWqwKEiSGra5PoWevPzyy3R2drJx48aqo5Rq+PDhjBs3jmHDhlUdRVKbaoui0NnZyciRIxk/fjwRUXWcUmQm69ato7OzkwkTJlQdR1KbaovbRxs3bqSjo6NtCwJARNDR0dH2rSFJ1WqLogC0dUHoMhh+RknVapuiIEl6/dqiT6G78dNuGtDrrZp5bK/7n3nmGebMmcNnP/vZrbruRz/6UebMmcPOO+/8euJJ7emi0VUnKOai9VUnGFBtWRRa7ZlnnuGb3/zma4rCpk2bGDp06BbPmzdvXtnRpNcY6D+ayrJqeNUJBieLwgCYNm0aDz30EAcccADDhg1jp512YuzYsSxdupQVK1bwsY99jNWrV7Nx40bOPfdcpk6dCvxhyo7nnnuOyZMnc/jhh3PnnXeyxx578JOf/IQddtih4p9M0mBjn8IAmDlzJnvvvTdLly7l0ksvZeHChXzlK19hxYoVAMyePZslS5awePFirrzyStatW/eaazz44IOcddZZLF++nJ133pkf/vCHrf4xJMmWQhkOOeSQzZ4luPLKK7nxxhsBWL16NQ8++CAdHR2bnTNhwgQOOOAAAA4++GBWrVrVsryS1MWiUIIdd9yxsXzbbbdx66238utf/5oRI0Zw5JFH9viswfbbb99YHjp0KC+++GJLskpSM28fDYCRI0eyYcOGHvetX7+eXXbZhREjRnD//fdz1113tTidJBXXli2FvoaQDrSOjg4OO+ww9ttvP3bYYQd22223xr5Jkybx7W9/m/3335+3v/3tvO9972tpNknaGm1ZFKowZ86cHrdvv/323HzzzT3u6+o3GDNmDMuWLWtsP//88wc8nyQV4e0jSVKDRUGS1GBRkCQ1WBQkSQ0WBUlSg0VBktTQnkNSB3rK3T6mxu3v1NkAV1xxBVOnTmXEiBH9TSdJA8aWwgDomjq7P6644gpeeOGFAU4kSf3Tni2FFmueOvvoo49m11135frrr+ell17ihBNO4OKLL+b555/npJNOorOzk02bNvHFL36RJ554gjVr1nDUUUcxZswYFixYUPWPImmQsygMgJkzZ7Js2TKWLl3K/PnzueGGG1i4cCGZyXHHHccvfvEL1q5dy+67785NN9VecLJ+/XpGjx7N5ZdfzoIFCxgzZkzFP4UkeftowM2fP5/58+dz4IEHctBBB3H//ffz4IMP8q53vYtbb72VL3zhC9xxxx2MHr2NvGpQ0qBSakshIiYBXweGAv+QmTO77d8LuAbYuX7MtMzcpt9RmZlMnz6dM8888zX7lixZwrx585g+fTrHHHMMM2bMqCChJG1ZaS2FiBgKXAVMBvYFTomIfbsddiFwfWYeCJwM9K+3tmLNU2d/5CMfYfbs2Tz33HMAPPbYYzz55JOsWbOGESNGcNppp3H++edzzz33vOZcSapaoZZCRLwfGN98fGb+nz5OOwRYmZkP169xHXA8sKLpmARG1ZdHA2sKpe5LH0NIB1rz1NmTJ0/m1FNP5dBDDwVgp5124tprr2XlypVccMEFDBkyhGHDhvGtb30LgKlTpzJ58mTGjh1rR7OkykVm9n5AxPeAvYGlwKb65szMc/o4bwowKTP/vL7+SeC9mXl20zFjgfnALsCOwIczc0kP15oKTAXYa6+9Dn700Uc323/ffffxzne+s9efo10Mpp9V5Rg/7aaqIxSyavipVUcopsV/hPZXRCzJzIl9HVekpTAR2Df7qh49ZOhhW/drnAJcnZmXRcShwPciYr/MfHWzkzJnAbMAJk6cuLU5JEkFFelTWAb8UT+u3Qns2bQ+jtfeHjoDuB4gM38NDAccmylJFSnSUhgDrIiIhcBLXRsz87g+zlsE7BMRE4DHqHUkd28P/jvwIeDqiHgntaKwtmD2zWQmET01TtrH1jfWJGnrFCkKF/Xnwpn5SkScDdxCbbjp7MxcHhGXAIszcy7weeA7EfE5areWPt2P21QMHz6cdevW0dHR0baFITNZt24dw4cPrzqKpDbWZ1HIzNv7e/H6Mwfzum2b0bS8Ajisv9fvMm7cODo7O1m7tl+NjG3G8OHDGTduXNUxJLWxLRaFiPhlZh4eERvYvIM4qI0+GrWFU1tu2LBhTJgwoeoYkrTN22JRyMzD699Hti6OJKlKRR9eOwg4nFqL4ZeZeW+pqSRJlehzSGpEzKA2P1EHtZFIV0fEhWUHkyS1XpGWwinAgZm5ESAiZgL3AF8uM5gkqfWKPLy2itrzA122Bx4qJY0kqVK9jT76BrU+hJeA5RHxs/r60cAvWxNPktRKvd0+Wlz/vgS4sWn7baWlkSRVqrchqde0MogkqXq+jlOS1GBRkCQ19FoUImJoRFzaqjCSpGr1WhQycxNwcLTr1KOSpM0UeXjtXuAnEfED4PmujZn5o9JSSZIqUaQovBlYB3ywaVsCFgVJajNF3qdweiuCSJKqV2RCvLdFxM8jYll9fX8nxJOk9lRkSOp3gOnAywCZ+Vtq71uWJLWZIkVhRGYu7LbtlTLCSJKqVaQoPBURe1N/JWdETAEeLzWVJKkSRUYfnQXMAt4REY8BjwCfKDWVJKkSRUYfPQx8OCJ2BIZk5obyY0mSqlBk9FFHRFwJ3AHcFhFfj4iO8qNJklqtSJ/CdcBa4E+BKfXlfy4zlCSpGoWeaM7MLzWtfzkiPlZWIElSdYq0FBZExMkRMaT+dRJwU9nBJEmtV6QonAnMofau5peo3U46LyI2RMSzZYaTJLVWkdFHI1sRRJJUPd+8JklqsChIkhosCpKkhiIPr+0dEdvXl4+MiHMiYufyo0mSWq1IS+GHwKaI+GPgu8AEaqORJEltpkhReDUzXwFOAK7IzM8BY8uNJUmqQpGi8HJEnAJ8Cvhpfduw8iJJkqpSpCicDhwKfCUzH4mICcC15caSJFWhyMNrK4BzmtYfAWaWGUqSVI0io48Oi4ifRcS/RcTDEfFIRDxc5OIRMSkiHoiIlRExbQvHnBQRKyJieUTYgS1JFSoyS+p3gc8BS4BNRS8cEUOBq4CjgU5gUUTMrbc8uo7ZB5gOHJaZT0fErlsTXpI0sIoUhfWZeXM/rn0IsLL+5jYi4jrgeGBF0zF/AVyVmU8DZOaT/fgcSdIAKTp19qURcWhEHNT1VeC8PYDVTeud9W3N3ga8LSJ+FRF3RcSkni4UEVMjYnFELF67dm2Bj5Yk9UeRlsJ7698nNm1L4IN9nBc9bMsePn8f4EhgHHBHROyXmc9sdlLmLGAWwMSJE7tfQ5I0QIqMPjqqn9fuBPZsWh8HrOnhmLsy82XgkYh4gFqRWNTPz5QkvQ5FRh+NjojLu27fRMRlETG6wLUXAftExISI2A44GZjb7ZgfA0fVP2cMtdtJhUY2SZIGXpE+hdnABuCk+tezwD/2dVJ9aoyzgVuA+4DrM3N5RFwSEcfVD7sFWBcRK4AFwAWZuW7rfwxJ0kAo0qewd2b+adP6xRGxtMjFM3MeMK/bthlNywmcV/+SJFWsSEvhxYg4vGslIg4DXiwvkiSpKkVaCn8JXFPvRwjgP4BPlxlKklSNIqOPlgLvjohR9fVnS08lSarEFotCRJyWmddGxHndtgOQmZeXnE2S1GK9tRR2rH8f2cM+HyCTpDa0xaKQmX9fX7w1M3/VvK/e2SxJajNFRh99o+A2SdI2rrc+hUOB9wNv6davMAoYWnYwSVLr9dansB2wU/2Y5n6FZ4EpZYaSJFWjtz6F24HbI+LqzHy0hZkkSRUp8vDaCxFxKfAnwPCujZnZ19TZkqRtTJGO5u8D9wMTgIuBVTi1tSS1pSJFoSMzvwu8nJm3Z+ZngPeVnEuSVIEit49ern9/PCKOpfainHHlRZIkVaVIUfhyfTK8z1N7PmEU8LlSU0mSKlGkKNydmeuB9dTfkiZJak9F+hTujIj5EXFGROxSeiJJUmX6LAqZuQ9wIbUhqUsi4qcRcVrpySRJLVekpUBmLszM84BDqL1k55pSU0mSKtFnUYiIURHxqYi4GbgTeJxacZAktZkiHc2/AX4MXJKZvy45jySpQr0WhYgYCtxYv3UkSWpzvd4+ysxNwLtblEWSVLEit4+WRsRc4AfA810bM/NHpaWSJFWiSFF4M7AOaJ4VNQGLgiS1mT6LQmae3oogkqTqFRmS+raI+HlELKuv7x8RF5YfTZLUakUeXvsOMJ36bKmZ+Vvg5DJDSZKqUaQojMjMhd22vVJGGElStYoUhaciYm9qnctExBRqTzVLktpMkdFHZwGzgHdExGPAI4AT4klSGyoy+uhh4MMRsSMwJDM3lB9LklSFIqOPzo2IUcALwNci4p6IOKb8aJKkVivSp/CZzHwWOAbYFTgdmFlqKklSJYoUhah//yjwj5n5m6ZtkqQ2UqQoLImI+dSKwi0RMRJ4tdxYkqQqFBl9dAZwAPBwZr4QER3UbiFJktpMkXc0vwqMB2ZExGXAEfWnmvsUEZMi4oGIWBkR03o5bkpEZERMLBpckjTwiow++ibw34DfAcuAMyPiqgLnDQWuAiYD+wKnRMS+PRw3EjgHuHvrokuSBlqR20cfAPbLzK4nmq+hViD6cgiwsv6cAxFxHXA8sKLbcV8C/g44v2hoSVI5inQ0PwDs1bS+J1Dk9tEewOqm9c76toaIOBDYMzN/2tuFImJqRCyOiMVr164t8NGSpP7YYkshIv6F2nxHo4H7IqJrUrxDgDsLXLunYavZdP0hwNeAT/d1ocycRW2qDSZOnJh9HC5J6qfebh/979d57U5qrYou44A1Tesjgf2A2yIC4I+AuRFxXGYufp2fLUnqhy0Whcy8vWs5InYD3lNfXZiZTxa49iJgn4iYADxG7R0MpzZdfz0wpukzbgPOtyBIUnWKjD46CVgI/BlwEnB3ffrsXmXmK8DZwC3AfcD1mbk8Ii6JiONeX2xJUhmKjD76K+A9Xa2DiHgLcCtwQ18nZuY8YF63bTO2cOyRBbJIkkpUZPTRkG63i9YVPE+StI0p0lL414i4Bfin+vrH6fbXvySpPRR5yc4FEXEicDi1YaazMvPG0pNJklquSEuBzPwR8KOSs0iSKmbfgCSpwaIgSWrYYlGIiJ/Xv3+1dXEkSVXqrU9hbER8ADiuPsPpZnMZZeY9pSaTJLVcb0VhBjCN2pxFl3fbl8AHywolSapGb3Mf3QDcEBFfzMwvtTCTJKkiRZ5T+FJ9rqIj6ptu6+v9B5KkbVORCfH+FjiX2hvTVgDn1rdJktpMkYfXjgUOyMxXofE6znuB6WUGkyS1XtHnFHZuWh5dRhBJUvWKtBT+Frg3IhZQG5Z6BLYSJKktFelo/qf6W9HeQ60ofCEz/1/ZwSRJrVd0QrzHgbklZ5EkVcy5jyRJDRYFSVJDr0UhIoZExLJWhZEkVavXolB/NuE3EbFXi/JIkipUpKN5LLA8IhYCz3dtzMzjSkslSapEkaJwcekpJElvCEWeU7g9It4K7JOZt0bECGBo+dEkSa1WZEK8vwBuAP6+vmkP4MdlhpIkVaPIkNSzgMOAZwEy80Fg1zJDSZKqUaQovJSZv+9aiYg3UXvzmiSpzRQpCrdHxP8CdoiIo4EfAP9SbixJUhWKFIVpwFrgd8CZwDzgwjJDSZKqUWT00av1F+vcTe220QOZ6e0jSWpDfRaFiDgW+DbwELWpsydExJmZeXPZ4SRJrVXk4bXLgKMycyVAROwN3ARYFCSpzRTpU3iyqyDUPQw8WVIeSVKFtthSiIgT64vLI2IecD21PoU/Axa1IJskqcV6u330X5uWnwA+UF9eC+xSWiJJUmW2WBQy8/RWBpEkVa/I6KMJwH8Hxjcf79TZktR+iow++jHwXWpPMb+6NRePiEnA16nNqvoPmTmz2/7zgD8HXqF2W+ozmfno1nyGJGngFCkKGzPzyq29cEQMBa4CjgY6gUURMTczVzQddi8wMTNfiIi/BP4O+PjWfpYkaWAUKQpfj4i/BuYDL3VtzMx7+jjvEGBlZj4MEBHXAccDjaKQmQuajr8LOK1gbklSCYoUhXcBnwQ+yB9uH2V9vTd7AKub1juB9/Zy/Bls4YG4iJgKTAXYay9fFy1JZSlSFE4A/lPz9NkFRQ/bepwzKSJOAybyh2Gvm5+UOQuYBTBx4kTnXZKkkhR5ovk3wM79uHYnsGfT+jhgTfeDIuLDwF8Bx2XmS933S5Jap0hLYTfg/ohYxOZ9Cn0NSV0E7FMf0voYcDJwavMBEXEgtdd8TspMp86QpIoVKQp/3Z8LZ+YrEXE2cAu1IamzM3N5RFwCLM7MucClwE7ADyIC4N99/kGSqlPkfQq39/fimTmP2kt5mrfNaFr+cH+vLUkaeEWeaN7AHzqItwOGAc9n5qgyg0mSWq9IS2Fk83pEfIzaMwiSpDZTZPTRZjLzx/T9jIIkaRtU5PbRiU2rQ6g9T+CzApLUhoqMPmp+r8IrwCpq01VIktpMkT4F36sgSYNEb6/jnLGlfUBm5pdKyCNJqlBvLYXne9i2I7WJ6zoAi4IktZneXsd5WddyRIwEzgVOB64DLtvSeZKkbVevfQoR8WbgPOATwDXAQZn5dCuCSZJar7c+hUuBE6lNWf2uzHyuZakkSZXo7eG1zwO7AxcCayLi2frXhoh4tjXxJEmt1FufwlY/7SxJ2rb5D78kqcGiIElqsChIkhqKzH2kfhg/7aaqIxSyauaxVUeQ9AZiURjsLhpddYJiLlpfdQJpUPD2kSSpwaIgSWqwKEiSGiwKkqQGi4IkqcGiIElqsChIkhosCpKkBouCJKnBoiBJarAoSJIaLAqSpAaLgiSpwaIgSWqwKEiSGiwKkqQGi4IkqcGiIElqsChIkhpKLQoRMSkiHoiIlRExrYf920fEP9f33x0R48vMI0nqXWlFISKGAlcBk4F9gVMiYt9uh50BPJ2Zfwx8DfhqWXkkSX0rs6VwCLAyMx/OzN8D1wHHdzvmeOCa+vINwIciIkrMJEnqxZtKvPYewOqm9U7gvVs6JjNfiYj1QAfwVPNBETEVmFpffS4iHigl8SAUMIZu/73fkC72b4XBxt/NAffWIgeVWRR6+i+V/TiGzJwFzBqIUNpcRCzOzIlV55C683ezGmXePuoE9mxaHwes2dIxEfEmYDTwHyVmkiT1osyisAjYJyImRMR2wMnA3G7HzAU+VV+eAvzfzHxNS0GS1Bql3T6q9xGcDdwCDAVmZ+byiLgEWJyZc4HvAt+LiJXUWggnl5VHW+RtOb1R+btZgfAPc0lSF59oliQ1WBQkSQ0WBUlSg0VhkImIt0fEoRExrD4VifSG4u9ltexoHkQi4kTgb4DH6l+Lgasz89lKg0lARLwtM/+tvjw0MzdVnWkwsqUwSETEMODjwBmZ+SHgJ9QeHPyfETGq0nAa9CLivwBLI2IOQGZussVQDYvC4DIK2Ke+fCPwU2A74FQnIlRVImJH4GzgfwC/j4hrwcJQFYvCIJGZLwOXAydGxH/OzFeBXwJLgcMrDadBLTOfBz4DzAHOB4Y3F4Yqsw1GFoXB5Q5gPvDJiDgiMzdl5hxgd+Dd1UbTYJaZazLzucx8CjgT2KGrMETEQRHxjmoTDh5lzpKqN5jM3BgR36c2E+30+v9oLwG7AY9XGk6qy8x1EXEmcGlE3E9tmpyjKo41aFgUBpnMfDoivgOsoPYX2UbgtMx8otpk0h9k5lMR8Vtqb248OjM7q840WDgkdRCrd+JlvX9BesOIiF2A64HPZ+Zvq84zmFgUJL0hRcTwzNxYdY7BxqIgSWpw9JEkqcGiIElqsChIkhosCpKkBouC1IuIeK6P/eMjYtlWXvPqiJjy+pJJ5bAoSJIaLApSARGxU0T8PCLuiYjfRcTxTbvfFBHXRMRvI+KGiBhRP+fgiLg9IpZExC0RMbai+FJhFgWpmI3ACZl5ELV5eC5rmm787cCszNwfeBb4bP39Fd8ApmTmwcBs4CsV5Ja2inMfScUE8DcRcQTwKrAHtYkEAVZn5q/qy9cC5wD/CuwH/KxeO4bipIPaBlgUpGI+AbwFODgzX46IVcDw+r7u0wIktSKyPDMPbV1E6fXz9pFUzGjgyXpBOAp4a9O+vSKi6x//U6i9vOgB4C1d2yNiWET8SUsTS/1gUZCK+T4wMSIWU2s13N+07z7gU/Wpnt8MfCszfw9MAb4aEb+h9oa797c4s7TVnBBPktRgS0GS1GBRkCQ1WBQkSQ0WBUlSg0VBktRgUZAkNVgUJEkN/x9PE8PdK4MIoAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "t1 = train.groupby(['label'])['label'].count() / len(train)\n",
    "t2 = test.groupby(['label'])['label'].count() / len(test)\n",
    "\n",
    "tmp = pd.concat([t1, t2], axis=1)\n",
    "tmp.columns = ['train', 'test']\n",
    "tmp.plot.bar()\n",
    "plt.xticks(rotation=45)\n",
    "plt.ylabel('Number of observations per bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "smote = SMOTE(ratio='minority')\n",
    "X_sm, y_sm = smote.fit_sample(X_train, y_train)\n",
    "# t1 = np.array([y_sm.sum() / y_sm.shape[0], 1- y_sm.sum() / y_sm.shape[0]])\n",
    "# t2 = y_test.groupby(y_test).count() / len(y_test)\n",
    "# tmp = pd.concat([t1, t2], axis=1)\n",
    "# tmp.columns = ['train', 'test']\n",
    "# tmp.plot.bar()\n",
    "# plt.xticks(rotation=45)y\n",
    "# plt.ylabel('Number of observations per bin')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transform data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "numeric_transformer = Pipeline(steps=[\n",
    "    ('scaler', MinMaxScaler())\n",
    "])\n",
    "catergorical_transformer = Pipeline(steps=[\n",
    "    ('onehot', OneHotCategoricalEncoder(\n",
    "                top_categories=None,\n",
    "                variables = features_categorical,\n",
    "                drop_last=False))\n",
    "])\n",
    "\n",
    "preprocessor = ColumnTransformer(\n",
    "    transformers=[\n",
    "        ('num', numeric_transformer, features_numerical),\n",
    "        ('cat', catergorical_transformer, features_categorical)\n",
    "    ]\n",
    "    )\n",
    "clf = Pipeline(steps=[\n",
    "    ('preprocessor', preprocessor),\n",
    "    ('classifier', RandomForestClassifier())\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {\"classifier__max_depth\": [3, None],\n",
    "              \"classifier__max_features\": [1, 3, 10],\n",
    "              \"classifier__min_samples_split\": [2, 3, 10]}\n",
    "grid_search = GridSearchCV(clf, param_grid, cv=5, iid=False, scoring='f1_micro')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:542: FutureWarning: From version 0.22, errors during fit will result in a cross validation score of NaN by default. Use error_score='raise' if you want an exception raised or error_score=np.nan to adopt the behavior from version 0.22.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Specifying the columns using strings is only supported for pandas DataFrames",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\compose\\_column_transformer.py\u001b[0m in \u001b[0;36m_get_column_indices\u001b[1;34m(X, key)\u001b[0m\n\u001b[0;32m    637\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 638\u001b[1;33m             \u001b[0mall_columns\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    639\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'numpy.ndarray' object has no attribute 'columns'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-74-ec6212a16846>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mgrid_search\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_sm\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_sm\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[0;32m    720\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mresults_container\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    721\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 722\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_run_search\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevaluate_candidates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    723\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    724\u001b[0m         \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mresults_container\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36m_run_search\u001b[1;34m(self, evaluate_candidates)\u001b[0m\n\u001b[0;32m   1189\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_run_search\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mevaluate_candidates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1190\u001b[0m         \u001b[1;34m\"\"\"Search all candidates in param_grid\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1191\u001b[1;33m         \u001b[0mevaluate_candidates\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mParameterGrid\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1192\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1193\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36mevaluate_candidates\u001b[1;34m(candidate_params)\u001b[0m\n\u001b[0;32m    709\u001b[0m                                \u001b[1;32mfor\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    710\u001b[0m                                in product(candidate_params,\n\u001b[1;32m--> 711\u001b[1;33m                                           cv.split(X, y, groups)))\n\u001b[0m\u001b[0;32m    712\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    713\u001b[0m                 \u001b[0mall_candidate_params\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcandidate_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m    915\u001b[0m             \u001b[1;31m# remaining jobs.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    916\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 917\u001b[1;33m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    918\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    919\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[1;34m(self, iterator)\u001b[0m\n\u001b[0;32m    757\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    758\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 759\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    760\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    761\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    714\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    715\u001b[0m             \u001b[0mjob_idx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 716\u001b[1;33m             \u001b[0mjob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    717\u001b[0m             \u001b[1;31m# A job can complete so quickly than its callback is\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    718\u001b[0m             \u001b[1;31m# called before we get here, causing self._jobs to\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[1;34m(self, func, callback)\u001b[0m\n\u001b[0;32m    180\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    181\u001b[0m         \u001b[1;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 182\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    183\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    184\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    547\u001b[0m         \u001b[1;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    548\u001b[0m         \u001b[1;31m# arguments in memory\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 549\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    550\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    551\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    223\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    224\u001b[0m             return [func(*args, **kwargs)\n\u001b[1;32m--> 225\u001b[1;33m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[0;32m    226\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    227\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    223\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    224\u001b[0m             return [func(*args, **kwargs)\n\u001b[1;32m--> 225\u001b[1;33m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[0;32m    226\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    227\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\u001b[0m in \u001b[0;36m_fit_and_score\u001b[1;34m(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, return_estimator, error_score)\u001b[0m\n\u001b[0;32m    526\u001b[0m             \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    527\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 528\u001b[1;33m             \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    529\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    530\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\pipeline.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[0;32m    263\u001b[0m             \u001b[0mThis\u001b[0m \u001b[0mestimator\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    264\u001b[0m         \"\"\"\n\u001b[1;32m--> 265\u001b[1;33m         \u001b[0mXt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfit_params\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    266\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_final_estimator\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    267\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_final_estimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\pipeline.py\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[0;32m    228\u001b[0m                 Xt, fitted_transformer = fit_transform_one_cached(\n\u001b[0;32m    229\u001b[0m                     \u001b[0mcloned_transformer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mXt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 230\u001b[1;33m                     **fit_params_steps[name])\n\u001b[0m\u001b[0;32m    231\u001b[0m                 \u001b[1;31m# Replace the transformer of the step with the fitted\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    232\u001b[0m                 \u001b[1;31m# transformer. This is necessary when loading the transformer\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\memory.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    340\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    341\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 342\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    343\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    344\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcall_and_shelve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\pipeline.py\u001b[0m in \u001b[0;36m_fit_transform_one\u001b[1;34m(transformer, X, y, weight, **fit_params)\u001b[0m\n\u001b[0;32m    612\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_fit_transform_one\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    613\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'fit_transform'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 614\u001b[1;33m         \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    615\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    616\u001b[0m         \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\compose\\_column_transformer.py\u001b[0m in \u001b[0;36mfit_transform\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    445\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_validate_transformers\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    446\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_validate_column_callables\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 447\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_validate_remainder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    448\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    449\u001b[0m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_fit_transform_one\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\compose\\_column_transformer.py\u001b[0m in \u001b[0;36m_validate_remainder\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    299\u001b[0m         \u001b[0mcols\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    300\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mcolumns\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_columns\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 301\u001b[1;33m             \u001b[0mcols\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_get_column_indices\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    302\u001b[0m         \u001b[0mremaining_idx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_columns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcols\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    303\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\compose\\_column_transformer.py\u001b[0m in \u001b[0;36m_get_column_indices\u001b[1;34m(X, key)\u001b[0m\n\u001b[0;32m    638\u001b[0m             \u001b[0mall_columns\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    639\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 640\u001b[1;33m             raise ValueError(\"Specifying the columns using strings is only \"\n\u001b[0m\u001b[0;32m    641\u001b[0m                              \"supported for pandas DataFrames\")\n\u001b[0;32m    642\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msix\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstring_types\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Specifying the columns using strings is only supported for pandas DataFrames"
     ]
    }
   ],
   "source": [
    "grid_search.fit(X_sm, y_sm)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "best = grid_search.best_estimator_\n",
    "y_pred = best.predict(X_test)\n",
    "cm = confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[ 113  381]\n",
      " [  32 4344]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaMAAAFgCAYAAAAW6RbuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xm8XPP9x/HXO4tIBAlBNQlBg6JERChBag3V4ld+jdrFWvrjp1pL9WdptbpRW9VaS1tLLZWqItagZBVLrCGWSJAgCAlZPr8/zvemk+veuTNx556Zue+nx3lkzvd8zznfc2fMZ77L+R5FBGZmZnnqkHcBzMzMHIzMzCx3DkZmZpY7ByMzM8udg5GZmeXOwcjMzHLnYFQlJHWV9A9JH0j62xc4zn6S7mnNsuVF0jaSXqiW80nqJykkdWqrMtUKSa9K2jG9PlXSFRU4xx8l/bS1j2vVQb7PqDySvgecAKwPfARMAs6OiEe+4HEPAH4AbBURC75wQaucpAD6R8SUvMvSHEmvAodFxL1pvR8wFejc2u+RpKuBaRFxWmset600/lu1wvEOTscb0hrHs+rnmlEZJJ0A/B74BbAasAbwB2CPVjj8msCL7SEQlcK1j8rx39aqUkR4KWEBVgTmAPsUydOFLFhNT8vvgS5p21BgGvBD4B1gBnBI2nYm8BkwP51jBHAG8OeCY/cDAuiU1g8GXiGrnU0F9itIf6Rgv62AccAH6d+tCrY9CPwMeDQd5x6gVzPX1lD+HxeUf09gN+BF4D3g1IL8g4HHgNkp70XAMmnb6HQtH6fr/W7B8U8C3gKua0hL+6yTzjEwrX8ZmAUMLeG9uwb4YXrdO537+2n9K+m4anS+64BFwNxUxh8XvAcHAa+n8/+kxPd/ifclpUU6/xHpvf8snesfzVxHAEcBLwHvAxfzn9aNDsBpwGvp/bkWWLHRZ2dEKvfogrRDgDfS8Y4CNgeeSu/bRQXnXge4H3g3XfdfgB4F218FdkyvzyB9dtP7PqdgWQCckbadDLxM9tl7FtgrpX8VmAcsTPvMTulXAz8vOOfhwJT0/o0EvlzK38pLdS65F6BWFmBY+h+pU5E8ZwGPA6sCqwD/Bn6Wtg1N+58FdCb7Ev8E6Jm2L/4fuJn1hi+PTsBywIfAemnb6sCG6fXBpC89YKX0P+IBab990/rKafuD6ctgXaBrWj+nmWtrKP//pfIfDswE/gosD2yYvkDWTvk3A7ZM5+0HPAccX3C8AL7SxPF/Rfal3pWC4JDyHJ6O0w24G/htie/doaQveOB76ZpvLNh2e0EZCs/3KukLttF7cHkq3ybAp8BXS3j/F78vTf0NaPRF28x1BHAH0IOsVj4TGFZwHVOAtYHuwK3AdY3KfS3ZZ6drQdofgWWBndP79/dU/t5kQW27dIyvADul92YVsoD2+6b+VjT67BbkGZDKvGla34fsR0UHsh8kHwOrF/l7Lf4bAduTBcWBqUwXAqNL+Vt5qc7FzXSlWxmYFcWb0fYDzoqIdyJiJlmN54CC7fPT9vkRcSfZr771lrI8i4CNJHWNiBkRMbmJPN8EXoqI6yJiQURcDzwPfKsgz58i4sWImAvcRPaF0Zz5ZP1j84EbgF7A+RHxUTr/ZGBjgIiYEBGPp/O+ClwKbFfCNZ0eEZ+m8iwhIi4n+6U7hiwA/6SF4zV4CNhGUgdgW+DXwNZp23ZpeznOjIi5EfEk8CRZUIKW3//WcE5EzI6I14EH+M/7tR9wbkS8EhFzgFOA4Y2a5M6IiI8b/W1/FhHzIuIesmBwfSr/m8DDwKYAETElIkal92YmcC4tv5+LSVqFLND9ICKeSMf8W0RMj4hFEXEj2Xs7uMRD7gdcFRETI+LTdL1fT/16DZr7W1kVcjAq3btArxba279M1kzS4LWUtvgYjYLZJ2S/YssSER+T/ZI8Cpgh6Z+S1i+hPA1l6l2w/lYZ5Xk3Iham1w1faG8XbJ/bsL+kdSXdIektSR+S9bP1KnJsgJkRMa+FPJcDGwEXpi+hFkXEy2SBfwCwDdkv5umS1mPpglFzf7OW3v/WUM65O5H1bTZ4o4njNX7/mns/V5V0g6Q30/v5Z1p+P0n7dgZuBv4aETcUpB8oaZKk2ZJmk72vJR2TRtebAvC7LP1n23LmYFS6x8iaMfYskmc62UCEBmuktKXxMVlzVIMvFW6MiLsjYieyGsLzZF/SLZWnoUxvLmWZynEJWbn6R8QKwKlk/TLFFB3aKak7WT/MlcAZklYqozwPAXuT9Vu9mdYPBHqSjYgsuzxNKPb+L/F+Slri/VyKc5Vy7gUsGVy+yDl+mfbfOL2f+9Py+9ngQrJ+ocUjBSWtSfaZPZas2bgH8EzBMVsq6xLXK2k5staLtvhsWwU4GJUoIj4g6y+5WNKekrpJ6ixpV0m/TtmuB06TtIqkXin/n5fylJOAbSWtIWlFsmYIACStJunb6X/AT8l+9S9s4hh3AutK+p6kTpK+C2xAVjOotOXJ+rXmpFrb0Y22v03Wv1GO84EJEXEY8E+y/g4AJJ0h6cEi+z5E9sU3Oq0/SDaU/pGC2l5j5Zax2Pv/JLChpAGSliXrV/ki52rq3P8raa0UtH9B1i/WWqMzlycNJpDUG/hRKTtJOpKs9vm9iFhUsGk5soAzM+U7hKxm1OBtoI+kZZo59F+BQ9LfswvZ9Y5JTcJWgxyMyhAR55LdY3Qa2f9Eb5B9wf09Zfk5MJ5sNNLTwMSUtjTnGgXcmI41gSUDSAeyUXnTyUYSbQd8v4ljvAvsnvK+SzYibPeImLU0ZSrTiWSDBT4i+wV8Y6PtZwDXpCaa/27pYJL2IBtEclRKOgEYKGm/tN6XbFRgcx4i+0JtCEaPkNVURje7R1YbOC2V8cSWykiR9z8iXiQb4HAvWd9I4/vSrgQ2SOf6O+W7imwE4Giy0ZXzyIJtazmTbLDAB2Q/BG4tcb99yYLsdElz0nJqRDwL/I6sxeFt4Gss+f7dT9YH+Zakz31eI+I+4KfALWSjNdcBhi/NhVl18E2vVhckTQJ2SAHYzGqMg5GZmeXOzXRmZpY7ByMzM8udg5GZmeWuXU2Y2KtXr1hjzX55F8Oq1MJF7j+14p6aNHFWRKySdzkAOq6wZsSCz01UUlTMnXl3RAyrUJG+kHYVjNZYsx+j/z0272JYlZozzxOmW3Gr9+jSeEaT3MSCuXRZr8W7IpYwb9LFpc5w0ebaVTAyM6sfAtVPT4uDkZlZLRKgUmdkqn4ORmZmtco1IzMzy51rRmZmli/3GZmZWTVwzcjMzHIlXDMyM7O8yTUjMzOrAnVUM6qfKzEzs5rlmpGZWa1yM52ZmeXLQ7vNzCxvng7IzMyqgmtGZmaWLzfTmZlZNejgZjozM8tTnc3AUD9XYmbW3kjlLSUfVh0lPSHpjrS+lqQxkl6SdKOkZVJ6l7Q+JW3vV3CMU1L6C5J2aemcDkZmZjUp9RmVs5TuOOC5gvVfAedFRH/gfWBESh8BvB8RXwHOS/mQtAEwHNgQGAb8QVLHYid0MDIzq1UVqBlJ6gN8E7girQvYHrg5ZbkG2DO93iOtk7bvkPLvAdwQEZ9GxFRgCjC42HkdjMzMalX5NaNeksYXLEc0cdTfAz8GFqX1lYHZEbEgrU8DeqfXvYE3ANL2D1L+xelN7NMkD2AwM6tFZfYDJbMiYlDzh9TuwDsRMUHS0IbkJrJGC9uK7dMkByMzs1rV+qPptga+LWk3YFlgBbKaUg9JnVLtpw8wPeWfBvQFpknqBKwIvFeQ3qBwnya5mc7MrFa1cp9RRJwSEX0ioh/ZAIT7I2I/4AFg75TtIOD29HpkWidtvz8iIqUPT6Pt1gL6A2OLnds1IzOzmtSmMzCcBNwg6efAE8CVKf1K4DpJU8hqRMMBImKypJuAZ4EFwDERsbDYCRyMzMxqVQUnSo2IB4EH0+tXaGI0XETMA/ZpZv+zgbNLPZ+b6czMLHeuGZmZ1aI6mw7IwcjMrCZ51m4zM6sGfriemZnlzjUjMzPLnWtGZmaWK7nPyMzMqoFrRmZmljc5GJmZWZ6Eg5GZmeVNNP2ghhrlYGRmVpPkmpGZmeXPwcjMzHLnYGRmZrlzMDIzs3zV2QCG+rl918zMapZrRmZmNUgeTWdmZtXAwcjMzHLnYGRmZrlzMDIzs3zV2Wg6ByMzsxpVTzUjD+02M6tBDaPpyllaPKa0rKSxkp6UNFnSmSn9aklTJU1Ky4CULkkXSJoi6SlJAwuOdZCkl9JyUEvnds3IzKxGVaBm9CmwfUTMkdQZeETSv9K2H0XEzY3y7wr0T8sWwCXAFpJWAk4HBgEBTJA0MiLeb+7ErhmZmdUqlbm0IDJz0mrntESRXfYArk37PQ70kLQ6sAswKiLeSwFoFDCs2LkdjMzMapFYmma6XpLGFyxHfO6wUkdJk4B3yALKmLTp7NQUd56kLimtN/BGwe7TUlpz6c1yM52ZWY1aima6WRExqFiGiFgIDJDUA7hN0kbAKcBbwDLAZcBJwFk0Xd+KIunNcs3IzKxGtfYAhkIRMRt4EBgWETNSU9ynwJ+AwSnbNKBvwW59gOlF0pvlYGRmVoMqNJpulVQjQlJXYEfg+dQPhLKD7Ak8k3YZCRyYRtVtCXwQETOAu4GdJfWU1BPYOaU1y810Zma1qvVvM1oduEZSR7LKyk0RcYek+yWtks44CTgq5b8T2A2YAnwCHAIQEe9J+hkwLuU7KyLeK3ZiByMzs1qk1h/aHRFPAZs2kb59M/kDOKaZbVcBV5V6bgcjM7Ma5RkYzMzMWpFrRmZmNaqeakYORmZmtap+YpGDkZlZraqnmlHV9BlJOljSl9Pr29LMsFMkfVAwU+xWRfZ/pGEmWTOzelfuPUbVHriqqWZ0MNmNVNMjYi8ASUOBEyNi9xzLZWZWlao9wJSjojUjSSdIeiYtx0vqJ+mZgu0nSjpD0t5kU43/JdWAuhY55k4pz9OSLpe0TCWvwcysWtVTzahiwUjSZmR3424BbAkcDvRsKm96RsZ4YL+IGBARc5s5Zjeym6i+ExFfA7oBn5t1ttE+RzTMUDtr5sylvh4zs6rTyo+QyFMla0ZDgNsi4uP0fIxbgW2+4DG/CrwUES+n9WuBbYvtEBGXRcSgiBjUa5VVvuDpzcyqRz3VjCrZZ9TUlfdgyQC4bCsc08ys/anAdEB5qmTNaDSwp6RukpYD9gL+BawqaeX0cKbCgQkfAcu3cMxngf6S1k7r+wMPtXK5zcyqngCpvKWaVaxmFBETJV0NjE1JV0TEOElnAWOAqcDzBbtcDfxR0lzg6031G0XEJ5JGALemWWXHAJdX6hrMzKpX9Te9laOiQ7sj4lzg3EZpFwAXNJH3FuCWRmkPkj3cqTDtHuCeJvYf8oULbGZWQ+ooFlXVfUZmZlYG14zMzCxfNdAPVA4HIzOzGiSgQ4f6iUYORmZmNaqeakZVM1GqmZm1X64ZmZnVKA9gMDOzfHkAg5mZ5S2bgaF+opH7jMzMalLrP1xP0rKSxkp6UtJkSWem9LUkjZH0kqQbGx7dI6lLWp+StvcrONYpKf0FSbu0dG4HIzOzGlWBuek+BbaPiE2AAcAwSVsCvwLOi4j+wPvAiJR/BPB+RHwFOC/lQ9IGwHBgQ2AY8Ic0hVuzHIzMzGpUa9eMIjMnrXZOSwDbAzen9GuAPdPrPdI6afsOyk60B3BDRHwaEVOBKcDgYud2MDIzq0Vl1opSLOrV8LDRtHzu4aSSOkqaBLwDjAJeBmZHxIKUZRrQO73uDbwBkLZ/AKxcmN7EPk3yAAYzsxq0lAMYZkXEoGIZImIhMEBSD+A2soeafi5bQTGa2tZcerNcMzIzq1GVfJ5RRMwme2rClkAPSQ2Vlz7A9PR6GtA3K4s6ASsC7xWmN7FPkxyMzMxqVAVG062SakRI6grsCDwHPADsnbIdBNyeXo9M66Tt90dEpPThabTdWkB//vNsuya5mc7MrEZV4Daj1YFr0si3DsBNEXGHpGeBGyT9HHgCuDLlvxK4TtIUshrRcICImCzpJrKncy8AjknNf81yMDIzq0Vq/ZteI+IpYNMm0l+hidFwETEP2KeZY50NnF3quR2MzMxqUDaAIe9StB4HIzOzmlRaP1CtcDAyM6tRdRSLPJrOzMzy55qRmVmNcjOdmZnly88zMjOzvPl5RlYzjj5iBGv1/RKDB268OO22W/7G5pt+jRW6dmLihPGL08ePG8tWgwey1eCBfH3zTRl5+215FNna0Lx589h1+63ZYetBbLflAH7zi7MAePih+9lp2y3YccjmfHvYN5j6yhQAHnv0YXbadgv6rNyNO26/Nc+iW9LaMzDkycGoju13wEHcNvLOJdK+uuFG/OXGm9l6yLZLpG+w4UaM/vdY/j12IreNvJPjjj2aBQsWYPWrS5cu3Dzybu57dDz3PjyOB+67hwnjxnDyCT/g4suv5t5HxvFfe3+X3//mHAD69OnL+X+4gr32Hp5zya1BJeema2tupqtjQ7bZltdefXWJtPXXb2oCXujWrdvi1/Pmzav6X1H2xUliue7dAZg/fz7z589f/At6zkcfAfDhhx+y2uqrA9B3zX4AdOjg37DVop7+P3UwssXGjR3D9488jDdef43LrrqGTp388ah3CxcuZJfttmTq1Jc55LCjGDhoML+94I/sv88eLNu1K92XX55/jno472JaU2qgtlMO/8SxxTYfvAXjnniaBx8dw7m/+RXz5s3Lu0hWYR07duTeR8YxcfIrPDFhPM8/O5nL/nABf/7b7Ux89hWG73cgZ/zkx3kX05ogyusvqvZalIORfc7663+Vbt2W49nJz+RdFGsjK/bowVZDtuX+e+/i2WeeYuCgbE7Mb++1D+PGPpZz6aw59dRn5GBkALw6deriAQuvv/YaL730AmukPgKrT7NmzeSD2bMBmDt3LqMfup/+667Phx9+yMtTXgRg9AP30X/d9fMsphXRQSprqWbuFKhjhxzwPR5++CHenTWL9dZZg1NPO52eK63Ej044jlkzZ7L3Xt9i44034e933MVj/36Ec3/7azp37kyHDh049/yL6NWrV96XYBX0zltvcdzRI1i4cCGLYhHf3nNvdhr2TX57/iUcduBwOqgDK/boyXkXXwrApInjOXT//2b27PcZddc/+c0vz+KhxyflfBXtW5XHl7Ioeyhf+zBws0Ex+t9FHzZo7diceR7KbsWt3qPLhIgYlHc5AFZc86ux5UlXl7XPPcdsWTXlb8w1IzOzGtWhjmpGDkZmZjWq2kfIlcPByMysRtVRLHIwMjOrRSK716heeGi3mZnlzjUjM7Ma1S4GMEhaodiOEfFh6xfHzMxKUgNT/JSjWDPdZOCZ9O/kRuueJ8bMLGetPR2QpL6SHpD0nKTJko5L6WdIelPSpLTsVrDPKZKmSHpB0i4F6cNS2hRJJ7d07mZrRhHRt+Wim5lZHgSVmOJnAfDDiJgoaXlggqRRadt5EfHbJcogbQAMBzYEvgzcK2ndtPliYCdgGjBO0siIeLa5E5fUZyRpOLB2RPxCUh9gtYiYUMYFmplZK2vtWBQRM4AZ6fVHkp4DehfZZQ/ghoj4FJgqaQowOG2bEhGvZOXUDSlvs8GoxdF0ki4CvgEckJI+Af7Y0n5mZlZZS/EIiV6SxhcsRxQ5dj9gU2BMSjpW0lOSrpLUM6X1Bt4o2G1aSmsuvVmlDO3eKiKOBOYBRMR7wDIl7GdmZhVSbn9RqkXNiohBBctlTR9b3YFbgOPTYLVLgHWAAWQ1p981ZG1i9yiS3qxSmunmS+rQcCBJKwOLStjPzMwqqBKPhZDUmSwQ/SUibgWIiLcLtl8O3JFWpwGF4wv6ANPT6+bSm1RKzejiVLBVJJ0JPAL8qoT9zMysglTm0uLxsra8K4HnIuLcgvTVC7LtxX9GVI8EhkvqImktoD8wFhgH9Je0lqRlyAY5jCx27hZrRhFxraQJwI4paZ+I8NBuM7OcVeA+o63Jxgc8LanhYVWnAvtKGkDWQvYqcCRAREyWdBPZwIQFwDERsTCV7VjgbqAjcFVETC524lJnYOgIzE8F8RRCZmY5y4Z2t+4xI+IRmq5E3Vlkn7OBs5tIv7PYfo2VMpruJ8D1ZGPI+wB/lXRKqScwM7MKKHMkXbXP1lBKzWh/YLOI+ARA0tnABOCXlSyYmZkVV+XxpSylBKPXGuXrBLxSmeKYmVmpqr22U45iE6WeR9ZH9AkwWdLdaX1nshF1ZmaWk0r0GeWpWM2oYcTcZOCfBemPV644ZmZWqnZRM4qIK9uyIGZm1n612GckaR2yYXsbAMs2pEfEus3uZGZmFVc/9aLS7hm6GvgT2XXvCtwE3FDBMpmZWQukbDqgcpZqVkow6hYRdwNExMsRcRrZLN5mZpaj1n64Xp5KGdr9aZqv6GVJRwFvAqtWtlhmZtaSdjGAocD/At2B/yHrO1oROLSShTIzs5bVUSwqaaLUhgcrfcR/HrBnZmY5EtXfD1SOYje93kaRhyFFxH9VpERmZtayGugHKkexmtFFbVaKNiKgU0dPOm5NW2voCXkXwaws7aLPKCLua8uCmJlZeerpp3WpzzMyM7MqItpJzcjMzKpbe5kodQmSukTEp5UsjJmZla6eglEpT3odLOlp4KW0vomkCyteMjMza1Y2q0L9POm1lP6vC4DdgXcBIuJJPB2QmVnuOqi8pZqV0kzXISJeaxRVF1aoPGZmVqIqr+yUpZRg9IakwUBI6gj8AHixssUyM7P2pJRmuqOBE4A1gLeBLVOamZnlJHvseOs+QkJSX0kPSHpO0mRJx6X0lSSNkvRS+rdnSpekCyRNkfSUpIEFxzoo5X9J0kEtnbuUueneAYa3eBVmZtamKnDT6wLghxExUdLywARJo4CDgfsi4hxJJwMnAyeRPeOuf1q2AC4BtpC0EnA6MIhsWrkJkkZGxPvNnbiUJ71eThNz1EXEEeVdo5mZtabW7jOKiBnAjPT6I0nPAb2BPYChKds1wINkwWgP4NqICOBxST0krZ7yjoqI97JyahQwDLi+uXOX0md0b8HrZYG9gDdKvDYzM6sALd3TW3tJGl+wfllEXNbM8fsBmwJjgNVSoCIiZkhqeKZdb5aMB9NSWnPpzSqlme7GRgW8DhjV0n5mZlZZS1EzmhURg1o+rroDtwDHR8SHRe5RampDFElv1tI0Oa4FrLkU+5mZWSuqxH1GkjqTBaK/RMStKfnt1PxG+vedlD4N6Fuwex9gepH05q+lhIK9L+m9tMwmqxWd2vIlmZlZpVRoNJ2AK4HnIuLcgk0jgYYRcQcBtxekH5hG1W0JfJCa8+4GdpbUM4282zmlNatoM10q2CbAmylpUeqoMjOznFXgptetyZ7o/bSkSSntVOAc4CZJI4DXgX3StjuB3YApwCfAIQAR8Z6knwHjUr6zGgYzNKdoMIqIkHRbRGxW/jWZmVnFVGCKn4h4hKb7ewB2aCJ/AMc0c6yrgKtKPXcpfUZjC29kMjOz6qAy/6tmzdaMJHWKiAXAEOBwSS8DH5NFzYgIBygzs5xkfUZ5l6L1FGumGwsMBPZso7KYmVkZ2kswEkBEvNxGZTEzszJU+zOKylEsGK0i6YTmNjYa9mdmZm2oPTXTdQS60/zICjMzy4vaz/OMZkTEWW1WEjMzK8tSzE1XtYoN7a6fqzQzs6pWrGb0uRuczMysOrSbPqOWpm4wM7N81VErXUnPMzIzs6ojOtRRb4qDkZlZDRL1VTOqwCPUrRrNmzePIV8fzOCBmzBwkw352ZmnA3DwAfux8YbrsdmAjTjysEOZP39+ziW1SuvQQTx2/Unccv5RAFxy+vcYc+PJjL3xFP76mxEs13WZJfLvteMA5j5xEQM3WGOJ9L5f6snMR3/H8Qe4ezkXZT7LqNr7lxyM2okuXbpw16j7GTvxScaMn8Q9d9/FmMcfZ/j39uPJZ55n/BNPM3feXP505RV5F9Uq7NjvfYMXpr69eP3Hv72VLb57DoO/+0veeOt9jh6+3eJt3bt14fv7DmXsU1M/d5xfn/gd7nl0cpuU2ZrW2s8zypODUTshie7duwMwf/58FsyfjySG7bobkpDEoEGDefPNaTmX1Cqp96o9GDZkQ/50278Xp3308bzFr5ft0pnCR5ad/v3dOffqe5n32YIljvOtoRszddosnn35rcoX2prU0ExXzlLNHIzakYULF7LFZgNY48ursv2OOzF4iy0Wb5s/fz7X/+U6dtplWI4ltEr7zY++w0/O/zuLFi35jMxLz9ifV+/9Bev1W40/3PAQAJus14c+X+rJvx5+Zom83ZZdhh8eshNnX3pnm5XbmuaakdWkjh07MmbCJKa8Oo3x48Yy+Zn/fMkcd+z32XqbbRkyZJscS2iVtOs2G/HOex/xxHNvfG7bkWf8mbV3/gnPT32LvXfeDEn8+sTvcNLvbv1c3p8e/U0u/PP9fDz3s7YothVRTzUjj6Zrh3r06MG22w3lnnvuYsONNuLsn53JzFkzufGSS/MumlXQ1wesze7bfY1hQzakyzKdWWG5Zbnq5wdy6GnXArBoUXDzPRP53wN35Pb7J7HBOqtzzxXHAbDayitw8++PZO/jL2XzjdZkrx0HcPbxe7Li8l1ZtCiY99l8/njj6Dwvr90R9VWbcDBqJ2bOnEnnzp3p0aMHc+fO5f777uWHPzqJP115BaPuuZt/3XMfHTrU00fbGvu/C0fyfxeOBGCbzfpz/IE7cOhp17J231688sYsAL657dd48dW3+XDOPPpuf/Life++/DhOOe82Jj77OjuO+P3i9J8cuRsff/KpA1Ee1H4eIWF15K0ZMzj80INYuHAhi2IR39n7v9ntm7vTfdlOrLHmmgwd8nUA9tjrvzj1tP/LubTWViRxxVkHsPxyXZHg6Rff5H9+cWPexbIS1U8oAhWOnKl3m202KB4dMz7vYliV6rn5sXkXwarcvEkXT4iIQXmXA2DtDTaOs64rbxDJAYP6Vk35G3PNyMysRtVTzcjByMysRtVRl1FdDcYwM2tHtPiG9VKXFo8oXSXpHUnPFKSdIelNSZPSslvBtlMkTZH0gqRdCtKHpbQpkk5ufJ6mOBiZmVmDq4Gm7nw/LyIGpOVOAEkbAMPIaXrGAAAQBElEQVSBDdM+f5DUUVJH4GJgV2ADYN+Utyg305mZ1aBK3GcUEaMl9Ssx+x7ADRHxKTBV0hRgcNo2JSJeAZB0Q8r7bLGDuWZkZlajlqKZrpek8QXLESWe6lhJT6VmvJ4prTdQOJ3HtJTWXHpRDkZmZjVKZS7ArIgYVLBcVsJpLgHWAQYAM4DfFZy+sSiSXpSb6czMalEbzcAQEYufNyLpcuCOtDoN6FuQtQ8wPb1uLr1ZrhmZmdWghj6jcpalOo+0esHqXkDDSLuRwHBJXSStBfQHxgLjgP6S1pK0DNkgh5Etncc1IzOzGtXaNSNJ1wNDyfqWpgGnA0MlDSBransVOBIgIiZLuolsYMIC4JiIWJiOcyxwN9ARuCoiWnwKo4ORmVmNau1GuojYt4nkK4vkPxs4u4n0O4Gy5ipyMDIzq1H1NAODg5GZWQ3K+ozqJxo5GJmZ1SjXjMzMLGdCrhmZmVneXDMyM7Ncuc/IzMzyJ9eMzMysCtRTMPJ0QGZmljvXjMzMapRH05mZWa4EdKifWORgZGZWq1wzMjOz3NXTAAYHIzOzGuWakZmZ5are+oxyGdot6d8tbL9N0iRJUyR9kF5PkrRVkX0eSQ+AMjNrB1T2f9Usl5pRRDQbVNL2vQAkDQVOjIjd26JcZmY1o85mYMirZjQn/TtU0oOSbpb0vKS/qIXn6EraKdWSnpZ0eXrGuplZu6Myl2pWDTMwbAocD2wArA1s3VxGSd2Aq4DvRMTXgG7AEW1RSDOzapL1GamspZpVQzAaGxHTImIRMAnoVyTvV4GXIuLltH4tsG2xg0s6QtJ4SeNnzprZKgU2M6sGrhm1rk8LXi+keD9W2X/PiLgsIgZFxKBVeq1SduHMzKpWHUWjaghG5XgW6C9p7bS+P/BQjuUxM8tNPY2mq6lgFBGfACOAWyU9TVarujzfUpmZ5UMqb2n5eLpK0juSnilIW0nSKEkvpX97pnRJuiDdgvOUpIEF+xyU8r8k6aBSriWXYBQR3dO/DxYO246IYyPi6oL1BxsP646IeyJiQER8LSIOi4jPUvqQiJjURpdgZpa7CrTSXQ0Ma5R2MnBfRPQH7kvrALsC/dNyBHAJZMELOB3YAhgMnN4QwIqpqZqRmZkVaOVoFBGjgfcaJe8BXJNeXwPsWZB+bWQeB3pIWh3YBRgVEe9FxPvAKD4f4D7HwcjMzIpZLSJmAKR/V03pvYE3CvJNS2nNpRfluenMzGpQVtkpe1BCL0njC9Yvi4jLvkARGosi6UU5GJmZ1aKlmw5oVkQMKnOftyWtHhEzUjPcOyl9GtC3IF8fYHpKH9oo/cGWTuJmOjOzGtVGtxmNBBpGxB0E3F6QfmAaVbcl8EFqxrsb2FlSzzRwYeeUVpRrRmZmtaqVbx2SdD1ZraaXpGlko+LOAW6SNAJ4HdgnZb8T2A2YAnwCHAIQEe9J+hkwLuU7KyIaD4r4HAcjM7Oa1Po3skbEvs1s2qGJvAEc08xxriKbR7RkDkZmZjWqyuc+LYuDkZlZDaqB6ebK4mBkZlar6igaORiZmdWoap/8tBwORmZmNcp9RmZmlrs6ikUORmZmNanORjA4GJmZ1Sj3GZmZWa6E+4zMzKwK1FEscjAyM6tZdRSNPGu3mZnlzjUjM7Ma5QEMZmaWOw9gMDOz3NVRLHIwMjOrWXUUjRyMzMxqUDYBQ/1EIwcjM7NaJPcZmZlZFaijWORgZGZWs+ooGjkYmZnVJLnPyMzM8uc+IzMzy1WdPc7Ic9OZmdUslbmUckjpVUlPS5okaXxKW0nSKEkvpX97pnRJukDSFElPSRq4tJfiYGRmVqNU5n9l+EZEDIiIQWn9ZOC+iOgP3JfWAXYF+qflCOCSpb0WByMzsxollbd8AXsA16TX1wB7FqRfG5nHgR6SVl+aEzgYmZnVqKVopeslaXzBckQThw3gHkkTCravFhEzANK/q6b03sAbBftOS2ll8wAGM7P2Y1ZB01tzto6I6ZJWBUZJer5I3qbqW7E0BXPNyMysFpXZRFdqM11ETE//vgPcBgwG3m5ofkv/vpOyTwP6FuzeB5i+NJfjYGRmVrNadzidpOUkLd/wGtgZeAYYCRyUsh0E3J5ejwQOTKPqtgQ+aGjOK5eb6czMapCoyE2vqwG3KTtwJ+CvEXGXpHHATZJGAK8D+6T8dwK7AVOAT4BDlvbEDkZmZjWqtWNRRLwCbNJE+rvADk2kB3BMa5zbwcjMrEZ5OiAzM8udJ0o1M7P81U8scjAyM6tVdRSLHIzMzGpRK0zxU1UcjMzMapT7jMzMLH/1E4scjMzMalUdxSIHIzOzWuU+IzMzy1nZD8yrag5GZmY1qEJz0+XGs3abmVnuHIzMzCx3bqYzM6tR9dRM52BkZlajPIDBzMzy5emAzMwsb6U9SLx2OBiZmdWqOopGDkZmZjXKfUZmZpY79xmZmVnu6igWORiZmdWsOopGDkZmZjXKfUZmZparepsoVRGRdxnajKSZwGt5l6OK9AJm5V0Iq2r+jCxpzYhYJe9CAEi6i+z9KcesiBhWifJ8Ue0qGNmSJI2PiEF5l8Oqlz8j1lY8a7eZmeXOwcjMzHLnYNS+XZZ3Aazq+TNibcJ9RmZmljvXjMzMLHcORmZmljsHo3ZKUue8y2DVS1KXvMtg7YuDUTskaQBwrqQv5V0Wqz6SNgIulbRm3mWx9sPBqH2aAqwD/FjSqnkXxqpLRDwDdAVOk7RG3uWx9sHBqB1RpmNEzAG+C6wJ/NQByeA/nw+AiPgu0Bk4S1LffEtm7YGDUTshqUNkFkpaISI+AvYHVgJOd0Bq3xp9PlYGiIiDgY+BX7iGZJXm+4zaGUkjgF2BV4GHgTuAa4B3gV9GxFv5lc7yJulw4NvA68DYiLhG0oXAcsBZEfFqnuWz+uWaUZ2TtLaknun1vsDxwMXAy8A+wKFpWQ84vqGZxtoHSes21IolfQf4H+AsYBwwRNKPI+IHZMHoREl+7IxVhINRHZO0G/A3YIWUtDLw64h4ALgOuBzYNm3bB7goIha2eUEtF+nzcQuwfEpaEbg0IsYBNwFXAAMk9Uh9SL+IiAX5lNbqnYNRnZK0C/B74JCIaHiG03yyX7erp0EMY8j6jPpHxEcRMS2n4lobS5+Pi4D9I+LllPwBcKyk9SPik4gYQ/ZDZl2AiJieT2mtPXCVuw6lL5pLyb5cuhdsuhz4EnCJpFOB9YEewHttXkjLTfp8XEL2+egDPAkQEbdI6gecJ+mXwKpkD297M5+SWnviAQx1RtK2wAXAKSnpWLKml5Fp+2rA94EtgABOjogn8yirtT1JW5P1Gf4Y+AQ4CfhbRFybtq8E7Ec2yGUu2aAFfz6s4hyM6oykQWTv6zhJvYBhwHDg8oi4vSBfV2BRRHyaU1EtB5LWB7pFxERJywM7kA1guSUirinI14Xs8zE/p6JaO+NmujqRAs9yETFe0jfSza2zJI0kqwEdLmlhRNwBEBFzcy2wtal0H9GiiHg+rSsiPpJ0L9nnY0T6fPwZwD9SrK05GNWP/sARkmaTNbFsDbwbER9K+gewCDhJ0oKIuCvPglrbSoFnUXq9GyDgcbLPxxxJ95EFpB9Kmh8RN+ZYXGunHIxqXPqiiYh4TNJw4CjggIh4t2Dbh5LuAhYAk/MtsbUVSSJrsm0IRIcAPyQbsPKEpFsj4qEUkB4g+3w8nV+JrT1zn1GdkLQJ0JtsAtSdgPOA0YX3DTU01eRURGtjkjo13Bck6ZtkP1S+TfYj9HSgC/CPiBid8ij8hWA58X1GdUDS2sDhwAYRcSHwD7LRdAMkHS/pVwAORO2HpFWAW9Pkp52AQcB2wOA0KOF84FNgeBphhwOR5ck1ozohaW9gCDA1Is6XdATZ7ArrAkdGxBO5FtDanKRuZH2H4yJitqSzgQ2A0yPiqTTM/0jgjxHxTp5lNXMwqmGS/hvoHRHnpfU9gZ2Bp8hucO0KdIqI2fmV0vKUPhMXARsCH5LVmDcDzk7Du910a1XBzXQ1RFKHgtddgWnAQZIOA4iIvwMzgSPIfvF+7EDUvqXPxGHAeLKpfX4JPEc2cq4L2Sg6s9x5NF0NKRgVdRSwJVkw+h1wcrpH5E9ko+VWI7uJ0V80RkTcJek4suHcX4+I0ySt7HuJrJq4ma7GpGn+zyR7MN4hZAHpS8CBwIPAxsC3IuLFvMpo1Sk12f0UGOQfKlZtHIxqTJrg9LOI+K2kZYCDgY3I5qNbD5jsB6BZcyR1TzO2m1UV9xnVnmeBbSRtEBGfRcRlZLWhiIh/OhBZMQ5EVq3cZ1R7HiS7Z2Q/SQ+SjZjrTjZSysysJrmZrgZJ+jLwHeBbwBzgTE/zb2a1zMGohqWbGhURH+ddFjOzL8LByMzMcucBDGZmljsHIzMzy52DkZmZ5c7ByMzMcudgZGZmuXMwstxIWihpkqRnJP0tDVVf2mMNlXRHev1tSScXydtD0veX4hxnSDqx1PRGea5Oz5wq9Vz9JD1TbhnNapWDkeVpbkQMiIiNgM/IHou9WHpKadmf0YgYGRHnFMnSAyg7GJlZ5TgYWbV4GPhKqhE8J+kPwESgr6SdJT0maWKqQXUHkDRM0vOSHgH+q+FAkg6WdFF6vZqk2yQ9mZatgHOAdVKt7Dcp348kjZP0lKQzC471E0kvSLqXbCLaoiQdno7zpKRbGtX2dpT0sKQXJe2e8neU9JuCcx/5Rf+QZrXIwchyJ6kTsCvwdEpaD7g2IjYFPgZOA3aMiIFkD4k7QdKyZE+z/RawDdljNJpyAfBQRGwCDCR73tPJwMupVvYjSTsD/YHBwABgM0nbStoMGA5sShbsNi/hcm6NiM3T+Z4DRhRs6wdsB3wT+GO6hhHABxGxeTr+4ZLWKuE8ZnXFE6VanrpKmpRePwxcCXwZeC0iHk/pWwIbAI9KAlgGeAxYH5gaES8BSPoz2RNuG9ue7FlPRMRC4ANJPRvl2TktT6T17mTBaXngtoj4JJ1jZAnXtJGkn5M1BXYH7i7YdlN6QOJLkl5J17AzsHFBf9KK6dx+HpW1Kw5Glqe5ETGgMCEFnMK59gSMioh9G+UbQOs9MlvALyPi0kbnOH4pznE1sGdEPCnpYGBowbbGx4p07h9ERGHQQlK/Ms9rVtPcTGfV7nFga0lfgWxyWEnrAs8Da0laJ+Xbt5n97wOOTvt2lLQC8BFZrafB3cChBX1RvSWtCowG9pLUVdLyZE2CLVkemCGpM7Bfo237SOqQyrw28EI699EpP5LWlbRcCecxqyuuGVlVi4iZqYZxvaQuKfm0iHhR0hHAPyXNAh4he+JtY8cBl0kaASwEjo6IxyQ9moZO/yv1G30VeCzVzOYA+0fEREk3ApOA18iaElvyU2BMyv80Swa9F4CHgNWAoyJinqQryPqSJio7+Uxgz9L+Omb1w7N2m5lZ7txMZ2ZmuXMwMjOz3DkYmZlZ7hyMzMwsdw5GZmaWOwcjMzPLnYORmZnl7v8B4nE4C5HqgFwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.utils.multiclass import unique_labels\n",
    "def plot_confusion_matrix(y_true, y_pred, classes,\n",
    "                          normalize=False,\n",
    "                          title=None,\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    if not title:\n",
    "        if normalize:\n",
    "            title = 'Normalized confusion matrix'\n",
    "        else:\n",
    "            title = 'Confusion matrix, without normalization'\n",
    "\n",
    "    # Compute confusion matrix\n",
    "    cm = confusion_matrix(y_true, y_pred)\n",
    "    # Only use the labels that appear in the data\n",
    "#     classes = classes[unique_labels(y_true, y_pred)]\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    fig, ax = plt.subplots(figsize=(10,5))\n",
    "    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    ax.figure.colorbar(im, ax=ax)\n",
    "    # We want to show all ticks...\n",
    "    ax.set(xticks=np.arange(cm.shape[1]),\n",
    "           yticks=np.arange(cm.shape[0]),\n",
    "           # ... and label them with the respective list entries\n",
    "           xticklabels=classes, yticklabels=classes,\n",
    "           title=title,\n",
    "           ylabel='True label',\n",
    "           xlabel='Predicted label')\n",
    "\n",
    "    # Rotate the tick labels and set their alignment.\n",
    "    plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\",\n",
    "             rotation_mode=\"anchor\")\n",
    "\n",
    "    # Loop over data dimensions and create text annotations.\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i in range(cm.shape[0]):\n",
    "        for j in range(cm.shape[1]):\n",
    "            ax.text(j, 0.5*i+0.25, format(cm[i, j], fmt),\n",
    "                    ha=\"center\", va=\"center\",\n",
    "                    color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "    fig.tight_layout()\n",
    "    return ax\n",
    "plot_confusion_matrix(y_test, y_pred, classes=['outTol', 'inTol'],\n",
    "                      title='Confusion matrix, without normalization')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Normalized confusion matrix\n",
      "[[0.22874494 0.77125506]\n",
      " [0.00731261 0.99268739]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2349732f630>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAFgCAYAAAB+GpTWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcHFW58PHfk4SQsIUlbFnY14DIElFRFjUgXDBwVRRElAsvKldQVFQURcQFL+4Kvl4QRRHZVCQs3qBeQEFAArIYlhCQJQuQRPYtZPLcP7omdCazZXp6umry++ZTn3RVnT719HRPP3POqToVmYkkSY0Y0uoAJEnVZzKRJDXMZCJJapjJRJLUMJOJJKlhJhNJUsNMJgMsIkZGxOUR8XREXNJAPYdFxNX9GVurRMTuEXFfWY4XEZtEREbEsIGKqVki4pSI+GXxeKOIeC4ihvbzMQb0/VM5mUy6EBHvi4hpxS/f3Ij4fUS8uR+qfjewPrBOZh7c10oy8/zM3Kcf4mmq4kt5i+7KZOZfMnPrgYqp4/Ei4qGImDRQx2+VzHwkM1fLzLZG6un4ng70+6dyMpl0IiI+CXwP+Dq1L/6NgB8BB/ZD9RsDMzJzUT/UVXmD4a///hI1/k6qmjLTpW4BRgHPAQd3U2ZlaslmTrF8D1i52LcXMAv4FPAEMBf4j2Lfl4GFwCvFMY4CTgF+WVf3JkACw4r1I4AHgWeBfwKH1W2/vu55uwG3AE8X/+9Wt+9a4CvADUU9VwOju3ht7fF/pi7+g4B/A2YA/wI+X1d+V+BG4Kmi7BnA8GLfn4vX8nzxet9bV/9ngceA89q3Fc/ZvDjGzsX6GGA+sFcv3rufA58qHo8tjv2fxfoWRb3R4XjnAYuBF4sYP1P3HnwQeKQ4/kndHPdc4EzgyuLnezOw+XK8N18r3psXizivBb4K/LWI6XJgHeB84Jmijk3q6vg+8Gix71Zg97p9p1B8vupe1zDgjUXd7ctLwEONvKd1x9y2eA1PAdOByb39WblUd2l5AGVbgH2BRRRf5l2UORW4CVgPWLf4pf9KsW+v4vmnAitR+xJ+AVir2L/kl7uL9fpf+FWLL4iti30bAtsVj4+gSCbA2sCTwOHF8w4t1tcp9l8LPABsBYws1r/RxWtrj//kIv6jgXnAr4DVge2KL57NivK7AG8ojrsJcA9wfF19CWzRSf3/RS0pj+zky+joop5VgKnAt3r53h0JXF48fl/xmi+q23dZXQz1x3sImNTJe3B2Ed9rgZeBbbs47rnUEtWuxc/hfODC5XhvHil+rsOKn/m1wExqiXUUcDe1RD6pKPML4Gd1x38/tWQzjNofMY8BIzp+vujwh0rd89uPeVoD7+msurpmAp8HhgNvpZY0tu7pZ+VS7cUm9bLWAeZn991QhwGnZuYTmTmPWovj8Lr9rxT7X8nMq6j9BdfXPuXFwPYRMTIz52bm9E7K7A/cn5nnZeaizLwAuBd4R12Zn2XmjMx8EbgY2LGbY74CfC0zXwEuBEYD38/MZ4vjTwd2AMjMWzPzpuK4DwH/DezZi9f0pcx8uYhnKZl5NnA/tb9aNwRO6qG+dtcBuxddRXsApwNvKvbtWexfHl/OzBcz8w7gDmpJpSu/zcy/FZ+b83n159ub9+bczJxe7H+l2PazzHwgM58Gfg88kJl/LOq/BNip/cmZ+cvMXFA8/9vUkvTyfN5+QK2lcVJRX1/e03ZvAFaj9sfKwsz8X+AKakm0XVc/K1WYyWRZC4DRPfTljwEerlt/uNi2pI4OyegFar9gyyUzn6fWjfARYG5EXBkR2/QinvaYxtatP7Yc8SzIVwdp27/sH6/b/2L78yNiq4i4IiIei4hnqI0zje6mboB5mflSD2XOBrYHfpiZL/dQFoDMfIBa4t4R2J3al9iciNiaviWT5fmZdVW2N+/No53U1/Hn3enPHyAiPhUR9xRnCD5FrTXT03vQ/twPU2tZvC8zFxfb+vKethsDPNpeV6GRz6IqwmSyrBupdeMc1E2ZOdQG0tttVGzri+epdee026B+Z2ZOzcy9qf2Ffi+1L9me4mmPaXYfY1oe/59aXFtm5hrUujeih+d0O1V1RKxGbRzqHOCUiFh7OeK5jtoZc8Mzc3ax/gFgLeD2vsTToN68N30+fkTsTm386T3UulLXpDY209N70P7crwAHFi2gdn15T9vNAcZ3OJFgoD6LaiGTSQfFL9XJwJkRcVBErBIRK0XEfhFxelHsAuALEbFuRIwuyv+yj4e8HdijuAZgFPC59h0RsX5ETI6IVan12T8HdHZa51XAVsXpzMMi4r3ABGp/mTfb6tTGdZ4rWk3HdNj/OLDZctb5feDWzPx/1AZqf9y+o7hu4tpunnsdcCy1gWKojQUcR218qatTYvsSY281+71ZndoY1DxgWEScDKzR05MiYjxwEfCBzJzRSZ19fU9vpvYH0meK35u9qHXpXdi7l6OqMpl0IjO/A3wS+AK1X9JHqX1B/a4o8lVgGnAncBdwW7GtL8f6A7Vf6jupnYlT/yUzhNqA6hxqg5Z7Av/ZSR0LgAOKsguonZF0QGbO70tMy+kEaoPdz1JrNV3UYf8pwM8j4qmIeE9PlUXEgdROgvhIsemTwM4RcVixPp7amU9duY7al2F7MrmeWsvvz10+A06j9sfBUxFxQk8xLo8BeG+mUhtTmUGtO+klOu826+ht1FrBvy6upXouItrH4/r8nmbmQmAysB+1s+B+RC1h3duH16YKiUxvjqXqiIjbgbcVX9KSSsJkIklqmN1ckqSGmUwkSQ0zmUiSGlaqSfbWXHudHDN2o1aHoQqb+fizrQ5BFbbo2SdY/OIzvb2mpqmGrrFx5qJlJojolXxx3tTM3LefQ+pWqZLJmLEb8Ysp17Y6DFXYQd+6ptUhqMKeuKRfzwxvSC56kZW37vFs+k69dPuZvZ2xoN+UKplIktoFVOiOBCYTSSqjAKIUPW69YjKRpLKyZSJJaliFWibVSXuSpNKyZSJJpeQAvCSpP1Som8tkIkllFNgykSQ1KmyZSJL6gS0TSVLDbJlIkhrj2VySpEZVbDqV6qQ9SVJp2TKRpLKym0uS1BjHTCRJ/WFIdcZMTCaSVEZeAS9J6hcVOpvLZCJJpeSYiSSpP1SoZVKdtCdJKi1bJpJUVnZzSZIaEk5BL0nqD7ZMJEkNs2UiSWqMpwZLkvqDLRNJUkOcTkWS1LhqdXNVJ1JJUmnZMpGksnLMRJLUsAp1c5lMJKmsbJlIkhoS1RqAN5lIUlnZMpEkNSpMJpKkRgTVSibV6ZCTJJWWLRNJKqMoloowmUhSKUWlurlMJpJUUiYTSVLDTCaSpIaZTCRJjXEAXpLUqHAAXpLUH6qUTLxoUZLUMFsmklRSVWqZmEwkqaRMJpKkxng2lySpP1SpZeIAvCSVUPupwX1Zeqw7Yt+IuC8iZkbEiZ3s3ygiromIv0fEnRHxbz3VaTKRpJJqRjKJiKHAmcB+wATg0IiY0KHYF4CLM3Mn4BDgRz3FajKRpLKKPi7d2xWYmZkPZuZC4ELgwA5lElijeDwKmNNTpY6ZSNLgMzoiptWtn5WZZxWPxwKP1u2bBby+w/NPAa6OiOOAVYFJPR3QZCJJZRQNDcDPz8yJXde8jOywfihwbmZ+OyLeCJwXEdtn5uKuDmgykaSSatLZXLOA8XXr41i2G+soYF+AzLwxIkYAo4EnuqrUMRNJKqkmnc11C7BlRGwaEcOpDbBP6VDmEeBtRQzbAiOAed1VastEkkqoWbMGZ+aiiDgWmAoMBX6amdMj4lRgWmZOAT4FnB0Rn6DWBXZEZnbsCluKyUSSyqpJ1yxm5lXAVR22nVz3+G7gTctTp8lEksqosQH4AWcykaSSMplIkhpWpWTi2VySpIbZMpGksqpOw8RkIkllVaVuLpOJJJVQb6eTL4umjpn0NGe+JKlrzbqfSTM0rWVSN2f+3tTmgrklIqYUF8NIknpgy6SmN3PmS5K60pz7mTRFM5NJZ3Pmj+1YKCI+FBHTImLak/9a0MRwJKlaqtTN1cxk0ps588nMszJzYmZOXGvtdZoYjiRVSJhM2vVmznxJ0iDQzGTSmznzJUmdCCCib0srNO1srq7mzG/W8SRpcKnWdSZNvWixsznzJUm9U6Fc4hXwklRWtkwkSY1p4fhHX5hMJKmEAhgypDrZxGQiSSVVpZaJN8eSJDXMlokklZQD8JKkxjgAL0lqVO0K+OpkE5OJJJWSV8BLkvpBhXKJyUSSysqWiSSpMQ7AS5IaVbUBeC9alCQ1zJaJJJVUhRomJhNJKqsqdXOZTCSppCqUS0wmklRKYctEktSg2tlcrY6i90wmklRKTqciSeoHFcolXmciSWqcLRNJKim7uSRJjXFuLklSo6o2N5fJRJJKymSi5fbX6/7It089kcWL2zjwPR/giGM+sdT+839yBpddfB5Dhw5lzbVHc/LpZ7Dh2I2YO/sRPnPM4bS1tbFo0SLe+4EP8a7DjmzRq1Cr7DVhPb787tcwdEhwwQ0Pc+Yf7l9q/5fetT27bbUuACNXGso6q6/Mdp++kt22HM2X3v2aJeU2X381PvrTaUy9c+6Axq/OVSiXmEzKoK2tjdO/dAJn/OJ3rL/BGD540FvYY9J+bLblNkvKbL3dDvzismsYMXIVfv3Lc/jBN77EaT/8GaPX3YBzLrma4SuvzAvPP8ch+76RPSbtx7rrb9jCV6SBNCTgq+95Le/74Q3MfepFrvzMXlx912Pc/9izS8p8+Tf/WPL4P/bcjO3GjwLgr/fP5+2nXQPAmqusxPWn7M119zwxsC9AXapSy8RTg0tg+h23Mn7jzRi30SasNHw4ex/wLq77w1VLlZn4xj0YMXIVAF6z00SeeGwOACsNH87wlVcGYOHChSxenAMbvFpux03W4qF5z/HIghd4pS257NZZ7LPDBl2WP3DiWC6bNmuZ7fvvNIZr7n6cl15pa2a46q1iAL4vSyuYTEpg3mNzWX/DsUvW199wDPMe77qb4bKLf8lue05asv7YnFkcut9uHPCm7fjAhz9uq2QFs+GaI5n75ItL1h976iU2XHNkp2XHrj2S8eusyg33zVtm3+RdxvG7TpKMWiOKK+D7srSCyaQEkmVbE119Hq763UXcc9ffOfzojy3ZtsGYcVzw+79y6TW3ceVvL2DBPLspVnTZRQP1wF3GcdXf59CxAbveGiuzzZg1uO5uPzvqG5NJCay3wRgenzt7yfrjc+cwer1lWxc3X38tPzvz23z7rAuWdG3VW3f9Ddlsy224/ZYbmxqvymXuUy+y4VqvtkQ2WHMEjz39YqdlJ+8yttPWxzt2Hsv/3DGXRXaTlordXFouE3bYmUceeoDZjz7EKwsX8ocrfsMek/Zbqsx90+/gtC8cz7fPuoC1R6+7ZPvjc2fz0ku1L45nnn6KO2+9mY0322JA41dr3fHwU2y63mqMX2cVVhoaHLjLOP5w12PLlNtsvdUYtcpwbv3nv5bZd+DEcZ2Oo6i1hkT0aWkFz+YqgWHDhvGZU77Jxz74LtoWtzH54Pez+Vbb8uPvfo1tX7MTe076N75/2sm8+PzznHjsB4Fa19Z3zr6Qh2bO4HtfP4mIIDM57Ojj2GKb7Vr8ijSQ2hYnX7z4Ts7/6G4MGRJcdOPDzJj7LCfsvw13PPLUksRy0MRxTLl12YQxbu1VGLPWSG6cOX+gQ1cPKnQyF5Fdda62wITX7JS/mHJtq8NQhR30rWtaHYIq7IlLTmDhEzNL8RU+auNt8w2fPbdPz736o2+4NTMndrU/IvYFvg8MBX6Smd/opMx7gFOABO7IzPd1d0xbJpJUUkOakNYiYihwJrA3MAu4JSKmZObddWW2BD4HvCkzn4yI9Xqq12QiSSXVpNN8dwVmZuaDxTEuBA4E7q4rczRwZmY+CZCZPZ7m5wC8JJVUA2dzjY6IaXXLh+qqHQs8Wrc+q9hWbytgq4i4ISJuKrrFumXLRJIGn/ndjJl01tzpOHg+DNgS2AsYB/wlIrbPzKe6OqAtE0kqoaC4Cr4P/3owCxhftz4OmNNJmcsy85XM/CdwH7Xk0iWTiSSV1JDo29KDW4AtI2LTiBgOHAJM6VDmd8BbACJiNLVurwe7q9RuLkkqoybNs5WZiyLiWGAqtVODf5qZ0yPiVGBaZk4p9u0TEXcDbcCnM3NBd/WaTCSppJp10WJmXgVc1WHbyXWPE/hksfRKl8kkItboIZhnensQSdLyCWjZ1Ch90V3LZDq1Ef76V9O+nsBGTYxLklZ4FcolXSeTzBzf1T5JUvMNujstRsQhEfH54vG4iNiluWFJ0oqtrxcslnYK+og4g9opYocXm14AftzMoCRJ1dKbs7l2y8ydI+LvAJn5r+LcZElSEw2WAfh2r0TEEIrL7SNiHWBxU6OSJPV8LXuJ9CaZnAn8Blg3Ir4MvAf4clOjkiRVagC+x2SSmb+IiFuBScWmgzPzH80NS5JWbLXrTFodRe/19gr4ocAr1Lq6nM9LkpqtSdOpNEtvzuY6CbgAGENtdslfRcTnmh2YJK3oqnRqcG9aJu8HdsnMFwAi4mvArcBpzQxMklZ0VWqZ9CaZPNyh3DB6mIpYktSYQTNmEhHfpTZG8gIwPSKmFuv7ANcPTHiSpCrormXSfsbWdODKuu03NS8cSVK7QdHNlZnnDGQgkqSlVSeV9GLMJCI2B74GTABGtG/PzK2aGJckrdAiqjWdSm+uGTkX+Bm1JLkfcDFwYRNjkiRRrVODe5NMVsnMqQCZ+UBmfoHiRvOSpOaJ4sLF5V1aoTenBr8ctegeiIiPALOB9ZobliSpQr1cvUomnwBWAz5GbexkFHBkM4OSpBVdEJUaM+nNRI83Fw+f5dUbZEmStER3Fy1eSnEPk85k5jubEpEkCVo4mN4X3bVMzhiwKAojhw9l+/GjBvqwGkQW/PWPrQ5BFbbouWdaHcJSBstFi38ayEAkSUur0v0+ens/E0nSAAoGSctEktRag2LW4I4iYuXMfLmZwUiSXlWlZNKbOy3uGhF3AfcX66+NiB82PTJJWoHVpkapzhXwvRnf+QFwALAAIDPvwOlUJKnphkTflpbE2psymflwh21tzQhGklRNvRkzeTQidgUyIoYCxwEzmhuWJKlCJ3P1KpkcQ62rayPgceCPxTZJUpPU7gFfnWzSm7m5ngAOGYBYJEl1BtVFixFxNp3M0ZWZH2pKRJIkYPB1c9VPdjQC+Hfg0eaEI0mC2mnBg62b66L69Yg4D/hD0yKSJAGDr2XS0abAxv0diCRpaVW6Ar43YyZP8uqYyRDgX8CJzQxKklQt3SaT4t7vr6V233eAxZnZ5Q2zJEn9o2qnBnd75lmROC7NzLZiMZFI0gCJ6NvSCr05jflvEbFz0yORJL2qj/NytWqcpbt7wA/LzEXAm4GjI+IB4Hlqra/MTBOMJDVRUJ1uru7GTP4G7AwcNECxSJIKtTGTVkfRe90lkwDIzAcGKBZJUp3BkkzWjYhPdrUzM7/ThHgkSYXBcg/4ocBqUKFOO0kaJAZTN9fczDx1wCKRJFVWj2MmkqQWaOE1I33RXTJ524BFIUlaxqC4Aj4z/zWQgUiSXtU+ZtKMixYjYt+IuC8iZkZEl3MtRsS7IyIjYmJPdVbpRl6StEJpxnQqETEUOBPYD5gAHBoREzoptzrwMeDm3sRqMpGkUgqG9HHpwa7AzMx8MDMXAhcCB3ZS7ivA6cBLvYnWZCJJJRQ01DIZHRHT6pb626yPZem75c4qtr167IidgPGZeUVv4+3LzbEkSc3W2KSN8zOzq3GOzmpdMiN8RAwBvgscsTwHtGVSEldP/R922G5rtttmC755+jeW2f/yyy/z/ve9l+222YLdd3s9Dz/0EAALFizg7ZPewug1V+P4jx07wFGrLPbebVvuuPSL/OOyL3HCf+y9zP6NNlyLq358HH+76HNMPfvjjF1vzSX7vvqxA5l2yeeZdsnnefc+zt+6ApgFjK9bHwfMqVtfHdgeuDYiHgLeAEzpaRDeZFICbW1tHP+xj3LZ5b/n73fezSUXXsA9d9+9VJlzf3oOa625FtPvnclxH/8EJ33+swCMGDGCk0/5Cqf917daEbpKYMiQ4HsnvocDj/0RO73rqxy87y5ss9kGS5U57RP/zvlX/o1d33saXz/r95x63GQA9n3zduy47Xhef8g32OPwb3H8Byex+qojWvEy1IkhEX1aenALsGVEbBoRw4FDgCntOzPz6cwcnZmbZOYmwE3A5Myc1m2sjb1U9Ydb/vY3Nt98CzbdbDOGDx/Owe89hCsuv2ypMldcfhmHHf5BAN75rndz7f/+icxk1VVX5U1vfjMjRvgFsKJ63fab8MCj83lo9gJeWdTGJVNv44C9dliqzDabbci1N98HwHW3zOCAvV4DwLabbcBfbr2ftrbFvPDSQu6aMYt9dtt2wF+DltXgmEmXiluLHAtMBe4BLs7M6RFxakRM7mu8JpMSmDNnNuPGvdrqHDt2HLNnz162zPhamWHDhrHGqFEsWLBgQONUOY1ZbxSzHn9yyfrsx59k7Lqjlipz14zZHPS2HQE48K2vZY3VRrL2qFW5c8Zs3v6mCYwcsRLrrLkqe07cinEbrDWg8atrTWqZkJlXZeZWmbl5Zn6t2HZyZk7ppOxePbVKwAH4UujsbsgdZwvtTRmtmDq7gVLHT8vnvnsp3/3swbx/8uu54baZzH78SRa1tfGnm+5ll+025ppzP8X8J5/j5jv/yaJFiwcmcPWoSr/iJpMSGDt2HLNmvXqm3uzZsxgzZsyyZR59lHHjxrFo0SKeefpp1l577YEOVSU0+4mnGLf+q62JseuvxZx5Ty9VZu68pznkhJ8AsOrI4Rz0th155rna5QOnnzOV08+ZCsC5Xz+CmY8+MUCRqztBtbqOqhTroDXxda9j5sz7eeif/2ThwoVcctGF7H/A0l2X+x8wmfPP+zkAv/3Nr9nzLW+1ZSIApk1/mC02WpeNx6zDSsOGcvDbd+bKa+9cqsw6a6665PPy6SPfzs8vuwmoDd6vPWpVALbfcgzbbzmGP95478C+AHUuar0PfVlawZZJCQwbNozvfv8M3rH/22lra+ODRxzJhO2249RTTmbnXSZywDsmc8SRR3HkEYez3TZbsNZaa3Pe+Rcuef7WW2zCs888w8KFC7l8yu+44qqr2XbCMrMjaJBqa1vMJ/7rYi7/0UcZOiT4+WU3cc+Dj/HFY/bntrsf4crr7mKPiVty6nGTyYTrb5vJ8addDMBKw4byx58eD8Czz73EkSf9nLY2u7nKokp/LkZnffGtsssuE/OGm3sc55G6tNbrvNZGfffyfRez+IUnSvEdvtmEHfLU867q03MPnzj+1m4uWmwKu7kkSQ2zm0uSSqoUTaReMplIUklV6Rwbk4kklVLrzszqC5OJJJVQ1a4zMZlIUknZMpEkNaw6qcRkIknlFLZMJEkNqtqYSZVilSSVlC0TSSopu7kkSQ2rTioxmUhSaVWoYWIykaQyqg3AVyebmEwkqaRsmUiSGhSELRNJUqOq1DLxOhNJUsNsmUhSCTkAL0lqXFSrm8tkIkklZTKRJDXMs7kkSQ0JYEh1conJRJLKypaJJKlhjplIkhpWpZaJFy1Kkhpmy0SSSsgBeElSP6jWRI9N6+aKiJ9GxBMR8Y9mHUOSBq3iCvi+LK3QzDGTc4F9m1i/JA1q0celFZrWzZWZf46ITZpVvyQNZrUxE7u5ei0iPhQR0yJi2rz581odjiSVRpVaJi1PJpl5VmZOzMyJ645et9XhSFJ5VCibtDyZSJKqz1ODJamkPDUYiIgLgBuBrSNiVkQc1axjSdJgVKVTg5t5NtehzapbklYE1WmX2M0lSeVVoWxiMpGkEqqdmFWdbGIykaQyauH4R1+YTCSppCqUS0wmklRaFcomXrQoSWqYyUSSSin6/K/HmiP2jYj7ImJmRJzYyf5PRsTdEXFnRPwpIjbuqU6TiSSVVDMuWoyIocCZwH7ABODQiJjQodjfgYmZuQPwa+D0nmI1mUhSCfV1jsdeDLPsCszMzAczcyFwIXBgfYHMvCYzXyhWbwLG9VSpyUSSyqo52WQs8Gjd+qxiW1eOAn7fU6WezSVJJdXARYujI2Ja3fpZmXnWkmqXlZ0eP+L9wERgz54OaDKRpJJq4KLF+Zk5sYt9s4DxdevjgDnLHjsmAScBe2bmyz0d0G4uSSqpJo2Z3AJsGRGbRsRw4BBgylLHjdgJ+G9gcmY+0ZtYTSaStALJzEXAscBU4B7g4sycHhGnRsTkotg3gdWASyLi9oiY0kV1S9jNJUll1MRb8GbmVcBVHbadXPd40vLWaTKRpJJy1mBJUkMCZw2WJPWDCuUSk4kklVaFsonJRJJKyjETSVLDHDORJDWsQrnEixYlSY2zZSJJZVWhponJRJJKqHYBfHWyiclEksqoF3dNLBOTiSSVVIVyiclEkkqrQtnEZCJJpRSOmUiSGueYiSSpIU28nUlTeNGiJKlhtkwkqawq1DQxmUhSSTkAL0lqmAPwkqSGVSiXmEwkqZScTkWS1D+qk01MJpJUQkG1WiZeZyJJapgtE0kqqQo1TEwmklRWVermMplIUkl50aIkqXHVySUmE0kqqwrlEpOJJJVReNGiJKk/OGYiSWpcdXKJFy1Kkhpny0SSSqpCDROTiSSVlQPwkqQGhQPwkqTGOGuwJGmFY8tEkkqqSi0Tk4kklVSVxkzs5pIkNcyWiSSVkXNzSZIaFXjRoiSpP1Qom5hMJKmkqjQAbzKRpJJyzESS1LAK5RKTiSSVVoWyiclEkkqqSmMmXrQoSWpYZGarY1giIuYBD7c6jhIbDcxvdRCqND9D3ds4M9dtdRAAEfE/1N6vvpifmfv2Zzw9KVUyUfciYlpmTmx1HKouP0NqFru5JEkNM5lIkhpmMqmWs1odgCrPz5CawjETSVLDbJlIkhpmMpEkNcxkIklqmMmkAiJi64h4Y0SsFBFDWx2PqsnPjprJAfiSi4h3Al8HZhfLNODczHympYGpMiJiq8ycUTwempltrY5Jg48tkxKLiJWA9wJHZebbgMuA8cBnImKNlganSoiIA4DbI+Iv9aK2AAAEYUlEQVRXAJnZZgtFzWAyKb81gC2Lx5cCVwDDgfdFVOnWORpoEbEqcCxwPLAwIn4JJhQ1h8mkxDLzFeA7wDsjYvfMXAxcD9wOvLmlwan0MvN54EjgV8AJwIj6hNLK2DT4mEzK7y/A1cDhEbFHZrZl5q+AMcBrWxuayi4z52Tmc5k5H/gwMLI9oUTEzhGxTWsj1GDhzbFKLjNfiojzgQQ+V/zyvwysD8xtaXCqlMxcEBEfBr4ZEfcCQ4G3tDgsDRImkwrIzCcj4mzgbmp/Xb4EvD8zH29tZKqazJwfEXcC+wF7Z+asVsekwcFTgyumGDjNYvxEWi4RsRZwMfCpzLyz1fFo8DCZSCuYiBiRmS+1Og4NLiYTSVLDPJtLktQwk4kkqWEmE0lSw0wmkqSGmUzUZxHRFhG3R8Q/IuKSiFilgbr2iogriseTI+LEbsquGRH/2YdjnBIRJ/R2e4cy50bEu5fjWJtExD+WN0apqkwmasSLmbljZm4PLAQ+Ur8zapb7M5aZUzLzG90UWRNY7mQiqXlMJuovfwG2KP4ivycifgTcBoyPiH0i4saIuK1owawGEBH7RsS9EXE98M72iiLiiIg4o3i8fkRcGhF3FMtuwDeAzYtW0TeLcp+OiFsi4s6I+HJdXSdFxH0R8Udg655eREQcXdRzR0T8pkNra1JE/CUiZhRTuxMRQyPim3XH/nCjP0ipikwmalhEDKM2PcddxaatgV9k5k7A88AXgEmZuTO1m3t9MiJGAGcD7wB2BzboovofANdl5muBnYHpwInAA0Wr6NMRsQ+1afp3BXYEdomIPSJiF+AQYCdqyep1vXg5v83M1xXHuwc4qm7fJsCewP7Aj4vXcBTwdGa+rqj/6IjYtBfHkQYV5+ZSI0ZGxO3F478A51CbzfjhzLyp2P4GYAJwQ3H7leHAjcA2wD8z836AYibbD3VyjLcCH4Al06Y/XUwJUm+fYvl7sb4ateSyOnBpZr5QHGNKL17T9hHxVWpdaasBU+v2XVxMY3N/RDxYvIZ9gB3qxlNGFcee0YtjSYOGyUSNeDEzd6zfUCSM5+s3AX/IzEM7lNuR2kzI/SGA0zLzvzsc4/g+HONc4KDMvCMijgD2qtvXsa4sjn1cZtYnHSJik+U8rlRpdnOp2W4C3hQRWwBExCoRsRVwL7BpRGxelDu0i+f/CTimeO7Q4nbFz1JrdbSbChxZNxYzNiLWA/4M/HtEjIyI1al1qfVkdWBuccvkwzrsOzgihhQxbwbcVxz7mKI8EbFVcYdDaYViy0RNlZnzir/wL4iIlYvNX8jMGRHxIeDKiJhP7Q6S23dSxceBsyLiKKANOCYzb4yIG4pTb39fjJtsC9xYtIyeozZF/20RcRG1O1M+TK0rridfBG4uyt/F0knrPuA6aveS+Uhxr5mfUBtLuS1qB58HHNS7n440eDjRoySpYXZzSZIaZjKRJDXMZCJJapjJRJLUMJOJJKlhJhNJUsNMJpKkhv0fMxRq6443AFsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_confusion_matrix(y_test, y_pred, classes=[0,1], normalize=True,\n",
    "                      title='Confusion matrix, with normalization')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sensitivity:  0.23\n",
      "Specitivity:  0.99\n",
      "Average:  0.61\n"
     ]
    }
   ],
   "source": [
    "sensitivity = cm[0,0] / cm.sum(axis=1)[0]\n",
    "specitivity = cm[1,1] / cm.sum(axis=1)[1]\n",
    "print('Sensitivity: ', sensitivity.round(2))\n",
    "print('Specitivity: ', specitivity.round(2))\n",
    "print('Average: ', np.mean([sensitivity, specitivity]).round(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}