Model_Training_and_Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, I have trained a machine learning model to predict potential loan defaulters.\n",
    "\n",
    "1. Data Prepocessing\n",
    "2. Training ML Model\n",
    "3. Evaluating Model on Test Data\n",
    "4. Learning Cuvre and Scope for Improvement"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports and Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:17:23.626113Z",
     "start_time": "2019-01-26T14:17:22.542966Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy.special import boxcox1p     \n",
    "import joblib\n",
    "%matplotlib inline\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# Importing custom classes and functions\n",
    "from lending_club.data_cleaning import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:17:24.305297Z",
     "start_time": "2019-01-26T14:17:23.628062Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.metrics import roc_auc_score,confusion_matrix,classification_report\n",
    "from sklearn.model_selection import learning_curve,StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:17:24.314227Z",
     "start_time": "2019-01-26T14:17:24.307246Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_performance_chart(data,y_pred,y_test):\n",
    "    picked=data.loc[y_pred==0]\n",
    "    grades=['A','B','C','D','E','F','G']\n",
    "    tmp=pd.DataFrame(columns=['ROI','% Picked','% Default'],index=grades)\n",
    "    for g in grades:\n",
    "        grade_picked=picked[picked['grade']==g]\n",
    "        if len(grade_picked.index)==0:\n",
    "            roi=None\n",
    "            default_rate=None\n",
    "            perc_picked=0\n",
    "        else:\n",
    "            num_picked=len(grade_picked.index)\n",
    "            num_total=len(data[data['grade']==g].index)\n",
    "            roi=sum(grade_picked['total_pymnt'])/sum(grade_picked['funded_amnt'])-1\n",
    "            l=y_test[list(grade_picked.index)]\n",
    "            default_rate=100*len(l[l==1])/len(l)\n",
    "            perc_picked=100*num_picked/num_total\n",
    "\n",
    "        tmp.loc[g,'% Picked']=perc_picked\n",
    "        tmp.loc[g,'ROI']=roi\n",
    "        tmp.loc[g,'% Default']=default_rate\n",
    "    return tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing\n",
    "\n",
    "* Selected only Fully Paid and Defaulted loans for training model.\n",
    "* Train-Test Split\n",
    "* Created a pipeline for data preprocessing. See 'Data Cleaning' notebook for detailed approach.\n",
    "* Standard Scaling\n",
    "* Over-Sampling using SMOTE : I have not used this in final model, as ML models I have used consider class imbalance.\n",
    "* I have not removed features with high VIF, as model is uderfitting and performing well after including these features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:17:35.446814Z",
     "start_time": "2019-01-26T14:17:24.317219Z"
    }
   },
   "outputs": [],
   "source": [
    "data=pd.read_csv('loan.csv')\n",
    "\n",
    "# Select only default or paid loans. Ignore current loans\n",
    "default=['Charged Off','Late (31-120 days)','Default',\n",
    "             'Does not meet the credit policy. Status:Charged Off','Late (16-30 days)',\n",
    "                 'In Grace Period']\n",
    "paid=['Fully Paid',\n",
    "              'Does not meet the credit policy. Status:Fully Paid']\n",
    "\n",
    "data['loan_status']=data['loan_status'].apply(lambda x: 'Default' if x in default else x)\n",
    "data['loan_status']=data['loan_status'].apply(lambda x: 'Fully Paid' if x in paid else x)\n",
    "data=data[(data['loan_status']=='Default') | (data['loan_status']=='Fully Paid')]\n",
    "y=data['loan_status'].apply(lambda x: 1 if x=='Default' else 0)\n",
    "data.drop('loan_status',axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:17:35.799915Z",
     "start_time": "2019-01-26T14:17:35.448764Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# Train-Test Split (Stratified)\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(data, y, test_size=0.2, stratify=y,random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:17:35.808895Z",
     "start_time": "2019-01-26T14:17:35.800915Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Creating pipeline for data preprocessing\n",
    "\n",
    "selected_featues=['loan_amnt', 'term', 'int_rate',\n",
    "                  'grade',#'sub_grade',\n",
    "       'emp_length', 'home_ownership', 'annual_inc', 'verification_status','issue_d', \n",
    "        'purpose', 'addr_state', 'dti',\n",
    "       'delinq_2yrs', 'earliest_cr_line', 'inq_last_6mths', 'open_acc',\n",
    "       'pub_rec', 'revol_bal', 'revol_util', 'total_acc',\n",
    "       'acc_now_delinq', 'tot_coll_amt', 'tot_cur_bal', 'total_rev_hi_lim']\n",
    "\n",
    "num_features= ['loan_amnt', 'int_rate', 'emp_length','annual_inc',\n",
    "       'dti','delinq_2yrs','inq_last_6mths','revol_bal', 'revol_util',\n",
    "       'total_acc','open_acc','pub_rec','acc_now_delinq','tot_coll_amt', 'tot_cur_bal', 'total_rev_hi_lim']\n",
    "\n",
    "dummy_cols=['term','grade','home_ownership','verification_status','purpose']\n",
    "\n",
    "states_to_keep=['FL','NY','LA','NV','IN','CA','IL','GA','TX']\n",
    "\n",
    "features_to_scale=['loan_amnt', 'int_rate', 'emp_length', 'annual_inc',\n",
    "       'dti', 'delinq_2yrs', 'earliest_cr_line',\n",
    "       'inq_last_6mths', 'open_acc', 'pub_rec', 'revol_bal', 'revol_util',\n",
    "       'total_acc', 'acc_now_delinq', 'tot_coll_amt', 'tot_cur_bal',\n",
    "       'total_rev_hi_lim']\n",
    "\n",
    "pipeline=Pipeline([\n",
    "    ('selector',DataFrameSelector(selected_featues)),\n",
    "    ('misc_fe',Misc_Feature_Engineering()),\n",
    "    ('total_rev_hi_lim_Imp',total_rev_hi_lim_Imputer()),\n",
    "    ('median_imp',Median_Imputer(num_features)),\n",
    "    ('dummy_vars',Dummy_Variables(states_to_keep=states_to_keep,cat_cols=dummy_cols)),\n",
    "    ('tranform_features',Transform_Skewed_Features())\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:17:53.225446Z",
     "start_time": "2019-01-26T14:17:35.810888Z"
    },
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X_train=pipeline.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:17:53.867732Z",
     "start_time": "2019-01-26T14:17:53.228411Z"
    }
   },
   "outputs": [],
   "source": [
    "# Scaling\n",
    "scaler=StandardScaler()\n",
    "X_train[features_to_scale]=scaler.fit_transform(X_train[features_to_scale])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:17:53.873693Z",
     "start_time": "2019-01-26T14:17:53.869694Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Over-sampling\n",
    "# columns=X_train.columns\n",
    "# from imblearn.over_sampling import SMOTE\n",
    "# smt=SMOTE(random_state=51,n_jobs=-1)\n",
    "# X_train,y_train=smt.fit_sample(X_train,y_train)\n",
    "# X_train=pd.DataFrame(X_train,columns=columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Model\n",
    "\n",
    "Models:\n",
    "1. Logistic Regression\n",
    "2. Random Forest\n",
    "3. LightGBM\n",
    "\n",
    "Final Model:\n",
    "* Hard voting of above three models (Defaulter if all three models predict as defaulter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:17:59.311079Z",
     "start_time": "2019-01-26T14:17:53.876679Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=10, class_weight='balanced', dual=False,\n",
       "          fit_intercept=True, intercept_scaling=1, max_iter=100,\n",
       "          multi_class='warn', n_jobs=-1, penalty='l2', random_state=51,\n",
       "          solver='liblinear', tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Linear Regression\n",
    "lr=LogisticRegression(C=10,tol=0.0001,random_state=51,n_jobs=-1,solver='liblinear',class_weight='balanced')\n",
    "lr.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:18:21.401663Z",
     "start_time": "2019-01-26T14:17:59.313074Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight='balanced',\n",
       "            criterion='gini', max_depth=14, max_features='auto',\n",
       "            max_leaf_nodes=None, min_impurity_decrease=1e-05,\n",
       "            min_impurity_split=None, min_samples_leaf=100,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=150, n_jobs=-1, oob_score=False, random_state=51,\n",
       "            verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Random Forest\n",
    "rfc=RandomForestClassifier(n_estimators=150,max_depth=14,max_leaf_nodes= None, \n",
    "                           min_impurity_decrease= 1e-05, min_samples_leaf= 100,\n",
    "                           class_weight='balanced',n_jobs=-1,random_state=51)\n",
    "rfc.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:18:23.885039Z",
     "start_time": "2019-01-26T14:18:21.403659Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(boosting_type='gbdt', class_weight={0: 1, 1: 4},\n",
       "        colsample_bytree=1, importance_type='split', learning_rate=0.1,\n",
       "        max_depth=7, min_child_samples=20, min_child_weight=0.001,\n",
       "        min_split_gain=0, n_estimators=100, n_jobs=-1, num_leaves=31,\n",
       "        objective=None, random_state=51, reg_alpha=13, reg_lambda=0.0,\n",
       "        silent=True, subsample=1.0, subsample_for_bin=200000,\n",
       "        subsample_freq=0)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LGBM\n",
    "lgb=LGBMClassifier(random_state=51, class_weight= {0: 1, 1: 4}, colsample_bytree=1, max_depth=7,\n",
    " min_child_samples=20, min_split_gain=0, n_estimators=100, reg_alpha=13)\n",
    "lgb.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predicting on Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:18:29.346461Z",
     "start_time": "2019-01-26T14:18:23.888032Z"
    }
   },
   "outputs": [],
   "source": [
    "# Preprocessing Test Data\n",
    "X_test_1=X_test.copy()\n",
    "X_test=pipeline.transform(X_test)\n",
    "X_test[features_to_scale]=scaler.transform(X_test[features_to_scale])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:18:29.868220Z",
     "start_time": "2019-01-26T14:18:29.348426Z"
    }
   },
   "outputs": [],
   "source": [
    "# Linear Regression\n",
    "y_test_pred_lr=lr.predict(X_test)\n",
    "\n",
    "# Random Forest\n",
    "y_test_pred_rfc=rfc.predict(X_test)\n",
    "\n",
    "# LightGBM\n",
    "y_test_pred_lgb=lgb.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:18:29.932049Z",
     "start_time": "2019-01-26T14:18:29.870216Z"
    }
   },
   "outputs": [],
   "source": [
    "# Voting\n",
    "result=pd.DataFrame(index=X_test.index)\n",
    "result['lr']=y_test_pred_lr\n",
    "result['rfc']=y_test_pred_rfc\n",
    "result['lgb']=y_test_pred_lgb\n",
    "result['count']=result.sum(axis=1)\n",
    "result['y_pred']=result['count'].apply(lambda x: 1 if x>=3 else 0)\n",
    "result['y']=y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:18:30.052891Z",
     "start_time": "2019-01-26T14:18:29.934045Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.70      0.77     41942\n",
      "           1       0.40      0.61      0.48     13486\n",
      "\n",
      "   micro avg       0.68      0.68      0.68     55428\n",
      "   macro avg       0.62      0.66      0.62     55428\n",
      "weighted avg       0.74      0.68      0.70     55428\n",
      "\n",
      "ROC-AUC: 0.655\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,result['y_pred']))\n",
    "print('ROC-AUC:',round(roc_auc_score(y_test,result['y_pred']),4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating Model (On Test Set)\n",
    "\n",
    "**Model is able to predict 61% of the defaulters.**\n",
    "\n",
    "Let us look at **Return on investment (ROI)** to see how model can help in real business.\n",
    "To calculate ROI, I've used total loan repayed and amount funded as loan. These variables leak future information and I have not used these variables for training. Data used to calculate ROI includes fully paid, charged off and loans in grace period. It does not include Current loans.\n",
    "\n",
    "**ROI = 'total_payment' / 'funded_amount'**\n",
    "\n",
    "**ROI without using model : -4.57**\n",
    "\n",
    "**ROI after using model : 2.22**\n",
    "\n",
    "### Grade-wise Analysis of ROI:\n",
    "A table below shows grade-wise analysis of loans. It shows ROI, % of loans selected and % of loans defaulted from selected loans.\n",
    "\n",
    "* ROI for every grade has improved.\n",
    "* ROI for grades C, D, E, G became positive after using model.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:18:30.244607Z",
     "start_time": "2019-01-26T14:18:30.054860Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(33,0.5,'Predicted Label')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAENCAYAAADwjPVkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHQRJREFUeJzt3Xm4VmW5x/Hvb+8tyqCGMiiDgInlkGUOR5tQS+Vo4qmc85RlUZZ5VLT0Ms00Kq3MCsvwOGVHcaiUlFTihJY5gEMakImoTCKKgNpBBfZ9/ljv3r4s9rC27HdYi9/Ha12u4Vnvuvcl3vvhfp/1PIoIzMysGBpqHYCZmXUfJ3UzswJxUjczKxAndTOzAnFSNzMrECd1M7MCcVI3MysQJ3UzswJxUjczK5CmWgeQRc/dT/Zrr7aee387vtYhWB3aa8SW2tDP6ErOWfXohA1+XndyT93MrEBy0VM3M6sq5be/66RuZpbW0FjrCN42J3UzszTVVZm8S5zUzczSXH4xMysQ99TNzArEPXUzswJxT93MrEA8+sXMrEBcfjEzKxCXX8zMCsQ9dTOzAnFSNzMrkEZ/UWpmVhyuqZuZFYjLL2ZmBeKeuplZgbinbmZWIO6pm5kViKcJMDMrEJdfzMwKxOUXM7MCcU/dzKxAnNTNzArEX5SamRWIa+pmZgXi8ouZWYG4p25mVhzKcVLP798xzMwqRFLmLcNnjZb0pKS5ks5qp81RkmZLmiXp+tS1LSQtkjQhS+zuqZuZpaihe3rqkhqBy4ADgYXADEmTI2J2WZuRwNnAByNiuaQBqY+5ELgn6zPdUzczS+nGnvrewNyImBcRbwKTgMNTbb4IXBYRywEiYmlZHHsAA4G7s8bupG5mltKVpC5prKSZZdvYso8aDCwoO15YOlduR2BHSfdJekDS6FIMDcCPgDO7ErvLL2ZmKV35ojQiJgIT2/uotm5JHTcBI4H9gCHAnyXtChwPTImIBV2Jx0ndzCyt+wa/LASGlh0PARa30eaBiFgNPCPpSZIkvy/wYUlfAfoAPSS9FhFtftnawkndzCylG4c0zgBGShoBLAKOAY5LtbkVOBa4RlI/knLMvIj4dFk8JwB7dpbQwUndzGw9DQ3d83VjRKyRdDJwF9AIXBURsyRdAMyMiMmlawdJmg2sBc6MiGVv95lO6mZmKd358lFETAGmpM6dV7YfwOmlrb3PuAa4JsvznNTNzNLy+0Kpk7qZWVqepwlwUjczS3FSNzMrkO6aJqAWnNTNzFLcUzczKxAndTOzAnFSNzMrECd1M7MiyW9Od1I3M0vrrmkCasFJ3cwsxeUX63YHfmAnfnjmETQ2NHDNrX/lh1dPXef6xeM+yUf22hGAXpv1oP9Wfdj2I1+vRahWYX+beT/X/eJHNDc3s9/owxlz9GfXuT7tjt8w9fe30NDQwGab9eLE/zqbwcO2Z83q1Vz50+/xzFNzaJA4/svj2Pm9e9Top8iZ/OZ0J/V61NAgLj3rKA49aQKLXljBX/7nTG6/5wn+MW9Ja5uv/+i3rfsnHTOK975rSC1CtQprXruWay+7mLO+O4Gt+g3gvFM+yx77fJjBw7ZvbbPvfgfz0UM/BcDD99/LrydeyjfG/5Q//eFWAL5/+Q2sXPEyP/jmqVzw02tyXVqoljz31KvyX1fSuyV9Q9JPJf2ktL9TNZ6dR3vtOpynF7zEs4uWsXrNWm6+6xE+vt9u7bY/avQe3HTnw1WM0Krl6SdnMXDbIQzYdjBNm2zCPqMO4uH7712nTa/efVr333h9VWtCWjT/GXZ5314AbPmOrejVpw/PPDWnesHnWDeuUVp1FU/qkr5BstiqgIdIJo0XcIOkTid83xgNGrAlC19Y3nq86IXlDO6/ZZttt9u2L8MGbc30GU9WKzyrouXLXmSr/gNbj7fqN4Dly15cr93UyTdz+uc+waQrf8ZnThoHwHbbj+SR++9h7do1LF2yiGef+gfLXnyharHnWZ6TejXKLycCu5SWamol6RJgFvD9KsSQK2qjoJde1LDFkQfvwa3THqO5ub0WlmfJVNspbeSRA8ccyYFjjuSvf7qTW2+4ii+fcT6jDj6MxQue4dyvfZZ+A7Zl5M670djYWPmgC8Bzv3SsGRgEPJc6v23pWptKK3KPBWgash9N/XapWID1ZtHSFQwZ2Lf1ePDAvix+cWWbbY84eA9O+/5N1QrNqmyrfgN4uax3/fJLS+m7Vf922+8z6iCu/tlFADQ2NnH8l95ad+Hbp53INoOGtnerlanHHnhW1aipnwpMk/QHSRNL253ANOC/2rspIiZGxJ4RsefGlNABZs56jh2268+wQVuzSVMjRx78fu6Y/vh67UYOG0DfLXrxwN+eqUGUVg3bv2tnlixewNIli1izejUP3HM379/nw+u0WbJofuv+Yw/dxzaDk8T9xuuv8/rrqwB44pEHaWhsXOcLVmufyy8diIg7Je0I7A0MJvnL40JgRkSsrfTz82jt2mZOu+gmfv/zr9LYIK697QHmzFvCuScdyiOz53PHPU8AcNToPbn5Ln9BWmSNjU189itncvE5p9Dc3Myogw5jyPB3csuvfsmIkTuxx74f4e7JNzPr0YdobGqid58t+NK4bwHwyoqXueicU2hoaKDv1v056cxv1/inyY86zNWZqc2aXZ3pufvJ9R+kVd29vx1f6xCsDu01YssNTskjz7wzc8556gej6+pXgMepm5mlNPiLUjOz4shz+cVJ3cwsxT11M7MCcU/dzKxA6nGoYlbtJnVJmcawR0S7LxCZmeVRjnN6hz31NbT/djok480D8HvHZlYoeZ7JsqOkPqJqUZiZ1ZFC9tQjIj1XS0tJZmBEPF/RqMzMaijPNfVMf8eQ9A5J1wOvA3NL58ZI+k4lgzMzqwUp+1ZvshaOLgdWAsOAN0vn7geOrkRQZma1tDFM6PVRYFBErJYUABHxoqQBlQvNzKw26jBXZ5Y1qa8E+gGttXRJ25Ufm5kVRZ7fKM1afvlv4DeS9gcaJO0LXEtSljEzK5Q8l1+yJvWLgJuAy4BNgKuA24CfVCguM7Oa6c4vSiWNlvSkpLltrcss6QRJL0p6rLR9oezadpLuljRH0mxJwzt7XqbySySTrl9a2szMCq27euCSGkk6wwdSWhxI0uSImJ1qemNEnNzGR/wKGB8RUyX1oYMlQFtknvtF0gHAsSTrjS4GJkXEtKz3m5nlRTdWVfYG5kbEvORzNQk4HEgn9TZi0M5AU0RMBYiI17I8MOs49dOBScDLwB3AMuB6SeOy3G9mlicNDcq8dWIwsKDseGHpXNqnJD0u6RZJLauD7wiskPRbSY9K+kGp59+hrD31ccABEfH3lhOSrgOmAj/K+BlmZrnQlfKLpLHA2LJTEyNiYsvlNm5Jz6n1e+CGiHhD0pdJBqEcQJKfPwzsDswHbgROAK7sKJ6uTL07N3U8r43gzMxyrytJvZTAJ7ZzeSEwtOx4CEn5uvz+ZWWHV5AMTGm599Gy0s2twD50ktTbLb9IamjZgPOBKyWNlNRT0o6lH+JbHX24mVkedePolxnASEkjJPUAjgEmr/ssbVt2OAaYU3ZvX0n9S8cHkKEWn3Xq3ZbQj02dO45kDLuZWWF01+iXiFgj6WTgLpJpyq+KiFmSLgBmRsRk4BRJY0hy7sskJRYiYq2kM4BpSgJ6mKQn3yFPvWtmltKd7xRFxBRgSurceWX7ZwNnt3PvVGC3rjyvS1PvmpltDPI8TUBXxqmPAUaRzAHT+hNHxGcqEJeZWc001OHr/1llHaf+LeCXpfZHkoxTPxhYUbnQzMxqY2OYT/3zwIERcRrwZunfhwHDKxWYmVmt5HlCr6zll3eUvXj0pqRNIuIhSaMqFZiZWa3kuKSeOak/LWmXiJgF/B04SdJyYHnlQjMzq42N4YvSbwJbl/bPAq4H+gBfqURQZma1pDbf7s+HrFPvTinbfwjYAUDS9hWKy8ysZnLcUe/S3C/rkLQp8BTJW1JmZoVRj1+AZvW2k3pJfn9yM7N25Dinb3BS9yyNZlY4eX75aEOTuplZ4RR29IukBbTfG8/vT21m1oEcd9Q77akfX5UozMzqSGHLLxFxT7UCMTOrF/lN6a6pm5mtZ2Me0mhmVjg5/p7USd3MLK2wo1/MzDZGhSy/SLqODC8XeeUjMyuaHHfUO1wkYy7wdGlbCfwHyTwvC0v3HY5XPjKzAirkIhkR8e2WfUl3AYdGxJ/Lzn0IOLey4ZmZVV/9persstbU9wEeSJ17ENi3e8MxM6u9xhzXX7KuUfoo8F1JPQFK/x4PPFapwMzMaqWQ5ZeUE0hWO1pZWsauLzAT+HSF4jIzq5k6zNWZZV356FngA5KGAoOA5yNifiUDMzOrlTzP/ZK1/IKkrYH9gFERMV/SIElDKhaZmVmNSNm3epOppy5pFPAbkpLLB4GLgZHAGcBhFYuuZPmMCZV+hOXQyFNvq3UIVocWTDh8gz+jHmvlWWWtqV8KHB0R00o1dUhGv+xdmbDMzGqncSNI6sMjYlppv+Ut0ze7cL+ZWW7keERj5pr6bEkHp859DHiim+MxM6u5BmXf6k3WnvY44HZJdwA9Jf2SpJa+4cUrM7M6U/iaekQ8IGk3kuXtrgIWAHtHxMJKBmdmVgv12APPKuvolzMi4ocko17Kz58eEZdUJDIzsxrJcUc9c039vHbOf7O7AjEzqxdNUuatM5JGS3pS0lxJZ3XQ7ghJIWnP0vEmkq6V9ISkOZLOzhR7J8EcUNptlLQ/605etj3wapaHmJnlSXf11CU1ApcBB5JMWz5D0uSImJ1qtzlwCslQ8RZHAptGxHsk9SIZsHJD6Q3/dnVWfrmy9O/NSGrpLQJ4AfhaJ/ebmeVON04TsDcwNyLmAUiaRDLAZHaq3YUk5e0zys4F0FtSE9CTZBj5K509sMOkHhEjSoH8yiscmdnGohtr6oNJBpa0WAj827rP0u7A0Ii4XVJ5Ur+F5BfA80Av4LSIeLmzB2atqV9SmsyrPJChkt6b8X4zs9zoyjh1SWMlzSzbxpZ9VFu/HlqXCZXUAPyYZNh42t7AWpJJFEcA4yRt31nsWcep/xoYkzrXA7gO2C3jZ5iZ5UJXFsmIiInAxHYuLwTKO8RDgMVlx5sDuwLTS2PjtwEmSxoDHAfcGRGrgaWS7gP2BOZ1FE/Wnvp2LTWhsh/kaWB4xvvNzHKjG98onQGMlDRCUg/gGGByy8WIWBkR/SJieEQMJ1lhbkxEzATmAwco0ZtkBbp/dBp7xp9xoaT3l58oHS9up72ZWW6pC/90JCLWACcDdwFzgJsiYpakC0q98Y5cBvQB/k7yy+HqiHi8s9izll9+DNwm6WLgaeCdJN/Sjs94v5lZbnTnG6URMQWYkjrX5rs/EbFf2f5rJMMauyTrNAFXSFoBnEhSH1oAjIuIW7r6QDOzelf4aQIAIuJm4OYKxmJmVhcKOaGXpP+MiOtK+59vr11EXNXeNTOzPGrMvNBn/emop34syZBFgP9sp02w7pumZma5l+eFp9tN6hFxSNn+/tUJx8ys9gpZUy+96dSpiGjuvnDMzGovxx31Dssvayh7nbUDjd0Ui5lZXWjoZPx5PesoqY8o2z8UOAL4HvAcMAz4BvCbyoVmZlYbheypR8RzLfuSTgf2jIgVpVP/lDQTmAn8orIhmplVV1OOi+pZx6lvSTL144qyc71K583MCqWQPfWUa4E/SrqU5G3SoSSrdFxbqcDMzGqlkEMaU74OzAWOJpnb93lgAnBFheIyM6uZHOf0zHO/NAOXlzYzs0LL8Qul2WIvzef7RUnTJD1eOvcRSUdVNjwzs+prkDJv9SbrL6QLSGZovALYrnRuIcmwRjOzQtkYkvoJwMcjYhJvvZD0DNDpenlmZnmjLmz1JusXpY3Aa6X9lqTep+ycmVlh1GEHPLOsPfU/AJdI2hSSGjtwIfD7SgVmZlYrkjJv9SZrUj+NZCjjSpIXjl7jrakCzMwKpaELW73ptPxS6pX3I5n7ZSuSZL4gIpZUODYzs5qoxy9As+o0qUdESHoC2DwilgJLKx+WmVnt1GNZJausf3t4FNixkoGYmdWLQpdfSqYDd0q6hmTul9Z51r1GqZkVTZ576lmT+gdJxqWPSp33GqVmVjj5TenZ537xGqVmttFoLGpPXVIv4JvArsAjwPci4o1qBGZmVis5zumd1vknAIcB/yAZ0vjDikdkZlZj6sI/9aazpP7vwEER8fXS/scrH5KZWW1J2bd601lNvXdEPA8QEQskefk6Myu8hjrsgWfVWVJvkrQ/b30ZnD4mIv63UsGZmdVCPfbAs+osqS9l3SGLy1LHgaffNbOCKew0ARExvEpxmJnVjYb85vTMLx+ZmW006nFUS1ZO6mZmKTmuvjip16v7/nwvF31/PM1rm/nEp47kxC+OXef6TTfewI03XE9jQwM9e/XivPMv5J077FCjaK2S9ttpAOcf8R4aG+CGv87n51OfWq/Nx3cfxGmHvJsgmLPoFb52zcMAXPeVfdh9+FbMmLeMz13+YLVDzy331K1brV27lu+Ov4BfXnE1AwcO5Lijj2C//Q9YJ2kfcuhhHHX0sQBM/99p/PDi7/GLiVfWKmSrkAbBd47ajeMm/JXnV6zi9jNHMfWJJTy15NXWNsP79+arB43kk5f8mZWrVrN1nx6t1y7/41x69mjk0x8aXoPo86s7a+qSRgM/IVkW9L8j4vvttDsCuBnYKyJmls6dDZwIrAVOiYi7OntePc4cudH7+xOPM3ToMIYMHcomPXow+pBDmf6naeu06dOnT+v+qlWrcj2rnLXvfcP78uxL/2L+sv9j9dpg8iOLOGi3bdZpc9wHhnHtvc+wctVqAJa99mbrtfv++RKvvbGmqjEXQYOUeeuIpEbgMpKXN3cGjpW0cxvtNgdOAR4sO7czcAywCzAa+Hnp8zpU0566pM9FxNW1jKEeLX3hBbbZ9q3/cQcMHMgTjz++XrtJ1/8P1/3qalavXs0VV11bzRCtSrbZcjMWL1/Vevz88lXsPrzvOm22H5D8gv/taR+isUH8eMqTTJ/jtWw2RDd2kfYG5kbEPABJk4DDgdmpdhcCFwNnlJ07HJhUmm/rGUlzS593f0cPrHVP/dvtXZA0VtJMSTOvvGJiNWOquXhruvpWbfXEjznu09xx5x859bQzuOLyX1QjNKuytv67p/90NDaKEQP6cNRP7uPkax7m4uPexxY9XVndEF3pqZfnqtJW/gXYYJI1KFosLJ1rJWl3YGhE3J4Ko9N721Lx//KS1u9ili4BA9u7LyImAhMBXl/TRpYrsIEDt2HJ828tAbv0hRcYMGBAu+1HH3Io4y88vwqRWbU9v2IVg/r2bD3etm9PXlj5+nptHn1mOWuagwXL/o+nl77GiP59+Nv8FdUOtzC60lMvz1UZP6o1n0lqAH4MnNDVe9tTjZ76QOAzJLM9prdlVXh+7uyy63uYP/9ZFi5cwOo33+TOKXcwav8D1mnz3HPPtu7fe890ths2rMpRWjX87bkVDO/fm6Fb92KTRjHm/YOZ+vi6a77f/bcl7LtjPwD69u7B9gP68Nyyf9Ui3OJQF7aOLQSGlh0PARaXHW9OMrX5dEnPAvsAkyXtmeHeNlXj72i3A30i4rH0BUnTq/D83GlqauLsc87jpLFfoLl5Lf/xiU+xww4juexnP2GXXXZlvwM+yqTrf80D99/PJk1NbL7FFlz43YtqHbZVwNrm4NybHufXX92XRokbH5jPP5e8yrhD383j81cw9YklTJ+zlI/s1J9p5xxAcwTjb53Fin8lX5r+5tQP8c6Bfei9aRMPXXgQZ17/KPfMebHGP1X968ZpAmYAIyWNABaRfPF5XMvFiFgJ9Gs5LuXEMyJipqRVwPWSLgEGASOBhzp7oCLqv7KxsZVfLJuRp95W6xCsDi2YcPgGZ+QZ81Zmzjl7bb9lh8+TdAhwKcmQxqsiYrykC4CZETE51XY6paReOj4H+DywBjg1Iv7QWTz+NsXMLK0bh79ExBRgSurcee203S91PB4Y35XnOambmaX4jVIzswLJ87t8TupmZik5zulO6mZmaXmedsNJ3cwsJcc53UndzCwtxzndSd3MbD05zupO6mZmKR7SaGZWIK6pm5kViJO6mVmBuPxiZlYg7qmbmRVIjnO6k7qZ2XpynNWd1M3MUrpxkYyqc1I3M0vJb0p3UjczW1+Os7qTuplZioc0mpkVSI5L6k7qZmZpOc7pTupmZmleJMPMrEBynNOd1M3M0nKc053UzczWk+Os7qRuZpbiIY1mZgXimrqZWYE0OKmbmRVJfrO6k7qZWYrLL2ZmBZLjnO6kbmaW5p66mVmBeJoAM7MCyW9Kd1I3M1tPjjvqNNQ6ADOzeqMu/NPpZ0mjJT0paa6ks9q4/mVJT0h6TNJfJO1cOn+gpIdL1x6WdECW2N1TNzNL66aeuqRG4DLgQGAhMEPS5IiYXdbs+oi4vNR+DHAJMBp4CTgsIhZL2hW4Cxjc2TPdUzczS1EXtk7sDcyNiHkR8SYwCTi8vEFEvFJ22BuI0vlHI2Jx6fwsYDNJm3b2QPfUzcxSGrqvqD4YWFB2vBD4t3QjSV8FTgd6AG2VWT4FPBoRb3T2QPfUzcxSpK5sGitpZtk2tvyj2vj4WO9ExGUR8U7gG8A3141FuwAXAV/KErt76mZmGyAiJgIT27m8EBhadjwEWNxOW0jKM79oOZA0BPgd8JmIeDpLPO6pm5mldKWn3okZwEhJIyT1AI4BJq/7LI0sOzwUeKp0/h3AHcDZEXFf1tjdUzczS+muRTIiYo2kk0lGrjQCV0XELEkXADMjYjJwsqSPAauB5cBnS7efDOwAnCvp3NK5gyJiaYexR6xX3qk7r69ZvwZlNvLU22odgtWhBRMO3+CM/MrrzZlzzhab1dfs6+6pm5ml5PmNUid1M7MUr1FqZlYg7qmbmRVIjnO6k7qZ2XpynNWd1M3MUrpxmoCqy8WQRnuLpLGlN9jMWvnPhbXwG6X5M7bzJrYR8p8LA5zUzcwKxUndzKxAnNTzx3VTa4v/XBjgL0rNzArFPXUzswJxUs+RzlYlt42PpKskLZX091rHYvXBST0nylYl/3dgZ+BYSTvXNiqrA9eQrDxvBjip50mnq5Lbxici7gVernUcVj+c1POjrVXJB9coFjOrU07q+ZFpVXIz27g5qedHV1clN7ONkJN6fnS6KrmZmZN6TkTEGpLVxe8C5gA3RcSs2kZltSbpBuB+4F2SFko6sdYxWW35jVIzswJxT93MrECc1M3MCsRJ3cysQJzUzcwKxEndzKxAnNRtoyPpO5Kuqfa9ZtXgpG5vm6TXyrZmSavKjj9dhef/WtL5lX6OWZ401ToAy6+I6NOyL+lZ4AsR8cf22ktqKr1EZWYV4p66VUypVHGjpBskvQocn+5dS/pY6RdCy/EQSb+T9KKkZyR99W0+e0LpDctXJM2Q9IFUk56Sbpb0qqSZkt7T3TGY1YKTulXaJ4DrgS2BGztqWFoI5HaSeW4GAwcCZ0r66Nt47oPAbsBWwC3AzZI2Lbv+yVJcLdd/J6mpm2Mwqzondau0v0TE7yOiOSJWddJ2H2CLiPhuRLwZEXOBK0kmL+uSiLguIl4ulXsuBrYAdihr8mBE/C4iVgM/KF3fqztjMKsF19St0hZ03qTVMGA7SSvKzjUC07v6UElfBz4PbEsy73xvoF9bcUXEWkmLgEHApt0Vg1ktOKlbpaVnjPsX0KvseJuy/QXAUxGx04Y8UNL+wOnAR4HZpdMrWXehkaFl7RtISi2LSf6f2OAYzGrF5RertseAQyX1lbQtcErZtfuBNyWNk7SZpEZJ75G0Rwef11Rq27L1ADYH1gAvAZsA55P01MvtLelwSZsAZwCvktTR304MZnXDSd2q7RqS+eCfA+4kWUAbaJ0z/hCSRbafJUnKvySpd7fnHGBV2XY3MAX4I/BU6XNeAZ5P3fc74HiSRZuPBj4ZEWveZgxmdcPzqZuZFYh76mZmBeKkbmZWIE7qZmYF4qRuZlYgTupmZgXipG5mViBO6mZmBeKkbmZWIE7qZmYF8v9hPm0EDqXLGAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Confusion Matrix\n",
    "cm = confusion_matrix(y_test, result['y_pred']).T\n",
    "cm = cm.astype('float')/cm.sum(axis=0)\n",
    "ax = sns.heatmap(cm, annot=True, cmap='Blues');\n",
    "ax.set_xlabel('True Label',size=12)\n",
    "ax.set_ylabel('Predicted Label',size=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:18:30.312459Z",
     "start_time": "2019-01-26T14:18:30.246583Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROI after filtering potential defaulters: 2.22\n",
      "ROI without filtering defaulters: -4.57\n"
     ]
    }
   ],
   "source": [
    "# Improvement in ROI after using model\n",
    "\n",
    "index=result[result['y_pred']==0].index\n",
    "tmp=X_test_1.loc[index][['total_pymnt','funded_amnt']]\n",
    "roi=(tmp['total_pymnt'].sum()/tmp['funded_amnt'].sum()-1)*100\n",
    "print(\"ROI after filtering potential defaulters:\",round(roi,2))\n",
    "\n",
    "tmp=X_test_1[['total_pymnt','funded_amnt']]\n",
    "roi=(tmp['total_pymnt'].sum()/tmp['funded_amnt'].sum()-1)*100\n",
    "print(\"ROI without filtering defaulters:\",round(roi,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:18:30.861745Z",
     "start_time": "2019-01-26T14:18:30.314453Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROI</th>\n",
       "      <th>% Picked</th>\n",
       "      <th>% Default</th>\n",
       "      <th>ROI_w/o_model</th>\n",
       "      <th>% Picked_w/o_model</th>\n",
       "      <th>% Default_w/o_model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A</th>\n",
       "      <td>0.0250712</td>\n",
       "      <td>99.9774</td>\n",
       "      <td>8.64002</td>\n",
       "      <td>0.025071</td>\n",
       "      <td>100</td>\n",
       "      <td>8.63806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>B</th>\n",
       "      <td>0.0200738</td>\n",
       "      <td>96.1866</td>\n",
       "      <td>16.1057</td>\n",
       "      <td>0.0152713</td>\n",
       "      <td>100</td>\n",
       "      <td>16.7564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C</th>\n",
       "      <td>0.017765</td>\n",
       "      <td>53.9985</td>\n",
       "      <td>18.7983</td>\n",
       "      <td>-0.0447373</td>\n",
       "      <td>100</td>\n",
       "      <td>25.9779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D</th>\n",
       "      <td>0.0349775</td>\n",
       "      <td>24.597</td>\n",
       "      <td>20.8825</td>\n",
       "      <td>-0.096151</td>\n",
       "      <td>100</td>\n",
       "      <td>34.7948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E</th>\n",
       "      <td>0.0703003</td>\n",
       "      <td>7.08117</td>\n",
       "      <td>22.2561</td>\n",
       "      <td>-0.143831</td>\n",
       "      <td>100</td>\n",
       "      <td>42.5518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F</th>\n",
       "      <td>-0.0234227</td>\n",
       "      <td>3.26508</td>\n",
       "      <td>22.0339</td>\n",
       "      <td>-0.16447</td>\n",
       "      <td>100</td>\n",
       "      <td>47.316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>G</th>\n",
       "      <td>0.0961287</td>\n",
       "      <td>2.6694</td>\n",
       "      <td>38.4615</td>\n",
       "      <td>-0.163535</td>\n",
       "      <td>100</td>\n",
       "      <td>50.1027</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         ROI % Picked % Default ROI_w/o_model % Picked_w/o_model  \\\n",
       "A  0.0250712  99.9774   8.64002      0.025071                100   \n",
       "B  0.0200738  96.1866   16.1057     0.0152713                100   \n",
       "C   0.017765  53.9985   18.7983    -0.0447373                100   \n",
       "D  0.0349775   24.597   20.8825     -0.096151                100   \n",
       "E  0.0703003  7.08117   22.2561     -0.143831                100   \n",
       "F -0.0234227  3.26508   22.0339      -0.16447                100   \n",
       "G  0.0961287   2.6694   38.4615     -0.163535                100   \n",
       "\n",
       "  % Default_w/o_model  \n",
       "A             8.63806  \n",
       "B             16.7564  \n",
       "C             25.9779  \n",
       "D             34.7948  \n",
       "E             42.5518  \n",
       "F              47.316  \n",
       "G             50.1027  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Grade-wise ROI\n",
    "df1=create_performance_chart(X_test_1,result['y_pred'],y_test)\n",
    "df2=create_performance_chart(X_test_1,np.zeros(len(y_test)),y_test)\n",
    "df2.columns=map(lambda x:x+'_w/o_model',df2.columns)\n",
    "pd.concat([df1,df2],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Curve and Scope for Improvement\n",
    "\n",
    "From learning curve, it is clear that model is the underfit. Even if we supply more data, it will improve model minimally.\n",
    "\n",
    "Also confusion matrix printed above shows that 30% of the fair loans were predicted as default.\n",
    "\n",
    "This model can be improved by adding more features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-26T14:19:12.194001Z",
     "start_time": "2019-01-26T14:18:30.862743Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1db1a5f2d68>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe8AAAD8CAYAAABevCxMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd0XOWd//H3d2ZUrF4sucm2FNsxbrjJxiaVQFggheWELDa9RZAQDks2u+uc8PuRwGYXkuwhkJAFAwYCCTZLwv5IYR0CCZtscJGwKbaxMa7CRe6Si2SN5vn9ca+kkSxZI3mk0Uif1zk6uveZZ+Z+r66tj+5zmznnEBERkeQRSHQBIiIi0j0KbxERkSSj8BYREUkyCm8REZEko/AWERFJMgpvERGRJKPwFhERSTIKbxERkSSj8BYREUkyoUQX0N7QoUNdaWlpossQERHpM1VVVfudc0Wx9u934V1aWkplZWWiyxAREekzZra9O/01bC4iIpJkFN4iIiJJRuEtIiKSZPrdMW8RERkcGhsbqa6upr6+PtGl9Jn09HRKSkpISUk5o89ReIuISEJUV1eTnZ1NaWkpZpbocnqdc44DBw5QXV1NWVnZGX2Whs1FRCQh6uvrKSwsHBTBDWBmFBYWxmWkQeEtIiIJM1iCu1m81ndAh7dzjn///UY21xxNdCkiIiJxM6DDu6augedW7WTB4hVs2luX6HJERKQfOXDgADNmzGDGjBkMHz6cUaNGtcyfPHkyps+44YYb2LhxYy9XeqqYwtvMLjKzjWa22cwWdfD6GDP7o5mtMbO3zewSv/2zZlZlZu/43z8T7xU4nWE56SytmEfAYOHiFWzYXduXixcRkX6ssLCQtWvXsnbtWm699VbuvPPOlvnU1FTAG8GNRCKdfsaTTz7JxIkT+6rkFl2Gt5kFgYeBi4HJwEIzm9yu213A8865mcAC4Kd++37gC865acB1wDPxKjxW44uzWHbLfFKCAa58bAXrdh3p6xJERCSJbN68malTp3Lrrbcya9Ysdu/eTUVFBeXl5UyZMoV77rmnpe/HP/5x1q5dSzgcJi8vj0WLFjF9+nTmz59PTU1Nr9UYy6Vic4HNzrktAGa2FLgUWB/VxwE5/nQusAvAObcmqs86IN3M0pxzDWdaeHeUDc1k2S3zuPKxlVz52EqevekcppXk9mUJIiJyGt/99TrW74rv6OjkkTnc/YUpPXrv+vXrefLJJ3nkkUcAuO+++ygoKCAcDnPeeedx+eWXM3ly2/3YI0eO8KlPfYr77ruPb3zjGyxZsoRFi04ZrI6LWIbNRwE7o+ar/bZo3wGuNrNq4HfA7R18zpeANX0d3M3GFmaytGIe2ekhrnx8BWt2HEpEGSIikgTGjRvHnDlzWuafe+45Zs2axaxZs9iwYQPr168/5T1Dhgzh4osvBmD27Nls27at1+qLZc+7o/PaXbv5hcBTzrl/N7P5wDNmNtU5FwEwsynA/cCFHS7ArAKoABgzZkystXfb6IIMlt0ynysfW8E1T6zi6RvnMHtsQa8tT0REYtPTPeTekpmZ2TL9/vvv8+CDD7Jq1Sry8vK4+uqrO7xWu/k4OUAwGCQcDvdafbHseVcDo6PmS/CHxaPcBDwP4Jx7A0gHhgKYWQnwInCtc+6DjhbgnFvsnCt3zpUXFcX8ONMeGZU3hGUV8ynKTuPaJ1axauvBXl2eiIgkt9raWrKzs8nJyWH37t0sX7480SXFFN6rgQlmVmZmqXgnpL3Urs8O4HwAM5uEF977zCwP+C3wLefc/8av7DMzPDedZRXzGJ6bznVLVvHXD/YnuiQREemnZs2axeTJk5k6dSpf+cpX+NjHPpbokjDn2o+Ad9DJu/TrR0AQWOKc+56Z3QNUOude8s8+fwzIwhtS/yfn3O/N7C7gW8D7UR93oXOu01PwysvLXWVlZc/XqBv21TVw1eMr2HHwOI9fO4ePTxjaJ8sVERHYsGEDkyZNSnQZfa6j9TazKudceayfEVN496W+DG+AA0cbuOrxlWzZf4zF18zm0xOL+2zZIiKDmcK7VXfDe0DfYS0WhVlpPPeVeYwvyqLiZ1W8umFvoksSERE5rUEf3gD5man84ivnMHF4Nrc+W8XydXsSXZKIiEinFN6+vIxUnr35HKaMzOW2n7/J797ZneiSREREOqTwjpI7JIVnbprL9NF53P7cGn79Vvsr4kRERBJP4d1OdnoKT984l9lj8rlj6RpeXFOd6JJERETaUHh3ICstxFM3zuGcskK+8fxbvFClABcRGWji8UhQgCVLlrBnT9+eKxXL7VEHpYzUEEuun0PFM5X84wtv0RSJcMWc3rt1q4iI9K3mR4ICfOc73yErK4tvfvOb3f6cJUuWMGvWLIYPHx7vEjulPe/TGJIa5LFry/nkhCL++Zfv8OyK7YkuSURE+sDTTz/N3LlzmTFjBl/72teIRCKEw2GuueYapk2bxtSpU3nooYdYtmwZa9eu5Yorruj2HvuZ0J53F9JTgiy+djZfe/ZN7vqvd2mKOK47tzTRZYmIDCwvL4I978T3M4dPg4vv6/bb3n33XV588UX++te/EgqFqKioYOnSpYwbN479+/fzzjtenYcPHyYvL48f//jH/OQnP2HGjBnxrf80tOcdg7RQkP+4ejYXTh7G3S+t4/E/b0l0SSIi0kv+8Ic/sHr1asrLy5kxYwavv/46H3zwAePHj2fjxo3ccccdLF++nNzc3ITVqD3vGKWGAjx81SzuWLqGf/ntBsIRx62fGpfoskREBoYe7CH3FuccN954I/fee+8pr7399tu8/PLLPPTQQ/zyl79k8eLFCahQe97dkhIM8NCCmXxh+kjue/k9fvLa+12/SUREksoFF1zA888/z/793hMnDxw4wI4dO9i3bx/OOb785S/z3e9+lzfffBOA7Oxs6urq+rRG7Xl3UygY4IG/m04oYPzw95sIRxx3nD8BM0t0aSIiEgfTpk3j7rvv5oILLiASiZCSksIjjzxCMBjkpptuwjmHmXH//fcDcMMNN3DzzTczZMgQVq1aRWpqaq/XOOifKtZTTRHHP//ybV6oqubr543nHy78qAJcRKQb9FSxVt19qpj2vHsoGDC+/6WzCQWMn/xxM42RCIsuOksBLiIivU7hfQYCAeNfL5tGKGg8+voWwk2Ouz43SQEuIiK9SuF9hgIB495LpxIKBHjiL1tpijju/sJkBbiISAyajx8PFvE6VK3wjgMz4+4vTCYUMB7/y1YamyLce+lUAoHB8w9SRKS70tPTOXDgAIWFhYMiwJ1zHDhwgPT09DP+LIV3nJgZ3/7cJELBAI+8/gFNEce/XjZNAS4i0omSkhKqq6vZt29fokvpM+np6ZSUlJzx5yi848jM+OeLJpISNH782mYamxzfv/xsggpwEZFTpKSkUFZWlugykpLCO87MjH+4cCKhQIAH/rCJpkiEH355OqGg7ocjIiLxofDuJXdcMIFQ0PjB8o00ObwbuyjARUQkDhTevei288YTChj/9vJ7hJsiPLRwJikKcBEROUNKkl52y6fGcdfnJvHyu3u47edvcjIcSXRJIiKS5GIKbzO7yMw2mtlmM1vUwetjzOyPZrbGzN42s0uiXvuW/76NZvY38Sw+Wdz8iY/w3S9O4ffr9/LVZ6toCDcluiQREUliXYa3mQWBh4GLgcnAQjOb3K7bXcDzzrmZwALgp/57J/vzU4CLgJ/6nzfoXHduKd+7bCqvvldDxc+qqG9UgIuISM/Esuc9F9jsnNvinDsJLAUubdfHATn+dC6wy5++FFjqnGtwzm0FNvufNyhddc5Y7v/SNP7n/X3c/HQlJ04qwEVEpPtiCe9RwM6o+Wq/Ldp3gKvNrBr4HXB7N947qFwxZww/uHw6//vBfm58ajXHT4YTXZKIiCSZWMK7ozuMtL8560LgKedcCXAJ8IyZBWJ8L2ZWYWaVZlY5GO60c/nsEh74uxms3HqA65es5miDAlxERGIXS3hXA6Oj5ktoHRZvdhPwPIBz7g0gHRga43txzi12zpU758qLiopirz6J/e3MUTy4YCZVOw5x3ZJV1NU3JrokERFJErGE92pggpmVmVkq3gloL7XrswM4H8DMJuGF9z6/3wIzSzOzMmACsCpexSe7L0wfyU8WzuStnYe5+olVHDmhABcRka51Gd7OuTDwdWA5sAHvrPJ1ZnaPmX3R7/YPwFfM7C3gOeB651mHt0e+Hvhv4DbnnM7SinLxtBH89KpZrN91hKsfX8nh4ycTXZKIiPRzFq9ni8ZLeXm5q6ysTHQZfe7VDXv56rNvMr44i2dvPoeCzNRElyQiIn3EzKqcc+Wx9tcd1vqJ8ycNY/G1s9m87yhXPraC/UcbEl2SiIj0UwrvfuTTE4tZct0cth04xsLFK6ipq090SSIi0g8pvPuZj08YypPXz6X60AkWLF7B3loFuIiItKXw7ofmjyvk6RvnsvdIPQsWr2D3kROJLklERPoRhXc/NbesgJ/dNJd9dQ1c8egKPjysABcREY/Cux+bPbaAZ28+h0PHT3LFo2+w8+DxRJckIiL9gMK7n5sxOo9f3DyPuvowVzz6BtsPHEt0SSIikmAK7yQwrSSXX3zlHE40NnHFoyvYsu9ooksSEZEEUngniSkjc3muYh6NTREWLF7B5hoFuIjIYKXwTiJnDc/huYp5RBwsWPwGm/bWJbokERFJAIV3kvnosGyWVswjYMaCxSvYsLs20SWJiEgfU3gnofHFWSy7ZT6pwQALH1vBux8eSXRJIiLShxTeSapsaCbLbplHZmqIKx9bwdvVhxNdkoiI9BGFdxIbW5jJ0op55AxJ4arHV7Jmx6FElyQiIn1A4Z3kRhdksOyW+RRkpnLNE6uo3HYw0SWJiEgvU3gPAKPyhrCsYj5F2Wlcu2QVK7ccSHRJIiLSixTeA8Tw3HSWVcxjRG461z+5mr9u3p/okkREpJcovAeQ4px0llbMZ3TBEG54ajV/fn9foksSEZFeoPAeYIqy03juK/MoG5rJTU9X8qeNNYkuSURE4kzhPQAVZnkBPqE4i4qfVfHqhr2JLklEROJI4T1A5Wem8oub53HWiGxufbaK5ev2JLokERGJE4X3AJabkcIzN53DlJG53PbzN/ndO7sTXZKIiMSBwnuAyx2SwjM3zWX66Dxuf24Nv35rV6JLEhGRM6TwHgSy01N4+sa5zB6Tzx1L1/DimupElyQiImdA4T1IZKWFeOrGOZxTVsg3nn+L/6zcmeiSRESkh2IKbzO7yMw2mtlmM1vUwesPmNla/2uTmR2Oeu37ZrbOzDaY2UNmZvFcAYldRmqIJdfP4ePjh/JPv3ybpat2JLokERHpgS7D28yCwMPAxcBkYKGZTY7u45y70zk3wzk3A/gx8Cv/vecCHwPOBqYCc4BPxXUNpFuGpAZ57NpyPjmhiEW/eodnVmxPdEkiItJNsex5zwU2O+e2OOdOAkuBS0/TfyHwnD/tgHQgFUgDUgBddJxg6SlBFl87m/PPKub//Ne7PPW/WxNdkoiIdEMs4T0KiD5AWu23ncLMxgJlwGsAzrk3gD8Cu/2v5c65DR28r8LMKs2sct8+3dKzL6SFgvzH1bO5cPIwvvPr9Tz+5y2JLklERGIUS3h3dIzaddJ3AfCCc64JwMzGA5OAErzA/4yZffKUD3NusXOu3DlXXlRUFFvlcsZSQwEevmoWl0wbzr/8dgP/8acPEl2SiIjEIBRDn2pgdNR8CdDZxcILgNui5i8DVjjnjgKY2cvAPOB/ul+q9IaUYICHFswkGHiL+//7PcJNEW4/f0KiyxIRkdOIZc97NTDBzMrMLBUvoF9q38nMJgL5wBtRzTuAT5lZyMxS8E5WO2XYXBIrFAzwwN9N57KZo/j3VzbxwCubcK6zwRUREUm0Lve8nXNhM/s6sBwIAkucc+vM7B6g0jnXHOQLgaWu7W/9F4DPAO/gDbX/t3Pu13FdA4mLUDDAD788nWDAePDV9wlHInzzwonoyj4Rkf4nlmFznHO/A37Xru3/tpv/TgfvawJuOYP6pA8FA8b3v3Q2KUHj4T9+QDjiWHTRWQpwEZF+JqbwlsEjEDC+97fTCAaMR1/fQrjJcdfnJinARUT6EYW3nCIQMO69dCqhQIAn/rKVpojj7i9MVoCLiPQTCm/pkJlx9xcmEwoYj/9lK41NEe69dCqBgAJcRCTRFN7SKTPj25+bRCgY4JHXP6Ap4vjXy6YpwEVEEkzhLadlZvzzRRNJCRo/fm0zjU2O719+NkEFuIhIwii8pUtmxj9cOJFQIMADf9hEUyTCD788nVBQT5QVEUkEhbfE7I4LJhAKGj9YvpFwxPHAFTNIUYCLiPQ5hbd0y23njScUMP7t5fdoijgeXDCT1JACXESkL+m3rnTbLZ8ax12fm8TL7+7htl+8SUO4KdEliYgMKgpv6ZGbP/ERvvvFKbyyfi9fffZN6hsV4CIifUXhLT123bmlfO+yqbz2Xg0Vz1QpwEVE+ojCW87IVeeM5f4vTePP7+/j5qcrOXFSAS4i0tsU3nLGrpgzhh9cPp3//WA/Nzy1imMN4USXJCIyoCm8JS4un13Cj66YwaqtB7n+yVUcVYCLiPQahbfEzaUzRvHQwpm8ueMw1z6xkrr6xkSXJCIyICm8Ja4+f/ZIfrJwJm9XH+HqJ1Zx5IQCXEQk3hTeEncXTxvBT6+axfpdR7j68ZUcPn4y0SWJiAwoCm/pFRdOGc4jV89m4546rnxsJQePKcBFROJF4S295vxJw1h87Ww27zvKlY+tYP/RhkSXJCIyICi8pVd9emIxS66bw7YDx1i4eAU1dfWJLklEJOkpvKXXfXzCUJ68fi7Vh06wYPEK9tYqwEVEzoTCW/rE/HGFPH3jXPYeqWfB4hXsPnIi0SWJiCQtc84luoY2ysvLXWVlZaLLkF5Stf0g1y1ZTcBgblkhc0rzKS/NZ+qoXNJCwUSXJyKSEGZW5Zwrj7W/nuctfWr22AKW3TKPJX/ZRtX2g/xhw14AUkMBppfkMntsAXNK85k9Np+8jNQEVysi0j/FtOdtZhcBDwJB4HHn3H3tXn8AOM+fzQCKnXN5/mtjgMeB0YADLnHObetsWdrzHlz21TVQtf0QldsOUrn9EO9+eIRwxPs3OaE4i/LS/JZAH1OQgZkluGIRkfjr7p53l+FtZkFgE/BZoBpYDSx0zq3vpP/twEzn3I3+/J+A7znnXjGzLCDinDve2fIU3oPbiZNNvFV9mKrth1i97SBV2w9RV+/dJ31oVlrLXvmc0gImj8whJajTNkQk+fXGsPlcYLNzbou/gKXApUCH4Q0sBO72+04GQs65VwCcc0djLUwGpyGpQeZ9pJB5HykEIBJxbKqpo3LboZZAf/ndPV7flCDTR+cyp7SA2WPzmTU2n5z0lESWLyLSJ2IJ71HAzqj5auCcjjqa2VigDHjNb/oocNjMfuW3/wFY5JzTQ58lJoGAcdbwHM4ansPV88YCsOdIPZXbD7YE+k//9AFNEYcZTByWTXlpfkugj8oboqF2ERlwYgnvjn7zdTbWvgB4ISqcQ8AngJnADmAZcD3wRJsFmFUAFQBjxoyJoSQZzIbnpvP5s0fy+bNHAnCsIczanYdbhtlffPNDnl2xw+ubk055aT7lY/MpLy3grOHZhDTULiJJLpbwrsY72axZCbCrk74LgNvavXdN1JD7fwHzaBfezrnFwGLwjnnHVLmILzMtxMfGD+Vj44cCEG6K8N6eOu9EOP9kuN+8vdvrmxpk5ph8P9ALmDkmj8w0XXQhIskllt9aq4EJZlYGfIgX0Fe272RmE4F84I127803syLn3D7gM4DORpNeFQoGmDoql6mjcrnu3FIAPjx8wjujfZsX6A+++j7OQTBgTBqRTfnYgpZAH56bntgVEBHpQpfh7ZwLm9nXgeV4l4otcc6tM7N7gErn3Et+14XAUhd1+rpzrsnMvgm8at6BxyrgsbivhUgXRuUNYdSMUVw6YxQAtfWNrNlxuCXQl67ewVN/3QZASf4QysfmM7vUu0Tto8XZBAI6bi4i/YfusCYCNDZFWL+rtmWYvXL7IfbVeU9By04PMWtMvn+ZWgEzRucxJFV3gxOR+In7dd59TeEt/YFzjp0HT7DaD/LKbQd5v8a70jEUMKaMymXO2PyWm8gUZacluGIRSWYKb5Fecvj4Sd7ccYjV2w5Rte0Qa6sPczIcAaC0MIPZ/nHzOaX5jCvK0iVqIhIzhbdIH2kIN/Huh7VUbT/oBfr2Qxw8dhKAvIwU77i5H+jTRuWSnqKhdhHpmMJbJEGcc2zZf4yqbYdabiKzZf8xAFKDAaaV5Lac0T57bD4FmXrwioh4FN4i/cj+o96DV5ofvvLOh0dobPL+z40rymy9RK20gNJCPXhFZLBSeIv0Y/WNTbxdfaTN7V2PnGgEYGhWKrPH5rcE+pSRuaSGdDc4kcFAz/MW6cfSU4LMLStgblkB4D14ZfO+o/7NY7xAX77Oe8Z5WijA9NF5zPGH2meNySc3Qw9eERHteYv0OzW19f7laYeo2n6Qd3fVtjx45aPF2cz2z2gvH1tASb4evCIyEGjYXGSAOX7Se/BK861d12w/RF2D94zzYTlpLSfAzSktYNIIPXhFJBlp2FxkgMlIDXHuuKGcO8578EpTxLFxT12bS9R++85uv2+QmWPyvEvUxuYzc0we2XrGuciAoz1vkQFg1+ETVG4/RNU2L9Df21NLxEHA4KzhOd6tXUu9QB+ZNyTR5YpIOxo2FxHq6hv9Z5x7x83X7DjM8ZNNgPeQltljWx+LOnF4NkE9eEUkoTRsLiJkp6fwiQlFfGJCEeA943zD7rqWM9pXbj3AS2/t8vqmhZg5Np/ZY/IpHZrBsJx0/yuNjFT9ihDpj7TnLTIIOeeoPnSiJcwrtx1iU00d7X8dZKeFGJbrBfmw7HSKc9IZnpPGsBxvelhOGsXZ6boeXeQMac9bRLpkZowuyGB0QQaXzSwB4GhDmD1HTrC3toE9R+rZW1dPTW0De2vr2VNbz8qtB6mpq2+5Q1y0wszUljAfHhXsw7LTGZ6bTnFOGoWZaRqeF4kThbeIAJCVFmJ8cTbji7M77ROJOA4dP8me2tZg31vbwN66evb6gb9uVy37jzacshcfDBhFWWnennx2WsvQfOswvTefOyRF166LdEHhLSIxCwSMwqw0CrPSmDKy837hpgj7jjZ4wV5bT42/9948v/3AcVZuPdhya9hoaaFAS5B7w/StIV/s78nreLwMdvrXLyJxFwoGGJE7hBG5p78srb6xyduDr6v3hupr66mp84fqj9Szflctr22o4URj0ynvzU4LUZyT5oV5dnqHw/Y6Hi8DlcJbRBImPSXImMIMxhRmdNrHOcfRhnDrEH2b795XrMfjh2WntzkBr3kPvzBLx+MluSi8RaRfMzOy01PITk+J6Xh88zH4mtp69hxpnd5b29D18fh2Q/XF/rH44ToeL/2MwltEBoTo4/GTyem0X7gpwv6jJ1vOoq+J2pPfU1vPjgPHWb3tIIePn3o8PjUUaDs0n+0P1ed6x+Obj81npulXq/Qu/QsTkUElFAwwPNc78W36afpFH4/vaKh+w65a/lhb03LnumjNx+Pbn0kf/b0oO420ULD3VlQGNIW3iEgHunc8vuGUM+pr/JPwVp3meHxBZirF2a0n3bUdtvfmCzJT9aQ4OYXCW0Skh9oej8/qtJ9zjkPHGzscqm/eq1/vH4+PdHDTy4zUIFlpIbLTQ2Slp5CTHmqdT0shKz0U1ebNZ6eHyI6az0wN6nj9ABJTeJvZRcCDQBB43Dl3X7vXHwDO82czgGLnXF7U6znABuBF59zX41G4iEiyMDMKMlMpyExl0ojYjsfvra1nb10DB442cLQ+zNGGMHX1YeoawtTVN7LnSD11fvtR//nupxMwyEwLkZOeEvWHgB/uaa1hH92W00GftFBAfwT0A12Gt5kFgYeBzwLVwGoze8k5t765j3Puzqj+twMz233MvcDrcalYRGSAij4e3x1NEcexk164e0HfSK0/XefP1/nT0fMHj51k+4HjfnsjDeFIl8tKCVqbMG/e289Ob51vDvzs0/TRoYAzE8ue91xgs3NuC4CZLQUuBdZ30n8hcHfzjJnNBoYB/w3EfNN1ERGJTTBg5KSnkJOeckafczIc8fbk68PU+QHfPH20Puz9QeDv+Tf/YVDXEObDwyfa/IHQ1NHYfztDUoKnDu+3GxFoHQno+A+AzNQQgUF6fX4s4T0K2Bk1Xw2c01FHMxsLlAGv+fMB4N+Ba4Dzz6hSERHpVamhAAUhb3i/p5xzNIQj1EYFfHPg17WbP9oQjhohaKSmrr71PSfDp1yP354ZZKV2cgig3Xxre9s/EnLSU5LyUEAs4d3RGnX2I10AvOCca7524mvA75xzO0/3gzGzCqACYMyYMTGUJCIi/ZGZkZ4SJD0lyGnuqdOliH8ooOVYf31r4NdFBX5d9HxDI4ePn2TnoeMtbR3dWre9UMBa/wDo4gTA5j5nj86lOLt7hzfiKZbwrgZGR82XALs66bsAuC1qfj7wCTP7GpAFpJrZUefcoug3OecWA4vBe553jLWLiMgAFQi0nsk/Irfnn9PYFOFYJ38A1LU7BND6h0Iju6NOCKyrbzzlUr/F18zmwinDz3Atey6W8F4NTDCzMuBDvIC+sn0nM5sI5ANvNLc5566Kev16oLx9cIuIiPSWlGCAvIxU8jLO/FBAdJiPLciMY5Xd12V4O+fCZvZ1YDnepWJLnHPrzOweoNI595LfdSGw1LmujlKIiIgkj+hDAUXZaYkuBwDrb1lbXl7uKisrE12GiIhInzGzKudczFdk6UI7ERGRJKPwFhERSTIKbxERkSSj8BYREUkyCm8REZEko/AWERFJMgpvERGRJKPwFhERSTIKbxERkSSj8BYREUkyCm8REZEko/AWERFJMgpvERGRJKPwFhERSTIKbxERkSSj8BYREUkyCm8REZEko/AWERFJMgpvERGRJKPwFhERSTIKbxERkSSj8BYREUkyCm8REZEko/AWERFJMgpvERGRJBNTeJvZRWa20cw2m9miDl7DIQdmAAAOZ0lEQVR/wMzW+l+bzOyw3z7DzN4ws3Vm9raZXRHvFRARERlsQl11MLMg8DDwWaAaWG1mLznn1jf3cc7dGdX/dmCmP3scuNY5976ZjQSqzGy5c+5wPFdCRERkMOkyvIG5wGbn3BYAM1sKXAqs76T/QuBuAOfcpuZG59wuM6sBigCFt4iIJE4kApFGiIShqREiTR3Mh6Pawm3nh02DrKKElR9LeI8CdkbNVwPndNTRzMYCZcBrHbw2F0gFPuh+mSIi0iec80OqOcTCnQRbJ6F2SltT1Oe0nw93sKwzWXYMy2qex53Zz2nBL+Csz8XlR94TsYS3ddDW2VovAF5wzjW1+QCzEcAzwHXOucgpCzCrACoAxowZE0NJIiL9kHNeODSdbPfVCOGG1ummhtO0++8PR013GHSnCbWOgq3TQG33uW1/ffedQAgCKd73YMifb24LQjAlqi3UOh9Kg0BmVFuwg89pfm/057T/3E6WE72s6PmhH03Mz8kXS3hXA6Oj5kuAXZ30XQDcFt1gZjnAb4G7nHMrOnqTc24xsBigvLz8DP8cEpEBKxLpIvii25tDtJNAbNPePmzbh2433hN35odHR4HURdiE0joOtY6CrdP5LkKswz7RARrDsgNBsI72E6UzsYT3amCCmZUBH+IF9JXtO5nZRCAfeCOqLRV4EfiZc+4/41KxiPSO5uHSWPYE2391GGQ9CddOArG5vTf2CoOp/lcKBNNap0Npfluq156aCcF8CKVGvSfqK3S6z0lt+1nN011+VqoXbCLtdBnezrmwmX0dWA4EgSXOuXVmdg9Q6Zx7ye+6EFjqnIvec/474JNAoZld77dd75xbG7c1EBlMnIPGE3DyGDQeg5PHofG4Px/9/bj/+rGo6fZ9O2g/0+OApzA/vLoKrBQ/HGMI0ZbPiW7vJPi6bE/RHp8kJWubtYlXXl7uKisrE12GSM9FIhA+0TYc4xa2x+lWwFoAUjIhNQNSMryATM1snU7J8F/z+7TsJcYYiF2Fq/YaRWJiZlXOufJY+8cybC4y8EQiXhDGFKCn22vtoE/j8e7VYsGOgzQtG7KHnz5sOw1k/3soTXuWIgOQwlv6r0ikhwHaSd/ovd/wie7VEgi1C0w/QNNzIXvE6QP0lLDNahu8wVQFrIh0i8Jb+kakCQ5tg73roGY9HNkZFbqdHIftScCmZp66V5qeBzkjTx+gnYVt8+eFUnvlxyIi0hMKb4kv5+BoDdSsg73roWaDN13zXlQYmz8cnNUalBkFkFJy+gDtaLg4uq8CVkQGCYW39FzD0ahw3tC6V338QGufzCIongzlN3jfh02GorO84BURkR5ReEvXmhrhwGY/nDd4Ab13HRze3tonJQOKJ8HES2DYFC+oiycn9N6/IiIDlcJbWjkHR6q9cK5Z7w97r4f9m1rvHGVBKBwPo2bBzGu8PeniSZBXCgE9Hl5EpC8ovAerE4faDnU3H59uONLaJ2eUt/c8/nwonuKF9NCPQkp64uoWERGF94DXWA/7N54a1HVRt6dPy/X2oKdd7u9JT4His2BIfuLqFhGRTim8B4pIBA5tbXtMumY9HPig9X7QwVQYOhHKPuGfPObvTeeM0nXGIiJJROGdjI7ui7oUy/++7722d/bKL/X2oCdf2nryWOE479aVIiKS1BTe/dnJY9710e2D+vj+1j4ZQ72h7lnXeXvRw6Z4l2KlZSWubhER6VUK7/6gKQwHP2h38tg6OLSdlodQpGR4oTzxotY96WFTIKs4oaWLiEjfU3j3JeegdlfbY9J713snlLVcihXwLsUaMQNmXOXtTRdPhvwyXYolIiKAwrv3nDjcevex5uula9ZDfdSlWNkjvSHvcee13n1s6ERdiiUiIqel8D5T4QbvJibRx6RrNkBtdWuftBwvnKd+qXXIu3iSdz9vERGRblJ4xyoS8W4HGn1MumYD7H+/9VKsQAoUTYSx57aePFY8GXJLdCmWiIjEjcK7I8f2Rx2Tbr6f9wbvUZXN8sZ64XzW51uDunC8LsUSEZFeN7jD++Rx2Leh7THpvevhWE1rn4xCb+951jX+yWP+3cfSshNXt4iIDGqDI7ybwnBwy6knjx3cSsulWKEhXihPuNDfk/ZvE5pVrCFvERHpVwZ2eDfUwZOXwL6N0NTgtVkACsbB8Glw9hWt10vnl0IgmNByRUREYjGwwzs1ywvlsk+2njxWNBFShiS6MhERkR4b2OFtBlc8k+gqRERE4kq37BIREUkyMYW3mV1kZhvNbLOZLerg9QfMbK3/tcnMDke9dp2Zve9/XRfP4kVERAajLofNzSwIPAx8FqgGVpvZS8659c19nHN3RvW/HZjpTxcAdwPleKd1V/nvPRTXtRARERlEYtnzngtsds5tcc6dBJYCl56m/0LgOX/6b4BXnHMH/cB+BbjoTAoWEREZ7GIJ71HAzqj5ar/tFGY2FigDXuvue0VERCQ2sYR3R3cocZ30XQC84Fzzzb5je6+ZVZhZpZlV7tu3L4aSREREBq9YwrsaGB01XwLs6qTvAlqHzGN+r3NusXOu3DlXXlRUFENJIiIig1cs4b0amGBmZWaWihfQL7XvZGYTgXzgjajm5cCFZpZvZvnAhX6biIiI9FCXZ5s758Jm9nW80A0CS5xz68zsHqDSOdcc5AuBpc45F/Xeg2Z2L94fAAD3OOcOnm55VVVV+81se09WppuGAvv7YDl9QevSP2ld+ietS/8zUNYDer4uY7vT2aKydlAxs0rnXHmi64gHrUv/pHXpn7Qu/c9AWQ/ou3XRHdZERESSjMJbREQkyQzm8F6c6ALiSOvSP2ld+ietS/8zUNYD+mhdBu0xbxERkWQ1mPe8RUREklLSh7eZjTazP5rZBjNbZ2Z3+O0FZvaK/zSzV/zrzDHPQ/4T0t42s1lRn9XhE9DMbLaZveO/5yEz6+jOcfFan6CZrTGz3/jzZWa20q9pmX+tPWaW5s9v9l8vjfqMb/ntG83sb6LaT/t0uDivR56ZvWBm7/nbZn4Sb5M7/X9b75rZc2aWnizbxcyWmFmNmb0b1dbr26GzZfTCuvzA/zf2tpm9aGZ5Ua916+fdk20az3WJeu2bZubMbKg/n3TbxW+/3f85rzOz70e1J9V2MbMZZrbCvKdmVprZXL89sdvFOZfUX8AIYJY/nQ1sAiYD3wcW+e2LgPv96UuAl/Fu3ToPWOm3FwBb/O/5/nS+/9oqYL7/npeBi3txfb4B/AL4jT//PLDAn34E+Ko//TXgEX96AbDMn54MvAWk4d1n/gO86/OD/vRHgFS/z+ReXI+ngZv96VQgLxm3Cd69+LcCQ6K2x/XJsl2ATwKzgHej2np9O3S2jF5YlwuBkD99f9S6dPvn3d1tGu918dtH491TYzswNIm3y3nAH4A0f744WbcL8Puon98lwJ/6w3aJ+y+7RH8B/w/v8aUbgRF+2whgoz/9KLAwqv9G//WFwKNR7Y/6bSOA96La2/SLc+0lwKvAZ4Df+Bt4P62/nOYDy/3p5cB8fzrk9zPgW8C3oj5zuf++lvf67W36xXk9cvACz9q1J+M2aX64ToH/c/4N3tPykma7AKW0/WXU69uhs2XEe13avXYZ8POOfo5d/bx78n+tN9YFeAGYDmyjNbyTbrvgBe4FHfRLuu3iL+eKqJ/lL/rDdkn6YfNo/rDJTGAlMMw5txvA/17sd+vsSWena6/uoL03/Aj4JyDizxcCh51z4Q6W3VKv//oRv3931683fATYBzxp3iGAx80skyTcJs65D4EfAjuA3Xg/5yqSc7s064vt0NkyetONeHsz0P116cn/tbgysy8CHzrn3mr3UjJul48Cn/CHs183szl+e9JtF+DvgR+Y2U683wXfar/8drX1yXYZMOFtZlnAL4G/d87Vnq5rB22uB+1xZWafB2qcc1XRzadZdr9cD18Ib+jpP5xzM4FjeENBnem36+Ife7oUb4hvJJAJXHya5ffbdYlB0tZuZt8GwsDPm5s66NbTden19TSzDODbwP/t6OVOlt+ft0sIb8h4HvCPwPP+8d2k2i6+rwJ3OudGA3cCT3Sx/D7ZLgMivM0sBS+4f+6c+5XfvNfMRvivjwBq/PbOnnR2uvaSDtrj7WPAF81sG7AUb+j8R0CemTXfgz562S31+q/nAge7WI9Ynw53pqqBaufcSn/+BbwwT7ZtAnABsNU5t8851wj8CjiX5NwuzfpiO3S2jLjzTwj6PHCV88cdu6i5o/b9dH+bxtM4vD8Q3/J/B5QAb5rZ8B6sS3/YLtXAr5xnFd5o4tAuau6P2wXgOrz/9wD/Ccxtv/x2tfXNdonH8Y5EfuH9NfMz4Eft2n9A2xMAvu9Pf462Jxms8tsL8I7T5vtfW4EC/7XVft/mkwwu6eV1+jStJ6z9J21P1viaP30bbU/WeN6fnkLbE0K24J0MEvKny2g9IWRKL67Dn4GJ/vR3/O2RdNsEOAdYB2T4y3oauD2ZtgunHsPr9e3Q2TJ6YV0uAtYDRe36dfvn3d1tGu91affaNlqPeSfjdrkV70FU4A2h7/RrSbrtAmwAPu1Pnw9U9YftEvdfdn39BXwcb+jhbWCt/3UJ3rGPV4H3/e/NPzwDHsY7s/EdoDzqs24ENvtfN0S1lwPv+u/5CXE4KaKLdfo0reH9EbwzFDf7/4ibz95M9+c3+69/JOr93/Zr3UjUWdj+z2WT/9q3e3kdZgCV/nb5L/8fcVJuE+C7wHv+8p7B+8WTFNsFeA7vWH0j3l/+N/XFduhsGb2wLpvxgqH5//4jPf1592SbxnNd2r2+jdbwTsbtkgo869fwJvCZZN0ueBlThfcHxUpgdn/YLrrDmoiISJIZEMe8RUREBhOFt4iISJJReIuIiCQZhbeIiEiSUXiLiIgkGYW3iIhIklF4i4iIJBmFt4iISJL5/7oVymz0LGHOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#learning curve\n",
    "lgb=LGBMClassifier(random_state=51, class_weight= {0: 1, 1: 4}, colsample_bytree=1, max_depth=7,\n",
    " min_child_samples=20, min_split_gain=0, n_estimators=100, reg_alpha=13)\n",
    "\n",
    "lc=learning_curve(lgb,X_train,y_train,cv=StratifiedKFold(random_state=51,n_splits=5),scoring='roc_auc',\n",
    "                  n_jobs=1,random_state=51)\n",
    "size=lc[0]\n",
    "train_score=[lc[1][i].mean() for i in range (0,5)]\n",
    "test_score=[lc[2][i].mean() for i in range (0,5)]\n",
    "fig=plt.figure(figsize=(8,4))\n",
    "plt.plot(size,train_score,label='Train')\n",
    "plt.plot(size,test_score,label=\"Test\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}