Stacking Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:29:21.186282Z",
     "start_time": "2019-08-20T10:29:21.172754Z"
    }
   },
   "outputs": [],
   "source": [
    "# standard imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import sqlite3\n",
    "from collections import Counter\n",
    "from tqdm.autonotebook import tqdm\n",
    "import warnings\n",
    "\n",
    "# processing and pre-processing imports\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "from sklearn.exceptions import DataConversionWarning\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "\n",
    "# ml model selection imports\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "# ml models\n",
    "from sklearn.neighbors import KNeighborsRegressor, KNeighborsClassifier\n",
    "from sklearn.linear_model import LinearRegression, Ridge, Lasso, LogisticRegression\n",
    "from sklearn.svm import LinearSVC, SVC, SVR\n",
    "from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor, GradientBoostingClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB, BernoulliNB, GaussianNB\n",
    "import xgboost as xgb\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "import machumachine\n",
    "\n",
    "# metric computation\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import confusion_matrix \n",
    "from sklearn.metrics import accuracy_score \n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn import metrics\n",
    "\n",
    "# ensemble models\n",
    "from mlxtend.classifier import StackingClassifier\n",
    "from mlxtend.classifier import StackingCVClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:29:21.303410Z",
     "start_time": "2019-08-20T10:29:21.296742Z"
    }
   },
   "outputs": [],
   "source": [
    "X = pd.read_pickle('results/2/X.pkl').values\n",
    "y = pd.read_pickle('results/2/target.pkl').values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:29:21.409047Z",
     "start_time": "2019-08-20T10:29:21.404268Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:29:21.559158Z",
     "start_time": "2019-08-20T10:29:21.547805Z"
    }
   },
   "outputs": [],
   "source": [
    "xgboost = xgb.XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
    "               colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
    "               learning_rate=0.1, max_delta_step=0, max_depth=3, max_features=3,\n",
    "               min_child_weight=1, min_samples_leaf=3, n_estimators=100, n_jobs=1, nthread=None,\n",
    "               objective='multi:softprob', random_state=0, reg_alpha=0,\n",
    "               reg_lambda=1, scale_pos_weight=1, seed=None, silent=None,\n",
    "               subsample=1, verbosity=1)\n",
    "lr = LogisticRegression(C=0.5, class_weight=None, dual=False, fit_intercept=True,\n",
    "                    intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
    "                    multi_class='warn', n_jobs=None, penalty='l1',\n",
    "                    random_state=None, solver='warn', tol=0.0001, verbose=0,\n",
    "                    warm_start=False)\n",
    "gbc = GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
    "                            learning_rate=0.5, loss='deviance', max_depth=3,\n",
    "                            max_features=5, max_leaf_nodes=None,\n",
    "                            min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "                            min_samples_leaf=5, min_samples_split=2,\n",
    "                            min_weight_fraction_leaf=0.0, n_estimators=400,\n",
    "                            n_iter_no_change=None, presort='auto',\n",
    "                            random_state=None, subsample=1.0, tol=0.0001,\n",
    "                            validation_fraction=0.1, verbose=0,\n",
    "                            warm_start=False)\n",
    "rf = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
    "                        max_depth=6, max_features=5, max_leaf_nodes=None,\n",
    "                        min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "                        min_samples_leaf=1, min_samples_split=2,\n",
    "                        min_weight_fraction_leaf=0.0, n_estimators=600,\n",
    "                        n_jobs=-1, oob_score=False, random_state=None, verbose=0,\n",
    "                        warm_start=False)\n",
    "knn = KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
    "                      metric_params=None, n_jobs=None, n_neighbors=11, p=2,\n",
    "                      weights='uniform')\n",
    "dt = DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=6,\n",
    "                        max_features=3, max_leaf_nodes=None,\n",
    "                        min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "                        min_samples_leaf=1, min_samples_split=2,\n",
    "                        min_weight_fraction_leaf=0.0, presort=False,\n",
    "                        random_state=None, splitter='best')\n",
    "\n",
    "models = [xgboost, lr, gbc, rf, knn, dt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:48:45.898356Z",
     "start_time": "2019-08-20T10:48:45.884135Z"
    }
   },
   "outputs": [],
   "source": [
    "# meta classifier\n",
    "mc = xgb.XGBClassifier() \n",
    "    \n",
    "# stacking classifier\n",
    "sclf = StackingCVClassifier(classifiers=models[:6],\n",
    "                            shuffle=False,\n",
    "                            use_probas=True,\n",
    "                            cv=5,\n",
    "                            meta_classifier=mc)\n",
    "\n",
    "\n",
    "classifier_array = models\n",
    "classifier_array.append(sclf)\n",
    "labels = [clf.__class__.__name__ for clf in classifier_array]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:48:46.546628Z",
     "start_time": "2019-08-20T10:48:46.542085Z"
    }
   },
   "outputs": [],
   "source": [
    "clf_dict = dict(zip(labels, classifier_array))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:49:03.900443Z",
     "start_time": "2019-08-20T10:48:47.093587Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBClassifier\n",
      "LogisticRegression\n",
      "GradientBoostingClassifier\n",
      "RandomForestClassifier\n",
      "KNeighborsClassifier\n",
      "DecisionTreeClassifier\n",
      "StackingCVClassifier\n"
     ]
    }
   ],
   "source": [
    "with warnings.catch_warnings():\n",
    "    warnings.filterwarnings('ignore')\n",
    "    for name in clf_dict.keys():\n",
    "        print(name)\n",
    "        ml_model = clf_dict[name]\n",
    "\n",
    "        ml_model.fit(X_train, y_train)\n",
    "\n",
    "        clf_dict[name] = ml_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:49:04.163830Z",
     "start_time": "2019-08-20T10:49:03.902634Z"
    }
   },
   "outputs": [],
   "source": [
    "results = pd.DataFrame()\n",
    "for key in clf_dict.keys():\n",
    "    # Make prediction on test set\n",
    "    y_pred = clf_dict[key].predict(X_test)\n",
    "    \n",
    "    # Save results in pandas dataframe object\n",
    "    results[f\"{key}\"] = y_pred\n",
    "\n",
    "# Add the test set to the results object\n",
    "results[\"Target\"] = y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:49:04.173256Z",
     "start_time": "2019-08-20T10:49:04.165839Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7073170731707317"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(results['Target'], results['StackingCVClassifier'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:49:04.185222Z",
     "start_time": "2019-08-20T10:49:04.175511Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBClassifier 0.7235772357723578\n",
      "LogisticRegression 0.6504065040650406\n",
      "GradientBoostingClassifier 0.6422764227642277\n",
      "RandomForestClassifier 0.573170731707317\n",
      "KNeighborsClassifier 0.5447154471544715\n",
      "DecisionTreeClassifier 0.46747967479674796\n",
      "StackingCVClassifier 0.7073170731707317\n"
     ]
    }
   ],
   "source": [
    "for i in results.columns[:-1]:\n",
    "    print(i, accuracy_score(results['Target'], results[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:49:04.190009Z",
     "start_time": "2019-08-20T10:49:04.186923Z"
    }
   },
   "outputs": [],
   "source": [
    "def meta_class(row):\n",
    "    return Counter(np.array(row)).most_common(1)[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:49:04.211947Z",
     "start_time": "2019-08-20T10:49:04.192012Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6991869918699187"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test, results.iloc[:,:-1].apply(meta_class, axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:49:04.216279Z",
     "start_time": "2019-08-20T10:49:04.213466Z"
    }
   },
   "outputs": [],
   "source": [
    "from itertools import combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T10:49:04.223523Z",
     "start_time": "2019-08-20T10:49:04.219601Z"
    }
   },
   "outputs": [],
   "source": [
    "combi_models = [[i[0],i[1],i[2]] for i in combinations(models,3)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Iterate Stacking for Multiple Model-Pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T11:04:15.851301Z",
     "start_time": "2019-08-20T10:49:04.225730Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "746be35cf27b478d864ff15317cb86a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=56), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "all_results = []\n",
    "for i in tqdm(combi_models):\n",
    "    sclf = StackingCVClassifier(classifiers=i,\n",
    "                            shuffle=False,\n",
    "                            use_probas=False,\n",
    "                            cv=5,\n",
    "                            meta_classifier=xgb.XGBClassifier())\n",
    "\n",
    "\n",
    "    classifier_array = i\n",
    "    classifier_array.append(sclf)\n",
    "    labels = [clf.__class__.__name__ for clf in classifier_array]\n",
    "    \n",
    "    with warnings.catch_warnings():\n",
    "        warnings.filterwarnings('ignore')\n",
    "        for name in clf_dict.keys():\n",
    "            ml_model = clf_dict[name]\n",
    "\n",
    "            ml_model.fit(X_train, y_train)\n",
    "\n",
    "            clf_dict[name] = ml_model\n",
    "            \n",
    "    results = pd.DataFrame()\n",
    "    for key in clf_dict.keys():\n",
    "        # Make prediction on test set\n",
    "        y_pred = clf_dict[key].predict(X_test)\n",
    "\n",
    "        # Save results in pandas dataframe object\n",
    "        results[f\"{key}\"] = y_pred\n",
    "\n",
    "    # Add the test set to the results object\n",
    "    results[\"Target\"] = y_test\n",
    "    \n",
    "    all_results.append(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T11:04:15.903699Z",
     "start_time": "2019-08-20T11:04:15.856467Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7317073170731707"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracies = [accuracy_score(y_test, i['StackingCVClassifier']) for i in all_results]\n",
    "np.array(accuracies).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T11:09:17.634393Z",
     "start_time": "2019-08-20T11:09:16.880814Z"
    }
   },
   "outputs": [],
   "source": [
    "new_acc = []\n",
    "for i in all_results:\n",
    "    if len(i.columns) > 4:\n",
    "        new_acc.append(accuracy_score(y_test, i.iloc[:,:-2].apply(meta_class, axis=1)))\n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T11:09:17.643596Z",
     "start_time": "2019-08-20T11:09:17.636209Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7398373983739838"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(new_acc).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T17:19:22.816203Z",
     "start_time": "2019-08-20T17:19:22.792274Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(np.array(new_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T17:20:36.438022Z",
     "start_time": "2019-08-20T17:20:36.382316Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.73      0.77       104\n",
      "           1       0.72      0.84      0.77       114\n",
      "           2       0.84      0.57      0.68        28\n",
      "\n",
      "    accuracy                           0.76       246\n",
      "   macro avg       0.79      0.71      0.74       246\n",
      "weighted avg       0.77      0.76      0.76       246\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, all_results[18].apply(meta_class, axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-20T17:21:08.388944Z",
     "start_time": "2019-08-20T17:21:08.361031Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[76, 28,  0],\n",
       "       [15, 96,  3],\n",
       "       [ 2, 10, 16]])"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, all_results[18].apply(meta_class, axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}