ML_SBS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:29:07.427329Z",
     "start_time": "2018-06-19T02:29:07.422844Z"
    }
   },
   "outputs": [],
   "source": [
    "## SBS sequential backward selection 序列后向选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:29:38.926883Z",
     "start_time": "2018-06-19T02:29:38.816777Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.base import clone\n",
    "from itertools import combinations\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "class SBS():\n",
    "    def __init__(self, estimator, k_features,\n",
    "                scoring = accuracy_score,\n",
    "                test_size = 0.25, random_state=1):\n",
    "        self.scoring = scoring\n",
    "        self.estimator = clone(estimator)\n",
    "        self.k_features = k_features\n",
    "        self.test_size = test_size\n",
    "        self.random_state = random_state\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = self.test_size,\n",
    "                                                            random_state = self.random_state)\n",
    "        dim = X_train.shape[1]\n",
    "        self.indices_ = tuple(range(dim))\n",
    "        self.subsets_ = [self.indices_]\n",
    "        score = self._calc_score(X_train, y_train,\n",
    "                                X_test, y_test, self.indices_)\n",
    "        self.scores_ = [score]\n",
    "        \n",
    "        while dim > self.k_features:\n",
    "            scores = []\n",
    "            subsets = []\n",
    "            \n",
    "            for p in combinations(self.indices_, r = dim-1):\n",
    "                score = self._calc_score(X_train, y_train,\n",
    "                                        X_test, y_test, p)\n",
    "                scores.append(score)\n",
    "                subsets.append(p)\n",
    "                \n",
    "            best = np.argmax(scores)\n",
    "            self.indices_ = subsets[best]\n",
    "            self.subsets_.append(self.indices_)\n",
    "            dim -= 1\n",
    "            \n",
    "            self.scores_.append(scores[best])\n",
    "        self.k_score_ = self.scores_[-1]\n",
    "        \n",
    "        return self\n",
    "    def transform(self, X):\n",
    "        return X[:, self.indices]\n",
    "       \n",
    "    def _calc_score(self, X_train, y_train,\n",
    "                   X_test, y_test, indices):\n",
    "        self.estimator.fit(X_train[:, indices], y_train)\n",
    "        y_pred = self.estimator.predict(X_test[:, indices])\n",
    "        score = self.scoring(y_test, y_pred)\n",
    "        return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:35:01.049425Z",
     "start_time": "2018-06-19T02:35:01.039400Z"
    }
   },
   "outputs": [],
   "source": [
    "# let's see our SBS implementation in action using the KNN classifer  from scikit-learn:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:37:23.290685Z",
     "start_time": "2018-06-19T02:37:21.759488Z"
    }
   },
   "outputs": [],
   "source": [
    "# 准备数据\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "df_wine = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data',header=None)\n",
    "df_wine.columns = ['Class label','Alcohol',\n",
    "                  'Malic acid','Ash',\n",
    "                  'Alcalinity of ash','Magnesium',\n",
    "                  'Total phenols','Flavanoids',\n",
    "                  'Nonflavanoid phenols',\n",
    "                  'Paranthocyanins',\n",
    "                  'Color intensity','Hue',\n",
    "                  'OD280/OD315 of diluted wines',\n",
    "                  'proline']\n",
    "# print('Class labels', np.unique(df_wine['Class label']))\n",
    "from sklearn.model_selection import train_test_split\n",
    "X,y = df_wine.iloc[:,1:].values, df_wine.iloc[:,0].values\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3,\n",
    "                                                   random_state=0)\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "stdsc = StandardScaler()\n",
    "X_train_std = stdsc.fit_transform(X_train)\n",
    "X_test_std = stdsc.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:37:23.367829Z",
     "start_time": "2018-06-19T02:37:23.292695Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<__main__.SBS at 0x1a11208cf8>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# KNN part\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "import matplotlib.pyplot as plt\n",
    "knn = KNeighborsClassifier(n_neighbors = 2)\n",
    "sbs = SBS(knn, k_features = 1)\n",
    "sbs.fit(X_train_std, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:38:25.972260Z",
     "start_time": "2018-06-19T02:38:25.527893Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl4VdW5x/HvmzAPCgjGyowigwMgEVGqhjqA1hacWuitQ7WX2kIdqrZwa9ViB1u11qrXikq1XitFpJYqFa2SQlWUiAwCBsOghCCIyBANQ5L3/nF29BCS7EPIzjk5+X2eZz/Ze+3pXZCT9+xhrWXujoiISE0ykh2AiIikPiULEREJpWQhIiKhlCxERCSUkoWIiIRSshARkVCRJQszm2pmm83snWrW9zWz181st5ndWGndSDPLN7MCM5sYVYwiIpKYKK8sHgNG1rB+K3ANcFd8oZllAg8A5wL9gbFm1j+iGEVEJAGRJQt3n0csIVS3frO7LwT2Vlo1BChw9zXuvgeYBoyKKk4REQnXJNkBVKEzsD5uuRA4uaoNzWwcMA6gZcuWg7t27Rp9dAehvLycjIz0eEyULnVJl3qA6pKqUr0uq1at2uLuncK2S8VkYVWUVdknibtPAaYAZGdne15eXpRxHbTc3FxycnKSHUadSJe6pEs9QHVJValeFzN7P5HtUjHdFQLxlwhdgKIkxSIiIqRmslgI9DaznmbWDBgDzEpyTCIijVpkt6HM7CkgB+hoZoXArUBTAHf/o5kdAeQBhwDlZnYd0N/dd5jZBGAOkAlMdfflUcUpIiLhIksW7j42ZP2HxG4xVbVuNjA7irhEROTApeJtKBERSTFKFiIiEkrJQkREQilZiIhIKCULEREJpWQhIiKhlCxERCSUkoWIiIRSshARkVBKFiIiEkrJQkREQilZiIhIKCULEREJpWQhIiKhlCxERCSUkoWIiIRSshARkVBKFiIiEiqyZGFmU81ss5m9U816M7M/mFmBmS01sxPj1pWZ2eJgmhVVjCIikpgoryweA0bWsP5coHcwjQMejFtX4u4Dg+nr0YUoIiKJiCxZuPs8YGsNm4wC/uwxC4B2ZvalqOIREZHaS+Yzi87A+rjlwqAMoIWZ5ZnZAjMbXf+hiYhIvCZJPLdVUebBz27uXmRmvYBXzGyZu6/e7wBm44jdwiIrK4vc3NzIgq0LxcXFKR9jotKlLulSD1BdUlW61CWZyaIQ6Bq33AUoAnD3ip9rzCwXGATslyzcfQowBSA7O9tzcnKijfgg5ebmkuoxJipd6pIu9QDVJVWlS12SeRtqFnBZ8FbUUGC7u280s/Zm1hzAzDoCw4AVSYxTRKTRi+zKwsyeAnKAjmZWCNwKNAVw9z8Cs4HzgALgM+A7wa79gIfMrJxYMrvD3ZUsRESSKLJk4e5jQ9Y7ML6K8teA46OKS0REDpxacIuISCglCxERCaVkISIioZQsREQklJKFiIiEUrIQEZFQShYiIhJKyUJEREIpWYiISCglCxERCaVkISIioZQsREQklJKFiIiEUrIQEZFQShYiIhJKyUJEREIpWYiISCglCxERCaVkISIioSIbg9vMpgLnA5vd/bgq1htwL3Ae8BlwhbsvCtZdDtwcbPoLd388qjgl9Tz79gbunJNP0bYSjmzXkptG9GH0oM6RnGPDthI6L3glknPEn0d1ObBzRFmX+qhHOoosWQCPAfcDf65m/blA72A6GXgQONnMOgC3AtmAA2+Z2Sx3/yTCWCVFPPv2BibNXEbJ3jIANmwrYeLMpewtK+drA46sk3P8Y0kRP/v7O+zaWx7ZOerrPKrLwZ9j0sxlAEoYIczdozu4WQ/guWquLB4Cct39qWA5H8ipmNz9e1VtV53s7GzPy8ury/DrXG5uLjk5OckOo05EVZdhd7zChm0ldX5ckZp0bteSVyd+JZJjp/rn3szecvfssO2ivLII0xlYH7dcGJRVV74fMxsHjAPIysoiNzc3kkDrSnFxccrHmKio6lJTorj4mKZ1co4Zq/ZGfo76Oo/qUjfn2LCtJLLPZrp87pOZLKyKMq+hfP9C9ynAFIhdWaRy9obU/4ZxIKKqS9Zr/2LTjt37lXdu15K7rqybb36vV3P1UpfnqK/zqC51c44Mg7KsfpzZL6tOzhMvXT73yXwbqhDoGrfcBSiqoVzS3J7Scppm7P8r2bJpJjeN6FNn57lpRB9aNs2M9Bz1dR7V5eDP0bxJBlltm3PV43lMfGYpxbtL6+x86SSZyWIWcJnFDAW2u/tGYA5wjpm1N7P2wDlBmaS5u1/Mp3BbCd8Z1oPO7VpixL5V/vrC4+v04ePoQZ359YXH07ldS4joHJXPo7oc2Dmoh3NU1OM3F51A7o+Hc/UZR/HXvPWcd+988tZtrbNzpg13j2QCngI2AnuJXS1cBVwNXB2sN+ABYDWwDMiO2/dKoCCYvpPI+QYPHuypbu7cuckOoc7UdV3mrdrs3X/ynE+aubROjxtG/yepKVl1eXPtx/7l37zsPSc+53f8c6Xv2lt60MdM9f8XIM8T+Bsb2TMLdx8bst6B8dWsmwpMjSIuST0fF+/mR9OX0PvwNvzsq/2THY40Yif16MA/rz2dXzy3ggdzVzP33c38fsxA+h5xSLJDSzq14JakcndufHoJ20v2ct+3BtGyWWb4TiIRatO8CXdcdAKPXJbNluLdfP2+V5kybzVl5dE1M2gIlCwkqf706jrm5n/ET8/rp29vklLO6p/FnOtOZ3jfTvxq9ruMfXgB67d+luywkkbJQpJmedF27vjnu5zV73AuO6V7ssMR2c9hbZrzx28P5q5LBrCiaAfn3juf6XnrK56tNipKFpIUn+0p5Zqn3qZdq6b89uIBxLoKE0k9ZsbFg7vwwnWncVznQ/jxjKWMe+ItthTv3x4onSlZSFLc/twK1mz5lHu+OZAOrZslOxyRUF3at+Iv3x3KzV/tx79XfcSIe+bx4vIPkx1WvVGykHo3e9lGnnpzPd87/SiGHd0x2eGIJCwjw/juab34x4Qvk3VIC8Y98RY/nrGEnbuq76okXShZSL3asK2Eic8sZUDXdtxwzjHJDkekVvoc0ZZnxw9j/PCjmPFWIefeO5831nyc7LAipWQh9aa0rJzrpy2m3OEPYwbSNFO/ftJwNWuSwU0j+vL01aeQmWGMeXgBv5q9kt2lZckOLRLJ7EhQ6lBDGNDl/rkFvLluK/d8cwDdD2ud7HBE6sTg7h2Yfc1p/HL2SqbMW8O8VR/xu28MZNWmnfUyKFV9UbJIA1UNGJRqA7osXLeVP7z8HhcM6swFg7okOxyROtW6eRN+dcHxnN0vix8/s5Tz75tPhhmlQUO+VPxMHijdB0gDd87J/zxRVCjZW8adc/KTFNG+tn+2l+umLaZL+1ZMHnVsssMRiczwvocz57rTadYk4/NEUSGVPpO1oSuLNFBUzYBB1ZXXJ3fnf/62jE07djHj+6fStkXdDcgjkoo6tG7G7mDY1spS4TNZW7qySANHBl06V9ameRP2llX9S1tfpuet5/llG/nROccwsGu7pMYiUl+q+0xmHdKiniOpO0oWaeCmEX1okrFvC+hMM3buLuWC/32V9zbtTEpcBZuLuW3WCoYdfRhXn35UUmIQSYaqBlkC2LlrLy+8szEJER08JYs0MLzP4WQatGya8fmALnd/YwB//PZgirbt4qv3/YdH/7OW8nrsNXN3aRnXPPU2LZpm8LtvDCQjQ915SONR1UBOk87tQ69Obbj6/xbxo+mL2dHAGvLpmUUaePz1dewuc54fP4xjjzx0n3WDu7dn0syl3P7cCl5euYk7Lxnw+S9wlH7zz3xWbNzBI5dlN+hLb5HaGj2oM6MHdd5nDO4rv9yL+15+jwdyV/PGmq3cdckATjnqsOQGmiBdWTRwn+4uZeqrazmz7+H7JQqATm2b8/Bl2fzmouNZsn4bI++Zx8xFhZH2mjk3fzNTX13L5ad056z+WZGdR6ShaZqZwY/O6cOMq0+hWZMMxj68gF88t4Jde1O/IZ+SRQP35Bvvs+2zvYz/ytHVbmNmfPOkbvzz2tPp+6W2/Gj6En7w5CK2frqnzuPZvHMXN05fQt8j2jLpvH51fnyRdDCoW3uev+bLXDq0O4/8Zy1fu+8/vLNhe7LDqlGkycLMRppZvpkVmNnEKtZ3N7OXzWypmeWaWZe4dWVmtjiYZkUZZ0O1a28ZD89fy7CjD+PEbu1Dt+92WCumjTuFief25V8rN3HOPfN45d1NdRZPeblzw/QlFO8u5b6xg2hRxQM+EYlp1awJt48+jsevHML2kr2MfuBV7n/lPUqT/AZjdSJLFmaWCTwAnAv0B8aaWeUBlu8C/uzuJwCTgV/HrStx94HB9PWo4mzIpuet56Oduxk/vPqrisoyM4yrzziKv4//Mh3bNOPKx/KYNHMZn+4uPeh4Hv3PWua/t4Vbvtaf3lltD/p4Io3BGcd04sXrT2fkcUdw14ur+MZDr7Nuy6fJDms/UV5ZDAEK3H2Nu+8BpgGjKm3TH3g5mJ9bxXqpxt6ych769xoGd2/PKb0O/AFZ/yMP4e8ThvG9M3oxbeEHnPeH+bz1/tZax7OscDu/nfMuI489gm8N6Vbr44g0Ru1aNeP+b53IvWMGUrC5mHPvnc+Tb7yfUiPyWVTBmNnFwEh3/26wfClwsrtPiNvmL8Ab7n6vmV0IPAN0dPePzawUWAyUAne4+7NVnGMcMA4gKytr8LRp0yKpS10pLi6mTZs2dXKs+YV7efSdPVw/uDkDOh3cS235W8t4eNluPi5xvtqrKaOPbrpfu43K4uuyq9S59bUS9pbD5FNb0qZZw3lNti7/T5JNdUlNB1qXrbvKeXTZbpZ/XM4JnTK58thmtGsR3ff64cOHv+Xu2aEbunskE3AJ8Ejc8qXAfZW2ORKYCbwN3AsUAodWrAt+9gLWAUfVdL7Bgwd7qps7d26dHKe0rNxz7pzr5907z8vLy+vkmDt37fUfP73Eu//kOT/39/P83Y07atw+vi43Tl/sPSY+56+v3lInsdSnuvo/SQWqS2qqTV3Kysr9sVfXep+bZ/vAn8/x55cW1X1gASDPE/ibHuVtqEKga9xyF6AofgN3L3L3C919EPDToGx7xbrg5xogFxgUYawNyvPLNrJ2y6dMGH50nY1d3aZ5E35z8Qk8fFk2m3fu4mv3/YeH560Jbcg3a0kRT79VyIThRzO0FrfDRGR/GRnG5af24PlrTqNbh1b84MlFXDftbbaXJK8hX5TJYiHQ28x6mlkzYAywz1tNZtbRzCpimARMDcrbm1nzim2AYcCKCGNtMMrLnQdeKeDow9sw4tgj6vz4Z/fPYs51p5PTpxO/nL2SsQ8voPCTz6rcdv3Wz/jpzGWc2K0d157Zu85jEWnsjurUhhnfP5XrzurNP5ZuZOTv5/FqwZakxBJZsnD3UmACMAdYCUx39+VmNtnMKt5uygHyzWwVkAX8MijvB+SZ2RJiD77vcHclC+BfKzeRv2kn44cfFVkXGoe1ac5Dlw7mzotPYHnRDkb+fj5P563f52FbWblz7bS3Abh3zCCaaNQ7kUg0zczgurOOYeb3T6Vls0z+65E3+Pk/ltd7Q75Iu/tw99nA7Eplt8TNzwBmVLHfa8DxUcbWELk7988toFuHVnzthCMjPZeZcUl2V4b2Oowbnl7CTTOW8tKKTZx+TEcezF3Dhm0lwGdcdkp3unZoFWksIgIDurbj+R+exm9eeJc/vbqO+e9tYdSAI5m2cH29jJCpr4MNyPz3trC0cDvfzzmq3r7Jd+3Qimn/PZSfntePl1du4uZnlweJIubpvEKefXtDvcQi0ti1bJbJbV8/lieuGsLmHbu4+6VVbNhWgvPFaHxRfR5D/+KY2QQzC28eLJG7f24BRxzSggtPrN9hGTMyjP8+vReHtWm+37qGPvqXSEN0Wu9OtG6+/42hKD+PiXw9PQJYaGbTg+47Gs5L9GnkzbVbeXPtVr53Ri+aN0lONxof7dxdZXlDHv1LpKH6cPuuKsuj+jyGJgt3vxnoDTwKXAG8Z2a/MjONZlOP7p9bwGGtmzHmpOS1jq5u9K/qykUkOvX9eUzoxnfQcOPDYCoF2gMzzOy3kUQl+1iyfhvzVn3EVaf1pGWz5HXOV9XoXy2bZnLTiD5Jikik8arvz2Po21Bmdg1wObAFeAS4yd33Bu0j3gN+HElk8rkH5hZwSIsmXDq0e1LjqHjL4s45+WzYVkLniN++EJHqxX8e6+NtqERene0IXOju78cXunu5mZ0fSVTyufwPd/Liik1cc2Zv2rZomuxwqhz9S0SSo+LzWB8SuQ01G/i8O1Iza2tmJwO4+8qoApOYB+YW0LpZJt85tUeyQxGRRiyRZPEgUBy3/GlQJhFbu+VTnltaxLeHdqd962bJDkdEGrFEkoV5XD8P7l5OxC2/JebB3AKaZmZw1Wk9kx2KiDRyiSSLNWZ2jZk1DaZrgTVRB9bYbdhWwsxFGxhzUlcOb9si2eGISCOXSLK4GjgV2ECs2/GTCQYckug89O/VAIw7Q81ZRCT5Qm8nuftmYt2LSz3ZvHMX0xau56ITu9BZDd5EJAUk0s6iBXAVcCzw+f0Qd78ywrgatUfnr6W0rJzv5+iqQkRSQyK3oZ4g1j/UCODfxEa82xllUI3ZJ5/u4YkF7/O1AUfSo2PrZIcjIgIkliyOdvefAZ+6++PAV9FYE5H502vr+GxPGT/IOTrZoYiIfC6RZFEx6Os2MzsOOBToEVlEjdjOXXt57NW1jDg2iz5HtE12OCIin0ukvcSUYDyLm4mNod0G+FmkUTVSTyx4nx27SpkwXONZi0hqqfHKIugscIe7f+Lu89y9l7sf7u4PJXLwYPyLfDMrMLOJVazvbmYvm9lSM8s1sy5x6y43s/eC6fIDrlkDU7KnjEfnr+WMYzpxfJdDkx2OiMg+akwWQWvtCbU5sJllAg8A5wL9gbFm1r/SZncBf3b3E4DJwK+DfTsAtxJr0zEEuDXdR+t76s0P+PjTPUz4ip5ViEjqSeSZxUtmdqOZdTWzDhVTAvsNAQrcfY277wGmAaMqbdMfeDmYnxu3fgTwkrtvdfdPgJeAkQmcs0HaXVrGlHlrGNKzAyf1SOSfVkSkfiXyzKKiPcX4uDIHeoXs1xlYH7dc0fo73hLgIuBe4AKgrZkdVs2++/XDa2bjCFqTZ2VlkZubGxJSchUXF1cZY+76vXy4Yw/fPoaUr0OF6urS0KRLPUB1SVXpUpdEWnDXthe7qsbq9krLNwL3m9kVwDxiXYqUJrgv7j4FmAKQnZ3tqT6+QlVjQJSWlfOzu3MZ0OVQxl80jIYyxHm6jGeRLvUA1SVVpUtdEmnBfVlV5e7+55BdC4GucctdgKJKxygCLgzO0wa4yN23m1khkFNp39ywWBuiWUuKWL+1hFvOP7bBJAoRaXwSuQ11Utx8C+BMYBEQliwWAr3NrCexK4YxwLfiNzCzjsDW4EH6JGBqsGoO8Ku4h9rnBOvTSnm587+5q+l7RFvO7Ht4ssMREalWIrehfhi/bGaHEusCJGy/UjObQOwPfyYw1d2Xm9lkIM/dZxG7evi1mTmx21Djg323mtntxBIOwGR337rfSRq4Ocs/pGBzMfeNHURGhq4qRCR11WYQo8+AhFqNuftsYsOyxpfdEjc/A5hRzb5T+eJKI+24O/fPLaBXx9acd/yXkh2OiEiNEnlm8Q++eLicQex11+lRBtUY5OZ/xPKiHfz24hPI1FWFiKS4RK4s7oqbLwXed/fCiOJpFNyd+155j87tWnLBoP3eCBYRSTmJJIsPgI3uvgvAzFqaWQ93XxdpZGns9TUfs+iDbdw+6liaZibSLlJEJLkS+Uv1NFAet1wWlEkt3f9KAZ3aNueS7K7hG4uIpIBEkkWToLsOAIL5ZtGFlN4WffAJr63+mHGn9aJF08xkhyMikpBEksVHZvb1igUzGwVsiS6k9PbAKwW0b9WUb53cLdmhiIgkLJFnFlcDT5rZ/cFyIVBlq26p2fs7ynj53c3ccPYxtG5em7eWRUSSI5FGeauBoUF3HObuGn/7AD379gbunJPPhm27MOCwtrqLJyINS+htKDP7lZm1c/did99pZu3N7Bf1EVw6ePbtDUyauYwN20qAWIOV2/+xkmff3pDcwEREDkAizyzOdfdtFQvB+BLnRRdSerlzTj4le8v2KSvZW8adc/KTFJGIyIFLJFlkmlnzigUzawk0r2F7iVNxRVFZUTXlIiKpKJGnrP8HvGxmfwqWvwM8Hl1I6aF4dymT/7G82vVHtmtZj9GIiBycRB5w/9bMlgJnERuU6AWge9SBNWRvrt3Kj6YvpmhbCWf1O5z/FGxh194v2jW2bJrJTSP6JDFCEZEDk+j7mx8Sa8X9DWAt8ExkETVgu0vL+N2Lq5gyfw1d27di+vdOIbtHh7i3oUro3K4lN43ow2j1CSUiDUi1ycLMjiE2YNFY4GPgr8RenR1eT7E1KCs37uD6vy7m3Q93MnZIN27+ar/P21KMHtSZ0YM6p83wiiLS+NR0ZfEuMB/4mrsXAJjZ9fUSVQNSVu5MmbeG372Uz6EtmzH1imy+0jcr2WGJiNSpmpLFRcSuLOaa2QvANGLPLCTwwcefccPTi1m47hPOPe4IfnnB8XRorQZ3IpJ+qk0W7v434G9m1hoYDVwPZJnZg8Df3P3Feoox5bg7f124ntufW0GGGb/7xgAuGNQZM+VSEUlPoe0s3P1Td3/S3c8HugCLgYmJHNzMRppZvpkVmNl++5hZNzOba2Zvm9lSMzsvKO9hZiVmtjiY/niA9YrMRzt3899/zmPizGWc0KUdL1x/Ohee2EWJQkTS2gH1ZufuW4GHgqlGZpYJPACcTazzwYVmNsvdV8RtdjMw3d0fNLP+xMbr7hGsW+3uAw8kvqi98M6H/M/fllG8u5Sfnd+f75zagwwNiSoijUCUXZ8OAQrcfQ2AmU0DRgHxycKBQ4L5Q4GiCOOptR279vLzWSt4ZlEhx3U+hHu+MZDeWW2THZaISL0xd4/mwGYXAyPd/bvB8qXAye4+IW6bLwEvAu2B1sBZ7v6WmfUAlgOrgB3Aze4+v4pzjAPGAWRlZQ2eNm1anddj5cdlPLJsN1t3Oecf1ZRRRzWlSS2vJoqLi2nTpk0dR5gc6VKXdKkHqC6pKtXrMnz48LfcPTtsuyivLKr6i1o5M40FHnP3u83sFOAJMzsO2Ah0c/ePzWww8KyZHevuO/Y5mPsUYApAdna212Ubhl17y7hrTj6P5q2lx2GtefjKAZzYrf1BHTOd2lmkS13SpR6guqSqdKlLlMmiEIgfZLoL+99mugoYCeDur5tZC6Cju28Gdgflb5nZauAYIC/CeD/3zobt/Gj6YlZtKubSod2ZdF5fWjXTYEUi0nhF+RdwIdDbzHoCG4i12fhWpW0+AM4EHjOzfkALYsO4dgK2unuZmfUCegNrogiyoiuOom0lfKldCwZ1bceLKzbRvlUzHvvOSeT0OTyK04qINCiRJQt3LzWzCcAcIBOY6u7LzWwykOfus4AbgIeDluEOXOHubmanA5PNrBQoA64O3sSqUxUDE1WMN1G0bRdF2z5kYNdD+dMVQ2ivBnYiIkC0Vxa4+2xir8PGl90SN78CGFbFfs9QD50VVjUwEcTaUihRiIh8IZHBj9JWdQMQFW3bVc+RiIiktkadLKobgEgDE4mI7KtRJ4ubRvShZdPMfco0MJGIyP4a9fugFQMQVbwNdaQGJhIRqVKjThbwxcBEIiJSvUZ9G0pERBKjZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCKVmIiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUJEmCzMbaWb5ZlZgZhOrWN/NzOaa2dtmttTMzotbNynYL9/MRkQZp4iI1CyyLsrNLBN4ADgbKAQWmtmsYNztCjcD0939QTPrT2y87h7B/BjgWOBI4F9mdoy77z9gtoiIRC7KK4shQIG7r3H3PcA0YFSlbRw4JJg/FCgK5kcB09x9t7uvBQqC44mISBJEOfhRZ2B93HIhcHKlbW4DXjSzHwKtgbPi9l1Qad/9Rigys3HAOICsrCxyc3PrIu7IFBcXp3yMiUqXuqRLPUB1SVXpUpcok4VVUeaVlscCj7n73WZ2CvCEmR2X4L64+xRgCkB2drbn5OQcXMQRy83NJdVjTFS61CVd6gGqS6pKl7pEmSwKga5xy1344jZThauAkQDu/rqZtQA6JriviIjUkyifWSwEeptZTzNrRuyB9axK23wAnAlgZv2AFsBHwXZjzKy5mfUEegNvRhiriIjUILIrC3cvNbMJwBwgE5jq7svNbDKQ5+6zgBuAh83semK3ma5wdweWm9l0YAVQCozXm1AiIskT5W0o3H02sddh48tuiZtfAQyrZt9fAr+MMj4REUmMWnCLiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCKVmIiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCKVmIiEgoJQsREQkVabIws5Fmlm9mBWY2sYr195jZ4mBaZWbb4taVxa2bFWWcIiJSs8jG4DazTOAB4GygEFhoZrOCcbcBcPfr47b/ITAo7hAl7j4wqvhERCRxUV5ZDAEK3H2Nu+8BpgGjath+LPBUhPGIiEgtRZksOgPr45YLg7L9mFl3oCfwSlxxCzPLM7MFZjY6ujBFRCRMZLehAKuizKvZdgwww93L4sq6uXuRmfUCXjGzZe6+ep8TmI0DxgFkZWWRm5tbB2FHp7i4OOVjTFS61CVd6gGqS6pKl7pEmSwKga5xy12Aomq2HQOMjy9w96Lg5xozyyX2PGN1pW2mAFMAsrOzPScnpy7ijkxubi6pHmOi0qUu6VIPUF1SVbrUJcrbUAuB3mbW08yaEUsI+73VZGZ9gPbA63Fl7c2seTDfERgGrKi8r4iI1I/IrizcvdTMJgBzgExgqrsvN7PJQJ67VySOscA0d4+/RdUPeMjMyokltDvi36ISEZH6FeVtKNx9NjBLwIwOAAAI3UlEQVS7UtktlZZvq2K/14Djo4xNREQSpxbcIiISSslCRERCKVmIiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCKVmIiEgoJQsREQmlZCEiIqGULEREJJSShYiIhFKyEBGRUEoWIiISSslCRERCKVmIiEioSJOFmY00s3wzKzCziVWsv8fMFgfTKjPbFrfucjN7L5gujzJOERGpWZOoDmxmmcADwNlAIbDQzGa5+4qKbdz9+rjtfwgMCuY7ALcC2YADbwX7fhJVvCIiUr0oryyGAAXuvsbd9wDTgFE1bD8WeCqYHwG85O5bgwTxEjAywlhFRKQGkV1ZAJ2B9XHLhcDJVW1oZt2BnsArNezbuYr9xgHjgsViM8s/yJij1hHYkuwg6ki61CVd6gGqS6pK9bp0T2SjKJOFVVHm1Ww7Bpjh7mUHsq+7TwGm1C68+mdmee6enew46kK61CVd6gGqS6pKl7pEeRuqEOgat9wFKKpm2zF8cQvqQPcVEZGIRZksFgK9zaynmTUjlhBmVd7IzPoA7YHX44rnAOeYWXszaw+cE5SJiEgSRHYbyt1LzWwCsT/ymcBUd19uZpOBPHevSBxjgWnu7nH7bjWz24klHIDJ7r41qljrUYO5ZZaAdKlLutQDVJdUlRZ1sbi/0SIiIlVSC24REQmlZCEiIqGULOqBmXU1s7lmttLMlpvZtcmO6WCYWaaZvW1mzyU7loNhZu3MbIaZvRv835yS7Jhqy8yuD3633jGzp8ysRbJjSpSZTTWzzWb2TlxZBzN7Keju56XgRZeUVk097gx+v5aa2d/MrF0yYzwYShb1oxS4wd37AUOB8WbWP8kxHYxrgZXJDqIO3Au84O59gQE00DqZWWfgGiDb3Y8j9kLJmORGdUAeY/8eGiYCL7t7b+DlYDnVPcb+9XgJOM7dTwBWAZPqO6i6omRRD9x9o7svCuZ3EvujtF+L9IbAzLoAXwUeSXYsB8PMDgFOBx4FcPc97r6t5r1SWhOgpZk1AVrRgNolufs8oPLbjqOAx4P5x4HR9RpULVRVD3d/0d1Lg8UFxNqMNUhKFvXMzHoQ6zDxjeRGUmu/B34MlCc7kIPUC/gI+FNwS+0RM2ud7KBqw903AHcBHwAbge3u/mJyozpoWe6+EWJftoDDkxxPXbgS+Geyg6gtJYt6ZGZtgGeA69x9R7LjOVBmdj6w2d3fSnYsdaAJcCLwoLsPAj6lYdzq2E9wP38Usf7VjgRam9m3kxuVxDOznxK7Hf1ksmOpLSWLemJmTYkliifdfWay46mlYcDXzWwdsV6Ev2Jm/5fckGqtECh094orvBnEkkdDdBaw1t0/cve9wEzg1CTHdLA2mdmXAIKfm5McT60F4/GcD/yXN+CGbUoW9cDMjNi98ZXu/rtkx1Nb7j7J3bu4ew9iD1BfcfcG+Q3W3T8E1gfdzQCcCayoYZdU9gEw1MxaBb9rZ9JAH9bHmQVUDHp2OfD3JMZSa2Y2EvgJ8HV3/yzZ8RwMJYv6MQy4lNg38YqRAc9LdlDCD4EnzWwpMBD4VZLjqZXg6mgGsAhYRuxz3WC6mDCzp4j1DdfHzArN7CrgDuBsM3uP2ABqdyQzxkRUU4/7gbbAS8Hn/o9JDfIgqLsPEREJpSsLEREJpWQhIiKhlCxERCSUkoWIiIRSshARkVBKFtIgmJmb2d1xyzea2W11dOzHzOziujhWyHkuCXq3nVvFujuDXmPvrMVxB+pVbImakoU0FLuBC82sY7IDiWdmmQew+VXAD9x9eBXrvgec6O431SKMgcABJQuL0edfEqZfFmkoSok1NLu+8orKVwZmVhz8zDGzf5vZdDNbZWZ3mNl/mdmbZrbMzI6KO8xZZjY/2O78YP/M4Bv/wmA8gu/FHXeumf2FWCO4yvGMDY7/jpn9Jii7Bfgy8MfKVw9mNgtoDbxhZt80s05m9kxw3oVmNizYboiZvRZ0fPiamfUxs2bAZOCbQaOvb5rZbWZ2Y9zx3zGzHsG00sz+l1gDvq5mdo6ZvW5mi8zs6aD/MoJ/qxVBve860P8sSUPurklTyk9AMXAIsA44FLgRuC1Y9xhwcfy2wc8cYBvwJaA5sAH4ebDuWuD3cfu/QOzLU29i/Ua1AMYBNwfbNAfyiHXWl0Os48GeVcR5JLHuNzoR66zwFWB0sC6X2JgTVdYvbv4vwJeD+W7EuokhqH+TYP4s4Jlg/grg/rj9bwNujFt+B+gRTOXA0KC8IzAPaB0s/wS4BegA5PNFo912yf7/15T8qUl4OhFJDe6+w8z+TGygn5IEd1voQVfXZrYaqOi6exkQfztouruXA++Z2RqgL3AOcELcVcuhxJLJHuBNd19bxflOAnLd/aPgnE8SGzfj2QTjhVgi6B/r5gmAQ8ysbXD+x82sN+BA0wM4ZoX33X1BMD8U6A+8GpyrGbHuKnYAu4BHzOx5oEGPiCh1Q8lCGprfE7uF8qe4slKCW6pBR3rN4tbtjpsvj1suZ9/f/8r93jhgwA/dfU78CjPLIXZlURWrpvxAZACnuPs+CdHM7gPmuvsFFhsXJbea/T//9wjED7EaH7cBL7n72MoHMLMhxDokHANMAL5yYFWQdKNnFtKguPtWYDqxh8UV1gGDg/lR1O4b9yVmlhE8x+hF7DbMHOD7QffymNkxFj5A0hvAGWbWMXj4PRb49wHG8iKxP9AE5x0YzB5K7FYaxG49VdhJrLO6CusIuls3sxOJ3TqrygJgmJkdHWzbKqhjG+BQd58NXEfsAbo0ckoW0hDdTex+e4WHif2BfhM4meq/9dckn9gf9X8CV7v7LmJDx64AFpnZO8BDhFyNB7e8JgFzgSXAInc/0O61rwGyg4fLK4Crg/LfAr82s1eJjbNdYS6x21aLzeybxMZN6WBmi4HvExv7uapYPyKWdJ6yWM+7C4jdfmsLPBeU/ZsqXiqQxke9zoqISChdWYiISCglCxERCaVkISIioZQsREQklJKFiIiEUrIQEZFQShYiIhLq/wFQzFlILUOOFgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a112089e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "k_feat = [len(k) for k in sbs.subsets_]\n",
    "plt.plot(k_feat, sbs.scores_, marker='o')\n",
    "plt.ylim([0.7, 1.1])\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Number of features')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:40:12.553730Z",
     "start_time": "2018-06-19T02:40:12.550996Z"
    }
   },
   "outputs": [],
   "source": [
    "# 看到，当k=5时，达到100%准确性，下面看看这5个feature都是什么"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:40:21.961733Z",
     "start_time": "2018-06-19T02:40:21.953077Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['Alcohol', 'Malic acid', 'Alcalinity of ash', 'Hue', 'proline'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "k5 = list(sbs.subsets_[8])\n",
    "print(df_wine.columns[1:][k5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:41:03.238490Z",
     "start_time": "2018-06-19T02:41:03.235867Z"
    }
   },
   "outputs": [],
   "source": [
    "# Next let's evaluate the performance of the KNN classifer on the original test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:41:25.097481Z",
     "start_time": "2018-06-19T02:41:25.086654Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy: 0.9838709677419355\n"
     ]
    }
   ],
   "source": [
    "knn.fit(X_train_std, y_train)\n",
    "print('Training accuracy:', knn.score(X_train_std, y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:41:37.782257Z",
     "start_time": "2018-06-19T02:41:37.777653Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.9444444444444444\n"
     ]
    }
   ],
   "source": [
    "print('Test accuracy:', knn.score(X_test_std, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:42:01.390258Z",
     "start_time": "2018-06-19T02:42:01.388124Z"
    }
   },
   "outputs": [],
   "source": [
    "# Now let's use the selected 5-feature subset and see how well KNN performs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:42:14.529699Z",
     "start_time": "2018-06-19T02:42:14.523610Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy: 0.9596774193548387\n"
     ]
    }
   ],
   "source": [
    "knn.fit(X_train_std[:,k5],y_train)\n",
    "print('Training accuracy:', knn.score(X_train_std[:,k5], y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:42:25.131354Z",
     "start_time": "2018-06-19T02:42:25.126338Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.9629629629629629\n"
     ]
    }
   ],
   "source": [
    "print('Test accuracy:', knn.score(X_test_std[:,k5], y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-06-19T02:43:03.943318Z",
     "start_time": "2018-06-19T02:43:03.940368Z"
    }
   },
   "outputs": [],
   "source": [
    "# 总结：Using fewer than half of the original features in the Wine dataset,\n",
    "# the prediction accuracy on the test set improved by almost 2 percent. \n",
    "# Also, we reduced overftting, which we can tell from the small gap between test (~96.3 percent) and training  (~96.0 percent) accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}