Competition/McKinseyAnalyticsPrediction/NumberOfVehiclesPrediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction Number of Vehicles\n",
    "## Given 20 months Date time, ID of junctions, and number of Vehicles to predict number of vehicles in next 4 months."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime as dt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "#warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read training and test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv', encoding = \"utf-8\", parse_dates=[\"DateTime\"],\n",
    "date_parser=lambda x: pd.to_datetime(x, format=\"%Y-%m-%d %H:%M:%S\"))\n",
    "test = pd.read_csv('test.csv', encoding = \"utf-8\", parse_dates=[\"DateTime\"],\n",
    "date_parser=lambda x: pd.to_datetime(x, format=\"%Y-%m-%d %H:%M:%S\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DateTime</th>\n",
       "      <th>Junction</th>\n",
       "      <th>Vehicles</th>\n",
       "      <th>ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2015-11-01 00:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>20151101001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2015-11-01 01:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>20151101011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2015-11-01 02:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>20151101021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2015-11-01 03:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>20151101031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2015-11-01 04:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>20151101041</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             DateTime  Junction  Vehicles           ID\n",
       "0 2015-11-01 00:00:00         1        15  20151101001\n",
       "1 2015-11-01 01:00:00         1        13  20151101011\n",
       "2 2015-11-01 02:00:00         1        10  20151101021\n",
       "3 2015-11-01 03:00:00         1         7  20151101031\n",
       "4 2015-11-01 04:00:00         1         9  20151101041"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DateTime</th>\n",
       "      <th>Junction</th>\n",
       "      <th>Vehicles</th>\n",
       "      <th>ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>48115</th>\n",
       "      <td>2017-06-30 19:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>20170630194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48116</th>\n",
       "      <td>2017-06-30 20:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>30</td>\n",
       "      <td>20170630204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48117</th>\n",
       "      <td>2017-06-30 21:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>20170630214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48118</th>\n",
       "      <td>2017-06-30 22:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>22</td>\n",
       "      <td>20170630224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48119</th>\n",
       "      <td>2017-06-30 23:00:00</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>20170630234</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 DateTime  Junction  Vehicles           ID\n",
       "48115 2017-06-30 19:00:00         4        11  20170630194\n",
       "48116 2017-06-30 20:00:00         4        30  20170630204\n",
       "48117 2017-06-30 21:00:00         4        16  20170630214\n",
       "48118 2017-06-30 22:00:00         4        22  20170630224\n",
       "48119 2017-06-30 23:00:00         4        12  20170630234"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.tail(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split DateTime to year, quarter, month, hour to make more features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DateTime</th>\n",
       "      <th>Junction</th>\n",
       "      <th>Vehicles</th>\n",
       "      <th>ID</th>\n",
       "      <th>year</th>\n",
       "      <th>quarter</th>\n",
       "      <th>month</th>\n",
       "      <th>hour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2015-11-01 00:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>20151101001</td>\n",
       "      <td>2015</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2015-11-01 01:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>20151101011</td>\n",
       "      <td>2015</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2015-11-01 02:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>20151101021</td>\n",
       "      <td>2015</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2015-11-01 03:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>20151101031</td>\n",
       "      <td>2015</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2015-11-01 04:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>20151101041</td>\n",
       "      <td>2015</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             DateTime  Junction  Vehicles           ID  year  quarter  month  \\\n",
       "0 2015-11-01 00:00:00         1        15  20151101001  2015        4     11   \n",
       "1 2015-11-01 01:00:00         1        13  20151101011  2015        4     11   \n",
       "2 2015-11-01 02:00:00         1        10  20151101021  2015        4     11   \n",
       "3 2015-11-01 03:00:00         1         7  20151101031  2015        4     11   \n",
       "4 2015-11-01 04:00:00         1         9  20151101041  2015        4     11   \n",
       "\n",
       "   hour  \n",
       "0     0  \n",
       "1     1  \n",
       "2     2  \n",
       "3     3  \n",
       "4     4  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['year'] = train['DateTime'].dt.year\n",
    "train['quarter'] = train['DateTime'].dt.quarter\n",
    "train['month'] = train['DateTime'].dt.month\n",
    "train['hour'] = train['DateTime'].dt.hour\n",
    "\n",
    "train.head(5)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DateTime</th>\n",
       "      <th>Junction</th>\n",
       "      <th>ID</th>\n",
       "      <th>year</th>\n",
       "      <th>quarter</th>\n",
       "      <th>month</th>\n",
       "      <th>hour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-07-01 00:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>20170701001</td>\n",
       "      <td>2017</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-07-01 01:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>20170701011</td>\n",
       "      <td>2017</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-07-01 02:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>20170701021</td>\n",
       "      <td>2017</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-07-01 03:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>20170701031</td>\n",
       "      <td>2017</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017-07-01 04:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>20170701041</td>\n",
       "      <td>2017</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             DateTime  Junction           ID  year  quarter  month  hour\n",
       "0 2017-07-01 00:00:00         1  20170701001  2017        3      7     0\n",
       "1 2017-07-01 01:00:00         1  20170701011  2017        3      7     1\n",
       "2 2017-07-01 02:00:00         1  20170701021  2017        3      7     2\n",
       "3 2017-07-01 03:00:00         1  20170701031  2017        3      7     3\n",
       "4 2017-07-01 04:00:00         1  20170701041  2017        3      7     4"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['year'] = test['DateTime'].dt.year\n",
    "test['quarter'] = test['DateTime'].dt.quarter\n",
    "test['month'] = test['DateTime'].dt.month\n",
    "test['hour'] = test['DateTime'].dt.hour\n",
    "\n",
    "test.head(5)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split data for training and validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X = train[['year','quarter', 'month', 'hour','Junction']]\n",
    "y = train['Vehicles']\n",
    "X_test = test[['year','quarter', 'month', 'hour','Junction']]\n",
    "X_train, X_validate, y_train, y_validate = train_test_split(X, y, test_size = 0.3, random_state=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>quarter</th>\n",
       "      <th>month</th>\n",
       "      <th>hour</th>\n",
       "      <th>Junction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>41605</th>\n",
       "      <td>2017</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5461</th>\n",
       "      <td>2016</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14844</th>\n",
       "      <td>2015</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38253</th>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7745</th>\n",
       "      <td>2016</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       year  quarter  month  hour  Junction\n",
       "41605  2017        2      4    13         3\n",
       "5461   2016        2      6    13         1\n",
       "14844  2015        4     11    12         2\n",
       "38253  2016        4     11    21         3\n",
       "7745   2016        3      9    17         1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## standardization\n",
    "### Due to year 2017 included in test set, so include year into the scope of standardization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "mmc = MinMaxScaler(feature_range=(0,1)) # default (0, 1) for sigmoid function\n",
    "mmc.fit(X_train)\n",
    "X_train_std = mmc.transform(X_train)\n",
    "X_validate_std = mmc.transform(X_validate)\n",
    "X_test_std = mmc.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.        , 0.33333333, 0.27272727, 0.56521739, 0.66666667],\n",
       "       [0.5       , 0.33333333, 0.45454545, 0.56521739, 0.        ],\n",
       "       [0.        , 1.        , 0.90909091, 0.52173913, 0.33333333],\n",
       "       ...,\n",
       "       [1.        , 0.        , 0.18181818, 1.        , 0.        ],\n",
       "       [1.        , 0.33333333, 0.45454545, 0.47826087, 0.        ],\n",
       "       [0.5       , 1.        , 0.90909091, 0.34782609, 0.66666667]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_std"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build a based line model: SVR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVR(C=200, cache_size=200, coef0=0.0, degree=3, epsilon=0.1, gamma='auto',\n",
       "  kernel='rbf', max_iter=-1, shrinking=True, tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# C and gamma are the parameters of SVR with kernel='rbf'\n",
    "# Below statement does not work and reason unknow. The main issue is the data of y_train.\n",
    "\n",
    "# C_range = np.logspace(2, 10, 13)\n",
    "# gamma_range = np.logspace(0.1, 3, 13)\n",
    "# param_grid = dict(gamma=gamma_range, C=C_range)\n",
    "# cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)\n",
    "# grid = GridSearchCV(svm.SVR(), param_grid=param_grid, cv=cv)\n",
    "# grid.fit(X_train_std, , y_train.values.ravel())\n",
    "\n",
    "# print(\"The best parameters are %s with a score of %0.2f\"\n",
    "#       % (grid.best_params_, grid.best_score_))\n",
    "\n",
    "# Bigger C takes longer time but better prediction. Be aware of overfitting.\n",
    "clf = svm.SVR(C=200)\n",
    "clf.fit(X_train_std, y_train) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = clf.predict(X_validate_std)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate the prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rmse=9.239314\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "mse = mean_squared_error(y_validate, y_pred)\n",
    "rmse = np.sqrt(mse)\n",
    "print ('rmse=%f'%rmse)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict the testing data set and output the data as file name submission.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = clf.predict(X_test_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {'ID' : pd.Series(test['ID']),\n",
    "      'Vehicles' : pd.Series(y_pred)\n",
    "    }\n",
    "df = pd.DataFrame(d)\n",
    "df['Vehicles'] = df['Vehicles'].astype(int)\n",
    "df.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build model: Try NN, LSTM, GRU for this prediction\n",
    "### Construct a 4 layers full connected deep learning neural network to perform prediction by early stop."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Clark\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.utils import np_utils\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten, TimeDistributed, LSTM, GRU, ConvLSTM2D\n",
    "from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, Callback\n",
    "\n",
    "def create_NN_model(optimizer='adam', init='normal'):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(512, input_dim=5, activation='relu', kernel_initializer=init))\n",
    "    #model.add(Dropout(0.5))\n",
    "    model.add(Dense(256, activation='relu', kernel_initializer=init))\n",
    "    model.add(Dense(64, activation='relu', kernel_initializer=init))\n",
    "    model.add(Dense(1, activation='relu', kernel_initializer=init))\n",
    "    model.compile(optimizer=optimizer, loss='mean_squared_error', metrics=['accuracy'])\n",
    "    return model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "33684/33684 [==============================] - 20s 580us/step - loss: 116.4375 - acc: 0.0626\n",
      "Epoch 2/5\n",
      "33684/33684 [==============================] - 20s 596us/step - loss: 85.2025 - acc: 0.0744\n",
      "Epoch 3/5\n",
      "33684/33684 [==============================] - 20s 582us/step - loss: 78.0850 - acc: 0.0805\n",
      "Epoch 4/5\n",
      "33684/33684 [==============================] - 22s 640us/step - loss: 74.9862 - acc: 0.0843\n",
      "Epoch 5/5\n",
      "33684/33684 [==============================] - 24s 723us/step - loss: 73.0994 - acc: 0.0839\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x27231b29240>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Parameters\n",
    "s_optimizer = 'adam' # rmsprop or adam\n",
    "s_init='uniform'\n",
    "n_epoches = 5\n",
    "n_batch_size = 6\n",
    "\n",
    "model_NN = create_NN_model(optimizer=s_optimizer, init=s_init)\n",
    "model_NN.fit(X_train_std, y_train.values.ravel(),\n",
    "          epochs=n_epoches,\n",
    "          batch_size=n_batch_size,\n",
    "          callbacks=[EarlyStopping(monitor='acc', min_delta=0.001, patience=2, verbose=1, mode='auto')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model_NN.predict(X_validate_std, batch_size=n_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rmse=8.977200\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "mse = mean_squared_error(y_validate, y_pred)\n",
    "rmse = np.sqrt(mse)\n",
    "print ('rmse=%f'%rmse)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict the testing data set and output the data as file name submission-nn.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_result = model_NN.predict(X_test_std, batch_size=n_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {'ID' : pd.Series(test['ID']),\n",
    "      'Vehicles' : pd.Series(y_result.ravel())\n",
    "    }\n",
    "df = pd.DataFrame(d)\n",
    "df['Vehicles'] = df['Vehicles'].astype(int)\n",
    "df.to_csv('submission-nn.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NN model auto tuning. We can try to auto tuning from parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -75.865369 using {'batch_size': 6, 'epochs': 3, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "-75.865369 (3.713737) with: {'batch_size': 6, 'epochs': 3, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "-75.965592 (3.578463) with: {'batch_size': 6, 'epochs': 3, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "-76.818805 (1.844773) with: {'batch_size': 6, 'epochs': 3, 'init': 'normal', 'optimizer': 'adam'}\n"
     ]
    }
   ],
   "source": [
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "\n",
    "# grid search epochs, batch size and optimizer\n",
    "# optimizers = ['rmsprop', 'adam']\n",
    "# init = ['glorot_uniform', 'normal', 'uniform']\n",
    "# epochs = [5, 10]\n",
    "# batches = [6, 12, 24]\n",
    "\n",
    "optimizers = ['adam']\n",
    "init = ['glorot_uniform', 'uniform', 'normal']\n",
    "epochs = [3]\n",
    "batches = [6]\n",
    "\n",
    "# create model\n",
    "model_NN = KerasRegressor(build_fn=create_NN_model, verbose=0)\n",
    "#Best: 0.091082 using {'batch_size': 6, 'epochs': 10, 'init': 'uniform', 'optimizer': 'adam'}\n",
    "\n",
    "param_grid = dict(optimizer=optimizers, epochs=epochs, batch_size=batches, init=init)\n",
    "grid = GridSearchCV(estimator=model_NN, param_grid=param_grid)\n",
    "grid_result = grid.fit(X_train_std, y_train.values.ravel())\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "https://machinelearningmastery.com/time-series-prediction-lstm-recurrent-neural-networks-python-keras/\n",
    "## No state of LSTM\n",
    "### stateful = False\n",
    "### If True, the last state for each sample at index i in a batch will be used as initial state for the sample of index i in the following batch.\n",
    "##  If you want to keep state memory of LSTM with Time windows approach (lags, horizon). You cannot shuffle your data.\n",
    "## If you want to control the state of LSTM\n",
    "\n",
    "### stateful = True, and change your training epochs to external loop. below example epochs = 100\n",
    "```python\n",
    "for i in range(100):\n",
    "\tmodel.fit(trainX, trainY, epochs=1, batch_size=batch_size, verbose=2, shuffle=False)\n",
    "\tmodel.reset_states()\n",
    "```\n",
    "\n",
    "### Construct a 4 layers deep learning LSTM network to perform prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_LSTM_model(optimizer='adam', init='normal'):\n",
    "    model = Sequential()\n",
    "    # batch_input_shape=(batch, timesteps/lags, features)\n",
    "    # input_shape=(timesteps/lags, features)\n",
    "    model.add(LSTM(512, input_shape=(1, 5), return_sequences=True, activation='tanh', recurrent_activation='hard_sigmoid', use_bias=True, \n",
    "                   kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros',\n",
    "                   unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, \n",
    "                   activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, \n",
    "                   dropout=0.0, recurrent_dropout=0.0, implementation=1, return_state=False, \n",
    "                   go_backwards=False, stateful=False, unroll=False))\n",
    "    #model.add(Dropout(0.5))\n",
    "    model.add(LSTM(256, return_sequences=True, activation='tanh', recurrent_activation='hard_sigmoid', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0.0, recurrent_dropout=0.0, implementation=1, return_state=False, go_backwards=False, stateful=False, unroll=False))\n",
    "    # The last layer of LSTM, return_sequences = False\n",
    "    model.add(LSTM(64, return_state=False, activation='tanh', recurrent_activation='hard_sigmoid', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', unit_forget_bias=True, kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0.0, recurrent_dropout=0.0, implementation=1, return_sequences=False, go_backwards=False, stateful=False, unroll=False))\n",
    "    model.add(Dense(1, activation='relu', kernel_initializer=init))\n",
    "    model.compile(optimizer=optimizer, loss='mean_squared_error', metrics=['accuracy'])\n",
    "    return model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(33684, 5)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_std.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "33684/33684 [==============================] - 458s 14ms/step - loss: 249.1206 - acc: 0.0474\n",
      "Epoch 2/5\n",
      "33684/33684 [==============================] - 449s 13ms/step - loss: 88.8254 - acc: 0.0713\n",
      "Epoch 3/5\n",
      "33684/33684 [==============================] - 448s 13ms/step - loss: 73.5931 - acc: 0.0825\n",
      "Epoch 4/5\n",
      "33684/33684 [==============================] - 441s 13ms/step - loss: 71.9012 - acc: 0.0866\n",
      "Epoch 5/5\n",
      "33684/33684 [==============================] - 441s 13ms/step - loss: 71.0807 - acc: 0.0864\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x27235e23c50>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Parameters\n",
    "s_optimizer = 'adam' # rmsprop or adam\n",
    "s_init='uniform'\n",
    "n_epoches = 5\n",
    "n_batch_size = 6\n",
    "\n",
    "# if timesteps/lags = 1\n",
    "X_3D_train_std = X_train_std.reshape(X_train_std.shape[0],1, X_train_std.shape[1])\n",
    "\n",
    "model_LSTM = create_LSTM_model(optimizer=s_optimizer, init=s_init)\n",
    "model_LSTM.fit(X_3D_train_std, y_train.values.ravel(),\n",
    "          epochs=n_epoches,\n",
    "          batch_size=n_batch_size,\n",
    "          callbacks=[EarlyStopping(monitor='acc', min_delta=0.001, patience=2, verbose=1, mode='auto')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model_LSTM.predict(X_validate_std.reshape(X_validate_std.shape[0],1, X_validate_std.shape[1]), batch_size=n_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rmse=8.401580\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "mse = mean_squared_error(y_validate, y_pred)\n",
    "rmse = np.sqrt(mse)\n",
    "print ('rmse=%f'%rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_result = model_LSTM.predict(X_test_std.reshape(X_test_std.shape[0],1, X_test_std.shape[1]), batch_size=n_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {'ID' : pd.Series(test['ID']),\n",
    "      'Vehicles' : pd.Series(y_result.ravel())\n",
    "    }\n",
    "df = pd.DataFrame(d)\n",
    "df['Vehicles'] = df['Vehicles'].astype(int)\n",
    "df.to_csv('submission-lstm.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Auto Tuning approach by grid search for LSTM model to get better parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -75.361852 using {'batch_size': 6, 'epochs': 3, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "-75.361852 (5.536853) with: {'batch_size': 6, 'epochs': 3, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "-370.314677 (416.153495) with: {'batch_size': 6, 'epochs': 3, 'init': 'normal', 'optimizer': 'adam'}\n",
      "-82.446533 (2.601921) with: {'batch_size': 6, 'epochs': 3, 'init': 'uniform', 'optimizer': 'adam'}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "\n",
    "# grid search epochs, batch size and optimizer\n",
    "# optimizers = ['rmsprop', 'adam']\n",
    "# init = ['glorot_uniform', 'normal', 'uniform']\n",
    "# epochs = [10, 20, 50]\n",
    "# batches = [5, 10, 20]\n",
    "\n",
    "optimizers = ['adam']\n",
    "init = ['glorot_uniform', 'normal', 'uniform']\n",
    "epochs = [3]\n",
    "batches = [6]\n",
    "\n",
    "# create model\n",
    "model_LSTM = KerasRegressor(build_fn=create_LSTM_model, verbose=0)\n",
    "\n",
    "param_grid = dict(optimizer=optimizers, epochs=epochs, batch_size=batches, init=init)\n",
    "grid = GridSearchCV(estimator=model_LSTM, param_grid=param_grid)\n",
    "grid_result = grid.fit(X_3D_train_std, y_train.values.ravel())\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gated Recurrent Unit (GRU) model\n",
    "\n",
    "### Construct a 4 layers deep learning GRU network to perform prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_GRU_model(optimizer='adam', init='normal'):\n",
    "    model = Sequential()\n",
    "    model.add(GRU(512, input_shape=(1, 5), return_sequences=True, activation='tanh', \n",
    "                  recurrent_activation='hard_sigmoid', use_bias=True, \n",
    "                  kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', \n",
    "                  kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, \n",
    "                  activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, \n",
    "                  dropout=0.0, recurrent_dropout=0.0, implementation=1, return_state=False, \n",
    "                  go_backwards=False, stateful=False, unroll=False))\n",
    "    #model.add(Dropout(0.5))\n",
    "    model.add(GRU(256, return_sequences=True, activation='tanh', recurrent_activation='hard_sigmoid', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0.0, recurrent_dropout=0.0, implementation=1, return_state=False, go_backwards=False, stateful=False, unroll=False))\n",
    "    model.add(GRU(64, return_sequences=False, activation='tanh', recurrent_activation='hard_sigmoid', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0.0, recurrent_dropout=0.0, implementation=1, return_state=False, go_backwards=False, stateful=False, unroll=False))\n",
    "    model.add(Dense(1, activation='relu', kernel_initializer=init))\n",
    "    model.compile(optimizer=optimizer, loss='mean_squared_error', metrics=['accuracy'])\n",
    "    return model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "33684/33684 [==============================] - 307s 9ms/step - loss: 242.6070 - acc: 0.0482\n",
      "Epoch 2/5\n",
      "33684/33684 [==============================] - 303s 9ms/step - loss: 76.8385 - acc: 0.0814\n",
      "Epoch 3/5\n",
      "33684/33684 [==============================] - 302s 9ms/step - loss: 73.3283 - acc: 0.0833\n",
      "Epoch 4/5\n",
      "33684/33684 [==============================] - 303s 9ms/step - loss: 71.4597 - acc: 0.0861\n",
      "Epoch 5/5\n",
      "33684/33684 [==============================] - 304s 9ms/step - loss: 70.3912 - acc: 0.0884\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x27213944588>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Parameters\n",
    "s_optimizer = 'adam' # rmsprop or adam\n",
    "s_init='uniform'\n",
    "n_epoches = 5\n",
    "n_batch_size = 6\n",
    "\n",
    "# if timesteps/lags = 1\n",
    "X_3D_train_std = X_train_std.reshape(X_train_std.shape[0],1, X_train_std.shape[1])\n",
    "\n",
    "model_GRU = create_GRU_model(optimizer=s_optimizer, init=s_init)\n",
    "model_GRU.fit(X_3D_train_std, y_train.values.ravel(),\n",
    "          epochs=n_epoches,\n",
    "          batch_size=n_batch_size,\n",
    "          callbacks=[EarlyStopping(monitor='acc', min_delta=0.001, patience=2, verbose=1, mode='auto')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model_GRU.predict(X_validate_std.reshape(X_validate_std.shape[0],1, X_validate_std.shape[1]), batch_size=n_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rmse=8.090658\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "mse = mean_squared_error(y_validate, y_pred)\n",
    "rmse = np.sqrt(mse)\n",
    "print ('rmse=%f'%rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_result = model_GRU.predict(X_test_std.reshape(X_test_std.shape[0],1, X_test_std.shape[1]), batch_size=n_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = {'ID' : pd.Series(test['ID']),\n",
    "      'Vehicles' : pd.Series(y_result.ravel())\n",
    "    }\n",
    "df = pd.DataFrame(d)\n",
    "df['Vehicles'] = df['Vehicles'].astype(int)\n",
    "df.to_csv('submission-gru.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -75.307071 using {'batch_size': 6, 'epochs': 3, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "-75.307071 (5.358852) with: {'batch_size': 6, 'epochs': 3, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "-361.536116 (394.932360) with: {'batch_size': 6, 'epochs': 3, 'init': 'normal', 'optimizer': 'adam'}\n",
      "-77.325250 (4.809811) with: {'batch_size': 6, 'epochs': 3, 'init': 'uniform', 'optimizer': 'adam'}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "\n",
    "# grid search epochs, batch size and optimizer\n",
    "# optimizers = ['rmsprop', 'adam']\n",
    "# init = ['glorot_uniform', 'normal', 'uniform']\n",
    "# epochs = [10, 20, 50]\n",
    "# batches = [5, 10, 20]\n",
    "\n",
    "optimizers = ['adam']\n",
    "init = ['glorot_uniform', 'normal', 'uniform']\n",
    "epochs = [3]\n",
    "batches = [6]\n",
    "\n",
    "# create model\n",
    "model = KerasRegressor(build_fn=create_GRU_model, verbose=0)\n",
    "\n",
    "param_grid = dict(optimizer=optimizers, epochs=epochs, batch_size=batches, init=init)\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid)\n",
    "grid_result = grid.fit(X_3D_train_std, y_train.values.ravel())\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GRU get better result in similar parameter settings."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## All those parameters still have a lot of space to fine-tune."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}