Research and Development/feature-engineering-pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setting up feature engineering pipeline with Scikit-learn Pipeline \n",
    "\n",
    "In this notebook we would wrap our feature engineering pipeline in Scikit-learn pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import joblib\n",
    "\n",
    "# scikit-learn\n",
    "\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import MinMaxScaler, Binarizer\n",
    "\n",
    "# feature-engine\n",
    "from feature_engine.imputation import(\n",
    "    AddMissingIndicator,\n",
    "    MeanMedianImputer,\n",
    "    CategoricalImputer\n",
    ")\n",
    "\n",
    "from feature_engine.encoding import (\n",
    "    RareLabelEncoder,\n",
    "    OrdinalEncoder\n",
    ")\n",
    "\n",
    "from feature_engine.transformation import (\n",
    "    LogTransformer,\n",
    "    YeoJohnsonTransformer\n",
    ")\n",
    "\n",
    "\n",
    "from feature_engine.selection import DropFeatures\n",
    "from feature_engine.wrappers import SklearnTransformerWrapper\n",
    "\n",
    "\n",
    "import preprocessors as pp\n",
    "\n",
    "pd.pandas.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1460, 81)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>LotConfig</th>\n",
       "      <th>LandSlope</th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>Condition1</th>\n",
       "      <th>Condition2</th>\n",
       "      <th>BldgType</th>\n",
       "      <th>HouseStyle</th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>OverallCond</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>RoofStyle</th>\n",
       "      <th>RoofMatl</th>\n",
       "      <th>Exterior1st</th>\n",
       "      <th>Exterior2nd</th>\n",
       "      <th>MasVnrType</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>ExterQual</th>\n",
       "      <th>ExterCond</th>\n",
       "      <th>Foundation</th>\n",
       "      <th>BsmtQual</th>\n",
       "      <th>BsmtCond</th>\n",
       "      <th>BsmtExposure</th>\n",
       "      <th>BsmtFinType1</th>\n",
       "      <th>BsmtFinSF1</th>\n",
       "      <th>BsmtFinType2</th>\n",
       "      <th>BsmtFinSF2</th>\n",
       "      <th>BsmtUnfSF</th>\n",
       "      <th>TotalBsmtSF</th>\n",
       "      <th>Heating</th>\n",
       "      <th>HeatingQC</th>\n",
       "      <th>CentralAir</th>\n",
       "      <th>Electrical</th>\n",
       "      <th>1stFlrSF</th>\n",
       "      <th>2ndFlrSF</th>\n",
       "      <th>LowQualFinSF</th>\n",
       "      <th>GrLivArea</th>\n",
       "      <th>BsmtFullBath</th>\n",
       "      <th>BsmtHalfBath</th>\n",
       "      <th>FullBath</th>\n",
       "      <th>HalfBath</th>\n",
       "      <th>BedroomAbvGr</th>\n",
       "      <th>KitchenAbvGr</th>\n",
       "      <th>KitchenQual</th>\n",
       "      <th>TotRmsAbvGrd</th>\n",
       "      <th>Functional</th>\n",
       "      <th>Fireplaces</th>\n",
       "      <th>FireplaceQu</th>\n",
       "      <th>GarageType</th>\n",
       "      <th>GarageYrBlt</th>\n",
       "      <th>GarageFinish</th>\n",
       "      <th>GarageCars</th>\n",
       "      <th>GarageArea</th>\n",
       "      <th>GarageQual</th>\n",
       "      <th>GarageCond</th>\n",
       "      <th>PavedDrive</th>\n",
       "      <th>WoodDeckSF</th>\n",
       "      <th>OpenPorchSF</th>\n",
       "      <th>EnclosedPorch</th>\n",
       "      <th>3SsnPorch</th>\n",
       "      <th>ScreenPorch</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>CollgCr</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>2Story</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>2003</td>\n",
       "      <td>2003</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>BrkFace</td>\n",
       "      <td>196.0</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>PConc</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>No</td>\n",
       "      <td>GLQ</td>\n",
       "      <td>706</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>150</td>\n",
       "      <td>856</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>856</td>\n",
       "      <td>854</td>\n",
       "      <td>0</td>\n",
       "      <td>1710</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>8</td>\n",
       "      <td>Typ</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>RFn</td>\n",
       "      <td>2</td>\n",
       "      <td>548</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>Veenker</td>\n",
       "      <td>Feedr</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>1Story</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>1976</td>\n",
       "      <td>1976</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>MetalSd</td>\n",
       "      <td>MetalSd</td>\n",
       "      <td>None</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>CBlock</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Gd</td>\n",
       "      <td>ALQ</td>\n",
       "      <td>978</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>284</td>\n",
       "      <td>1262</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>1262</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1262</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>TA</td>\n",
       "      <td>6</td>\n",
       "      <td>Typ</td>\n",
       "      <td>1</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>1976.0</td>\n",
       "      <td>RFn</td>\n",
       "      <td>2</td>\n",
       "      <td>460</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>298</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>CollgCr</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>2Story</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>2001</td>\n",
       "      <td>2002</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>BrkFace</td>\n",
       "      <td>162.0</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>PConc</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Mn</td>\n",
       "      <td>GLQ</td>\n",
       "      <td>486</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>434</td>\n",
       "      <td>920</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>920</td>\n",
       "      <td>866</td>\n",
       "      <td>0</td>\n",
       "      <td>1786</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>6</td>\n",
       "      <td>Typ</td>\n",
       "      <td>1</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>2001.0</td>\n",
       "      <td>RFn</td>\n",
       "      <td>2</td>\n",
       "      <td>608</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>70</td>\n",
       "      <td>RL</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Corner</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>Crawfor</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>2Story</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>1915</td>\n",
       "      <td>1970</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>Wd Sdng</td>\n",
       "      <td>Wd Shng</td>\n",
       "      <td>None</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>BrkTil</td>\n",
       "      <td>TA</td>\n",
       "      <td>Gd</td>\n",
       "      <td>No</td>\n",
       "      <td>ALQ</td>\n",
       "      <td>216</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>540</td>\n",
       "      <td>756</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>961</td>\n",
       "      <td>756</td>\n",
       "      <td>0</td>\n",
       "      <td>1717</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>7</td>\n",
       "      <td>Typ</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>Detchd</td>\n",
       "      <td>1998.0</td>\n",
       "      <td>Unf</td>\n",
       "      <td>3</td>\n",
       "      <td>642</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>272</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>Gtl</td>\n",
       "      <td>NoRidge</td>\n",
       "      <td>Norm</td>\n",
       "      <td>Norm</td>\n",
       "      <td>1Fam</td>\n",
       "      <td>2Story</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>Gable</td>\n",
       "      <td>CompShg</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>VinylSd</td>\n",
       "      <td>BrkFace</td>\n",
       "      <td>350.0</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>PConc</td>\n",
       "      <td>Gd</td>\n",
       "      <td>TA</td>\n",
       "      <td>Av</td>\n",
       "      <td>GLQ</td>\n",
       "      <td>655</td>\n",
       "      <td>Unf</td>\n",
       "      <td>0</td>\n",
       "      <td>490</td>\n",
       "      <td>1145</td>\n",
       "      <td>GasA</td>\n",
       "      <td>Ex</td>\n",
       "      <td>Y</td>\n",
       "      <td>SBrkr</td>\n",
       "      <td>1145</td>\n",
       "      <td>1053</td>\n",
       "      <td>0</td>\n",
       "      <td>2198</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>Gd</td>\n",
       "      <td>9</td>\n",
       "      <td>Typ</td>\n",
       "      <td>1</td>\n",
       "      <td>TA</td>\n",
       "      <td>Attchd</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>RFn</td>\n",
       "      <td>3</td>\n",
       "      <td>836</td>\n",
       "      <td>TA</td>\n",
       "      <td>TA</td>\n",
       "      <td>Y</td>\n",
       "      <td>192</td>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "0   1          60       RL         65.0     8450   Pave   NaN      Reg   \n",
       "1   2          20       RL         80.0     9600   Pave   NaN      Reg   \n",
       "2   3          60       RL         68.0    11250   Pave   NaN      IR1   \n",
       "3   4          70       RL         60.0     9550   Pave   NaN      IR1   \n",
       "4   5          60       RL         84.0    14260   Pave   NaN      IR1   \n",
       "\n",
       "  LandContour Utilities LotConfig LandSlope Neighborhood Condition1  \\\n",
       "0         Lvl    AllPub    Inside       Gtl      CollgCr       Norm   \n",
       "1         Lvl    AllPub       FR2       Gtl      Veenker      Feedr   \n",
       "2         Lvl    AllPub    Inside       Gtl      CollgCr       Norm   \n",
       "3         Lvl    AllPub    Corner       Gtl      Crawfor       Norm   \n",
       "4         Lvl    AllPub       FR2       Gtl      NoRidge       Norm   \n",
       "\n",
       "  Condition2 BldgType HouseStyle  OverallQual  OverallCond  YearBuilt  \\\n",
       "0       Norm     1Fam     2Story            7            5       2003   \n",
       "1       Norm     1Fam     1Story            6            8       1976   \n",
       "2       Norm     1Fam     2Story            7            5       2001   \n",
       "3       Norm     1Fam     2Story            7            5       1915   \n",
       "4       Norm     1Fam     2Story            8            5       2000   \n",
       "\n",
       "   YearRemodAdd RoofStyle RoofMatl Exterior1st Exterior2nd MasVnrType  \\\n",
       "0          2003     Gable  CompShg     VinylSd     VinylSd    BrkFace   \n",
       "1          1976     Gable  CompShg     MetalSd     MetalSd       None   \n",
       "2          2002     Gable  CompShg     VinylSd     VinylSd    BrkFace   \n",
       "3          1970     Gable  CompShg     Wd Sdng     Wd Shng       None   \n",
       "4          2000     Gable  CompShg     VinylSd     VinylSd    BrkFace   \n",
       "\n",
       "   MasVnrArea ExterQual ExterCond Foundation BsmtQual BsmtCond BsmtExposure  \\\n",
       "0       196.0        Gd        TA      PConc       Gd       TA           No   \n",
       "1         0.0        TA        TA     CBlock       Gd       TA           Gd   \n",
       "2       162.0        Gd        TA      PConc       Gd       TA           Mn   \n",
       "3         0.0        TA        TA     BrkTil       TA       Gd           No   \n",
       "4       350.0        Gd        TA      PConc       Gd       TA           Av   \n",
       "\n",
       "  BsmtFinType1  BsmtFinSF1 BsmtFinType2  BsmtFinSF2  BsmtUnfSF  TotalBsmtSF  \\\n",
       "0          GLQ         706          Unf           0        150          856   \n",
       "1          ALQ         978          Unf           0        284         1262   \n",
       "2          GLQ         486          Unf           0        434          920   \n",
       "3          ALQ         216          Unf           0        540          756   \n",
       "4          GLQ         655          Unf           0        490         1145   \n",
       "\n",
       "  Heating HeatingQC CentralAir Electrical  1stFlrSF  2ndFlrSF  LowQualFinSF  \\\n",
       "0    GasA        Ex          Y      SBrkr       856       854             0   \n",
       "1    GasA        Ex          Y      SBrkr      1262         0             0   \n",
       "2    GasA        Ex          Y      SBrkr       920       866             0   \n",
       "3    GasA        Gd          Y      SBrkr       961       756             0   \n",
       "4    GasA        Ex          Y      SBrkr      1145      1053             0   \n",
       "\n",
       "   GrLivArea  BsmtFullBath  BsmtHalfBath  FullBath  HalfBath  BedroomAbvGr  \\\n",
       "0       1710             1             0         2         1             3   \n",
       "1       1262             0             1         2         0             3   \n",
       "2       1786             1             0         2         1             3   \n",
       "3       1717             1             0         1         0             3   \n",
       "4       2198             1             0         2         1             4   \n",
       "\n",
       "   KitchenAbvGr KitchenQual  TotRmsAbvGrd Functional  Fireplaces FireplaceQu  \\\n",
       "0             1          Gd             8        Typ           0         NaN   \n",
       "1             1          TA             6        Typ           1          TA   \n",
       "2             1          Gd             6        Typ           1          TA   \n",
       "3             1          Gd             7        Typ           1          Gd   \n",
       "4             1          Gd             9        Typ           1          TA   \n",
       "\n",
       "  GarageType  GarageYrBlt GarageFinish  GarageCars  GarageArea GarageQual  \\\n",
       "0     Attchd       2003.0          RFn           2         548         TA   \n",
       "1     Attchd       1976.0          RFn           2         460         TA   \n",
       "2     Attchd       2001.0          RFn           2         608         TA   \n",
       "3     Detchd       1998.0          Unf           3         642         TA   \n",
       "4     Attchd       2000.0          RFn           3         836         TA   \n",
       "\n",
       "  GarageCond PavedDrive  WoodDeckSF  OpenPorchSF  EnclosedPorch  3SsnPorch  \\\n",
       "0         TA          Y           0           61              0          0   \n",
       "1         TA          Y         298            0              0          0   \n",
       "2         TA          Y           0           42              0          0   \n",
       "3         TA          Y           0           35            272          0   \n",
       "4         TA          Y         192           84              0          0   \n",
       "\n",
       "   ScreenPorch  PoolArea PoolQC Fence MiscFeature  MiscVal  MoSold  YrSold  \\\n",
       "0            0         0    NaN   NaN         NaN        0       2    2008   \n",
       "1            0         0    NaN   NaN         NaN        0       5    2007   \n",
       "2            0         0    NaN   NaN         NaN        0       9    2008   \n",
       "3            0         0    NaN   NaN         NaN        0       2    2006   \n",
       "4            0         0    NaN   NaN         NaN        0      12    2008   \n",
       "\n",
       "  SaleType SaleCondition  SalePrice  \n",
       "0       WD        Normal     208500  \n",
       "1       WD        Normal     181500  \n",
       "2       WD        Normal     223500  \n",
       "3       WD       Abnorml     140000  \n",
       "4       WD        Normal     250000  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('train.csv')\n",
    "\n",
    "print(data.shape)\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['MSSubClass'] = data['MSSubClass'].astype('O')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1314, 79), (146, 79))"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    data.drop(['Id', 'SalePrice'], axis=1),\n",
    "    data['SalePrice'],\n",
    "    test_size=0.1,\n",
    "    random_state=0\n",
    ")\n",
    "\n",
    "\n",
    "X_train.shape, X_test.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np.log(y_train)\n",
    "y_test = np.log(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# categorical variables with NA in train set\n",
    "CATEGORICAL_VARS_WITH_NA_FREQUENT = ['MasVnrType',\n",
    "                                     'BsmtQual',\n",
    "                                     'BsmtCond',\n",
    "                                     'BsmtExposure',\n",
    "                                     'BsmtFinType1',\n",
    "                                     'BsmtFinType2',\n",
    "                                     'Electrical',\n",
    "                                     'GarageType',\n",
    "                                     'GarageFinish',\n",
    "                                     'GarageQual',\n",
    "                                     'GarageCond']\n",
    "\n",
    "\n",
    "CATEGORICAL_VARS_WITH_NA_MISSING = [\n",
    "    'Alley', 'FireplaceQu', 'PoolQC', 'Fence', 'MiscFeature']\n",
    "\n",
    "\n",
    "# numerical variables with NA in train set\n",
    "NUMERICAL_VARS_WITH_NA = ['LotFrontage', 'MasVnrArea', 'GarageYrBlt']\n",
    "\n",
    "\n",
    "TEMPORAL_VARS = ['YearBuilt', 'YearRemodAdd', 'GarageYrBlt']\n",
    "REF_VAR = \"YrSold\"\n",
    "\n",
    "\n",
    "# variables to log transform\n",
    "NUMERICALS_LOG_VARS = [\"LotFrontage\", \"1stFlrSF\", \"GrLivArea\"]\n",
    "\n",
    "NUMERICALS_YEO_VARS = ['LotArea']\n",
    "\n",
    "\n",
    "BINARIZE_VARS = [\n",
    "    'BsmtFinSF2', 'LowQualFinSF', 'EnclosedPorch',\n",
    "    '3SsnPorch', 'ScreenPorch', 'MiscVal'\n",
    "]\n",
    "\n",
    "# variables to map\n",
    "QUAL_VARS = ['ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond',\n",
    "             'HeatingQC', 'KitchenQual', 'FireplaceQu',\n",
    "             'GarageQual', 'GarageCond',\n",
    "             ]\n",
    "\n",
    "EXPOSURE_VARS = ['BsmtExposure']\n",
    "\n",
    "FINISH_VARS = ['BsmtFinType1', 'BsmtFinType2']\n",
    "\n",
    "GARAGE_VARS = ['GarageFinish']\n",
    "\n",
    "FENCE_VARS = ['Fence']\n",
    "\n",
    "# categorical variables to encode\n",
    "CATEGORICAL_VARS = [\n",
    "    'MSZoning',\n",
    "    'Street',\n",
    "    'Alley',\n",
    "    'LotShape',\n",
    "    'LandContour',\n",
    "    'Utilities',\n",
    "    'LotConfig',\n",
    "    'LandSlope',\n",
    "    'Neighborhood',\n",
    "    'Condition1',\n",
    "    'Condition2',\n",
    "    'BldgType',\n",
    "    'HouseStyle',\n",
    "    'RoofStyle',\n",
    "    'RoofMatl',\n",
    "    'Exterior1st',\n",
    "    'Exterior2nd',\n",
    "    'MasVnrType',\n",
    "    'Foundation',\n",
    "    'Heating',\n",
    "    'CentralAir',\n",
    "    'Electrical',\n",
    "    'Functional',\n",
    "    'GarageType',\n",
    "    'PavedDrive',\n",
    "    'PoolQC',\n",
    "    'MiscFeature',\n",
    "    'SaleType',\n",
    "    'SaleCondition',\n",
    "    'MSSubClass']\n",
    "\n",
    "\n",
    "QUAL_MAPPINGS = {'Po': 1, 'Fa': 2, 'TA': 3,\n",
    "                 'Gd': 4, 'Ex': 5, 'Missing': 0, 'NA': 0}\n",
    "\n",
    "EXPOSURE_MAPPINGS = {'No': 1, 'Mn': 2, 'Av': 3, 'Gd': 4}\n",
    "\n",
    "FINISH_MAPPINGS = {'Missing': 0, 'NA': 0, 'Unf': 1,\n",
    "                   'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ': 6}\n",
    "\n",
    "GARAGE_MAPPINGS = {'Missing': 0, 'NA': 0, 'Unf': 1, 'RFn': 2, 'Fin': 3}\n",
    "\n",
    "FENCE_MAPPINGS = {'Missing': 0, 'NA': 0,\n",
    "                  'MnWw': 1, 'GdWo': 2, 'MnPrv': 3, 'GdPrv': 4}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up pipeline\n",
    "\n",
    "price_pipe = Pipeline([\n",
    "\n",
    "    # impute categorical variable with string 'missing'\n",
    "    ('missing_imputation', CategoricalImputer(\n",
    "        imputation_method='missing', variables=CATEGORICAL_VARS_WITH_NA_MISSING\n",
    "    )),\n",
    "\n",
    "    ('frequent_imputation', CategoricalImputer(\n",
    "        imputation_method='frequent', variables=CATEGORICAL_VARS_WITH_NA_FREQUENT\n",
    "    )),\n",
    "\n",
    "    # add missing indicator\n",
    "    ('missing_indicator', AddMissingIndicator(variables=NUMERICAL_VARS_WITH_NA)),\n",
    "\n",
    "    # Impute numerical variables with mean\n",
    "    ('mean_imputation', MeanMedianImputer(\n",
    "        imputation_method='mean', variables=NUMERICAL_VARS_WITH_NA\n",
    "    )),\n",
    "\n",
    "    \n",
    "    # ----- TEMPORAL VARIABLE ------\n",
    "\n",
    "    ('elapsed_time', pp.TemporalVariableTransformer(\n",
    "        variables=TEMPORAL_VARS, reference_variable=REF_VAR\n",
    "    )),\n",
    "\n",
    "    ('drop_features', DropFeatures(features_to_drop=[REF_VAR])), \n",
    "\n",
    "\n",
    "    # ==== VARIABLE TRANSFORMATION =====\n",
    "    ('log', LogTransformer(variables=NUMERICALS_LOG_VARS)),\n",
    "\n",
    "    ('yeojohnson', YeoJohnsonTransformer(variables=NUMERICALS_YEO_VARS)),\n",
    "\n",
    "    ('binarizer', SklearnTransformerWrapper(\n",
    "        transformer=Binarizer(threshold=0), variables=BINARIZE_VARS\n",
    "    )),\n",
    "\n",
    "    # ---- MAPPER -----\n",
    "    ('mapper_qual', pp.Mapper(\n",
    "        variables=QUAL_VARS, mappings=QUAL_MAPPINGS\n",
    "    )),\n",
    "\n",
    "    ('mapper_exposure', pp.Mapper(\n",
    "        variables=EXPOSURE_VARS, mappings=EXPOSURE_MAPPINGS\n",
    "    )),\n",
    "\n",
    "    ('mapper_finish', pp.Mapper(\n",
    "        variables=FINISH_VARS, mappings=FINISH_MAPPINGS\n",
    "    )),\n",
    "\n",
    "    ('mapping_garage', pp.Mapper(\n",
    "        variables=GARAGE_VARS, mappings=GARAGE_MAPPINGS\n",
    "    )),\n",
    "\n",
    "    ('mapper_fence', pp.Mapper(\n",
    "        variables=FENCE_VARS, mappings=FENCE_MAPPINGS\n",
    "    )),\n",
    "\n",
    "    # ------- CATEGORICAL ENCODING  -------\n",
    "    ('rare_label_encoder', RareLabelEncoder(\n",
    "        tol=0.01, n_categories=1, variables=CATEGORICAL_VARS\n",
    "    )),\n",
    "\n",
    "    ('categorical_encoder', OrdinalEncoder(\n",
    "        encoding_method='ordered', variables=CATEGORICAL_VARS\n",
    "    )),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arnold/Desktop/MlOp/env/lib/python3.8/site-packages/scipy/stats/morestats.py:1557: RuntimeWarning: divide by zero encountered in log\n",
      "  loglike = -n_samples / 2 * np.log(trans.var(axis=0))\n",
      "/home/arnold/Desktop/MlOp/env/lib/python3.8/site-packages/scipy/optimize/optimize.py:2621: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  w = xb - ((xb - xc) * tmp2 - (xb - xa) * tmp1) / denom\n",
      "/home/arnold/Desktop/MlOp/env/lib/python3.8/site-packages/scipy/optimize/optimize.py:2214: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  tmp1 = (x - w) * (fx - fv)\n",
      "/home/arnold/Desktop/MlOp/env/lib/python3.8/site-packages/scipy/optimize/optimize.py:2215: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  tmp2 = (x - v) * (fx - fw)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('missing_imputation',\n",
       "                 CategoricalImputer(variables=['Alley', 'FireplaceQu', 'PoolQC',\n",
       "                                               'Fence', 'MiscFeature'])),\n",
       "                ('frequent_imputation',\n",
       "                 CategoricalImputer(imputation_method='frequent',\n",
       "                                    variables=['MasVnrType', 'BsmtQual',\n",
       "                                               'BsmtCond', 'BsmtExposure',\n",
       "                                               'BsmtFinType1', 'BsmtFinType2',\n",
       "                                               'Electrical', 'GarageType',\n",
       "                                               'GarageFinish', 'GarageQual',\n",
       "                                               'GarageCon...\n",
       "                 OrdinalEncoder(variables=['MSZoning', 'Street', 'Alley',\n",
       "                                           'LotShape', 'LandContour',\n",
       "                                           'Utilities', 'LotConfig',\n",
       "                                           'LandSlope', 'Neighborhood',\n",
       "                                           'Condition1', 'Condition2',\n",
       "                                           'BldgType', 'HouseStyle',\n",
       "                                           'RoofStyle', 'RoofMatl',\n",
       "                                           'Exterior1st', 'Exterior2nd',\n",
       "                                           'MasVnrType', 'Foundation',\n",
       "                                           'Heating', 'CentralAir',\n",
       "                                           'Electrical', 'Functional',\n",
       "                                           'GarageType', 'PavedDrive', 'PoolQC',\n",
       "                                           'MiscFeature', 'SaleType',\n",
       "                                           'SaleCondition', 'MSSubClass']))])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "price_pipe.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = price_pipe.transform(X_train)\n",
    "X_test = price_pipe.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[var for var in X_train.columns if X_train[var].isnull().sum() > 0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('env': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "8957aee21f29590461af2a48414c66c4d86b7c449264058c3c62786bbf5308d6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}