getting_data_ready.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.Getting the data ready to be used with ML model\n",
    "\n",
    "Three main things to do:\n",
    "\n",
    "1. Split the data into features and labels (usually `X` & `y`)\n",
    "2. Filling (Imputing) or disregarding missing values\n",
    "3. Converting non-numerical values into numerical values (feature encoding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing Essentials\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the data\n",
    "car_sales_missing = pd.read_csv('data/car-sales-extended-missing-data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15323.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19943.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13434.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14043.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>Black</td>\n",
       "      <td>35820.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>32042.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>NaN</td>\n",
       "      <td>White</td>\n",
       "      <td>155144.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5716.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>66604.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>31570.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>215883.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4001.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>Blue</td>\n",
       "      <td>248360.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>12732.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Make Colour  Odometer (KM)  Doors    Price\n",
       "0     Honda  White        35431.0    4.0  15323.0\n",
       "1       BMW   Blue       192714.0    5.0  19943.0\n",
       "2     Honda  White        84714.0    4.0  28343.0\n",
       "3    Toyota  White       154365.0    4.0  13434.0\n",
       "4    Nissan   Blue       181577.0    3.0  14043.0\n",
       "..      ...    ...            ...    ...      ...\n",
       "995  Toyota  Black        35820.0    4.0  32042.0\n",
       "996     NaN  White       155144.0    3.0   5716.0\n",
       "997  Nissan   Blue        66604.0    4.0  31570.0\n",
       "998   Honda  White       215883.0    4.0   4001.0\n",
       "999  Toyota   Blue       248360.0    4.0  12732.0\n",
       "\n",
       "[1000 rows x 5 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#getting familiar with the data\n",
    "car_sales_missing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to display all the rows and columns as sometimes they are omitted in the notebook: \n",
    "\n",
    "def display_all(df):\n",
    "    with pd.option_context(\"display.max_rows\", 1000, \"display.max_columns\", 1000): \n",
    "        display(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 Dealing with missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             49\n",
       "Colour           50\n",
       "Odometer (KM)    50\n",
       "Doors            50\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checking the missing values in data\n",
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             0.049\n",
       "Colour           0.050\n",
       "Odometer (KM)    0.050\n",
       "Doors            0.050\n",
       "Price            0.050\n",
       "dtype: float64"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Percantage of missing data in each column\n",
    "car_sales_missing.isna().sum()/car_sales_missing.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop the rows with no labels, i.e drop the rows where the value in target variable is nan\n",
    "car_sales_missing.dropna(subset=['Price'], inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## There are 2 ways to deal with missinge values\n",
    "\n",
    "### Option 1: Fill missing values (Imputation)\n",
    "\n",
    "- Numerical data can be replaced with mean, mode or median depending on the requirement and type of data\n",
    "- Categorical data can be replaced with `missing` which will act as another category\n",
    "- Generally the missing values are replaced in feature variables, and the rows where there exists a nan value in target variable is dropped\n",
    "- While replacing the missing values it should be noted that the missing values in training and test sets should be replaced seperately (Data from the future (Test data) should not be used to replace values in the past (Training data) ) \n",
    "\n",
    "### Option 2: Remove the rows and columns with null values\n",
    "\n",
    "- Remove all the rows with even a single null value, since it is not possible for a ML model to process null value\n",
    "- Remove the columns which has more null values than a given threshold (The threshold may vary for each dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filling missing values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dealing with missing values in categorical features\n",
    "Replacing nan values in categorical features with `missing`, making it another category.\n",
    "\n",
    "Even if we don't replace the missing values in categorical features, the code (`ML.txt` line 39) will replace the nan values with -1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 950 entries, 0 to 999\n",
      "Data columns (total 5 columns):\n",
      " #   Column         Non-Null Count  Dtype  \n",
      "---  ------         --------------  -----  \n",
      " 0   Make           903 non-null    object \n",
      " 1   Colour         904 non-null    object \n",
      " 2   Odometer (KM)  902 non-null    float64\n",
      " 3   Doors          903 non-null    float64\n",
      " 4   Price          950 non-null    float64\n",
      "dtypes: float64(3), object(2)\n",
      "memory usage: 44.5+ KB\n"
     ]
    }
   ],
   "source": [
    "car_sales_missing.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "car_sales_missing['Make'].fillna('missing', inplace = True)\n",
    "car_sales_missing['Colour'].fillna('missing', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Make', 'Colour']"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Converting features with object dtype to categorical features\n",
    "categorical = []\n",
    "for col in car_sales_missing.columns[:]:\n",
    "    if car_sales_missing[col].dtype == 'object' : categorical.append(col)  # pandas treat \"str\" as \"object\"\n",
    "categorical # list of all the variables which are strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in categorical: car_sales_missing[col] = car_sales_missing[col].astype(\"category\").cat.as_ordered()\n",
    "for col in categorical : car_sales_missing[col] = car_sales_missing[col].cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    4.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Replacing missing values in `Doors` column with the mode\n",
    "car_sales_missing['Doors'].mode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "car_sales_missing['Doors'].fillna(4, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dealing with missing values in numerical features\n",
    "\n",
    "To fill the missing numerical data the dataset is first split into train and test and then the missing values are filled individually"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split into feature and target variables\n",
    "X = car_sales_missing.drop('Price', axis=1)\n",
    "y = car_sales_missing['Price']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((760, 4), (190, 4), (760,), (190,))"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checking the shape of train and test sets\n",
    "X_train.shape, X_test.shape, y_train.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\KIIT\\Desktop\\sample_project_1\\env\\lib\\site-packages\\pandas\\core\\generic.py:6245: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._update_inplace(new_data)\n"
     ]
    }
   ],
   "source": [
    "# Replacing nan values in 'Odometer (KM)' column with the average of other values in the column\n",
    "X_train['Odometer (KM)'].fillna(X_train['Odometer (KM)'].mean(), inplace = True)\n",
    "X_test['Odometer (KM)'].fillna(X_test['Odometer (KM)'].mean(), inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             0\n",
       "Colour           0\n",
       "Odometer (KM)    0\n",
       "Doors            0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checking the count of nan values in train and test data\n",
    "X_train.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             0\n",
       "Colour           0\n",
       "Odometer (KM)    0\n",
       "Doors            0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 760 entries, 347 to 715\n",
      "Data columns (total 4 columns):\n",
      " #   Column         Non-Null Count  Dtype  \n",
      "---  ------         --------------  -----  \n",
      " 0   Make           760 non-null    int8   \n",
      " 1   Colour         760 non-null    int8   \n",
      " 2   Odometer (KM)  760 non-null    float64\n",
      " 3   Doors          760 non-null    float64\n",
      "dtypes: float64(2), int8(2)\n",
      "memory usage: 19.3 KB\n"
     ]
    }
   ],
   "source": [
    "# Checking the dtype of columns\n",
    "X_train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Working with missing values using Scikit-Learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "car_sales_missing = pd.read_csv('data/car-sales-extended-missing-data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Make</th>\n",
       "      <th>Colour</th>\n",
       "      <th>Odometer (KM)</th>\n",
       "      <th>Doors</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>35431.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15323.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BMW</td>\n",
       "      <td>Blue</td>\n",
       "      <td>192714.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19943.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Honda</td>\n",
       "      <td>White</td>\n",
       "      <td>84714.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28343.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Toyota</td>\n",
       "      <td>White</td>\n",
       "      <td>154365.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13434.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Nissan</td>\n",
       "      <td>Blue</td>\n",
       "      <td>181577.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14043.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Make Colour  Odometer (KM)  Doors    Price\n",
       "0   Honda  White        35431.0    4.0  15323.0\n",
       "1     BMW   Blue       192714.0    5.0  19943.0\n",
       "2   Honda  White        84714.0    4.0  28343.0\n",
       "3  Toyota  White       154365.0    4.0  13434.0\n",
       "4  Nissan   Blue       181577.0    3.0  14043.0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "car_sales_missing.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             49\n",
       "Colour           50\n",
       "Odometer (KM)    50\n",
       "Doors            50\n",
       "Price            50\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check missing values\n",
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             47\n",
       "Colour           46\n",
       "Odometer (KM)    48\n",
       "Doors            47\n",
       "Price             0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop the rows with no labels\n",
    "car_sales_missing.dropna(subset=[\"Price\"], inplace=True)\n",
    "car_sales_missing.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dividing the data into training and test sets\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split into X & y\n",
    "X = car_sales_missing.drop(\"Price\", axis=1)\n",
    "y = car_sales_missing[\"Price\"]\n",
    "\n",
    "# Split data into train and test\n",
    "np.random.seed(42)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filling the training and test values separately to ensure training data stays with the training data and test data stays with the test data.\n",
    "\n",
    "Note: We use fit_transform() on the training data and transform() on the testing data. In essence, we learn the patterns in the training set and transform it via imputation (fit, then transform). Then we take those same patterns and fill the test set (transform only)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['Honda', 'White', 4.0, 71934.0],\n",
       "       ['Toyota', 'Red', 4.0, 162665.0],\n",
       "       ['Honda', 'White', 4.0, 42844.0],\n",
       "       ...,\n",
       "       ['Toyota', 'White', 4.0, 196225.0],\n",
       "       ['Honda', 'Blue', 4.0, 133117.0],\n",
       "       ['Honda', 'missing', 4.0, 150582.0]], dtype=object)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "# Filling categorical values with 'missing' and numerical values with mean\n",
    "cat_imputer = SimpleImputer(strategy = 'constant', fill_value = 'missing')\n",
    "door_imputer = SimpleImputer(strategy = 'constant',  fill_value = 4)\n",
    "num_imputer = SimpleImputer(strategy = 'mean')\n",
    "\n",
    "# Define columns\n",
    "cat_columns = [\"Make\", \"Colour\"]\n",
    "door_columns = [\"Doors\"]\n",
    "num_columns = [\"Odometer (KM)\"]\n",
    "\n",
    "# Create an imputer: Something that fills the missing values\n",
    "imputer = ColumnTransformer([\n",
    "    (\"cat_imputer\", cat_imputer, cat_columns),\n",
    "    (\"door_imputer\", door_imputer, door_columns),\n",
    "    (\"num_imputer\", num_imputer, num_columns)\n",
    "])\n",
    "\n",
    "# Filling the train and test sets seperately\n",
    "filled_X_train = imputer.fit_transform(X_train) # fit_transform imputes the missing values from the training set and fills them simultaneously\n",
    "filled_X_test = imputer.transform(X_test) # tranform takes the imputing missing values from the training set and fills the test set with them\n",
    "\n",
    "# Check filled X_train\n",
    "filled_X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             0\n",
       "Colour           0\n",
       "Doors            0\n",
       "Odometer (KM)    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checking the missing values in new datasets\n",
    "\n",
    "# Get our transformed data array's back into DataFrame's\n",
    "car_sales_filled_train = pd.DataFrame(filled_X_train, \n",
    "                                      columns=[\"Make\", \"Colour\", \"Doors\", \"Odometer (KM)\"])\n",
    "\n",
    "car_sales_filled_test = pd.DataFrame(filled_X_test, \n",
    "                                     columns=[\"Make\", \"Colour\", \"Doors\", \"Odometer (KM)\"])\n",
    "\n",
    "# Check missing data in training set\n",
    "car_sales_filled_train.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Make             0\n",
       "Colour           0\n",
       "Doors            0\n",
       "Odometer (KM)    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check missing data in test set\n",
    "car_sales_filled_test.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So the data has no missing values, but still we need to convert some of the data into numbers, so using one hot encoding.\n",
    "Again keeping the training and testing datasets seperate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 7.19340e+04],\n",
       "       [0.00000e+00, 0.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 1.62665e+05],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 4.28440e+04],\n",
       "       ...,\n",
       "       [0.00000e+00, 0.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 1.96225e+05],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 1.33117e+05],\n",
       "       [0.00000e+00, 1.00000e+00, 0.00000e+00, ..., 1.00000e+00,\n",
       "        0.00000e+00, 1.50582e+05]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import OneHotEncoder class from sklearn\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "# one hot encode the features\n",
    "categorical_features = [\"Make\", \"Colour\", \"Doors\"]\n",
    "one_hot = OneHotEncoder()\n",
    "transformer = ColumnTransformer([(\"one_hot\", \n",
    "                                 one_hot, \n",
    "                                 categorical_features)],\n",
    "                                 remainder=\"passthrough\")\n",
    "\n",
    "# Fill train and test values separately\n",
    "transformed_X_train = transformer.fit_transform(car_sales_filled_train) # fit and transform the training data\n",
    "transformed_X_test = transformer.transform(car_sales_filled_test) # transform the test data\n",
    "\n",
    "# Check transformed and filled X_train\n",
    "transformed_X_train.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit a model\n",
    "\n",
    "The data is finally ready and can be fitted in a model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.21229043336119102"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now we've transformed X, let's see if we can fit a model\n",
    "np.random.seed(42)\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# Setup model\n",
    "model = RandomForestRegressor()\n",
    "\n",
    "# Make sure to use transformed (filled and one-hot encoded X data)\n",
    "model.fit(transformed_X_train, y_train)\n",
    "\n",
    "# Evaluating on the test data\n",
    "model.score(transformed_X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}