sklearn - Train Test Split.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Test Split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import the relevant libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# In this lesson we will explore the train_test_split module\n",
    "# Therefore we need no more than the module itself and NumPy\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate some data we are going to split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's generate a new data frame 'a' which will contain all integers from 1 to 100\n",
    "# The method np.arange works like the built-in method 'range' with the difference it creates an array\n",
    "a = np.arange(1,101)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,  13,\n",
       "        14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,  26,\n",
       "        27,  28,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,  39,\n",
       "        40,  41,  42,  43,  44,  45,  46,  47,  48,  49,  50,  51,  52,\n",
       "        53,  54,  55,  56,  57,  58,  59,  60,  61,  62,  63,  64,  65,\n",
       "        66,  67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,  78,\n",
       "        79,  80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,  91,\n",
       "        92,  93,  94,  95,  96,  97,  98,  99, 100])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's check it out\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513,\n",
       "       514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526,\n",
       "       527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539,\n",
       "       540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552,\n",
       "       553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565,\n",
       "       566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578,\n",
       "       579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591,\n",
       "       592, 593, 594, 595, 596, 597, 598, 599, 600])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Similarly, let's create another ndarray 'b', which will contain integers from 501 to 600\n",
    "# We have intentionally picked these numbers so we can easily compare the two\n",
    "# Obviously, the difference between the elements of the two arrays is 500 for any two corresponding elements\n",
    "b = np.arange(501,601)\n",
    "b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split the data\n",
    "Full documentation: https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([87, 32, 90,  1,  2,  8, 51, 73, 22, 95,  4, 57, 27, 58, 48, 99, 96,\n",
       "        74, 72, 29, 76, 64,  3, 12, 53,  6, 18, 16, 65, 66, 63, 46, 39, 17,\n",
       "        91, 25, 15, 78, 83, 19, 45, 68, 33, 98, 97, 14, 44, 86, 80, 34, 70,\n",
       "        47, 54, 93, 94, 85, 42, 60, 92, 41, 61, 71, 89, 23, 21, 11, 84, 13,\n",
       "        82, 59, 49, 79, 36, 55,  5]),\n",
       " array([ 24,  56,  40,   9,  69,  75,  10,  28,  38,  30,  62,  67, 100,\n",
       "         88,  37,  20,   7,  31,  77,  43,  35,  26,  81,  52,  50])]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's check out how this works\n",
    "train_test_split(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# There are several different arguments we can set when we employ this method\n",
    "# Most often, we have inputs and targets, so we have to split 2 different arrays\n",
    "# we are simulating this situation by splitting 'a' and 'b'\n",
    "\n",
    "# You can specify the 'test_size' or the 'train_size' (but the latter is deprecated and will be removed)\n",
    "# essentially the two have the same meaning \n",
    "# Common splits are 75-25, 80-20, 85-15, 90-10\n",
    "\n",
    "# Finally, you should always employ a 'random_state'\n",
    "# In this way you ensure that when you are splitting the data you will always get the SAME random shuffle\n",
    "\n",
    "# Note 2 arrays will be split into 4\n",
    "# The order is train1, test1, train2, test2 \n",
    "# It is very useful to store them in 4 variables, so we can later use them\n",
    "a_train, a_test, b_train, b_test = train_test_split(a, b, test_size=0.2, random_state=365)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore the result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((80,), (20,))"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's check the shapes\n",
    "# Basically, we are checking how does the 'test_size' work\n",
    "a_train.shape, a_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 25,  32,  99,  73,  91,  66,   3,  59,  94,   1,   8,  15,  90,\n",
       "        54,  31,  20,  77,  82,  30,  35,  95,  42,  38,   7,  11,  50,\n",
       "        21,  48,   2,  17,  10,  58,  68,  43,  41,  16,  88,  72,  79,\n",
       "       100,  80,  39,  24,  86,  22,  23,  62,  76,  18,  47,  55,  26,\n",
       "        60,  19,  71,  64,  51,  63,  65,  28,  12,  78,  13,  44,  75,\n",
       "        87,  40,   4,  29,  49,  37,  57,  27,  74,   6,  45,  92,  34,\n",
       "        53,  83])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Explore manually\n",
    "a_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 9, 69, 81, 56, 33, 93, 84, 61, 46, 89, 85, 67, 97,  5, 70, 36, 98,\n",
       "       96, 14, 52])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Explore manually\n",
    "a_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((80,), (20,))"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b_train.shape, b_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([525, 532, 599, 573, 591, 566, 503, 559, 594, 501, 508, 515, 590,\n",
       "       554, 531, 520, 577, 582, 530, 535, 595, 542, 538, 507, 511, 550,\n",
       "       521, 548, 502, 517, 510, 558, 568, 543, 541, 516, 588, 572, 579,\n",
       "       600, 580, 539, 524, 586, 522, 523, 562, 576, 518, 547, 555, 526,\n",
       "       560, 519, 571, 564, 551, 563, 565, 528, 512, 578, 513, 544, 575,\n",
       "       587, 540, 504, 529, 549, 537, 557, 527, 574, 506, 545, 592, 534,\n",
       "       553, 583])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([509, 569, 581, 556, 533, 593, 584, 561, 546, 589, 585, 567, 597,\n",
       "       505, 570, 536, 598, 596, 514, 552])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b_test"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}