playground.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear Regression Playground\n",
    "Some experiments with the Linear Regression model to get a better intuition about it."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import datetime as dt\n",
    "import pathlib\n",
    "import pprint\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LinearRegression, Lasso, Ridge, ElasticNet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sklearn Linear model docs:\n",
    "\n",
    "https://scikit-learn.org/stable/modules/classes.html#module-sklearn.linear_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "My DS libraries course project notebook - helping with how to use certain methods:\n",
    "\n",
    "https://github.com/shatandv/GB_Python_DS_Libs_Contd_2020_09_17/blob/lesson4/course_project/AShataev_solution.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "CWD = pathlib.Path('.')\n",
    "DATA_DIR = CWD / 'data'\n",
    "US_DATA_PATH = DATA_DIR / 'USvideos.csv'\n",
    "US_CATS_PATH = DATA_DIR / 'US_category_id.json'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's read some data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read CSV file -> DF:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(US_DATA_PATH, delimiter=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 40949 entries, 0 to 40948\n",
      "Data columns (total 16 columns):\n",
      " #   Column                  Non-Null Count  Dtype \n",
      "---  ------                  --------------  ----- \n",
      " 0   video_id                40949 non-null  object\n",
      " 1   trending_date           40949 non-null  object\n",
      " 2   title                   40949 non-null  object\n",
      " 3   channel_title           40949 non-null  object\n",
      " 4   category_id             40949 non-null  int64 \n",
      " 5   publish_time            40949 non-null  object\n",
      " 6   tags                    40949 non-null  object\n",
      " 7   views                   40949 non-null  int64 \n",
      " 8   likes                   40949 non-null  int64 \n",
      " 9   dislikes                40949 non-null  int64 \n",
      " 10  comment_count           40949 non-null  int64 \n",
      " 11  thumbnail_link          40949 non-null  object\n",
      " 12  comments_disabled       40949 non-null  bool  \n",
      " 13  ratings_disabled        40949 non-null  bool  \n",
      " 14  video_error_or_removed  40949 non-null  bool  \n",
      " 15  description             40379 non-null  object\n",
      "dtypes: bool(3), int64(5), object(8)\n",
      "memory usage: 4.2+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category_id</th>\n",
       "      <th>views</th>\n",
       "      <th>likes</th>\n",
       "      <th>dislikes</th>\n",
       "      <th>comment_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>40949.000000</td>\n",
       "      <td>4.094900e+04</td>\n",
       "      <td>4.094900e+04</td>\n",
       "      <td>4.094900e+04</td>\n",
       "      <td>4.094900e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>19.972429</td>\n",
       "      <td>2.360785e+06</td>\n",
       "      <td>7.426670e+04</td>\n",
       "      <td>3.711401e+03</td>\n",
       "      <td>8.446804e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>7.568327</td>\n",
       "      <td>7.394114e+06</td>\n",
       "      <td>2.288853e+05</td>\n",
       "      <td>2.902971e+04</td>\n",
       "      <td>3.743049e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.490000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>17.000000</td>\n",
       "      <td>2.423290e+05</td>\n",
       "      <td>5.424000e+03</td>\n",
       "      <td>2.020000e+02</td>\n",
       "      <td>6.140000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>24.000000</td>\n",
       "      <td>6.818610e+05</td>\n",
       "      <td>1.809100e+04</td>\n",
       "      <td>6.310000e+02</td>\n",
       "      <td>1.856000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>25.000000</td>\n",
       "      <td>1.823157e+06</td>\n",
       "      <td>5.541700e+04</td>\n",
       "      <td>1.938000e+03</td>\n",
       "      <td>5.755000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>43.000000</td>\n",
       "      <td>2.252119e+08</td>\n",
       "      <td>5.613827e+06</td>\n",
       "      <td>1.674420e+06</td>\n",
       "      <td>1.361580e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        category_id         views         likes      dislikes  comment_count\n",
       "count  40949.000000  4.094900e+04  4.094900e+04  4.094900e+04   4.094900e+04\n",
       "mean      19.972429  2.360785e+06  7.426670e+04  3.711401e+03   8.446804e+03\n",
       "std        7.568327  7.394114e+06  2.288853e+05  2.902971e+04   3.743049e+04\n",
       "min        1.000000  5.490000e+02  0.000000e+00  0.000000e+00   0.000000e+00\n",
       "25%       17.000000  2.423290e+05  5.424000e+03  2.020000e+02   6.140000e+02\n",
       "50%       24.000000  6.818610e+05  1.809100e+04  6.310000e+02   1.856000e+03\n",
       "75%       25.000000  1.823157e+06  5.541700e+04  1.938000e+03   5.755000e+03\n",
       "max       43.000000  2.252119e+08  5.613827e+06  1.674420e+06   1.361580e+06"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video_id</th>\n",
       "      <th>trending_date</th>\n",
       "      <th>title</th>\n",
       "      <th>channel_title</th>\n",
       "      <th>category_id</th>\n",
       "      <th>publish_time</th>\n",
       "      <th>tags</th>\n",
       "      <th>views</th>\n",
       "      <th>likes</th>\n",
       "      <th>dislikes</th>\n",
       "      <th>comment_count</th>\n",
       "      <th>thumbnail_link</th>\n",
       "      <th>comments_disabled</th>\n",
       "      <th>ratings_disabled</th>\n",
       "      <th>video_error_or_removed</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2kyS6SvSYSE</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>WE WANT TO TALK ABOUT OUR MARRIAGE</td>\n",
       "      <td>CaseyNeistat</td>\n",
       "      <td>22</td>\n",
       "      <td>2017-11-13T17:13:01.000Z</td>\n",
       "      <td>SHANtell martin</td>\n",
       "      <td>748374</td>\n",
       "      <td>57527</td>\n",
       "      <td>2966</td>\n",
       "      <td>15954</td>\n",
       "      <td>https://i.ytimg.com/vi/2kyS6SvSYSE/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>SHANTELL'S CHANNEL - https://www.youtube.com/s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1ZAPwfrtAFY</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>The Trump Presidency: Last Week Tonight with J...</td>\n",
       "      <td>LastWeekTonight</td>\n",
       "      <td>24</td>\n",
       "      <td>2017-11-13T07:30:00.000Z</td>\n",
       "      <td>last week tonight trump presidency|\"last week ...</td>\n",
       "      <td>2418783</td>\n",
       "      <td>97185</td>\n",
       "      <td>6146</td>\n",
       "      <td>12703</td>\n",
       "      <td>https://i.ytimg.com/vi/1ZAPwfrtAFY/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>One year after the presidential election, John...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5qpjK5DgCt4</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>Racist Superman | Rudy Mancuso, King Bach &amp; Le...</td>\n",
       "      <td>Rudy Mancuso</td>\n",
       "      <td>23</td>\n",
       "      <td>2017-11-12T19:05:24.000Z</td>\n",
       "      <td>racist superman|\"rudy\"|\"mancuso\"|\"king\"|\"bach\"...</td>\n",
       "      <td>3191434</td>\n",
       "      <td>146033</td>\n",
       "      <td>5339</td>\n",
       "      <td>8181</td>\n",
       "      <td>https://i.ytimg.com/vi/5qpjK5DgCt4/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>WATCH MY PREVIOUS VIDEO ▶ \\n\\nSUBSCRIBE ► http...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>puqaWrEC7tY</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>Nickelback Lyrics: Real or Fake?</td>\n",
       "      <td>Good Mythical Morning</td>\n",
       "      <td>24</td>\n",
       "      <td>2017-11-13T11:00:04.000Z</td>\n",
       "      <td>rhett and link|\"gmm\"|\"good mythical morning\"|\"...</td>\n",
       "      <td>343168</td>\n",
       "      <td>10172</td>\n",
       "      <td>666</td>\n",
       "      <td>2146</td>\n",
       "      <td>https://i.ytimg.com/vi/puqaWrEC7tY/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Today we find out if Link is a Nickelback amat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d380meD0W0M</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>I Dare You: GOING BALD!?</td>\n",
       "      <td>nigahiga</td>\n",
       "      <td>24</td>\n",
       "      <td>2017-11-12T18:01:41.000Z</td>\n",
       "      <td>ryan|\"higa\"|\"higatv\"|\"nigahiga\"|\"i dare you\"|\"...</td>\n",
       "      <td>2095731</td>\n",
       "      <td>132235</td>\n",
       "      <td>1989</td>\n",
       "      <td>17518</td>\n",
       "      <td>https://i.ytimg.com/vi/d380meD0W0M/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>I know it's been a while since we did this sho...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      video_id trending_date  \\\n",
       "0  2kyS6SvSYSE      17.14.11   \n",
       "1  1ZAPwfrtAFY      17.14.11   \n",
       "2  5qpjK5DgCt4      17.14.11   \n",
       "3  puqaWrEC7tY      17.14.11   \n",
       "4  d380meD0W0M      17.14.11   \n",
       "\n",
       "                                               title          channel_title  \\\n",
       "0                 WE WANT TO TALK ABOUT OUR MARRIAGE           CaseyNeistat   \n",
       "1  The Trump Presidency: Last Week Tonight with J...        LastWeekTonight   \n",
       "2  Racist Superman | Rudy Mancuso, King Bach & Le...           Rudy Mancuso   \n",
       "3                   Nickelback Lyrics: Real or Fake?  Good Mythical Morning   \n",
       "4                           I Dare You: GOING BALD!?               nigahiga   \n",
       "\n",
       "   category_id              publish_time  \\\n",
       "0           22  2017-11-13T17:13:01.000Z   \n",
       "1           24  2017-11-13T07:30:00.000Z   \n",
       "2           23  2017-11-12T19:05:24.000Z   \n",
       "3           24  2017-11-13T11:00:04.000Z   \n",
       "4           24  2017-11-12T18:01:41.000Z   \n",
       "\n",
       "                                                tags    views   likes  \\\n",
       "0                                    SHANtell martin   748374   57527   \n",
       "1  last week tonight trump presidency|\"last week ...  2418783   97185   \n",
       "2  racist superman|\"rudy\"|\"mancuso\"|\"king\"|\"bach\"...  3191434  146033   \n",
       "3  rhett and link|\"gmm\"|\"good mythical morning\"|\"...   343168   10172   \n",
       "4  ryan|\"higa\"|\"higatv\"|\"nigahiga\"|\"i dare you\"|\"...  2095731  132235   \n",
       "\n",
       "   dislikes  comment_count                                  thumbnail_link  \\\n",
       "0      2966          15954  https://i.ytimg.com/vi/2kyS6SvSYSE/default.jpg   \n",
       "1      6146          12703  https://i.ytimg.com/vi/1ZAPwfrtAFY/default.jpg   \n",
       "2      5339           8181  https://i.ytimg.com/vi/5qpjK5DgCt4/default.jpg   \n",
       "3       666           2146  https://i.ytimg.com/vi/puqaWrEC7tY/default.jpg   \n",
       "4      1989          17518  https://i.ytimg.com/vi/d380meD0W0M/default.jpg   \n",
       "\n",
       "   comments_disabled  ratings_disabled  video_error_or_removed  \\\n",
       "0              False             False                   False   \n",
       "1              False             False                   False   \n",
       "2              False             False                   False   \n",
       "3              False             False                   False   \n",
       "4              False             False                   False   \n",
       "\n",
       "                                         description  \n",
       "0  SHANTELL'S CHANNEL - https://www.youtube.com/s...  \n",
       "1  One year after the presidential election, John...  \n",
       "2  WATCH MY PREVIOUS VIDEO ▶ \\n\\nSUBSCRIBE ► http...  \n",
       "3  Today we find out if Link is a Nickelback amat...  \n",
       "4  I know it's been a while since we did this sho...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read JSON:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(US_CATS_PATH, 'r', encoding='utf-8') as file:\n",
    "    categories = json.loads(file.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "pp = pprint.PrettyPrinter(indent=2)\n",
    "# pp.pprint(categories)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's add a 'category' column with the name of a category."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_id_name = {int(item['id']): item['snippet']['title'] for item in categories['items']}\n",
    "# pp.pprint(cat_id_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['category'] = df['category_id'].apply(lambda x: cat_id_name[x])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pick out features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's pick out only numerical, date and bool features:\n",
    "- trending_date,\n",
    "- publish_time,\n",
    "- category_id,\n",
    "- views,\n",
    "- likes,\n",
    "- dislikes,\n",
    "- comment_count,\n",
    "- comments_disabled,\n",
    "- ratings_disabled,\n",
    "- video_error_or_removed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_features = ['trending_date', 'publish_time', 'category_id', 'views', 'likes', 'dislikes', \n",
    "                'comment_count', 'comments_disabled', 'ratings_disabled', 'video_error_or_removed']\n",
    "df = df[num_features]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleaning data a bit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert category_id and boolean features to uint8:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[[\n",
    "    'category_id',\n",
    "    'comments_disabled', \n",
    "    'ratings_disabled', \n",
    "    'video_error_or_removed'\n",
    "]] = df[['category_id', 'comments_disabled', 'ratings_disabled', 'video_error_or_removed']].astype(np.uint8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert trending_date, publish_time to datetime64[ns]:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['trending_date'] = pd.to_datetime(df['trending_date'], format='%y.%d.%m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['publish_time'] = pd.to_datetime(df['publish_time'], format='%Y-%m-%dT%H:%M:%S.%fZ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>trending_date</th>\n",
       "      <th>publish_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-11-14</td>\n",
       "      <td>2017-11-13 17:13:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-11-14</td>\n",
       "      <td>2017-11-13 07:30:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-11-14</td>\n",
       "      <td>2017-11-12 19:05:24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-11-14</td>\n",
       "      <td>2017-11-13 11:00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2017-11-14</td>\n",
       "      <td>2017-11-12 18:01:41</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  trending_date        publish_time\n",
       "0    2017-11-14 2017-11-13 17:13:01\n",
       "1    2017-11-14 2017-11-13 07:30:00\n",
       "2    2017-11-14 2017-11-12 19:05:24\n",
       "3    2017-11-14 2017-11-13 11:00:04\n",
       "4    2017-11-14 2017-11-12 18:01:41"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['trending_date', 'publish_time']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 40949 entries, 0 to 40948\n",
      "Data columns (total 10 columns):\n",
      " #   Column                  Non-Null Count  Dtype         \n",
      "---  ------                  --------------  -----         \n",
      " 0   trending_date           40949 non-null  datetime64[ns]\n",
      " 1   publish_time            40949 non-null  datetime64[ns]\n",
      " 2   category_id             40949 non-null  uint8         \n",
      " 3   views                   40949 non-null  int64         \n",
      " 4   likes                   40949 non-null  int64         \n",
      " 5   dislikes                40949 non-null  int64         \n",
      " 6   comment_count           40949 non-null  int64         \n",
      " 7   comments_disabled       40949 non-null  uint8         \n",
      " 8   ratings_disabled        40949 non-null  uint8         \n",
      " 9   video_error_or_removed  40949 non-null  uint8         \n",
      "dtypes: datetime64[ns](2), int64(4), uint8(4)\n",
      "memory usage: 2.0 MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add some new features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hour number of publish_time:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['publish_hour'] = df['publish_time'].dt.hour"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Age of the video in days:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['video_age_days'] = (dt.datetime.now() - df['publish_time']).dt.days.astype(np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAPnklEQVR4nO3dcaid913H8ffHZHZFLWvtbQhJNRGiLC1sM5cYGYpaWTM3l4oUMtQGLARKhQmCpu4P8Y9ApyBStB1BR1OcKwGdDRt1C9E6hLjsduvWpl3odZ3tJaG56xA7lGi6r3/c3/QhObn33PTm3Nz+3i94eJ7zfX6/c57f6cOnz/2d55ykqpAk9eH7VvsAJEmTY+hLUkcMfUnqiKEvSR0x9CWpI+tX+wCWcvPNN9eWLVtW+zAkaU15+umnv1VVUxfXr/nQ37JlCzMzM6t9GJK0piT5t1F1p3ckqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakj1/w3cteiLQc+u2qv/c0HP7Bqry3p2ueVviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcRv5EpXyG9eay3ySl+SOjJW6Cf5ZpJnkzyTZKbVbkpyLMmLbX3joP0DSWaTnE5y56C+oz3PbJKHkmTlhyRJupzlXOn/fFW9u6qm2+MDwPGq2gYcb49Jsh3YC9wG7AYeTrKu9XkE2A9sa8vuNz8ESdK43sz0zh7gcNs+DNw1qD9eVeer6iVgFtiZZCNwQ1WdqKoCHhv0kSRNwLihX8DnkzydZH+rbaiqswBtfUurbwJeGfSda7VNbfvi+iWS7E8yk2Rmfn5+zEOUJC1l3Lt33ltVZ5LcAhxL8vVF2o6ap69F6pcWqw4BhwCmp6dHtpEkLd9YoV9VZ9r6XJJPAzuBV5NsrKqzbermXGs+B9w66L4ZONPqm0fUpTdlNW+dlNaaJad3kvxAkh/63jbwPuA54CiwrzXbBzzRto8Ce5Ncl2QrCx/YnmxTQK8n2dXu2rln0EeSNAHjXOlvAD7d7q5cD/x1Vf19ki8BR5LcC7wM3A1QVaeSHAGeBy4A91fVG+257gMeBa4HnmyLJGlClgz9qvoG8K4R9deAOy7T5yBwcER9Brh9+YcpSVoJ/gyDpGueP3mxcvwZBknqiFf6ksbmnVJrn1f6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I64j+XqBXhP6MnrQ1e6UtSRwx9SeqIoS9JHTH0Jakjhr4kdcS7d95ivItG0mK80pekjowd+knWJflKks+0xzclOZbkxba+cdD2gSSzSU4nuXNQ35Hk2bbvoSRZ2eFIkhaznCv9jwAvDB4fAI5X1TbgeHtMku3AXuA2YDfwcJJ1rc8jwH5gW1t2v6mjlyQty1ihn2Qz8AHgLwblPcDhtn0YuGtQf7yqzlfVS8AssDPJRuCGqjpRVQU8NugjSZqAca/0/xT4XeC7g9qGqjoL0Na3tPom4JVBu7lW29S2L65fIsn+JDNJZubn58c8REnSUpYM/SQfBM5V1dNjPueoefpapH5psepQVU1X1fTU1NSYLytJWso4t2y+F/hQkl8C3g7ckOSvgFeTbKyqs23q5lxrPwfcOui/GTjT6ptH1CUtk7fm6koteaVfVQ9U1eaq2sLCB7T/UFW/DhwF9rVm+4An2vZRYG+S65JsZeED25NtCuj1JLvaXTv3DPpIkibgzXw560HgSJJ7gZeBuwGq6lSSI8DzwAXg/qp6o/W5D3gUuB54si2SpAlZVuhX1VPAU237NeCOy7Q7CBwcUZ8Bbl/uQUqSVobfyJWkjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6siSoZ/k7UlOJvlqklNJ/rDVb0pyLMmLbX3joM8DSWaTnE5y56C+I8mzbd9DSXJ1hiVJGmWcK/3zwC9U1buAdwO7k+wCDgDHq2obcLw9Jsl2YC9wG7AbeDjJuvZcjwD7gW1t2b1yQ5EkLWXJ0K8F32kP39aWAvYAh1v9MHBX294DPF5V56vqJWAW2JlkI3BDVZ2oqgIeG/SRJE3AWHP6SdYleQY4Bxyrqi8CG6rqLEBb39KabwJeGXSfa7VNbfviuiRpQsYK/ap6o6reDWxm4ar99kWaj5qnr0Xqlz5Bsj/JTJKZ+fn5cQ5RkjSGZd29U1X/DjzFwlz8q23KhrY+15rNAbcOum0GzrT65hH1Ua9zqKqmq2p6ampqOYcoSVrEOHfvTCV5R9u+HvhF4OvAUWBfa7YPeKJtHwX2JrkuyVYWPrA92aaAXk+yq921c8+gjyRpAtaP0WYjcLjdgfN9wJGq+kySE8CRJPcCLwN3A1TVqSRHgOeBC8D9VfVGe677gEeB64En2yJJmpAlQ7+qvga8Z0T9NeCOy/Q5CBwcUZ8BFvs8QJJ0FfmNXEnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUkXF+T3/N2nLgs6t9CJJ0TfFKX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4sGfpJbk3yj0leSHIqyUda/aYkx5K82NY3Dvo8kGQ2yekkdw7qO5I82/Y9lCRXZ1iSpFHGudK/APxOVb0T2AXcn2Q7cAA4XlXbgOPtMW3fXuA2YDfwcJJ17bkeAfYD29qyewXHIklawpKhX1Vnq+rLbft14AVgE7AHONyaHQbuatt7gMer6nxVvQTMAjuTbARuqKoTVVXAY4M+kqQJWNacfpItwHuALwIbquosLPyPAbilNdsEvDLoNtdqm9r2xfVRr7M/yUySmfn5+eUcoiRpEWOHfpIfBP4G+O2q+o/Fmo6o1SL1S4tVh6pquqqmp6amxj1ESdISxgr9JG9jIfA/WVV/28qvtikb2vpcq88Btw66bwbOtPrmEXVJ0oSMc/dOgL8EXqiqPxnsOgrsa9v7gCcG9b1JrkuylYUPbE+2KaDXk+xqz3nPoI8kaQLWj9HmvcBvAM8meabVfh94EDiS5F7gZeBugKo6leQI8DwLd/7cX1VvtH73AY8C1wNPtkWSNCFLhn5V/TOj5+MB7rhMn4PAwRH1GeD25RygJGnl+I1cSeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOLBn6ST6R5FyS5wa1m5IcS/JiW9842PdAktkkp5PcOajvSPJs2/dQkqz8cCRJixnnSv9RYPdFtQPA8araBhxvj0myHdgL3Nb6PJxkXevzCLAf2NaWi59TknSVLRn6VfUF4NsXlfcAh9v2YeCuQf3xqjpfVS8Bs8DOJBuBG6rqRFUV8NigjyRpQq50Tn9DVZ0FaOtbWn0T8Mqg3VyrbWrbF9dHSrI/yUySmfn5+Ss8REnSxVb6g9xR8/S1SH2kqjpUVdNVNT01NbViBydJvbvS0H+1TdnQ1udafQ64ddBuM3Cm1TePqEuSJuhKQ/8osK9t7wOeGNT3JrkuyVYWPrA92aaAXk+yq921c8+gjyRpQtYv1SDJp4CfA25OMgf8AfAgcCTJvcDLwN0AVXUqyRHgeeACcH9VvdGe6j4W7gS6HniyLZKkCVoy9Kvqw5fZdcdl2h8EDo6ozwC3L+voJEkrym/kSlJHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4secumJPVsy4HPrsrrfvPBD1yV5/VKX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOTDz0k+xOcjrJbJIDk359SerZREM/yTrgz4H3A9uBDyfZPsljkKSeTfpKfycwW1XfqKr/Bh4H9kz4GCSpW+sn/HqbgFcGj+eAn7q4UZL9wP728DtJTk/g2K5FNwPfWu2DWEW9jx98D7odfz72f5tX+h786KjipEM/I2p1SaHqEHDo6h/OtS3JTFVNr/ZxrJbexw++B72PH1b+PZj09M4ccOvg8WbgzISPQZK6NenQ/xKwLcnWJN8P7AWOTvgYJKlbE53eqaoLSX4L+BywDvhEVZ2a5DGsMb1PcfU+fvA96H38sMLvQaoumVKXJL1F+Y1cSeqIoS9JHTH0JyjJJ5KcS/LcoHZTkmNJXmzrGwf7Hmg/V3E6yZ2D+o4kz7Z9DyUZdSvsNWc540+yJcl/JXmmLR8f9FmT44fLvgd3JzmV5LtJpi9q38M5MHL8nZ0Df5zk60m+luTTSd4x2Ley50BVuUxoAX4W+EnguUHtj4ADbfsA8LG2vR34KnAdsBX4V2Bd23cS+GkWvvfwJPD+1R7bVRj/lmG7i55nTY5/kffgncBPAE8B04N6L+fA5cbf0znwPmB92/7Y1cwBr/QnqKq+AHz7ovIe4HDbPgzcNag/XlXnq+olYBbYmWQjcENVnaiF//KPDfpc05Y5/pHW8vhh9HtQVS9U1ahvnXdxDiwy/pHW8vjhsu/B56vqQnv4Lyx8hwmuwjlg6K++DVV1FqCtb2n1UT9ZsaktcyPqa9Xlxg+wNclXkvxTkp9ptbfa+BfTyzmwmB7Pgd9k4codrsI5MOmfYdD4LveTFWP9lMVbwFngR6rqtSQ7gL9Lchv9jB88B7o7B5J8FLgAfPJ7pRHN3tQ54JX+6nu1/an2vT9bz7X65X6yYo7//9NvWF+rRo6//Tn7Wtt+moW5zB/nrTf+xfRyDozU2zmQZB/wQeDX2pQNXIVzwNBffUeBfW17H/DEoL43yXVJtgLbgJNtCuT1JLvap/X3DPqsRSPHn2QqC//+Akl+jIXxf+MtOP7F9HIOjNTTOZBkN/B7wIeq6j8Hu1b+HFjtT7J7WoBPsfAn6/+w8H/qe4EfBo4DL7b1TYP2H2Xh6uY0g0/mgWngubbvz2jfrL7Wl+WMH/hV4BQLdy58GfjltT7+Rd6DX2nb54FXgc91dg6MHH9n58AsC3P3z7Tl41frHPBnGCSpI07vSFJHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUkf8FtKy5ccL1+yoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(df.loc[df['video_age_days'] < 1200, 'video_age_days'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Like/Dislike Ratio = likes / dislikes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['like_dislike_ratio'] = df['likes'] / df['dislikes']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Replace the like_dislike_ratio 'inf' values with 10000 - a big value:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "inf            213\n",
       "1438.750000      1\n",
       "553.000000       1\n",
       "510.000000       1\n",
       "533.921053       1\n",
       "552.000000       1\n",
       "667.285714       1\n",
       "587.000000       1\n",
       "1493.750000      1\n",
       "532.000000       1\n",
       "533.642857       1\n",
       "615.354167       1\n",
       "448.166667       1\n",
       "1404.500000      1\n",
       "440.412371       1\n",
       "713.555556       1\n",
       "453.750000       1\n",
       "657.333333       1\n",
       "508.750000       1\n",
       "1468.750000      1\n",
       "1737.333333      1\n",
       "843.200000       1\n",
       "454.038514       1\n",
       "495.333333       1\n",
       "732.947368       1\n",
       "436.031102       1\n",
       "525.500000       1\n",
       "506.257299       1\n",
       "437.000000       1\n",
       "725.175000       1\n",
       "538.000000       1\n",
       "1363.750000      1\n",
       "489.750000       1\n",
       "Name: like_dislike_ratio, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df['like_dislike_ratio'] >= df['like_dislike_ratio'].quantile(0.994), 'like_dislike_ratio'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['like_dislike_ratio'] = df['like_dislike_ratio'].replace(np.inf, 2000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Look at data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category_id</th>\n",
       "      <th>views</th>\n",
       "      <th>likes</th>\n",
       "      <th>dislikes</th>\n",
       "      <th>comment_count</th>\n",
       "      <th>comments_disabled</th>\n",
       "      <th>ratings_disabled</th>\n",
       "      <th>video_error_or_removed</th>\n",
       "      <th>publish_hour</th>\n",
       "      <th>video_age_days</th>\n",
       "      <th>like_dislike_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>40949.000000</td>\n",
       "      <td>4.094900e+04</td>\n",
       "      <td>4.094900e+04</td>\n",
       "      <td>4.094900e+04</td>\n",
       "      <td>4.094900e+04</td>\n",
       "      <td>40949.000000</td>\n",
       "      <td>40949.000000</td>\n",
       "      <td>40949.000000</td>\n",
       "      <td>40949.000000</td>\n",
       "      <td>40949.000000</td>\n",
       "      <td>40779.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>19.972429</td>\n",
       "      <td>2.360785e+06</td>\n",
       "      <td>7.426670e+04</td>\n",
       "      <td>3.711401e+03</td>\n",
       "      <td>8.446804e+03</td>\n",
       "      <td>0.015458</td>\n",
       "      <td>0.004127</td>\n",
       "      <td>0.000562</td>\n",
       "      <td>13.766685</td>\n",
       "      <td>1086.600869</td>\n",
       "      <td>54.185345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>7.568327</td>\n",
       "      <td>7.394114e+06</td>\n",
       "      <td>2.288853e+05</td>\n",
       "      <td>2.902971e+04</td>\n",
       "      <td>3.743049e+04</td>\n",
       "      <td>0.123368</td>\n",
       "      <td>0.064110</td>\n",
       "      <td>0.023693</td>\n",
       "      <td>6.424996</td>\n",
       "      <td>161.677963</td>\n",
       "      <td>150.026073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.490000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>963.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>17.000000</td>\n",
       "      <td>2.423290e+05</td>\n",
       "      <td>5.424000e+03</td>\n",
       "      <td>2.020000e+02</td>\n",
       "      <td>6.140000e+02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>1022.000000</td>\n",
       "      <td>13.353674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>24.000000</td>\n",
       "      <td>6.818610e+05</td>\n",
       "      <td>1.809100e+04</td>\n",
       "      <td>6.310000e+02</td>\n",
       "      <td>1.856000e+03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>1076.000000</td>\n",
       "      <td>29.291667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>25.000000</td>\n",
       "      <td>1.823157e+06</td>\n",
       "      <td>5.541700e+04</td>\n",
       "      <td>1.938000e+03</td>\n",
       "      <td>5.755000e+03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>1132.000000</td>\n",
       "      <td>57.105736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>43.000000</td>\n",
       "      <td>2.252119e+08</td>\n",
       "      <td>5.613827e+06</td>\n",
       "      <td>1.674420e+06</td>\n",
       "      <td>1.361580e+06</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>5307.000000</td>\n",
       "      <td>2000.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        category_id         views         likes      dislikes  comment_count  \\\n",
       "count  40949.000000  4.094900e+04  4.094900e+04  4.094900e+04   4.094900e+04   \n",
       "mean      19.972429  2.360785e+06  7.426670e+04  3.711401e+03   8.446804e+03   \n",
       "std        7.568327  7.394114e+06  2.288853e+05  2.902971e+04   3.743049e+04   \n",
       "min        1.000000  5.490000e+02  0.000000e+00  0.000000e+00   0.000000e+00   \n",
       "25%       17.000000  2.423290e+05  5.424000e+03  2.020000e+02   6.140000e+02   \n",
       "50%       24.000000  6.818610e+05  1.809100e+04  6.310000e+02   1.856000e+03   \n",
       "75%       25.000000  1.823157e+06  5.541700e+04  1.938000e+03   5.755000e+03   \n",
       "max       43.000000  2.252119e+08  5.613827e+06  1.674420e+06   1.361580e+06   \n",
       "\n",
       "       comments_disabled  ratings_disabled  video_error_or_removed  \\\n",
       "count       40949.000000      40949.000000            40949.000000   \n",
       "mean            0.015458          0.004127                0.000562   \n",
       "std             0.123368          0.064110                0.023693   \n",
       "min             0.000000          0.000000                0.000000   \n",
       "25%             0.000000          0.000000                0.000000   \n",
       "50%             0.000000          0.000000                0.000000   \n",
       "75%             0.000000          0.000000                0.000000   \n",
       "max             1.000000          1.000000                1.000000   \n",
       "\n",
       "       publish_hour  video_age_days  like_dislike_ratio  \n",
       "count  40949.000000    40949.000000        40779.000000  \n",
       "mean      13.766685     1086.600869           54.185345  \n",
       "std        6.424996      161.677963          150.026073  \n",
       "min        0.000000      963.000000            0.000000  \n",
       "25%       10.000000     1022.000000           13.353674  \n",
       "50%       15.000000     1076.000000           29.291667  \n",
       "75%       18.000000     1132.000000           57.105736  \n",
       "max       23.000000     5307.000000         2000.000000  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's predict comment_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comment count is a good predictor of engagement."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['trending_date', 'publish_time', 'category_id', 'views', 'likes',\n",
       "       'dislikes', 'comment_count', 'comments_disabled', 'ratings_disabled',\n",
       "       'video_error_or_removed', 'publish_hour', 'video_age_days',\n",
       "       'like_dislike_ratio'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.sort_values(['trending_date', 'publish_time'], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>trending_date</th>\n",
       "      <th>publish_time</th>\n",
       "      <th>category_id</th>\n",
       "      <th>views</th>\n",
       "      <th>likes</th>\n",
       "      <th>dislikes</th>\n",
       "      <th>comment_count</th>\n",
       "      <th>comments_disabled</th>\n",
       "      <th>ratings_disabled</th>\n",
       "      <th>video_error_or_removed</th>\n",
       "      <th>publish_hour</th>\n",
       "      <th>video_age_days</th>\n",
       "      <th>like_dislike_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>2017-11-14</td>\n",
       "      <td>2011-03-27 04:31:25</td>\n",
       "      <td>24</td>\n",
       "      <td>95085</td>\n",
       "      <td>909</td>\n",
       "      <td>52</td>\n",
       "      <td>193</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>3599</td>\n",
       "      <td>17.480769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2017-11-14</td>\n",
       "      <td>2011-05-29 17:03:12</td>\n",
       "      <td>1</td>\n",
       "      <td>50867</td>\n",
       "      <td>715</td>\n",
       "      <td>238</td>\n",
       "      <td>246</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>3536</td>\n",
       "      <td>3.004202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>2017-11-14</td>\n",
       "      <td>2012-06-10 19:24:38</td>\n",
       "      <td>27</td>\n",
       "      <td>6412</td>\n",
       "      <td>49</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>3158</td>\n",
       "      <td>2000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>2017-11-14</td>\n",
       "      <td>2015-05-21 07:55:00</td>\n",
       "      <td>24</td>\n",
       "      <td>192609</td>\n",
       "      <td>1345</td>\n",
       "      <td>24</td>\n",
       "      <td>126</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2083</td>\n",
       "      <td>56.041667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>2017-11-14</td>\n",
       "      <td>2017-03-31 21:46:53</td>\n",
       "      <td>27</td>\n",
       "      <td>773</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>1403</td>\n",
       "      <td>2000.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    trending_date        publish_time  category_id   views  likes  dislikes  \\\n",
       "139    2017-11-14 2011-03-27 04:31:25           24   95085    909        52   \n",
       "29     2017-11-14 2011-05-29 17:03:12            1   50867    715       238   \n",
       "155    2017-11-14 2012-06-10 19:24:38           27    6412     49         0   \n",
       "135    2017-11-14 2015-05-21 07:55:00           24  192609   1345        24   \n",
       "160    2017-11-14 2017-03-31 21:46:53           27     773      2         0   \n",
       "\n",
       "     comment_count  comments_disabled  ratings_disabled  \\\n",
       "139            193                  0                 0   \n",
       "29             246                  0                 0   \n",
       "155              2                  0                 0   \n",
       "135            126                  0                 0   \n",
       "160              0                  0                 0   \n",
       "\n",
       "     video_error_or_removed  publish_hour  video_age_days  like_dislike_ratio  \n",
       "139                       0             4            3599           17.480769  \n",
       "29                        0            17            3536            3.004202  \n",
       "155                       0            19            3158         2000.000000  \n",
       "135                       0             7            2083           56.041667  \n",
       "160                       0            21            1403         2000.000000  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}