examples/03_features_preprocessing_and_lightFM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-10T16:01:45.639135Z",
     "start_time": "2020-02-10T16:01:45.612577Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%config Completer.use_jedi = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:18:00.221718Z",
     "iopub.status.busy": "2024-02-29T10:18:00.221299Z",
     "iopub.status.idle": "2024-02-29T10:18:11.844001Z",
     "shell.execute_reply": "2024-02-29T10:18:11.842302Z",
     "shell.execute_reply.started": "2024-02-29T10:18:00.221684Z"
    }
   },
   "outputs": [],
   "source": [
    "! pip install rs-datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The notebook contains an example of features preprocessing with PySpark for RePlay LightFM model wrapper and includes:\n",
    "1. Data loading and reindexing\n",
    "2. Features preprocessing with pyspark\n",
    "3. Building LightFM model based on interaction matrix and features\n",
    "4. Model evaluation\n",
    "\n",
    "*Note: to run this notebook, you will need an __experimental__ version of the RePlay*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:18:37.992732Z",
     "iopub.status.busy": "2024-02-29T10:18:37.991564Z",
     "iopub.status.idle": "2024-02-29T10:18:37.998663Z",
     "shell.execute_reply": "2024-02-29T10:18:37.997377Z",
     "shell.execute_reply.started": "2024-02-29T10:18:37.992694Z"
    }
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "from optuna.exceptions import ExperimentalWarning\n",
    "warnings.filterwarnings(\"ignore\", category=UserWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=ExperimentalWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:18:39.010506Z",
     "iopub.status.busy": "2024-02-29T10:18:39.009029Z",
     "iopub.status.idle": "2024-02-29T10:19:07.073871Z",
     "shell.execute_reply": "2024-02-29T10:19:07.072177Z",
     "shell.execute_reply.started": "2024-02-29T10:18:39.010467Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type='text/css'>\n",
       ".datatable table.frame { margin-bottom: 0; }\n",
       ".datatable table.frame thead { border-bottom: none; }\n",
       ".datatable table.frame tr.coltypes td {  color: #FFFFFF;  line-height: 6px;  padding: 0 0.5em;}\n",
       ".datatable .bool    { background: #DDDD99; }\n",
       ".datatable .object  { background: #565656; }\n",
       ".datatable .int     { background: #5D9E5D; }\n",
       ".datatable .float   { background: #4040CC; }\n",
       ".datatable .str     { background: #CC4040; }\n",
       ".datatable .time    { background: #40CC40; }\n",
       ".datatable .row_index {  background: var(--jp-border-color3);  border-right: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  font-size: 9px;}\n",
       ".datatable .frame tbody td { text-align: left; }\n",
       ".datatable .frame tr.coltypes .row_index {  background: var(--jp-border-color0);}\n",
       ".datatable th:nth-child(2) { padding-left: 12px; }\n",
       ".datatable .hellipsis {  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .vellipsis {  background: var(--jp-layout-color0);  color: var(--jp-cell-editor-border-color);}\n",
       ".datatable .na {  color: var(--jp-cell-editor-border-color);  font-size: 80%;}\n",
       ".datatable .sp {  opacity: 0.25;}\n",
       ".datatable .footer { font-size: 9px; }\n",
       ".datatable .frame_dimensions {  background: var(--jp-border-color3);  border-top: 1px solid var(--jp-border-color0);  color: var(--jp-ui-font-color3);  display: inline-block;  opacity: 0.6;  padding: 1px 10px 1px 5px;}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.sql.types import IntegerType\n",
    "from pyspark.sql.functions import array_contains, col, explode, split, substring\n",
    "\n",
    "from replay.experimental.preprocessing.data_preparator import Indexer, DataPreparator\n",
    "from replay.metrics import HitRate, NDCG, MAP, Coverage, Experiment\n",
    "from replay.experimental.models import LightFMWrap\n",
    "from replay.utils.session_handler import State\n",
    "from replay.splitters import TimeSplitter\n",
    "from replay.utils.spark_utils import get_log_info\n",
    "from rs_datasets import MovieLens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-10T15:59:09.227179Z",
     "start_time": "2020-02-10T15:59:06.427348Z"
    },
    "execution": {
     "iopub.execute_input": "2024-02-29T10:19:15.882063Z",
     "iopub.status.busy": "2024-02-29T10:19:15.881255Z",
     "iopub.status.idle": "2024-02-29T10:19:23.007381Z",
     "shell.execute_reply": "2024-02-29T10:19:23.006337Z",
     "shell.execute_reply.started": "2024-02-29T10:19:15.882024Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "spark = State().session\n",
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:19:23.010458Z",
     "iopub.status.busy": "2024-02-29T10:19:23.009763Z",
     "iopub.status.idle": "2024-02-29T10:19:23.017916Z",
     "shell.execute_reply": "2024-02-29T10:19:23.016824Z",
     "shell.execute_reply.started": "2024-02-29T10:19:23.010422Z"
    }
   },
   "outputs": [],
   "source": [
    "spark.sparkContext.setLogLevel('ERROR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:19:23.020408Z",
     "iopub.status.busy": "2024-02-29T10:19:23.019790Z",
     "iopub.status.idle": "2024-02-29T10:19:23.028389Z",
     "shell.execute_reply": "2024-02-29T10:19:23.027463Z",
     "shell.execute_reply.started": "2024-02-29T10:19:23.020380Z"
    }
   },
   "outputs": [],
   "source": [
    "K=10\n",
    "SEED=1234"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Data loading"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use MovieLens 10m dataset from rs_datasets package, which contains a list of recommendations datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:19:37.245858Z",
     "iopub.status.busy": "2024-02-29T10:19:37.245362Z",
     "iopub.status.idle": "2024-02-29T10:19:47.344223Z",
     "shell.execute_reply": "2024-02-29T10:19:47.342882Z",
     "shell.execute_reply.started": "2024-02-29T10:19:37.245815Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65.6MB [00:01, 56.5MB/s]                            \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ratings\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>122</td>\n",
       "      <td>5.0</td>\n",
       "      <td>838985046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>185</td>\n",
       "      <td>5.0</td>\n",
       "      <td>838983525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>231</td>\n",
       "      <td>5.0</td>\n",
       "      <td>838983392</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  rating  timestamp\n",
       "0        1      122     5.0  838985046\n",
       "1        1      185     5.0  838983525\n",
       "2        1      231     5.0  838983392"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "items\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   item_id                    title  \\\n",
       "0        1         Toy Story (1995)   \n",
       "1        2           Jumanji (1995)   \n",
       "2        3  Grumpier Old Men (1995)   \n",
       "\n",
       "                                        genres  \n",
       "0  Adventure|Animation|Children|Comedy|Fantasy  \n",
       "1                   Adventure|Children|Fantasy  \n",
       "2                               Comedy|Romance  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tags\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>tag</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15</td>\n",
       "      <td>4973</td>\n",
       "      <td>excellent!</td>\n",
       "      <td>1215184630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20</td>\n",
       "      <td>1747</td>\n",
       "      <td>politics</td>\n",
       "      <td>1188263867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20</td>\n",
       "      <td>1747</td>\n",
       "      <td>satire</td>\n",
       "      <td>1188263867</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id         tag   timestamp\n",
       "0       15     4973  excellent!  1215184630\n",
       "1       20     1747    politics  1188263867\n",
       "2       20     1747      satire  1188263867"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data = MovieLens(\"10m\")\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 Convert interaction log to RePlay format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:19:50.968549Z",
     "iopub.status.busy": "2024-02-29T10:19:50.966934Z",
     "iopub.status.idle": "2024-02-29T10:19:50.974383Z",
     "shell.execute_reply": "2024-02-29T10:19:50.973386Z",
     "shell.execute_reply.started": "2024-02-29T10:19:50.968478Z"
    }
   },
   "outputs": [],
   "source": [
    "preparator = DataPreparator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:21:42.887962Z",
     "iopub.status.busy": "2024-02-29T10:21:42.887511Z",
     "iopub.status.idle": "2024-02-29T10:21:55.113954Z",
     "shell.execute_reply": "2024-02-29T10:21:55.112828Z",
     "shell.execute_reply.started": "2024-02-29T10:21:42.887929Z"
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "log = preparator.transform(columns_mapping={'user_id': 'user_id',\n",
    "                                      'item_id': 'item_id',\n",
    "                                      'relevance': 'rating',\n",
    "                                      'timestamp': 'timestamp'\n",
    "                                     }, \n",
    "                           data=data.ratings)\n",
    "item_features = preparator.transform(columns_mapping={'item_id': 'item_id'}, \n",
    "                           data=data.items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:21:55.115819Z",
     "iopub.status.busy": "2024-02-29T10:21:55.115488Z",
     "iopub.status.idle": "2024-02-29T10:21:55.478494Z",
     "shell.execute_reply": "2024-02-29T10:21:55.477502Z",
     "shell.execute_reply.started": "2024-02-29T10:21:55.115791Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------+---------+-------------------+\n",
      "|user_id|item_id|relevance|          timestamp|\n",
      "+-------+-------+---------+-------------------+\n",
      "|      1|    122|      5.0|1996-08-02 11:24:06|\n",
      "|      1|    185|      5.0|1996-08-02 10:58:45|\n",
      "+-------+-------+---------+-------------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "log.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:21:57.602937Z",
     "iopub.status.busy": "2024-02-29T10:21:57.602497Z",
     "iopub.status.idle": "2024-02-29T10:21:57.731162Z",
     "shell.execute_reply": "2024-02-29T10:21:57.730179Z",
     "shell.execute_reply.started": "2024-02-29T10:21:57.602907Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+----------------+--------------------+\n",
      "|item_id|           title|              genres|\n",
      "+-------+----------------+--------------------+\n",
      "|      1|Toy Story (1995)|Adventure|Animati...|\n",
      "|      2|  Jumanji (1995)|Adventure|Childre...|\n",
      "+-------+----------------+--------------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "item_features.show(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='indexing'></a>\n",
    "### 1.2. Indexing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert given users' and items' identifiers (\\_id) to integers starting at zero without gaps (\\_idx) with Indexer class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:22:00.375499Z",
     "iopub.status.busy": "2024-02-29T10:22:00.375093Z",
     "iopub.status.idle": "2024-02-29T10:22:00.381378Z",
     "shell.execute_reply": "2024-02-29T10:22:00.380047Z",
     "shell.execute_reply.started": "2024-02-29T10:22:00.375469Z"
    }
   },
   "outputs": [],
   "source": [
    "indexer = Indexer(user_col='user_id', item_col='item_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:22:01.958496Z",
     "iopub.status.busy": "2024-02-29T10:22:01.958084Z",
     "iopub.status.idle": "2024-02-29T10:22:09.573415Z",
     "shell.execute_reply": "2024-02-29T10:22:09.572502Z",
     "shell.execute_reply.started": "2024-02-29T10:22:01.958464Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 258 ms, sys: 18.9 ms, total: 277 ms\n",
      "Wall time: 7.61 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "indexer.fit(users=log.select('user_id'),\n",
    "           items=log.select('item_id').union(item_features.select('item_id')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:23:59.493504Z",
     "iopub.status.busy": "2024-02-29T10:23:59.493062Z",
     "iopub.status.idle": "2024-02-29T10:24:03.481468Z",
     "shell.execute_reply": "2024-02-29T10:24:03.479977Z",
     "shell.execute_reply.started": "2024-02-29T10:23:59.493471Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------+---------+-------------------+\n",
      "|user_idx|item_idx|relevance|          timestamp|\n",
      "+--------+--------+---------+-------------------+\n",
      "|   65232|    1057|      5.0|1996-08-02 11:24:06|\n",
      "|   65232|      76|      5.0|1996-08-02 10:58:45|\n",
      "+--------+--------+---------+-------------------+\n",
      "only showing top 2 rows\n",
      "\n",
      "CPU times: user 283 ms, sys: 21.4 ms, total: 304 ms\n",
      "Wall time: 3.98 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "log_replay = indexer.transform(df=log)\n",
    "log_replay.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:24:03.483485Z",
     "iopub.status.busy": "2024-02-29T10:24:03.483110Z",
     "iopub.status.idle": "2024-02-29T10:24:03.960886Z",
     "shell.execute_reply": "2024-02-29T10:24:03.959852Z",
     "shell.execute_reply.started": "2024-02-29T10:24:03.483458Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+----------------+--------------------+\n",
      "|item_idx|           title|              genres|\n",
      "+--------+----------------+--------------------+\n",
      "|      11|Toy Story (1995)|Adventure|Animati...|\n",
      "|     117|  Jumanji (1995)|Adventure|Childre...|\n",
      "+--------+----------------+--------------------+\n",
      "only showing top 2 rows\n",
      "\n",
      "CPU times: user 51.9 ms, sys: 4.19 ms, total: 56.1 ms\n",
      "Wall time: 472 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "item_features_replay = indexer.transform(df=item_features)\n",
    "item_features_replay.show(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3. Data split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-10T15:59:50.986401Z",
     "start_time": "2020-02-10T15:59:42.042998Z"
    },
    "execution": {
     "iopub.execute_input": "2024-02-29T10:26:25.978742Z",
     "iopub.status.busy": "2024-02-29T10:26:25.978372Z",
     "iopub.status.idle": "2024-02-29T10:27:28.248498Z",
     "shell.execute_reply": "2024-02-29T10:27:28.246535Z",
     "shell.execute_reply.started": "2024-02-29T10:26:25.978719Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train info:\n",
      " total lines: 8000043, total users: 59522, total items: 8989\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test info:\n",
      " total lines: 249418, total users: 3196, total items: 8180\n"
     ]
    }
   ],
   "source": [
    "# train/test split \n",
    "train_spl = TimeSplitter(\n",
    "    time_threshold=0.2,\n",
    "    drop_cold_items=True,\n",
    "    drop_cold_users=True,\n",
    "    item_column = 'item_idx',\n",
    "    query_column = 'user_idx'\n",
    ")\n",
    "\n",
    "\n",
    "train, test = train_spl.split(log_replay)\n",
    "print('train info:\\n', get_log_info(train, user_col='user_idx', item_col='item_idx'))\n",
    "print('test info:\\n', get_log_info(test, user_col='user_idx', item_col='item_idx'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:27:28.252703Z",
     "iopub.status.busy": "2024-02-29T10:27:28.250452Z",
     "iopub.status.idle": "2024-02-29T10:27:28.263130Z",
     "shell.execute_reply": "2024-02-29T10:27:28.261848Z",
     "shell.execute_reply.started": "2024-02-29T10:27:28.252669Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.is_cached"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Features preprocessing with pyspark"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:28:15.324571Z",
     "iopub.status.busy": "2024-02-29T10:28:15.324135Z",
     "iopub.status.idle": "2024-02-29T10:28:15.491658Z",
     "shell.execute_reply": "2024-02-29T10:28:15.490422Z",
     "shell.execute_reply.started": "2024-02-29T10:28:15.324535Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+----+\n",
      "|item_idx|year|\n",
      "+--------+----+\n",
      "|      11|1995|\n",
      "|     117|1995|\n",
      "+--------+----+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "year = item_features_replay.withColumn('year', substring(col('title'), -5, 4).astype(IntegerType())).select('item_idx', 'year')\n",
    "year.show(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Genres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:28:16.935207Z",
     "iopub.status.busy": "2024-02-29T10:28:16.934839Z",
     "iopub.status.idle": "2024-02-29T10:28:16.961541Z",
     "shell.execute_reply": "2024-02-29T10:28:16.960405Z",
     "shell.execute_reply.started": "2024-02-29T10:28:16.935180Z"
    }
   },
   "outputs": [],
   "source": [
    "genres = (\n",
    "    item_features_replay\n",
    "    .select(\n",
    "        \"item_idx\",\n",
    "        split(\"genres\", \"\\|\").alias(\"genres\")\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:28:19.211351Z",
     "iopub.status.busy": "2024-02-29T10:28:19.210619Z",
     "iopub.status.idle": "2024-02-29T10:28:19.299396Z",
     "shell.execute_reply": "2024-02-29T10:28:19.298634Z",
     "shell.execute_reply.started": "2024-02-29T10:28:19.211308Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+\n",
      "|item_idx|              genres|\n",
      "+--------+--------------------+\n",
      "|      11|[Adventure, Anima...|\n",
      "|     117|[Adventure, Child...|\n",
      "|     274|   [Comedy, Romance]|\n",
      "|    1382|[Comedy, Drama, R...|\n",
      "|     320|            [Comedy]|\n",
      "|      89|[Action, Crime, T...|\n",
      "|     252|   [Comedy, Romance]|\n",
      "|    2179|[Adventure, Child...|\n",
      "|    1018|            [Action]|\n",
      "|      51|[Action, Adventur...|\n",
      "|     139|[Comedy, Drama, R...|\n",
      "|    1112|    [Comedy, Horror]|\n",
      "|    2403|[Animation, Child...|\n",
      "|     682|             [Drama]|\n",
      "|    1348|[Action, Adventur...|\n",
      "|     189|      [Crime, Drama]|\n",
      "|     111|[Comedy, Drama, R...|\n",
      "|     880|[Comedy, Drama, T...|\n",
      "|     129|            [Comedy]|\n",
      "|    1039|[Action, Comedy, ...|\n",
      "+--------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "genres.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:28:20.990178Z",
     "iopub.status.busy": "2024-02-29T10:28:20.989795Z",
     "iopub.status.idle": "2024-02-29T10:28:21.368462Z",
     "shell.execute_reply": "2024-02-29T10:28:21.367549Z",
     "shell.execute_reply.started": "2024-02-29T10:28:20.990153Z"
    }
   },
   "outputs": [],
   "source": [
    "genres_list = (\n",
    "    genres.select(explode(\"genres\").alias(\"genre\"))\n",
    "    .distinct().filter('genre <> \"(no genres listed)\"')\n",
    "    .toPandas()[\"genre\"].tolist()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:28:22.694618Z",
     "iopub.status.busy": "2024-02-29T10:28:22.694248Z",
     "iopub.status.idle": "2024-02-29T10:28:22.703658Z",
     "shell.execute_reply": "2024-02-29T10:28:22.701225Z",
     "shell.execute_reply.started": "2024-02-29T10:28:22.694590Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Children',\n",
       " 'Crime',\n",
       " 'Sci-Fi',\n",
       " 'Musical',\n",
       " 'Animation',\n",
       " 'Mystery',\n",
       " 'Action',\n",
       " 'Documentary',\n",
       " 'Fantasy',\n",
       " 'Drama',\n",
       " 'IMAX',\n",
       " 'Comedy',\n",
       " 'Horror',\n",
       " 'Adventure',\n",
       " 'Western',\n",
       " 'Romance',\n",
       " 'Thriller',\n",
       " 'War',\n",
       " 'Film-Noir']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genres_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:28:32.741706Z",
     "iopub.status.busy": "2024-02-29T10:28:32.741274Z",
     "iopub.status.idle": "2024-02-29T10:28:33.883816Z",
     "shell.execute_reply": "2024-02-29T10:28:33.882815Z",
     "shell.execute_reply.started": "2024-02-29T10:28:32.741679Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10681"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_features = genres\n",
    "for genre in genres_list:\n",
    "    item_features = item_features.withColumn(\n",
    "        genre,\n",
    "        array_contains(col(\"genres\"), genre).astype(IntegerType())\n",
    "    )\n",
    "item_features = item_features.drop(\"genres\").cache()\n",
    "item_features.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:28:35.623456Z",
     "iopub.status.busy": "2024-02-29T10:28:35.621698Z",
     "iopub.status.idle": "2024-02-29T10:28:36.229844Z",
     "shell.execute_reply": "2024-02-29T10:28:36.228874Z",
     "shell.execute_reply.started": "2024-02-29T10:28:35.623379Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10681"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_features = item_features.join(year, on='item_idx', how='inner')\n",
    "item_features.cache()\n",
    "item_features.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:28:37.967856Z",
     "iopub.status.busy": "2024-02-29T10:28:37.967419Z",
     "iopub.status.idle": "2024-02-29T10:28:38.140294Z",
     "shell.execute_reply": "2024-02-29T10:28:38.138873Z",
     "shell.execute_reply.started": "2024-02-29T10:28:37.967825Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------+-----+------+-------+---------+-------+------+-----------+-------+-----+----+------+------+---------+-------+-------+--------+---+---------+----+\n",
      "|item_idx|Children|Crime|Sci-Fi|Musical|Animation|Mystery|Action|Documentary|Fantasy|Drama|IMAX|Comedy|Horror|Adventure|Western|Romance|Thriller|War|Film-Noir|year|\n",
      "+--------+--------+-----+------+-------+---------+-------+------+-----------+-------+-----+----+------+------+---------+-------+-------+--------+---+---------+----+\n",
      "|      11|       1|    0|     0|      0|        1|      0|     0|          0|      1|    0|   0|     1|     0|        1|      0|      0|       0|  0|        0|1995|\n",
      "|     117|       1|    0|     0|      0|        0|      0|     0|          0|      1|    0|   0|     0|     0|        1|      0|      0|       0|  0|        0|1995|\n",
      "+--------+--------+-----+------+-------+---------+-------+------+-----------+-------+-----+----+------+------+---------+-------+-------+--------+---+---------+----+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "item_features.show(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Building LightFM model based on interaction matrix and item features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:28:47.363702Z",
     "iopub.status.busy": "2024-02-29T10:28:47.363319Z",
     "iopub.status.idle": "2024-02-29T10:28:47.370154Z",
     "shell.execute_reply": "2024-02-29T10:28:47.368344Z",
     "shell.execute_reply.started": "2024-02-29T10:28:47.363674Z"
    }
   },
   "outputs": [],
   "source": [
    "model_feat = LightFMWrap(random_state=SEED, loss='warp', no_components=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:28:48.170316Z",
     "iopub.status.busy": "2024-02-29T10:28:48.169929Z",
     "iopub.status.idle": "2024-02-29T10:32:29.861487Z",
     "shell.execute_reply": "2024-02-29T10:32:29.859272Z",
     "shell.execute_reply.started": "2024-02-29T10:28:48.170286Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/replay/utils/spark_utils.py:43: SparkCollectToMasterWarning: Spark Data Frame is collected to master node, this may lead to OOM exception for larger dataset. To remove this warning set allow_collect_to_master=True in the recommender constructor.\n",
      "  warnings.warn(\n",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11min 16s, sys: 1.02 s, total: 11min 17s\n",
      "Wall time: 3min 41s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "model_feat.fit(train, item_features=item_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:32:29.864720Z",
     "iopub.status.busy": "2024-02-29T10:32:29.864301Z",
     "iopub.status.idle": "2024-02-29T10:35:56.993594Z",
     "shell.execute_reply": "2024-02-29T10:35:56.992150Z",
     "shell.execute_reply.started": "2024-02-29T10:32:29.864688Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 349 ms, sys: 59 ms, total: 408 ms\n",
      "Wall time: 3min 27s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "31960"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "recs = model_feat.predict(\n",
    "    log=train,\n",
    "    k=K,\n",
    "    users=test.select('user_idx').distinct(),\n",
    "    item_features=item_features,\n",
    "    filter_seen_items=True,\n",
    ")\n",
    "recs.cache()\n",
    "recs.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Model evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-10T16:07:28.942205Z",
     "start_time": "2020-02-10T16:07:26.281475Z"
    },
    "execution": {
     "iopub.execute_input": "2024-02-29T10:35:56.996014Z",
     "iopub.status.busy": "2024-02-29T10:35:56.995565Z",
     "iopub.status.idle": "2024-02-29T10:35:57.005615Z",
     "shell.execute_reply": "2024-02-29T10:35:57.003965Z",
     "shell.execute_reply.started": "2024-02-29T10:35:56.995975Z"
    },
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "metrics = Experiment(\n",
    "    [NDCG(K), MAP(K), HitRate([1, K]), Coverage(K)],\n",
    "    test,\n",
    "    train,\n",
    "    query_column = \"user_idx\",\n",
    "    item_column = \"item_idx\",\n",
    "    rating_column = \"relevance\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-29T10:35:57.010163Z",
     "iopub.status.busy": "2024-02-29T10:35:57.009791Z",
     "iopub.status.idle": "2024-02-29T10:36:27.839190Z",
     "shell.execute_reply": "2024-02-29T10:36:27.838220Z",
     "shell.execute_reply.started": "2024-02-29T10:35:57.010139Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 133 ms, sys: 32.3 ms, total: 165 ms\n",
      "Wall time: 30.8 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NDCG@10</th>\n",
       "      <th>MAP@10</th>\n",
       "      <th>HitRate@1</th>\n",
       "      <th>HitRate@10</th>\n",
       "      <th>Coverage@10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>LightFM_item_features</th>\n",
       "      <td>0.27179</td>\n",
       "      <td>0.185185</td>\n",
       "      <td>0.336671</td>\n",
       "      <td>0.659262</td>\n",
       "      <td>0.108688</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       NDCG@10    MAP@10  HitRate@1  HitRate@10  Coverage@10\n",
       "LightFM_item_features  0.27179  0.185185   0.336671    0.659262     0.108688"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "metrics.add_result(\"LightFM_item_features\", recs)\n",
    "metrics.results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [],
   "isGpuEnabled": false,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}