examples/movielens/movielens-1m.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from prediction_flow.features import Number, Category, Sequence, Features\n",
    "from prediction_flow.transformers.column import (\n",
    "    StandardScaler, CategoryEncoder, SequenceEncoder)\n",
    "\n",
    "from prediction_flow.pytorch.data import Dataset\n",
    "from prediction_flow.pytorch import WideDeep, DeepFM, DNN, DIN, DIEN, AttentionGroup\n",
    "\n",
    "from prediction_flow.pytorch.functions import fit, predict, create_dataloader_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(\"./ml-1m/train.csv\")\n",
    "valid_df = pd.read_csv(\"./ml-1m/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.movieId = train_df.movieId.astype(str)\n",
    "valid_df.movieId = valid_df.movieId.astype(str)\n",
    "\n",
    "train_df.histHighRatedMovieIds = train_df.histHighRatedMovieIds.astype(str)\n",
    "valid_df.histHighRatedMovieIds = valid_df.histHighRatedMovieIds.astype(str)\n",
    "\n",
    "train_df.negHistMovieIds = train_df.negHistMovieIds.astype(str)\n",
    "valid_df.negHistMovieIds = valid_df.negHistMovieIds.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "userId                     int64\n",
       "movieId                   object\n",
       "ratting                    int64\n",
       "timestamp                  int64\n",
       "last                     float64\n",
       "histHighRatedMovieIds     object\n",
       "negHistMovieIds           object\n",
       "gender                    object\n",
       "age                        int64\n",
       "occupation                 int64\n",
       "zipCode                   object\n",
       "title                     object\n",
       "genres                    object\n",
       "label                      int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "userId                     int64\n",
       "movieId                   object\n",
       "ratting                    int64\n",
       "timestamp                  int64\n",
       "last                     float64\n",
       "histHighRatedMovieIds     object\n",
       "negHistMovieIds           object\n",
       "gender                    object\n",
       "age                        int64\n",
       "occupation                 int64\n",
       "zipCode                   object\n",
       "title                     object\n",
       "genres                    object\n",
       "label                      int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>ratting</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>last</th>\n",
       "      <th>histHighRatedMovieIds</th>\n",
       "      <th>negHistMovieIds</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>occupation</th>\n",
       "      <th>zipCode</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>48</td>\n",
       "      <td>5</td>\n",
       "      <td>978824351</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1545|527|595|588|1|2355|2294|783|1566|1907</td>\n",
       "      <td>393|2562|3289|3442|2808|2259|295|1190|3321|3470</td>\n",
       "      <td>F</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>48067</td>\n",
       "      <td>Pocahontas (1995)</td>\n",
       "      <td>Animation|Children's|Musical|Romance</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2030</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>977809337</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3928|1934|2099|3345|1013|1101|3524|3429|3751|2138</td>\n",
       "      <td>3142|1743|525|1746|3323|935|2183|3556|3343|1542</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>4</td>\n",
       "      <td>77345</td>\n",
       "      <td>Pocahontas (1995)</td>\n",
       "      <td>Animation|Children's|Musical|Romance</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4877</td>\n",
       "      <td>48</td>\n",
       "      <td>2</td>\n",
       "      <td>962767153</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2078|1032|2081|616|364|2033|3034|2394|3745|3615</td>\n",
       "      <td>2354|747|3478|2638|311|1204|3370|807|29|1380</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>4</td>\n",
       "      <td>94703</td>\n",
       "      <td>Pocahontas (1995)</td>\n",
       "      <td>Animation|Children's|Musical|Romance</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>1917</td>\n",
       "      <td>3</td>\n",
       "      <td>978300174</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3418|349|1527|2353|1370|648|368|736|2002|1544</td>\n",
       "      <td>2117|2560|3480|3678|2798|1222|3872|3503|3913|3494</td>\n",
       "      <td>M</td>\n",
       "      <td>56</td>\n",
       "      <td>16</td>\n",
       "      <td>70072</td>\n",
       "      <td>Armageddon (1998)</td>\n",
       "      <td>Action|Adventure|Sci-Fi|Thriller</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5122</td>\n",
       "      <td>1917</td>\n",
       "      <td>5</td>\n",
       "      <td>962213528</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3078|2858|2907|1584|2571|32|788|1573|2428|1580</td>\n",
       "      <td>1860|194|742|2379|1966|3397|3195|678|2240|2196</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>20009</td>\n",
       "      <td>Armageddon (1998)</td>\n",
       "      <td>Action|Adventure|Sci-Fi|Thriller</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId movieId  ratting  timestamp  last  \\\n",
       "0       1      48        5  978824351   1.0   \n",
       "1    2030      48        3  977809337   1.0   \n",
       "2    4877      48        2  962767153   1.0   \n",
       "3       2    1917        3  978300174   1.0   \n",
       "4    5122    1917        5  962213528   1.0   \n",
       "\n",
       "                               histHighRatedMovieIds  \\\n",
       "0         1545|527|595|588|1|2355|2294|783|1566|1907   \n",
       "1  3928|1934|2099|3345|1013|1101|3524|3429|3751|2138   \n",
       "2    2078|1032|2081|616|364|2033|3034|2394|3745|3615   \n",
       "3      3418|349|1527|2353|1370|648|368|736|2002|1544   \n",
       "4     3078|2858|2907|1584|2571|32|788|1573|2428|1580   \n",
       "\n",
       "                                     negHistMovieIds gender  age  occupation  \\\n",
       "0    393|2562|3289|3442|2808|2259|295|1190|3321|3470      F    1          10   \n",
       "1    3142|1743|525|1746|3323|935|2183|3556|3343|1542      M   25           4   \n",
       "2       2354|747|3478|2638|311|1204|3370|807|29|1380      M   25           4   \n",
       "3  2117|2560|3480|3678|2798|1222|3872|3503|3913|3494      M   56          16   \n",
       "4     1860|194|742|2379|1966|3397|3195|678|2240|2196      M   25           0   \n",
       "\n",
       "  zipCode              title                                genres  label  \n",
       "0   48067  Pocahontas (1995)  Animation|Children's|Musical|Romance      1  \n",
       "1   77345  Pocahontas (1995)  Animation|Children's|Musical|Romance      0  \n",
       "2   94703  Pocahontas (1995)  Animation|Children's|Musical|Romance      0  \n",
       "3   70072  Armageddon (1998)      Action|Adventure|Sci-Fi|Thriller      0  \n",
       "4   20009  Armageddon (1998)      Action|Adventure|Sci-Fi|Thriller      1  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>ratting</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>last</th>\n",
       "      <th>histHighRatedMovieIds</th>\n",
       "      <th>negHistMovieIds</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>occupation</th>\n",
       "      <th>zipCode</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>2294</td>\n",
       "      <td>4</td>\n",
       "      <td>978226678</td>\n",
       "      <td>1.0</td>\n",
       "      <td>590|480|2302|2268|529|349|524|16|1653|2278</td>\n",
       "      <td>876|1953|2644|3540|2794|90|3291|406|2282|2406</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>17</td>\n",
       "      <td>61614</td>\n",
       "      <td>Antz (1998)</td>\n",
       "      <td>Animation|Children's</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15</td>\n",
       "      <td>3510</td>\n",
       "      <td>5</td>\n",
       "      <td>978361393</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1892|648|1422|1597|3489|1196|260|1210|3461|2115</td>\n",
       "      <td>322|135|1324|3928|2808|3220|2992|3937|652|3671</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>7</td>\n",
       "      <td>22903</td>\n",
       "      <td>Frequency (2000)</td>\n",
       "      <td>Drama|Thriller</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2165</td>\n",
       "      <td>3510</td>\n",
       "      <td>5</td>\n",
       "      <td>974982139</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1095|2959|2416|3552|16|1266|3210|104|3147|3148</td>\n",
       "      <td>1936|1248|2241|985|3097|2282|3550|2690|3731|718</td>\n",
       "      <td>M</td>\n",
       "      <td>25</td>\n",
       "      <td>7</td>\n",
       "      <td>32836</td>\n",
       "      <td>Frequency (2000)</td>\n",
       "      <td>Drama|Thriller</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2319</td>\n",
       "      <td>3510</td>\n",
       "      <td>4</td>\n",
       "      <td>974476547</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2987|1240|2243|1302|2716|1961|1096|3408|3481|3148</td>\n",
       "      <td>392|2782|1615|3720|781|1511|3225|2273|2655|3798</td>\n",
       "      <td>M</td>\n",
       "      <td>35</td>\n",
       "      <td>7</td>\n",
       "      <td>48104</td>\n",
       "      <td>Frequency (2000)</td>\n",
       "      <td>Drama|Thriller</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16</td>\n",
       "      <td>2701</td>\n",
       "      <td>2</td>\n",
       "      <td>978174795</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2683|2699|2355|2975|2369|3175|2761|2581|2394|2724</td>\n",
       "      <td>3128|1830|3281|163|1334|585|3580|1776|1392|774</td>\n",
       "      <td>F</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>20670</td>\n",
       "      <td>Wild Wild West (1999)</td>\n",
       "      <td>Action|Sci-Fi|Western</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId movieId  ratting  timestamp  last  \\\n",
       "0       9    2294        4  978226678   1.0   \n",
       "1      15    3510        5  978361393   1.0   \n",
       "2    2165    3510        5  974982139   1.0   \n",
       "3    2319    3510        4  974476547   1.0   \n",
       "4      16    2701        2  978174795   1.0   \n",
       "\n",
       "                               histHighRatedMovieIds  \\\n",
       "0         590|480|2302|2268|529|349|524|16|1653|2278   \n",
       "1    1892|648|1422|1597|3489|1196|260|1210|3461|2115   \n",
       "2     1095|2959|2416|3552|16|1266|3210|104|3147|3148   \n",
       "3  2987|1240|2243|1302|2716|1961|1096|3408|3481|3148   \n",
       "4  2683|2699|2355|2975|2369|3175|2761|2581|2394|2724   \n",
       "\n",
       "                                   negHistMovieIds gender  age  occupation  \\\n",
       "0    876|1953|2644|3540|2794|90|3291|406|2282|2406      M   25          17   \n",
       "1   322|135|1324|3928|2808|3220|2992|3937|652|3671      M   25           7   \n",
       "2  1936|1248|2241|985|3097|2282|3550|2690|3731|718      M   25           7   \n",
       "3  392|2782|1615|3720|781|1511|3225|2273|2655|3798      M   35           7   \n",
       "4   3128|1830|3281|163|1334|585|3580|1776|1392|774      F   35           0   \n",
       "\n",
       "  zipCode                  title                 genres  label  \n",
       "0   61614            Antz (1998)   Animation|Children's      1  \n",
       "1   22903       Frequency (2000)         Drama|Thriller      1  \n",
       "2   32836       Frequency (2000)         Drama|Thriller      1  \n",
       "3   48104       Frequency (2000)         Drama|Thriller      1  \n",
       "4   20670  Wild Wild West (1999)  Action|Sci-Fi|Western      0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4832, 14) (1208, 14)\n"
     ]
    }
   ],
   "source": [
    "print(train_df.shape, valid_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "number_features = [\n",
    "    Number('age', StandardScaler())\n",
    "]\n",
    "\n",
    "category_features = [\n",
    "    Category('gender', CategoryEncoder(min_cnt=1)),\n",
    "    Category('movieId', CategoryEncoder(min_cnt=1)),\n",
    "    Category('occupation', CategoryEncoder(min_cnt=1)),\n",
    "    Category('zipCode', CategoryEncoder(min_cnt=1))\n",
    "]\n",
    "\n",
    "sequence_features = [\n",
    "    Sequence('genres', SequenceEncoder(sep='|', min_cnt=1)),\n",
    "    Sequence('histHighRatedMovieIds', SequenceEncoder(sep='|', min_cnt=1)),\n",
    "    Sequence('negHistMovieIds', SequenceEncoder(sep='|', min_cnt=1))\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "features, train_loader, valid_loader = create_dataloader_fn(\n",
    "    number_features, category_features, sequence_features, 64, train_df, 'label', valid_df, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluation(df, dataloader):\n",
    "    preds = predict(model, dataloader)\n",
    "    return roc_auc_score(df['label'], preds.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "din_attention_groups = [\n",
    "    AttentionGroup(\n",
    "        name='group1',\n",
    "        pairs=[{'ad': 'movieId', 'pos_hist': 'histHighRatedMovieIds'}],\n",
    "        hidden_layers=[16, 8], att_dropout=0.1)]\n",
    "\n",
    "gru_attention_groups = [\n",
    "    AttentionGroup(\n",
    "        name='group1',\n",
    "        pairs=[{'ad': 'movieId', 'pos_hist': 'histHighRatedMovieIds'}],\n",
    "        hidden_layers=[16, 8], att_dropout=0.1, gru_type='GRU')]\n",
    "\n",
    "aigru_attention_groups = [\n",
    "    AttentionGroup(\n",
    "        name='group1',\n",
    "        pairs=[{'ad': 'movieId', 'pos_hist': 'histHighRatedMovieIds'}],\n",
    "        hidden_layers=[16, 8], att_dropout=0.1, gru_type='AIGRU')]\n",
    "\n",
    "agru_attention_groups = [\n",
    "    AttentionGroup(\n",
    "        name='group1',\n",
    "        pairs=[{'ad': 'movieId', 'pos_hist': 'histHighRatedMovieIds'}],\n",
    "        hidden_layers=[16, 8], att_dropout=0.1, gru_type='AGRU')]\n",
    "\n",
    "augru_attention_groups = [\n",
    "    AttentionGroup(\n",
    "        name='group1',\n",
    "        pairs=[{'ad': 'movieId', 'pos_hist': 'histHighRatedMovieIds'}],\n",
    "        hidden_layers=[16, 8], att_dropout=0.1, gru_type='AUGRU')]\n",
    "\n",
    "augru_attention_groups_with_neg = [\n",
    "    AttentionGroup(\n",
    "        name='group1',\n",
    "        pairs=[{'ad': 'movieId', 'pos_hist': 'histHighRatedMovieIds', 'neg_hist': 'negHistMovieIds'}],\n",
    "        hidden_layers=[16, 8], att_dropout=0.1, gru_type='AUGRU')]\n",
    "\n",
    "models = [\n",
    "    DNN(features, 2, 16, (32, 16), final_activation='sigmoid', dropout=0.3),\n",
    "    WideDeep(\n",
    "        features,\n",
    "        wide_features=['genres', 'movieId'],\n",
    "        deep_features=['age', 'gender', 'movieId', 'occupation',\n",
    "         'zipCode', 'histHighRatedMovieIds'],\n",
    "        cross_features=[('movieId', 'histHighRatedMovieIds')],\n",
    "        num_classes=2, embedding_size=16, hidden_layers=(32, 16),\n",
    "        final_activation='sigmoid', dropout=0.3),\n",
    "    DeepFM(features, 2, 16, (32, 16), final_activation='sigmoid', dropout=0.3),\n",
    "    DIN(features, din_attention_groups, 2, 16, (32, 16), final_activation='sigmoid', dropout=0.3),\n",
    "    DIEN(features, gru_attention_groups, 2, 16, (32, 16), final_activation='sigmoid', dropout=0.3),\n",
    "    DIEN(features, aigru_attention_groups, 2, 16, (32, 16), final_activation='sigmoid', dropout=0.3),\n",
    "    DIEN(features, agru_attention_groups, 2, 16, (32, 16), final_activation='sigmoid', dropout=0.3),\n",
    "    DIEN(features, augru_attention_groups, 2, 16, (32, 16), final_activation='sigmoid', dropout=0.3),\n",
    "    DIEN(features, augru_attention_groups_with_neg, 2, 16, (32, 16),\n",
    "         final_activation='sigmoid', dropout=0.3, use_negsampling=True)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DNN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (embedding:histHighRatedMovieIds): Embedding(2917, 16, padding_idx=0)\n",
      "  (embedding:negHistMovieIds): Embedding(3886, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:histHighRatedMovieIds): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=113, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): ReLU()\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): ReLU()\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "831469956f7e4a1091851ab48b7257ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70be7c268783442cbc0af34eca0b545b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec887320b7944b2cb689519cb9331071",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WideDeep(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (embedding:histHighRatedMovieIds): Embedding(2917, 16, padding_idx=0)\n",
      "  (embedding:negHistMovieIds): Embedding(3886, 16, padding_idx=0)\n",
      "  (pooling:genres): SumPooling()\n",
      "  (pooling:histHighRatedMovieIds): SumPooling()\n",
      "  (pooling:negHistMovieIds): SumPooling()\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=81, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): ReLU()\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): ReLU()\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=64, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a68ddc1657148b6937f80b34d43e0c5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "352ca9c2aeea48bbbafa3ef760969a77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1905f1e143e4a029776b5b4eed3ab9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DeepFM(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (embedding:histHighRatedMovieIds): Embedding(2917, 16, padding_idx=0)\n",
      "  (embedding:negHistMovieIds): Embedding(3886, 16, padding_idx=0)\n",
      "  (embedding:age): Linear(in_features=1, out_features=16, bias=False)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:histHighRatedMovieIds): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (linear): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=1, out_features=1, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(1, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (fm): FM()\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=129, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): ReLU()\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): ReLU()\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=18, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb26dce33bf944c1a7e46196551f0e07",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f48f657732a49b8a6a8ab735acd195c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "862a4249a5b348cd8c6fbdfd66baa07a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DIN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (embedding:histHighRatedMovieIds): Embedding(2917, 16, padding_idx=0)\n",
      "  (embedding:negHistMovieIds): Embedding(3886, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (attention_pooling:group1): Attention(\n",
      "    (mlp): MLP(\n",
      "      (_sequential): Sequential(\n",
      "        (dense0): Linear(in_features=64, out_features=16, bias=True)\n",
      "        (batchnorm0): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (activation0): PReLU(num_parameters=1)\n",
      "        (dropout0): Dropout(p=0.1)\n",
      "        (dense1): Linear(in_features=16, out_features=8, bias=True)\n",
      "        (batchnorm1): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (activation1): PReLU(num_parameters=1)\n",
      "        (dropout1): Dropout(p=0.1)\n",
      "      )\n",
      "    )\n",
      "    (fc): Linear(in_features=8, out_features=1, bias=True)\n",
      "  )\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=113, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): PReLU(num_parameters=1)\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): PReLU(num_parameters=1)\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e1d82391a2747beac878208889f6769",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7a7ec74e5034f9baf8c6d743b172073",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "77145256e8d64498a9a243277458d804",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DIEN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (embedding:histHighRatedMovieIds): Embedding(2917, 16, padding_idx=0)\n",
      "  (embedding:negHistMovieIds): Embedding(3886, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (attention_pooling:group1): Interest(\n",
      "    (interest_extractor): GRU(16, 16, batch_first=True)\n",
      "    (interest_evolution): GRU(16, 16, batch_first=True)\n",
      "    (attention): Attention(\n",
      "      (mlp): MLP(\n",
      "        (_sequential): Sequential(\n",
      "          (dense0): Linear(in_features=64, out_features=16, bias=True)\n",
      "          (batchnorm0): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation0): PReLU(num_parameters=1)\n",
      "          (dropout0): Dropout(p=0.1)\n",
      "          (dense1): Linear(in_features=16, out_features=8, bias=True)\n",
      "          (batchnorm1): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation1): PReLU(num_parameters=1)\n",
      "          (dropout1): Dropout(p=0.1)\n",
      "        )\n",
      "      )\n",
      "      (fc): Linear(in_features=8, out_features=1, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=113, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): PReLU(num_parameters=1)\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): PReLU(num_parameters=1)\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dcc85db2ed014edaaa9f66a1879f3af4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "470168d07f65459ca4ebb373ef129cd6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "282decf6d6f44949bd7709ffb78cf8b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DIEN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (embedding:histHighRatedMovieIds): Embedding(2917, 16, padding_idx=0)\n",
      "  (embedding:negHistMovieIds): Embedding(3886, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (attention_pooling:group1): Interest(\n",
      "    (interest_extractor): GRU(16, 16, batch_first=True)\n",
      "    (attention): Attention(\n",
      "      (mlp): MLP(\n",
      "        (_sequential): Sequential(\n",
      "          (dense0): Linear(in_features=64, out_features=16, bias=True)\n",
      "          (batchnorm0): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation0): PReLU(num_parameters=1)\n",
      "          (dropout0): Dropout(p=0.1)\n",
      "          (dense1): Linear(in_features=16, out_features=8, bias=True)\n",
      "          (batchnorm1): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation1): PReLU(num_parameters=1)\n",
      "          (dropout1): Dropout(p=0.1)\n",
      "        )\n",
      "      )\n",
      "      (fc): Linear(in_features=8, out_features=1, bias=True)\n",
      "    )\n",
      "    (interest_evolution): GRU(16, 16, batch_first=True)\n",
      "  )\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=113, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): PReLU(num_parameters=1)\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): PReLU(num_parameters=1)\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d97eb65ca78644049ccccf6d0dd57983",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eacd0a73a504479bad06e31112e96c79",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a50dba839c2343bd85fb3aa0e4dcb7cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DIEN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (embedding:histHighRatedMovieIds): Embedding(2917, 16, padding_idx=0)\n",
      "  (embedding:negHistMovieIds): Embedding(3886, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (attention_pooling:group1): Interest(\n",
      "    (interest_extractor): GRU(16, 16, batch_first=True)\n",
      "    (attention): Attention(\n",
      "      (mlp): MLP(\n",
      "        (_sequential): Sequential(\n",
      "          (dense0): Linear(in_features=64, out_features=16, bias=True)\n",
      "          (batchnorm0): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation0): PReLU(num_parameters=1)\n",
      "          (dropout0): Dropout(p=0.1)\n",
      "          (dense1): Linear(in_features=16, out_features=8, bias=True)\n",
      "          (batchnorm1): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation1): PReLU(num_parameters=1)\n",
      "          (dropout1): Dropout(p=0.1)\n",
      "        )\n",
      "      )\n",
      "      (fc): Linear(in_features=8, out_features=1, bias=True)\n",
      "    )\n",
      "    (interest_evolution): DynamicGRU(\n",
      "      (rnn): AttentionGRUCell()\n",
      "    )\n",
      "  )\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=113, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): PReLU(num_parameters=1)\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): PReLU(num_parameters=1)\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70b0ee4430f3431f9bd4398fbfedaaa0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78051aa6160d44cc893578d6797f2d37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52b3df51d1ff4d8fbc7d05c4bc02e9ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DIEN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (embedding:histHighRatedMovieIds): Embedding(2917, 16, padding_idx=0)\n",
      "  (embedding:negHistMovieIds): Embedding(3886, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (attention_pooling:group1): Interest(\n",
      "    (interest_extractor): GRU(16, 16, batch_first=True)\n",
      "    (attention): Attention(\n",
      "      (mlp): MLP(\n",
      "        (_sequential): Sequential(\n",
      "          (dense0): Linear(in_features=64, out_features=16, bias=True)\n",
      "          (batchnorm0): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation0): PReLU(num_parameters=1)\n",
      "          (dropout0): Dropout(p=0.1)\n",
      "          (dense1): Linear(in_features=16, out_features=8, bias=True)\n",
      "          (batchnorm1): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation1): PReLU(num_parameters=1)\n",
      "          (dropout1): Dropout(p=0.1)\n",
      "        )\n",
      "      )\n",
      "      (fc): Linear(in_features=8, out_features=1, bias=True)\n",
      "    )\n",
      "    (interest_evolution): DynamicGRU(\n",
      "      (rnn): AttentionUpdateGateGRUCell()\n",
      "    )\n",
      "  )\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=113, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): PReLU(num_parameters=1)\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): PReLU(num_parameters=1)\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c696778c3e9241dca290ba69ed94da05",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bbb3e33644364131a5c9cd19e920679e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "833c1011ec2b455f8aa8da2e9fc4b5ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DIEN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (embedding:histHighRatedMovieIds): Embedding(2917, 16, padding_idx=0)\n",
      "  (embedding:negHistMovieIds): Embedding(3886, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (attention_pooling:group1): Interest(\n",
      "    (interest_extractor): GRU(16, 16, batch_first=True)\n",
      "    (auxiliary_net): AuxiliaryNet(\n",
      "      (_sequential): Sequential(\n",
      "        (dense0): Linear(in_features=32, out_features=100, bias=True)\n",
      "        (activation0): Sigmoid()\n",
      "        (dense1): Linear(in_features=100, out_features=50, bias=True)\n",
      "        (activation1): Sigmoid()\n",
      "        (final_layer): Linear(in_features=50, out_features=1, bias=True)\n",
      "      )\n",
      "    )\n",
      "    (attention): Attention(\n",
      "      (mlp): MLP(\n",
      "        (_sequential): Sequential(\n",
      "          (dense0): Linear(in_features=64, out_features=16, bias=True)\n",
      "          (batchnorm0): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation0): PReLU(num_parameters=1)\n",
      "          (dropout0): Dropout(p=0.1)\n",
      "          (dense1): Linear(in_features=16, out_features=8, bias=True)\n",
      "          (batchnorm1): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation1): PReLU(num_parameters=1)\n",
      "          (dropout1): Dropout(p=0.1)\n",
      "        )\n",
      "      )\n",
      "      (fc): Linear(in_features=8, out_features=1, bias=True)\n",
      "    )\n",
      "    (interest_evolution): DynamicGRU(\n",
      "      (rnn): AttentionUpdateGateGRUCell()\n",
      "    )\n",
      "  )\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=97, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): PReLU(num_parameters=1)\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): PReLU(num_parameters=1)\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31d8775c7f5d496f95621414a7d3564d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7108c034d43e4397bb90aa540b04ff27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e44937f2d1424e688797ace230bb1ca2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "scores = []\n",
    "for model in models:\n",
    "    print(model)\n",
    "    loss_func = nn.BCELoss()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-5)\n",
    "    fit(10, model, loss_func, optimizer,\n",
    "        train_loader, valid_loader, notebook=True, auxiliary_loss_rate=0.1)\n",
    "    scores.append(evaluation(valid_df, valid_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.5678126351433473, 0.561144856709187, 0.5784686439168888, 0.5822279255266528, 0.5906340183893881, 0.5961033491524658, 0.601482231034707, 0.5828921594952953, 0.6221356676681995]\n"
     ]
    }
   ],
   "source": [
    "print(scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# share embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'movieId'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features.category_features[1].name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2idx = features.category_features[1].column_flow.transformers[0].word2idx\n",
    "idx2word = features.category_features[1].column_flow.transformers[0].idx2word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "number_features = [\n",
    "    Number('age', StandardScaler())\n",
    "]\n",
    "\n",
    "category_features = [\n",
    "    Category('gender', CategoryEncoder(min_cnt=1)),\n",
    "    Category('movieId',\n",
    "             CategoryEncoder(word2idx=word2idx, idx2word=idx2word),\n",
    "             embedding_name='movieId'),\n",
    "    Category('occupation', CategoryEncoder(min_cnt=1)),\n",
    "    Category('zipCode', CategoryEncoder(min_cnt=1))\n",
    "]\n",
    "\n",
    "sequence_features = [\n",
    "    Sequence('genres', SequenceEncoder(sep='|', min_cnt=1)),\n",
    "    Sequence('histHighRatedMovieIds',\n",
    "             SequenceEncoder(sep='|', word2idx=word2idx, idx2word=idx2word),\n",
    "             embedding_name='movieId'),\n",
    "    Sequence('negHistMovieIds',\n",
    "             SequenceEncoder(sep='|', word2idx=word2idx, idx2word=idx2word),\n",
    "             embedding_name='movieId')\n",
    "]\n",
    "\n",
    "features, train_loader, valid_loader = create_dataloader_fn(\n",
    "    number_features, category_features, sequence_features, 64, train_df, 'label', valid_df, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "din_attention_groups = [\n",
    "    AttentionGroup(\n",
    "        name='group1',\n",
    "        pairs=[{'ad': 'movieId', 'pos_hist': 'histHighRatedMovieIds'}],\n",
    "        hidden_layers=[16, 8], att_dropout=0.1)]\n",
    "\n",
    "gru_attention_groups = [\n",
    "    AttentionGroup(\n",
    "        name='group1',\n",
    "        pairs=[{'ad': 'movieId', 'pos_hist': 'histHighRatedMovieIds'}],\n",
    "        hidden_layers=[16, 8], att_dropout=0.1, gru_type='GRU')]\n",
    "\n",
    "aigru_attention_groups = [\n",
    "    AttentionGroup(\n",
    "        name='group1',\n",
    "        pairs=[{'ad': 'movieId', 'pos_hist': 'histHighRatedMovieIds'}],\n",
    "        hidden_layers=[16, 8], att_dropout=0.1, gru_type='AIGRU')]\n",
    "\n",
    "agru_attention_groups = [\n",
    "    AttentionGroup(\n",
    "        name='group1',\n",
    "        pairs=[{'ad': 'movieId', 'pos_hist': 'histHighRatedMovieIds'}],\n",
    "        hidden_layers=[16, 8], att_dropout=0.1, gru_type='AGRU')]\n",
    "\n",
    "augru_attention_groups = [\n",
    "    AttentionGroup(\n",
    "        name='group1',\n",
    "        pairs=[{'ad': 'movieId', 'pos_hist': 'histHighRatedMovieIds'}],\n",
    "        hidden_layers=[16, 8], att_dropout=0.1, gru_type='AUGRU')]\n",
    "\n",
    "augru_attention_groups_with_neg = [\n",
    "    AttentionGroup(\n",
    "        name='group1',\n",
    "        pairs=[{'ad': 'movieId', 'pos_hist': 'histHighRatedMovieIds', 'neg_hist': 'negHistMovieIds'}],\n",
    "        hidden_layers=[16, 8], att_dropout=0.1, gru_type='AUGRU')]\n",
    "\n",
    "models = [\n",
    "    DNN(features, 2, 16, (32, 16), final_activation='sigmoid', dropout=0.3),\n",
    "    WideDeep(\n",
    "        features,\n",
    "        wide_features=['genres', 'movieId'],\n",
    "        deep_features=['age', 'gender', 'movieId', 'occupation',\n",
    "         'zipCode', 'histHighRatedMovieIds'],\n",
    "        cross_features=[('movieId', 'histHighRatedMovieIds')],\n",
    "        num_classes=2, embedding_size=16, hidden_layers=(32, 16),\n",
    "        final_activation='sigmoid', dropout=0.3),\n",
    "    DeepFM(features, 2, 16, (32, 16), final_activation='sigmoid', dropout=0.3),\n",
    "    DIN(features, din_attention_groups, 2, 16, (32, 16),\n",
    "        final_activation='sigmoid', dropout=0.3),\n",
    "    DIEN(features, gru_attention_groups, 2, 16, (32, 16),\n",
    "         final_activation='sigmoid', dropout=0.3),\n",
    "    DIEN(features, aigru_attention_groups, 2, 16, (32, 16),\n",
    "         final_activation='sigmoid', dropout=0.3),\n",
    "    DIEN(features, agru_attention_groups, 2, 16, (32, 16),\n",
    "         final_activation='sigmoid', dropout=0.3),\n",
    "    DIEN(features, augru_attention_groups, 2, 16, (32, 16),\n",
    "         final_activation='sigmoid', dropout=0.3),\n",
    "    DIEN(features, augru_attention_groups_with_neg, 2, 16, (32, 16),\n",
    "         final_activation='sigmoid', dropout=0.3, use_negsampling=True)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DNN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:histHighRatedMovieIds): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=113, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): ReLU()\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): ReLU()\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f9010733297493ea78ba46812182054",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01175f757e084a6384902a7ebcbfd920",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb74d118f23e4175b4d41059332f51a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WideDeep(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (pooling:genres): SumPooling()\n",
      "  (pooling:histHighRatedMovieIds): SumPooling()\n",
      "  (pooling:negHistMovieIds): SumPooling()\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=81, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): ReLU()\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): ReLU()\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=64, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ecbe8a4d41eb4125bd4dd5b4ff3612c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ad94d96988f4ff58756abce393050a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "357da662c62741ffa7dafe32465d2659",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DeepFM(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (embedding:age): Linear(in_features=1, out_features=16, bias=False)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:histHighRatedMovieIds): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (linear): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=1, out_features=1, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(1, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (fm): FM()\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=129, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): ReLU()\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): ReLU()\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=18, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd8900aace9144d5b1c4156bdd9251a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc122117cc334ce582f9e24f93c2e829",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd371a2492bc4318a65ba2f35c2d3059",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DIN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (attention_pooling:group1): Attention(\n",
      "    (mlp): MLP(\n",
      "      (_sequential): Sequential(\n",
      "        (dense0): Linear(in_features=64, out_features=16, bias=True)\n",
      "        (batchnorm0): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (activation0): PReLU(num_parameters=1)\n",
      "        (dropout0): Dropout(p=0.1)\n",
      "        (dense1): Linear(in_features=16, out_features=8, bias=True)\n",
      "        (batchnorm1): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (activation1): PReLU(num_parameters=1)\n",
      "        (dropout1): Dropout(p=0.1)\n",
      "      )\n",
      "    )\n",
      "    (fc): Linear(in_features=8, out_features=1, bias=True)\n",
      "  )\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=113, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): PReLU(num_parameters=1)\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): PReLU(num_parameters=1)\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c939612beed4cbc8edb8b8ef1cdbfad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97de87bd551244a3991693d082069182",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e7bbc075c804633ae7f3d65699fc1db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DIEN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (attention_pooling:group1): Interest(\n",
      "    (interest_extractor): GRU(16, 16, batch_first=True)\n",
      "    (interest_evolution): GRU(16, 16, batch_first=True)\n",
      "    (attention): Attention(\n",
      "      (mlp): MLP(\n",
      "        (_sequential): Sequential(\n",
      "          (dense0): Linear(in_features=64, out_features=16, bias=True)\n",
      "          (batchnorm0): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation0): PReLU(num_parameters=1)\n",
      "          (dropout0): Dropout(p=0.1)\n",
      "          (dense1): Linear(in_features=16, out_features=8, bias=True)\n",
      "          (batchnorm1): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation1): PReLU(num_parameters=1)\n",
      "          (dropout1): Dropout(p=0.1)\n",
      "        )\n",
      "      )\n",
      "      (fc): Linear(in_features=8, out_features=1, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=113, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): PReLU(num_parameters=1)\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): PReLU(num_parameters=1)\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8fc5b94a55a546f9bc7760c3fc72c951",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8200e8f56a21459c868f5fcb1f32f647",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b5baad31075455b88dd9f58fc3fb843",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DIEN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (attention_pooling:group1): Interest(\n",
      "    (interest_extractor): GRU(16, 16, batch_first=True)\n",
      "    (attention): Attention(\n",
      "      (mlp): MLP(\n",
      "        (_sequential): Sequential(\n",
      "          (dense0): Linear(in_features=64, out_features=16, bias=True)\n",
      "          (batchnorm0): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation0): PReLU(num_parameters=1)\n",
      "          (dropout0): Dropout(p=0.1)\n",
      "          (dense1): Linear(in_features=16, out_features=8, bias=True)\n",
      "          (batchnorm1): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation1): PReLU(num_parameters=1)\n",
      "          (dropout1): Dropout(p=0.1)\n",
      "        )\n",
      "      )\n",
      "      (fc): Linear(in_features=8, out_features=1, bias=True)\n",
      "    )\n",
      "    (interest_evolution): GRU(16, 16, batch_first=True)\n",
      "  )\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=113, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): PReLU(num_parameters=1)\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): PReLU(num_parameters=1)\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9394c664cd8f4ed0acec5e1ecfa85b5f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ba18df54787443389c181442022ae44",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4fa5693f2fd34b9c9a6944afc04589aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DIEN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (attention_pooling:group1): Interest(\n",
      "    (interest_extractor): GRU(16, 16, batch_first=True)\n",
      "    (attention): Attention(\n",
      "      (mlp): MLP(\n",
      "        (_sequential): Sequential(\n",
      "          (dense0): Linear(in_features=64, out_features=16, bias=True)\n",
      "          (batchnorm0): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation0): PReLU(num_parameters=1)\n",
      "          (dropout0): Dropout(p=0.1)\n",
      "          (dense1): Linear(in_features=16, out_features=8, bias=True)\n",
      "          (batchnorm1): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation1): PReLU(num_parameters=1)\n",
      "          (dropout1): Dropout(p=0.1)\n",
      "        )\n",
      "      )\n",
      "      (fc): Linear(in_features=8, out_features=1, bias=True)\n",
      "    )\n",
      "    (interest_evolution): DynamicGRU(\n",
      "      (rnn): AttentionGRUCell()\n",
      "    )\n",
      "  )\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=113, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): PReLU(num_parameters=1)\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): PReLU(num_parameters=1)\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f43cb2b457f4574b245d60dd8c5bb9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2094794637c64b5aa0f73379738ccfd6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f581733677744679efe63cb6e19248a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DIEN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (pooling:negHistMovieIds): MaxPooling()\n",
      "  (attention_pooling:group1): Interest(\n",
      "    (interest_extractor): GRU(16, 16, batch_first=True)\n",
      "    (attention): Attention(\n",
      "      (mlp): MLP(\n",
      "        (_sequential): Sequential(\n",
      "          (dense0): Linear(in_features=64, out_features=16, bias=True)\n",
      "          (batchnorm0): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation0): PReLU(num_parameters=1)\n",
      "          (dropout0): Dropout(p=0.1)\n",
      "          (dense1): Linear(in_features=16, out_features=8, bias=True)\n",
      "          (batchnorm1): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation1): PReLU(num_parameters=1)\n",
      "          (dropout1): Dropout(p=0.1)\n",
      "        )\n",
      "      )\n",
      "      (fc): Linear(in_features=8, out_features=1, bias=True)\n",
      "    )\n",
      "    (interest_evolution): DynamicGRU(\n",
      "      (rnn): AttentionUpdateGateGRUCell()\n",
      "    )\n",
      "  )\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=113, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): PReLU(num_parameters=1)\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): PReLU(num_parameters=1)\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66351b87983a4319b644c22af8c49f19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54578698463f4bc5a54faec43fdc1dbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47737ede0b1f4ca7b30181206dfbf1e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DIEN(\n",
      "  (embedding:gender): Embedding(4, 16, padding_idx=0)\n",
      "  (embedding:movieId): Embedding(1718, 16, padding_idx=0)\n",
      "  (embedding:occupation): Embedding(23, 16, padding_idx=0)\n",
      "  (embedding:zipCode): Embedding(2957, 16, padding_idx=0)\n",
      "  (embedding:genres): Embedding(20, 16, padding_idx=0)\n",
      "  (pooling:genres): MaxPooling()\n",
      "  (attention_pooling:group1): Interest(\n",
      "    (interest_extractor): GRU(16, 16, batch_first=True)\n",
      "    (auxiliary_net): AuxiliaryNet(\n",
      "      (_sequential): Sequential(\n",
      "        (dense0): Linear(in_features=32, out_features=100, bias=True)\n",
      "        (activation0): Sigmoid()\n",
      "        (dense1): Linear(in_features=100, out_features=50, bias=True)\n",
      "        (activation1): Sigmoid()\n",
      "        (final_layer): Linear(in_features=50, out_features=1, bias=True)\n",
      "      )\n",
      "    )\n",
      "    (attention): Attention(\n",
      "      (mlp): MLP(\n",
      "        (_sequential): Sequential(\n",
      "          (dense0): Linear(in_features=64, out_features=16, bias=True)\n",
      "          (batchnorm0): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation0): PReLU(num_parameters=1)\n",
      "          (dropout0): Dropout(p=0.1)\n",
      "          (dense1): Linear(in_features=16, out_features=8, bias=True)\n",
      "          (batchnorm1): BatchNorm1d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "          (activation1): PReLU(num_parameters=1)\n",
      "          (dropout1): Dropout(p=0.1)\n",
      "        )\n",
      "      )\n",
      "      (fc): Linear(in_features=8, out_features=1, bias=True)\n",
      "    )\n",
      "    (interest_evolution): DynamicGRU(\n",
      "      (rnn): AttentionUpdateGateGRUCell()\n",
      "    )\n",
      "  )\n",
      "  (mlp): MLP(\n",
      "    (_sequential): Sequential(\n",
      "      (dense0): Linear(in_features=97, out_features=32, bias=True)\n",
      "      (batchnorm0): BatchNorm1d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation0): PReLU(num_parameters=1)\n",
      "      (dropout0): Dropout(p=0.3)\n",
      "      (dense1): Linear(in_features=32, out_features=16, bias=True)\n",
      "      (batchnorm1): BatchNorm1d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (activation1): PReLU(num_parameters=1)\n",
      "      (dropout1): Dropout(p=0.3)\n",
      "    )\n",
      "  )\n",
      "  (final_layer): Linear(in_features=16, out_features=1, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2469f7f95934f25b075edc25131c3c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='training routine', max=10, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3dc56df1063d4704aed286c2d4e12078",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='train', max=76, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1f5139df42448cbb9c7722d86272807",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='valid', max=19, style=ProgressStyle(description_width='initia…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "scores = []\n",
    "for model in models:\n",
    "    print(model)\n",
    "    loss_func = nn.BCELoss()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-5)\n",
    "    fit(10, model, loss_func, optimizer,\n",
    "        train_loader, valid_loader, notebook=True, auxiliary_loss_rate=0.1)\n",
    "    scores.append(evaluation(valid_df, valid_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.5732876189614773, 0.5810577431308315, 0.5488579415530639, 0.5971491643371369, 0.5990090194493359, 0.5756873408311688, 0.6166408981573868, 0.5890624690848553, 0.6236704721148927]\n"
     ]
    }
   ],
   "source": [
    "print(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}