HW2_.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import lightgbm as lgb\n",
    "from tqdm import tqdm\n",
    "import optuna\n",
    "import pickle\n",
    "import pyarrow.parquet as pq\n",
    "import pyarrow as pa\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "def isrc_to_year(isrc):\n",
    "    if type(isrc) == str:\n",
    "        if int(isrc[5:7]) > 17:\n",
    "            return 1900 + int(isrc[5:7])\n",
    "        else:\n",
    "            return 2000 + int(isrc[5:7])\n",
    "    else:\n",
    "        return np.nan"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ПРЕПРОЦЕССИНГ"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### загрузим все данные"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "                                    \n",
    "songs = pd.read_csv('songs.csv')\n",
    "members = pd.read_csv('members.csv',\n",
    "                     parse_dates=['registration_init_time','expiration_date'])\n",
    "\n",
    "songs_extra = pd.read_csv('song_extra_info.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### составим датасеты"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "songs_mi = songs.loc[:, ['song_id', 'artist_name', 'genre_ids', 'song_length', 'language']]\n",
    "train = train.merge(songs_mi, on='song_id', how='left')\n",
    "\n",
    "\n",
    "members['registration_year'] = members.registration_init_time.apply(lambda x: x.year)\n",
    "members['registration_month'] = members.registration_init_time.apply(lambda x: x.month)\n",
    "members['registration_day'] = members.registration_init_time.apply(lambda x: x.day)\n",
    "\n",
    "members['expiration_year'] = members.expiration_date.apply(lambda x: x.year)\n",
    "members['expiration_month'] = members.expiration_date.apply(lambda x: x.month)\n",
    "members['expiration_day'] = members.expiration_date.apply(lambda x: x.day)\n",
    "\n",
    "members = members.drop(['registration_init_time'], axis=1)\n",
    "members = members.drop(['expiration_date'], axis=1)\n",
    "\n",
    "\n",
    "songs_extra['song_year'] = songs_extra['isrc'].apply(isrc_to_year)\n",
    "songs_extra.drop(['isrc'], axis = 1, inplace = True)\n",
    "\n",
    "train = train.merge(members, on='msno', how='left')\n",
    "train = train.merge(songs_extra, on = 'song_id', how = 'left')\n",
    "\n",
    "# заменим хеши на инты\n",
    "song_dict = dict([(j, i) for i, j in enumerate(train.song_id.unique())])\n",
    "user_dict = dict([(j, i) for i, j in enumerate(train.msno.unique())])\n",
    "\n",
    "train.msno = train.msno.apply(lambda x: user_dict[x])\n",
    "train.song_id = train.song_id.apply(lambda x: song_dict[x])\n",
    "\n",
    "\n",
    "# посмотрим, сколько песен слушали юзеры и сколько юзеров слушали песню\n",
    "train['counter'] = 1\n",
    "user_cnt = train.loc[:, ['msno', 'counter']].groupby('msno').sum()\n",
    "song_cnt = train.loc[:, ['song_id', 'counter']].groupby('song_id').sum()\n",
    "\n",
    "train = train.merge(user_cnt, left_on='msno', right_on=user_cnt.index, suffixes=('', '_user'))\n",
    "train = train.merge(song_cnt, left_on='song_id', right_on=song_cnt.index, suffixes=('', '_song'))\n",
    "train.drop(columns=['counter'], inplace=True)\n",
    "\n",
    "train['song_name_length'] = train.name.apply(lambda x: len(str(x)))\n",
    "train['song_words_count'] = train.name.apply(lambda x: len(str(x).split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "gender                2961479\n",
       "song_year              577858\n",
       "source_screen_name     414804\n",
       "genre_ids              118455\n",
       "source_system_tab       24849\n",
       "source_type             21539\n",
       "name                     1457\n",
       "language                  150\n",
       "artist_name               114\n",
       "song_length               114\n",
       "bd                          0\n",
       "song_id                     0\n",
       "target                      0\n",
       "city                        0\n",
       "song_words_count            0\n",
       "song_name_length            0\n",
       "registered_via              0\n",
       "registration_year           0\n",
       "registration_month          0\n",
       "registration_day            0\n",
       "expiration_year             0\n",
       "expiration_month            0\n",
       "expiration_day              0\n",
       "counter_user                0\n",
       "counter_song                0\n",
       "msno                        0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# посмотрим, что у нас по nan\n",
    "train.isna().sum().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# language, song_length просто дропнем, их не много\n",
    "train = train[train['song_length'].notna() & train['language'].notna()]\n",
    "\n",
    "# неизвестные имена сделаем неизвестными\n",
    "train.loc[train['name'].isna(), ['name']] = '<unknown>'\n",
    "train.loc[train['artist_name'].isna(), ['artist_name']] = '<unknown>'\n",
    "\n",
    "# все остальное заменим на самые частовстречаемые\n",
    "train = train.apply(lambda x: x.fillna(x.value_counts().index[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "song_words_count      0\n",
       "song_name_length      0\n",
       "song_id               0\n",
       "source_system_tab     0\n",
       "source_screen_name    0\n",
       "source_type           0\n",
       "target                0\n",
       "artist_name           0\n",
       "genre_ids             0\n",
       "song_length           0\n",
       "language              0\n",
       "city                  0\n",
       "bd                    0\n",
       "gender                0\n",
       "registered_via        0\n",
       "registration_year     0\n",
       "registration_month    0\n",
       "registration_day      0\n",
       "expiration_year       0\n",
       "expiration_month      0\n",
       "expiration_day        0\n",
       "name                  0\n",
       "song_year             0\n",
       "counter_user          0\n",
       "counter_song          0\n",
       "msno                  0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# нанов больше нет\n",
    "train.isna().sum().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# добавим полное название песни\n",
    "train['full_name'] = train.loc[:, ['name', 'artist_name']].apply(lambda x: x['name'] + ' | ' +  x['artist_name'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.to_parquet('dataset.parquet')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Часть 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df = pq.read_table('dataset.parquet').to_pandas()\n",
    "df = all_df.loc[:, ['msno', 'song_id', 'artist_name', 'genre_ids', 'name', 'target', 'full_name']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# выделим только таргет единица, как понравившееся пользователю\n",
    "df = df.loc[df.target == 1].drop('target', axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 14917, 167154, 149139,  62953])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# разные айдишники на одну песню - супер странно\n",
    "df.loc[(df.artist_name == 'Eminem') & (df.name == 'Lose Yourself')].song_id.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# будем идентифицировать песню по исполнителю и названию\n",
    "# составим дикты соответсвия\n",
    "full_name_to_int_dict = dict([(j, i) for i, j in enumerate(df.full_name.unique())])\n",
    "int_to_full_name_dict = {j:i for i, j in full_name_to_int_dict.items()}\n",
    "df.full_name = df.full_name.apply(lambda x: full_name_to_int_dict[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                      \r"
     ]
    }
   ],
   "source": [
    "# составим плейлисты юзеров \n",
    "play_lists = []\n",
    "for u_id in tqdm(df.msno.unique(), position=0,leave=False):\n",
    "    songs = df.loc[df.msno == u_id].full_name.values.tolist()\n",
    "    play_lists.append(songs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "lengths = []\n",
    "for i in range(len(play_lists)):\n",
    "    lengths.append(len(play_lists[i]))\n",
    "    play_lists[i] = list(map(str, play_lists[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "137.00409397705897"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# средняя длина предложения\n",
    "np.mean(lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# теперь займемся обучением модели\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models.callbacks import CallbackAny2Vec\n",
    "import logging\n",
    "from time import time\n",
    "import multiprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word2Vec(vocab=0, size=128, alpha=0.025)\n"
     ]
    }
   ],
   "source": [
    "logging.basicConfig(format=\"%(asctime)s : %(levelname)s : %(message)s\", level=logging.INFO)\n",
    "\n",
    "class Callback(CallbackAny2Vec):\n",
    "    def __init__(self):\n",
    "        self.epoch = 1\n",
    "        self.training_loss = []\n",
    "\n",
    "    def on_epoch_end(self, model):\n",
    "        loss = model.get_latest_training_loss()\n",
    "        if self.epoch == 1:\n",
    "            current_loss = loss\n",
    "        else:\n",
    "            current_loss = loss - self.loss_previous_step\n",
    "        print(f\"Loss after epoch {self.epoch}: {current_loss}\")\n",
    "        self.training_loss.append(current_loss)\n",
    "        self.epoch += 1\n",
    "        self.loss_previous_step = loss\n",
    "        \n",
    "# обучим word2vec\n",
    "model = Word2Vec(\n",
    "    size = 128,\n",
    "    window = 10,\n",
    "    min_count = 1,\n",
    "    sg = 0,\n",
    "    negative = 20,\n",
    "    workers = multiprocessing.cpu_count())\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-11-03 13:37:08,060 : INFO : collecting all words and their counts\n",
      "2020-11-03 13:37:08,062 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2020-11-03 13:37:08,487 : INFO : PROGRESS: at sentence #10000, processed 2384931 words, keeping 145453 word types\n",
      "2020-11-03 13:37:08,691 : INFO : PROGRESS: at sentence #20000, processed 3550266 words, keeping 181877 word types\n",
      "2020-11-03 13:37:08,727 : INFO : collected 194949 word types from a corpus of 3714592 raw words and 27113 sentences\n",
      "2020-11-03 13:37:08,727 : INFO : Loading a fresh vocabulary\n",
      "2020-11-03 13:37:09,481 : INFO : effective_min_count=1 retains 194949 unique words (100% of original 194949, drops 0)\n",
      "2020-11-03 13:37:09,482 : INFO : effective_min_count=1 leaves 3714592 word corpus (100% of original 3714592, drops 0)\n",
      "2020-11-03 13:37:09,928 : INFO : deleting the raw counts dictionary of 194949 items\n",
      "2020-11-03 13:37:09,931 : INFO : sample=0.001 downsamples 4 most-common words\n",
      "2020-11-03 13:37:09,932 : INFO : downsampling leaves estimated 3712978 word corpus (100.0% of prior 3714592)\n",
      "2020-11-03 13:37:10,346 : INFO : estimated required memory for 194949 words and 128 dimensions: 297102276 bytes\n",
      "2020-11-03 13:37:10,347 : INFO : resetting layer weights\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time to build vocab: 37.92 seconds\n"
     ]
    }
   ],
   "source": [
    "logging.disable(logging.NOTSET) # enable logging\n",
    "t = time()\n",
    "\n",
    "model.build_vocab(play_lists)\n",
    "\n",
    "print(f\"Time to build vocab: {round((time() - t), 2)} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss after epoch 1: 2961812.75\n",
      "Loss after epoch 2: 1639997.75\n",
      "Loss after epoch 3: 1411400.5\n",
      "Loss after epoch 4: 1359397.0\n",
      "Loss after epoch 5: 1291124.0\n",
      "Loss after epoch 6: 1223655.0\n",
      "Loss after epoch 7: 1209704.0\n",
      "Loss after epoch 8: 1182948.0\n",
      "Loss after epoch 9: 1152730.0\n",
      "Loss after epoch 10: 1146496.0\n",
      "Loss after epoch 11: 1131422.0\n",
      "Loss after epoch 12: 1096061.0\n",
      "Loss after epoch 13: 990844.0\n",
      "Loss after epoch 14: 999838.0\n",
      "Loss after epoch 15: 978050.0\n",
      "Loss after epoch 16: 990216.0\n",
      "Loss after epoch 17: 985506.0\n",
      "Loss after epoch 18: 970248.0\n",
      "Loss after epoch 19: 960458.0\n",
      "Loss after epoch 20: 962980.0\n",
      "Loss after epoch 21: 954052.0\n",
      "Loss after epoch 22: 948720.0\n",
      "Loss after epoch 23: 942266.0\n",
      "Loss after epoch 24: 934652.0\n",
      "Loss after epoch 25: 928194.0\n",
      "Loss after epoch 26: 949374.0\n",
      "Loss after epoch 27: 934416.0\n",
      "Loss after epoch 28: 924682.0\n",
      "Loss after epoch 29: 924436.0\n",
      "Loss after epoch 30: 828348.0\n",
      "Loss after epoch 31: 682964.0\n",
      "Loss after epoch 32: 696460.0\n",
      "Loss after epoch 33: 676736.0\n",
      "Loss after epoch 34: 687332.0\n",
      "Loss after epoch 35: 669604.0\n",
      "Loss after epoch 36: 672992.0\n",
      "Loss after epoch 37: 662372.0\n",
      "Loss after epoch 38: 654664.0\n",
      "Loss after epoch 39: 655760.0\n",
      "Loss after epoch 40: 652760.0\n",
      "Loss after epoch 41: 654000.0\n",
      "Loss after epoch 42: 648408.0\n",
      "Loss after epoch 43: 647348.0\n",
      "Loss after epoch 44: 638704.0\n",
      "Loss after epoch 45: 650372.0\n",
      "Loss after epoch 46: 641700.0\n",
      "Loss after epoch 47: 621356.0\n",
      "Loss after epoch 48: 634332.0\n",
      "Loss after epoch 49: 631152.0\n",
      "Loss after epoch 50: 635948.0\n",
      "Loss after epoch 51: 625440.0\n",
      "Loss after epoch 52: 602904.0\n",
      "Loss after epoch 53: 610196.0\n",
      "Loss after epoch 54: 613920.0\n",
      "Loss after epoch 55: 621088.0\n",
      "Loss after epoch 56: 615064.0\n",
      "Loss after epoch 57: 594284.0\n",
      "Loss after epoch 58: 623588.0\n",
      "Loss after epoch 59: 600796.0\n",
      "Loss after epoch 60: 604464.0\n",
      "Loss after epoch 61: 590968.0\n",
      "Loss after epoch 62: 591004.0\n",
      "Loss after epoch 63: 582876.0\n",
      "Loss after epoch 64: 592372.0\n",
      "Loss after epoch 65: 585860.0\n",
      "Loss after epoch 66: 578508.0\n",
      "Loss after epoch 67: 570896.0\n",
      "Loss after epoch 68: 552612.0\n",
      "Loss after epoch 69: 550404.0\n",
      "Loss after epoch 70: 561680.0\n",
      "Loss after epoch 71: 553716.0\n",
      "Loss after epoch 72: 550328.0\n",
      "Loss after epoch 73: 529940.0\n",
      "Loss after epoch 74: 530524.0\n",
      "Loss after epoch 75: 543496.0\n",
      "Loss after epoch 76: 531460.0\n",
      "Loss after epoch 77: 522668.0\n",
      "Loss after epoch 78: 512616.0\n",
      "Loss after epoch 79: 515428.0\n",
      "Loss after epoch 80: 511048.0\n",
      "Loss after epoch 81: 505476.0\n",
      "Loss after epoch 82: 496248.0\n",
      "Loss after epoch 83: 482384.0\n",
      "Loss after epoch 84: 478828.0\n",
      "Loss after epoch 85: 470564.0\n",
      "Loss after epoch 86: 461948.0\n",
      "Loss after epoch 87: 185276.0\n",
      "Loss after epoch 88: 167160.0\n",
      "Loss after epoch 89: 161704.0\n",
      "Loss after epoch 90: 158112.0\n",
      "Loss after epoch 91: 146632.0\n",
      "Loss after epoch 92: 136632.0\n",
      "Loss after epoch 93: 131552.0\n",
      "Loss after epoch 94: 123584.0\n",
      "Loss after epoch 95: 116776.0\n",
      "Loss after epoch 96: 110416.0\n",
      "Loss after epoch 97: 102280.0\n",
      "Loss after epoch 98: 96304.0\n",
      "Loss after epoch 99: 91960.0\n",
      "Loss after epoch 100: 88000.0\n",
      "Time to train the model: 868.46 seconds\n"
     ]
    }
   ],
   "source": [
    "logging.disable(logging.INFO) # disable logging\n",
    "callback = Callback() # instead, print out loss for each epoch\n",
    "t = time()\n",
    "\n",
    "model.train(play_lists,\n",
    "            total_examples = model.corpus_count,\n",
    "            epochs = 100,\n",
    "            compute_loss = True,\n",
    "            callbacks = [callback]) \n",
    "\n",
    "print(f\"Time to train the model: {round((time() - t), 2)} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6424"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_name_to_int_dict['W.T.P. | Eminem']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('6427', 0.7944604158401489)\n",
      "\t        artist_name genre_ids             name\n",
      "3444582      Eminem      1259  Won't Back Down\n",
      "\n",
      "('6426', 0.7898609042167664)\n",
      "\t        artist_name genre_ids    name\n",
      "3444534      Eminem      1259  So Bad\n",
      "\n",
      "('6422', 0.7023847103118896)\n",
      "\t        artist_name genre_ids     name\n",
      "3444269      Eminem      1259  No Love\n",
      "\n",
      "('6421', 0.6940373778343201)\n",
      "\t        artist_name genre_ids        name\n",
      "3444201      Eminem      1259  25 To Life\n",
      "\n",
      "('6428', 0.6697537302970886)\n",
      "\t        artist_name genre_ids     name\n",
      "3444625      Eminem      1259  On Fire\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# посмотрим на симиляры\n",
    "for ms in model.wv.most_similar('6424')[:5]:\n",
    "    print(ms, end='\\n\\t')\n",
    "    found = df.loc[(df.artist_name == 'Eminem') & (df.full_name == int(ms[0])), ['artist_name', 'genre_ids', 'name']].drop_duplicates()\n",
    "    print(found)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                        \r"
     ]
    }
   ],
   "source": [
    "### эмбединги песен у нас есть, ембединг юзера будет средним эмбедингом его песен из плейлиста\n",
    "user_emb = dict()\n",
    "counter = 0\n",
    "for user_id in tqdm(df.msno.unique(), position=0, leave=False):\n",
    "    songs_embeds = [model.wv[str(i)] for i in play_lists[counter]]\n",
    "    user_emb[user_id] = np.array(songs_embeds).mean(axis = 0)\n",
    "    counter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                             \r"
     ]
    }
   ],
   "source": [
    "# посчитаем скалярное произведение между эмбедингом юзера и песней, которую он слушал \n",
    "new_feature = []\n",
    "for row in tqdm(all_df[['msno', 'full_name']].to_numpy(), position=0,leave=False):\n",
    "    user, song = row\n",
    "    \n",
    "\n",
    "    u_emb = user_emb[user] if user in user_emb else np.zeros(128)\n",
    "        \n",
    "    if song in full_name_to_int_dict and str(full_name_to_int_dict[song]) in model.wv:\n",
    "        s_emb = model.wv[str(full_name_to_int_dict[song])]\n",
    "    else:\n",
    "        s_emb = np.zeros(128)\n",
    "        \n",
    "    new_feature.append((u_emb * s_emb).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"emb_dists.txt\", \"wb\") as fp:  \n",
    "       pickle.dump(new_feature, fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ЧАСТЬ 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df = pq.read_table('dataset.parquet').to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['msno', 'song_id', 'source_system_tab', 'source_screen_name',\n",
       "       'source_type', 'target', 'artist_name', 'genre_ids', 'song_length',\n",
       "       'language', 'city', 'bd', 'gender', 'registered_via',\n",
       "       'registration_year', 'registration_month', 'registration_day',\n",
       "       'expiration_year', 'expiration_month', 'expiration_day', 'name',\n",
       "       'song_year', 'counter_user', 'counter_song', 'song_name_length',\n",
       "       'song_words_count', 'full_name'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df.drop(columns = ['msno', 'song_id', 'artist_name', 'name', 'full_name'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in all_df.columns:\n",
    "    if col not in ['target', 'counter_song', 'counter_user', 'song_name_length', 'song_words_count']:\n",
    "        all_df[col] = all_df[col].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = all_df.drop(['target'], axis=1)\n",
    "y = all_df['target']\n",
    "kf = KFold(n_splits=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### сделаем небольшой серч оптимальных параметров"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective(trial):\n",
    "    param = {\n",
    "        'objective': 'binary',\n",
    "        'boosting': \"dart\",\n",
    "        'learning_rate': trial.suggest_float('learning_rate', 0.15, 0.35, step = 0.05),\n",
    "        'max_depth': trial.suggest_int('max_depth', 4, 8),\n",
    "        'boosting_type': 'dart',\n",
    "        \"lambda_l1\":trial.suggest_float('lambda_l1', 0.0, 1.0, step = 0.1),\n",
    "        \"lambda_l2\":trial.suggest_float('lambda_l2', 0.0, 1.0, step = 0.1),\n",
    "        \"metric\":'auc',\n",
    "        \"bagging_freq\": 1,\n",
    "        \"seed\": 0,\n",
    "        \"nthread\": -1\n",
    "        \n",
    "    }\n",
    "    \n",
    "    lgbnumround = 50\n",
    "    evals = {}\n",
    "    gbm = lgb.train(param, \n",
    "                    lgbtrain, \n",
    "                    lgbnumround, \n",
    "                    valid_sets = [lgbtrain, lgbval], \n",
    "                    verbose_eval = None,\n",
    "                    evals_result=evals)\n",
    "    auc = np.array(evals['valid_1']['auc'])\n",
    "    return np.max(auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start iteration\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2020-11-03 12:20:30,296]\u001b[0m A new study created in memory with name: no-name-177524b8-e562-4f0c-8273-a09e559b5f99\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training LGBM model...\n",
      "prepare model\n",
      "optimizing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2020-11-03 12:21:45,427]\u001b[0m Trial 0 finished with value: 0.7098320505279004 and parameters: {'learning_rate': 0.35, 'max_depth': 5, 'lambda_l1': 0.7000000000000001, 'lambda_l2': 0.7000000000000001}. Best is trial 0 with value: 0.7098320505279004.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:23:01,137]\u001b[0m Trial 1 finished with value: 0.7117915954433376 and parameters: {'learning_rate': 0.25, 'max_depth': 8, 'lambda_l1': 0.5, 'lambda_l2': 0.8}. Best is trial 1 with value: 0.7117915954433376.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:24:14,474]\u001b[0m Trial 2 finished with value: 0.7038394460991109 and parameters: {'learning_rate': 0.15, 'max_depth': 6, 'lambda_l1': 0.9, 'lambda_l2': 0.0}. Best is trial 1 with value: 0.7117915954433376.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:25:33,450]\u001b[0m Trial 3 finished with value: 0.7076576996093258 and parameters: {'learning_rate': 0.2, 'max_depth': 7, 'lambda_l1': 0.7000000000000001, 'lambda_l2': 0.4}. Best is trial 1 with value: 0.7117915954433376.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:26:43,723]\u001b[0m Trial 4 finished with value: 0.7084141917561899 and parameters: {'learning_rate': 0.2, 'max_depth': 7, 'lambda_l1': 0.30000000000000004, 'lambda_l2': 0.1}. Best is trial 1 with value: 0.7117915954433376.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:28:02,344]\u001b[0m Trial 5 finished with value: 0.7051634254289505 and parameters: {'learning_rate': 0.15, 'max_depth': 8, 'lambda_l1': 0.8, 'lambda_l2': 0.7000000000000001}. Best is trial 1 with value: 0.7117915954433376.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:29:18,474]\u001b[0m Trial 6 finished with value: 0.7049437065824448 and parameters: {'learning_rate': 0.15, 'max_depth': 8, 'lambda_l1': 0.30000000000000004, 'lambda_l2': 0.7000000000000001}. Best is trial 1 with value: 0.7117915954433376.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:30:10,617]\u001b[0m Trial 7 finished with value: 0.6971830925203848 and parameters: {'learning_rate': 0.15, 'max_depth': 4, 'lambda_l1': 0.8, 'lambda_l2': 0.8}. Best is trial 1 with value: 0.7117915954433376.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:31:03,493]\u001b[0m Trial 8 finished with value: 0.7013119265692485 and parameters: {'learning_rate': 0.25, 'max_depth': 4, 'lambda_l1': 0.2, 'lambda_l2': 0.30000000000000004}. Best is trial 1 with value: 0.7117915954433376.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:32:12,034]\u001b[0m Trial 9 finished with value: 0.7084511256173971 and parameters: {'learning_rate': 0.2, 'max_depth': 8, 'lambda_l1': 1.0, 'lambda_l2': 0.8}. Best is trial 1 with value: 0.7117915954433376.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start iteration\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2020-11-03 12:32:16,821]\u001b[0m A new study created in memory with name: no-name-7c49b9d2-21a9-413e-accc-79498e398dfd\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training LGBM model...\n",
      "prepare model\n",
      "optimizing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2020-11-03 12:33:13,644]\u001b[0m Trial 0 finished with value: 0.7111628875307786 and parameters: {'learning_rate': 0.30000000000000004, 'max_depth': 4, 'lambda_l1': 0.8, 'lambda_l2': 0.7000000000000001}. Best is trial 0 with value: 0.7111628875307786.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:34:20,134]\u001b[0m Trial 1 finished with value: 0.7187925249025774 and parameters: {'learning_rate': 0.30000000000000004, 'max_depth': 6, 'lambda_l1': 0.6000000000000001, 'lambda_l2': 0.4}. Best is trial 1 with value: 0.7187925249025774.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:35:29,361]\u001b[0m Trial 2 finished with value: 0.7149347393682349 and parameters: {'learning_rate': 0.2, 'max_depth': 7, 'lambda_l1': 0.7000000000000001, 'lambda_l2': 0.1}. Best is trial 1 with value: 0.7187925249025774.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:36:40,795]\u001b[0m Trial 3 finished with value: 0.7164348792910947 and parameters: {'learning_rate': 0.2, 'max_depth': 8, 'lambda_l1': 0.7000000000000001, 'lambda_l2': 0.2}. Best is trial 1 with value: 0.7187925249025774.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:37:34,995]\u001b[0m Trial 4 finished with value: 0.7097748991509469 and parameters: {'learning_rate': 0.25, 'max_depth': 4, 'lambda_l1': 0.5, 'lambda_l2': 0.30000000000000004}. Best is trial 1 with value: 0.7187925249025774.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:38:40,453]\u001b[0m Trial 5 finished with value: 0.7159488401136549 and parameters: {'learning_rate': 0.30000000000000004, 'max_depth': 5, 'lambda_l1': 0.30000000000000004, 'lambda_l2': 1.0}. Best is trial 1 with value: 0.7187925249025774.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:39:45,399]\u001b[0m Trial 6 finished with value: 0.7159466849814081 and parameters: {'learning_rate': 0.30000000000000004, 'max_depth': 5, 'lambda_l1': 0.6000000000000001, 'lambda_l2': 0.8}. Best is trial 1 with value: 0.7187925249025774.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:40:54,042]\u001b[0m Trial 7 finished with value: 0.7167831156541578 and parameters: {'learning_rate': 0.25, 'max_depth': 6, 'lambda_l1': 0.4, 'lambda_l2': 0.6000000000000001}. Best is trial 1 with value: 0.7187925249025774.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:41:47,807]\u001b[0m Trial 8 finished with value: 0.7111628007277504 and parameters: {'learning_rate': 0.30000000000000004, 'max_depth': 4, 'lambda_l1': 0.8, 'lambda_l2': 0.8}. Best is trial 1 with value: 0.7187925249025774.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:42:55,398]\u001b[0m Trial 9 finished with value: 0.7113781633000735 and parameters: {'learning_rate': 0.15, 'max_depth': 6, 'lambda_l1': 1.0, 'lambda_l2': 0.7000000000000001}. Best is trial 1 with value: 0.7187925249025774.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start iteration\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2020-11-03 12:43:00,029]\u001b[0m A new study created in memory with name: no-name-53f96dd1-05f9-4dd0-84fd-ff6e062bcc6a\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training LGBM model...\n",
      "prepare model\n",
      "optimizing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2020-11-03 12:43:58,040]\u001b[0m Trial 0 finished with value: 0.7074574337498178 and parameters: {'learning_rate': 0.2, 'max_depth': 4, 'lambda_l1': 0.6000000000000001, 'lambda_l2': 0.5}. Best is trial 0 with value: 0.7074574337498178.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:45:07,505]\u001b[0m Trial 1 finished with value: 0.7163038086937794 and parameters: {'learning_rate': 0.25, 'max_depth': 6, 'lambda_l1': 0.1, 'lambda_l2': 0.6000000000000001}. Best is trial 1 with value: 0.7163038086937794.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:46:01,439]\u001b[0m Trial 2 finished with value: 0.7117599102253058 and parameters: {'learning_rate': 0.35, 'max_depth': 4, 'lambda_l1': 0.2, 'lambda_l2': 0.1}. Best is trial 1 with value: 0.7163038086937794.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:47:10,408]\u001b[0m Trial 3 finished with value: 0.7119215852529962 and parameters: {'learning_rate': 0.15, 'max_depth': 7, 'lambda_l1': 0.7000000000000001, 'lambda_l2': 0.6000000000000001}. Best is trial 1 with value: 0.7163038086937794.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:48:17,458]\u001b[0m Trial 4 finished with value: 0.7110229315149208 and parameters: {'learning_rate': 0.15, 'max_depth': 6, 'lambda_l1': 0.2, 'lambda_l2': 0.5}. Best is trial 1 with value: 0.7163038086937794.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:49:25,321]\u001b[0m Trial 5 finished with value: 0.7134782049357412 and parameters: {'learning_rate': 0.2, 'max_depth': 6, 'lambda_l1': 0.4, 'lambda_l2': 0.7000000000000001}. Best is trial 1 with value: 0.7163038086937794.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:50:34,866]\u001b[0m Trial 6 finished with value: 0.7121806470755698 and parameters: {'learning_rate': 0.15, 'max_depth': 8, 'lambda_l1': 0.8, 'lambda_l2': 1.0}. Best is trial 1 with value: 0.7163038086937794.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:51:37,206]\u001b[0m Trial 7 finished with value: 0.709315956609259 and parameters: {'learning_rate': 0.15, 'max_depth': 5, 'lambda_l1': 0.1, 'lambda_l2': 0.1}. Best is trial 1 with value: 0.7163038086937794.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:52:46,514]\u001b[0m Trial 8 finished with value: 0.7121812565920569 and parameters: {'learning_rate': 0.15, 'max_depth': 8, 'lambda_l1': 0.8, 'lambda_l2': 0.5}. Best is trial 1 with value: 0.7163038086937794.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:53:57,428]\u001b[0m Trial 9 finished with value: 0.7185010866558978 and parameters: {'learning_rate': 0.30000000000000004, 'max_depth': 7, 'lambda_l1': 0.8, 'lambda_l2': 1.0}. Best is trial 9 with value: 0.7185010866558978.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start iteration\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2020-11-03 12:54:02,048]\u001b[0m A new study created in memory with name: no-name-ec922830-687b-4bcb-bfb5-50f1ac0be9b6\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training LGBM model...\n",
      "prepare model\n",
      "optimizing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2020-11-03 12:55:01,775]\u001b[0m Trial 0 finished with value: 0.7118640948508781 and parameters: {'learning_rate': 0.35, 'max_depth': 4, 'lambda_l1': 0.1, 'lambda_l2': 0.7000000000000001}. Best is trial 0 with value: 0.7118640948508781.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:56:18,686]\u001b[0m Trial 1 finished with value: 0.7173261030419887 and parameters: {'learning_rate': 0.25, 'max_depth': 8, 'lambda_l1': 0.7000000000000001, 'lambda_l2': 0.5}. Best is trial 1 with value: 0.7173261030419887.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:57:25,774]\u001b[0m Trial 2 finished with value: 0.7206094553990315 and parameters: {'learning_rate': 0.35, 'max_depth': 7, 'lambda_l1': 1.0, 'lambda_l2': 0.0}. Best is trial 2 with value: 0.7206094553990315.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:58:26,490]\u001b[0m Trial 3 finished with value: 0.715244856966806 and parameters: {'learning_rate': 0.30000000000000004, 'max_depth': 5, 'lambda_l1': 0.5, 'lambda_l2': 1.0}. Best is trial 2 with value: 0.7206094553990315.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 12:59:33,429]\u001b[0m Trial 4 finished with value: 0.7140955090860741 and parameters: {'learning_rate': 0.2, 'max_depth': 7, 'lambda_l1': 0.1, 'lambda_l2': 0.7000000000000001}. Best is trial 2 with value: 0.7206094553990315.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:00:38,098]\u001b[0m Trial 5 finished with value: 0.7106878554653575 and parameters: {'learning_rate': 0.15, 'max_depth': 6, 'lambda_l1': 0.9, 'lambda_l2': 0.5}. Best is trial 2 with value: 0.7206094553990315.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:01:27,399]\u001b[0m Trial 6 finished with value: 0.7053360057082128 and parameters: {'learning_rate': 0.15, 'max_depth': 4, 'lambda_l1': 0.9, 'lambda_l2': 0.4}. Best is trial 2 with value: 0.7206094553990315.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:02:32,263]\u001b[0m Trial 7 finished with value: 0.713375045939524 and parameters: {'learning_rate': 0.2, 'max_depth': 6, 'lambda_l1': 0.0, 'lambda_l2': 0.6000000000000001}. Best is trial 2 with value: 0.7206094553990315.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:03:39,891]\u001b[0m Trial 8 finished with value: 0.7171742577958456 and parameters: {'learning_rate': 0.25, 'max_depth': 7, 'lambda_l1': 0.7000000000000001, 'lambda_l2': 0.8}. Best is trial 2 with value: 0.7206094553990315.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:04:47,752]\u001b[0m Trial 9 finished with value: 0.7208923903253939 and parameters: {'learning_rate': 0.35, 'max_depth': 8, 'lambda_l1': 1.0, 'lambda_l2': 0.9}. Best is trial 9 with value: 0.7208923903253939.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start iteration\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2020-11-03 13:04:52,237]\u001b[0m A new study created in memory with name: no-name-16122028-8caa-4c58-be9f-2ac2110d355b\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training LGBM model...\n",
      "prepare model\n",
      "optimizing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2020-11-03 13:06:02,978]\u001b[0m Trial 0 finished with value: 0.7090396956551177 and parameters: {'learning_rate': 0.2, 'max_depth': 6, 'lambda_l1': 0.0, 'lambda_l2': 0.7000000000000001}. Best is trial 0 with value: 0.7090396956551177.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:07:08,555]\u001b[0m Trial 1 finished with value: 0.7134012943408412 and parameters: {'learning_rate': 0.30000000000000004, 'max_depth': 6, 'lambda_l1': 0.8, 'lambda_l2': 0.2}. Best is trial 1 with value: 0.7134012943408412.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:08:21,502]\u001b[0m Trial 2 finished with value: 0.7155996961718344 and parameters: {'learning_rate': 0.30000000000000004, 'max_depth': 7, 'lambda_l1': 0.4, 'lambda_l2': 1.0}. Best is trial 2 with value: 0.7155996961718344.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:09:46,448]\u001b[0m Trial 3 finished with value: 0.7072535467037864 and parameters: {'learning_rate': 0.15, 'max_depth': 8, 'lambda_l1': 0.6000000000000001, 'lambda_l2': 0.5}. Best is trial 2 with value: 0.7155996961718344.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:11:06,308]\u001b[0m Trial 4 finished with value: 0.715894918094139 and parameters: {'learning_rate': 0.30000000000000004, 'max_depth': 8, 'lambda_l1': 0.30000000000000004, 'lambda_l2': 1.0}. Best is trial 4 with value: 0.715894918094139.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:12:16,436]\u001b[0m Trial 5 finished with value: 0.7096256599512459 and parameters: {'learning_rate': 0.2, 'max_depth': 7, 'lambda_l1': 0.8, 'lambda_l2': 0.0}. Best is trial 4 with value: 0.715894918094139.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:13:28,439]\u001b[0m Trial 6 finished with value: 0.7120234924897915 and parameters: {'learning_rate': 0.25, 'max_depth': 6, 'lambda_l1': 0.1, 'lambda_l2': 1.0}. Best is trial 4 with value: 0.715894918094139.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:14:41,249]\u001b[0m Trial 7 finished with value: 0.7147761736039628 and parameters: {'learning_rate': 0.30000000000000004, 'max_depth': 7, 'lambda_l1': 0.2, 'lambda_l2': 1.0}. Best is trial 4 with value: 0.715894918094139.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:15:42,330]\u001b[0m Trial 8 finished with value: 0.710730721585935 and parameters: {'learning_rate': 0.30000000000000004, 'max_depth': 5, 'lambda_l1': 0.8, 'lambda_l2': 0.6000000000000001}. Best is trial 4 with value: 0.715894918094139.\u001b[0m\n",
      "\u001b[32m[I 2020-11-03 13:16:49,380]\u001b[0m Trial 9 finished with value: 0.7109315543837942 and parameters: {'learning_rate': 0.35, 'max_depth': 5, 'lambda_l1': 0.4, 'lambda_l2': 0.8}. Best is trial 4 with value: 0.715894918094139.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "search_params = []\n",
    "for train_index, test_index in kf.split(X):\n",
    "    print(\"Start iteration\")\n",
    "    # test\n",
    "    X_test = X.iloc[test_index]\n",
    "    y_test = y.iloc[test_index]\n",
    "    \n",
    "    XX = X.iloc[train_index]\n",
    "    yy = y.iloc[train_index]\n",
    "    \n",
    "    X_train, X_val, y_train, y_val = train_test_split(\n",
    "                                     XX, yy, test_size=0.2, random_state=42, stratify=yy)\n",
    "    \n",
    "    print('Training LGBM model...')\n",
    "    \n",
    "    print('prepare model')\n",
    "    lgbtrain = lgb.Dataset(X_train, y_train)\n",
    "    lgbval = lgb.Dataset(X_val, y_val)\n",
    "    \n",
    "    print('optimizing')\n",
    "    \n",
    "    study = optuna.create_study(direction='maximize')\n",
    "    study.optimize(objective, n_trials=10)\n",
    "    \n",
    "    search_params.append(study.best_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.30000000000000004, 8, 0.5, 1.0)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "lr, md, l1, l2 = [], [], [], []\n",
    "for par in search_params:\n",
    "    lr.append(par['learning_rate'])\n",
    "    md.append(par['max_depth'])\n",
    "    l1.append(par['lambda_l1'])\n",
    "    l2.append(par['lambda_l2'])\n",
    "    \n",
    "# получили параметры\n",
    "Counter(lr).most_common()[0][0], Counter(md).most_common()[0][0], Counter(l1).most_common()[0][0], Counter(l2).most_common()[0][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### теперь обучим модель, посмотрим на скор"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start iteration\n",
      "Training LGBM model...\n",
      "prepare model\n",
      "learning...\n",
      "[10]\ttraining's auc: 0.727915\tvalid_1's auc: 0.722392\n",
      "[20]\ttraining's auc: 0.73816\tvalid_1's auc: 0.729779\n",
      "[30]\ttraining's auc: 0.743872\tvalid_1's auc: 0.733293\n",
      "[40]\ttraining's auc: 0.748675\tvalid_1's auc: 0.736091\n",
      "[50]\ttraining's auc: 0.753476\tvalid_1's auc: 0.738577\n",
      "Start iteration\n",
      "Training LGBM model...\n",
      "prepare model\n",
      "learning...\n",
      "[10]\ttraining's auc: 0.730258\tvalid_1's auc: 0.726342\n",
      "[20]\ttraining's auc: 0.739902\tvalid_1's auc: 0.73352\n",
      "[30]\ttraining's auc: 0.742915\tvalid_1's auc: 0.734844\n",
      "[40]\ttraining's auc: 0.751306\tvalid_1's auc: 0.741302\n",
      "[50]\ttraining's auc: 0.753944\tvalid_1's auc: 0.742281\n",
      "Start iteration\n",
      "Training LGBM model...\n",
      "prepare model\n",
      "learning...\n",
      "[10]\ttraining's auc: 0.729907\tvalid_1's auc: 0.724619\n",
      "[20]\ttraining's auc: 0.741004\tvalid_1's auc: 0.733134\n",
      "[30]\ttraining's auc: 0.748524\tvalid_1's auc: 0.738783\n",
      "[40]\ttraining's auc: 0.753681\tvalid_1's auc: 0.741869\n",
      "[50]\ttraining's auc: 0.757773\tvalid_1's auc: 0.743896\n",
      "Start iteration\n",
      "Training LGBM model...\n",
      "prepare model\n",
      "learning...\n",
      "[10]\ttraining's auc: 0.729619\tvalid_1's auc: 0.72415\n",
      "[20]\ttraining's auc: 0.740555\tvalid_1's auc: 0.732255\n",
      "[30]\ttraining's auc: 0.746964\tvalid_1's auc: 0.736645\n",
      "[40]\ttraining's auc: 0.752138\tvalid_1's auc: 0.740104\n",
      "[50]\ttraining's auc: 0.755689\tvalid_1's auc: 0.741841\n",
      "Start iteration\n",
      "Training LGBM model...\n",
      "prepare model\n",
      "learning...\n",
      "[10]\ttraining's auc: 0.72502\tvalid_1's auc: 0.720336\n",
      "[20]\ttraining's auc: 0.735824\tvalid_1's auc: 0.728396\n",
      "[30]\ttraining's auc: 0.741358\tvalid_1's auc: 0.731583\n",
      "[40]\ttraining's auc: 0.748075\tvalid_1's auc: 0.73575\n",
      "[50]\ttraining's auc: 0.750957\tvalid_1's auc: 0.736864\n"
     ]
    }
   ],
   "source": [
    "k_fold_results = []\n",
    "for train_index, test_index in kf.split(X):\n",
    "    print(\"Start iteration\")\n",
    "    # test\n",
    "    X_test = X.iloc[test_index]\n",
    "    y_test = y.iloc[test_index]\n",
    "    \n",
    "    XX = X.iloc[train_index]\n",
    "    yy = y.iloc[train_index]\n",
    "    \n",
    "    X_train, X_val, y_train, y_val = train_test_split(\n",
    "                                     XX, yy, test_size=0.2, random_state=42, stratify=yy)\n",
    "    \n",
    "    print('Training LGBM model...')\n",
    "    \n",
    "    print('prepare model')\n",
    "    lgbtrain = lgb.Dataset(X_train, y_train)\n",
    "    lgbval = lgb.Dataset(X_val, y_val)\n",
    "\n",
    "\n",
    "    params = {}\n",
    "    params['learning_rate'] = 0.3\n",
    "    params['application'] = 'binary'\n",
    "    params['max_depth'] = 8\n",
    "    params['num_leaves'] = 2**8\n",
    "    params['lambda_l1'] = 0.5\n",
    "    params['lambda_l2'] = 1.0\n",
    "    params['verbosity'] = 0\n",
    "    params['metric'] = 'auc'\n",
    "    \n",
    "    print('learning...')\n",
    "    evals = {}\n",
    "    gbm = lgb.train(params, \n",
    "                      train_set=lgbtrain, \n",
    "                      num_boost_round=50, \n",
    "                      valid_sets=[lgbtrain, lgbval],\n",
    "                      verbose_eval=10,\n",
    "                      evals_result=evals,)\n",
    "    \n",
    "    y_pred = gbm.predict(X_test)\n",
    "    scr = roc_auc_score(y_test, y_pred)\n",
    "    \n",
    "    df_feat = pd.DataFrame(data=gbm.feature_importance(importance_type='gain'), index=gbm.feature_name(), columns=['importance'])\n",
    "    df_feat = df_feat.sort_values(by='importance', ascending=False)\n",
    "    \n",
    "    \n",
    "    k_fold_results.append((gbm, scr, df_feat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "imps = []\n",
    "auc_score = []\n",
    "models = []\n",
    "for rez in k_fold_results:\n",
    "    imps.append(rez[-1].sort_values('importance', ascending=False))\n",
    "    auc_score.append(rez[1])\n",
    "    models.append(rez[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>source_type</th>\n",
       "      <td>586850.746804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>counter_song</th>\n",
       "      <td>338298.725428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>song_length</th>\n",
       "      <td>169590.544187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>counter_user</th>\n",
       "      <td>141293.571653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>registration_day</th>\n",
       "      <td>124443.048590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>expiration_day</th>\n",
       "      <td>118182.034976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bd</th>\n",
       "      <td>115117.126807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_screen_name</th>\n",
       "      <td>107842.496414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_system_tab</th>\n",
       "      <td>40406.107944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>registration_year</th>\n",
       "      <td>31605.048764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city</th>\n",
       "      <td>29988.301859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>registration_month</th>\n",
       "      <td>25852.095968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>genre_ids</th>\n",
       "      <td>19781.701336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>expiration_month</th>\n",
       "      <td>16690.981862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>expiration_year</th>\n",
       "      <td>12524.791487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>registered_via</th>\n",
       "      <td>6008.558939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>song_year</th>\n",
       "      <td>4672.939569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>language</th>\n",
       "      <td>4664.260911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>song_name_length</th>\n",
       "      <td>1834.818424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gender</th>\n",
       "      <td>862.732890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>song_words_count</th>\n",
       "      <td>396.827509</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       importance\n",
       "source_type         586850.746804\n",
       "counter_song        338298.725428\n",
       "song_length         169590.544187\n",
       "counter_user        141293.571653\n",
       "registration_day    124443.048590\n",
       "expiration_day      118182.034976\n",
       "bd                  115117.126807\n",
       "source_screen_name  107842.496414\n",
       "source_system_tab    40406.107944\n",
       "registration_year    31605.048764\n",
       "city                 29988.301859\n",
       "registration_month   25852.095968\n",
       "genre_ids            19781.701336\n",
       "expiration_month     16690.981862\n",
       "expiration_year      12524.791487\n",
       "registered_via        6008.558939\n",
       "song_year             4672.939569\n",
       "language              4664.260911\n",
       "song_name_length      1834.818424\n",
       "gender                 862.732890\n",
       "song_words_count       396.827509"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(sum(imps) / len(imps)).sort_values('importance', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7130743127788632"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "one_auc = np.mean(auc_score)\n",
    "one_auc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ЧАСТЬ 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df = pq.read_table('dataset.parquet').to_pandas()\n",
    "with open(\"emb_dists.txt\", \"rb\") as fp:   \n",
    "       new_feature = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df.drop(columns = ['msno', 'song_id', 'artist_name', 'name', 'full_name'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in all_df.columns:\n",
    "    if col not in ['target', 'counter_song', 'counter_user', 'song_name_length', 'song_words_count']:\n",
    "        all_df[col] = all_df[col].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "# добавим фичу из эмбедингов\n",
    "all_df['emb_dists'] = new_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = all_df.drop(['target'], axis=1)\n",
    "y = all_df['target']\n",
    "kf = KFold(n_splits=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start iteration\n",
      "Training LGBM model...\n",
      "prepare model\n",
      "learning...\n",
      "[10]\ttraining's auc: 0.772108\tvalid_1's auc: 0.767193\n",
      "[20]\ttraining's auc: 0.784129\tvalid_1's auc: 0.77667\n",
      "[30]\ttraining's auc: 0.790766\tvalid_1's auc: 0.781378\n",
      "[40]\ttraining's auc: 0.795266\tvalid_1's auc: 0.784036\n",
      "[50]\ttraining's auc: 0.800229\tvalid_1's auc: 0.786863\n",
      "Start iteration\n",
      "Training LGBM model...\n",
      "prepare model\n",
      "learning...\n",
      "[10]\ttraining's auc: 0.777946\tvalid_1's auc: 0.774984\n",
      "[20]\ttraining's auc: 0.791456\tvalid_1's auc: 0.78567\n",
      "[30]\ttraining's auc: 0.796954\tvalid_1's auc: 0.789271\n",
      "[40]\ttraining's auc: 0.800277\tvalid_1's auc: 0.790995\n",
      "[50]\ttraining's auc: 0.804188\tvalid_1's auc: 0.793286\n",
      "Start iteration\n",
      "Training LGBM model...\n",
      "prepare model\n",
      "learning...\n",
      "[10]\ttraining's auc: 0.779132\tvalid_1's auc: 0.774382\n",
      "[20]\ttraining's auc: 0.789768\tvalid_1's auc: 0.782577\n",
      "[30]\ttraining's auc: 0.797951\tvalid_1's auc: 0.788759\n",
      "[40]\ttraining's auc: 0.800483\tvalid_1's auc: 0.789803\n",
      "[50]\ttraining's auc: 0.803698\tvalid_1's auc: 0.791462\n",
      "Start iteration\n",
      "Training LGBM model...\n",
      "prepare model\n",
      "learning...\n",
      "[10]\ttraining's auc: 0.776611\tvalid_1's auc: 0.772206\n",
      "[20]\ttraining's auc: 0.789814\tvalid_1's auc: 0.78252\n",
      "[30]\ttraining's auc: 0.795374\tvalid_1's auc: 0.785858\n",
      "[40]\ttraining's auc: 0.800942\tvalid_1's auc: 0.789623\n",
      "[50]\ttraining's auc: 0.805558\tvalid_1's auc: 0.792452\n",
      "Start iteration\n",
      "Training LGBM model...\n",
      "prepare model\n",
      "learning...\n",
      "[10]\ttraining's auc: 0.75761\tvalid_1's auc: 0.752761\n",
      "[20]\ttraining's auc: 0.768652\tvalid_1's auc: 0.761247\n",
      "[30]\ttraining's auc: 0.776405\tvalid_1's auc: 0.766765\n",
      "[40]\ttraining's auc: 0.780766\tvalid_1's auc: 0.769166\n",
      "[50]\ttraining's auc: 0.786503\tvalid_1's auc: 0.772606\n"
     ]
    }
   ],
   "source": [
    "k_fold_results = []\n",
    "for train_index, test_index in kf.split(X):\n",
    "    print(\"Start iteration\")\n",
    "    # test\n",
    "    X_test = X.iloc[test_index]\n",
    "    y_test = y.iloc[test_index]\n",
    "    \n",
    "    XX = X.iloc[train_index]\n",
    "    yy = y.iloc[train_index]\n",
    "    \n",
    "    X_train, X_val, y_train, y_val = train_test_split(\n",
    "                                     XX, yy, test_size=0.2, random_state=42, stratify=yy)\n",
    "    \n",
    "    print('Training LGBM model...')\n",
    "    \n",
    "    print('prepare model')\n",
    "    lgbtrain = lgb.Dataset(X_train, y_train)\n",
    "    lgbval = lgb.Dataset(X_val, y_val)\n",
    "\n",
    "\n",
    "    params = {}\n",
    "    params['learning_rate'] = 0.3\n",
    "    params['application'] = 'binary'\n",
    "    params['max_depth'] = 8\n",
    "    params['num_leaves'] = 2**8\n",
    "    params['lambda_l1'] = 0.5\n",
    "    params['lambda_l2'] = 1.0\n",
    "    params['verbosity'] = 0\n",
    "    params['metric'] = 'auc'\n",
    "    \n",
    "    print('learning...')\n",
    "    evals = {}\n",
    "    gbm = lgb.train(params, \n",
    "                      train_set=lgbtrain, \n",
    "                      num_boost_round=50, \n",
    "                      valid_sets=[lgbtrain, lgbval],\n",
    "                      verbose_eval=10,\n",
    "                      evals_result=evals,)\n",
    "    \n",
    "    y_pred = gbm.predict(X_test)\n",
    "    scr = roc_auc_score(y_test, y_pred)\n",
    "    \n",
    "    df_feat = pd.DataFrame(data=gbm.feature_importance(importance_type='gain'), index=gbm.feature_name(), columns=['importance'])\n",
    "    df_feat = df_feat.sort_values(by='importance', ascending=False)\n",
    "    \n",
    "    \n",
    "    k_fold_results.append((gbm, scr, df_feat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "imps = []\n",
    "auc_score = []\n",
    "models = []\n",
    "for rez in k_fold_results:\n",
    "    imps.append(rez[-1].sort_values('importance', ascending=False))\n",
    "    auc_score.append(rez[1])\n",
    "    models.append(rez[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>emb_dists</th>\n",
       "      <td>933222.758434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_type</th>\n",
       "      <td>510738.988739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>counter_song</th>\n",
       "      <td>300752.419416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>song_length</th>\n",
       "      <td>274674.080472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>counter_user</th>\n",
       "      <td>207224.594504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_screen_name</th>\n",
       "      <td>118807.610065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>registration_day</th>\n",
       "      <td>89490.949817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>expiration_day</th>\n",
       "      <td>85655.890956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bd</th>\n",
       "      <td>82546.804894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source_system_tab</th>\n",
       "      <td>56707.701243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>registration_year</th>\n",
       "      <td>25892.108803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city</th>\n",
       "      <td>22840.434104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>genre_ids</th>\n",
       "      <td>20971.813381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>registration_month</th>\n",
       "      <td>18701.198527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>song_year</th>\n",
       "      <td>17140.184943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>expiration_month</th>\n",
       "      <td>16121.770447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>expiration_year</th>\n",
       "      <td>12116.899383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>language</th>\n",
       "      <td>6429.690987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>registered_via</th>\n",
       "      <td>2989.451714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>song_name_length</th>\n",
       "      <td>1898.495577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gender</th>\n",
       "      <td>853.901129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>song_words_count</th>\n",
       "      <td>294.080535</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       importance\n",
       "emb_dists           933222.758434\n",
       "source_type         510738.988739\n",
       "counter_song        300752.419416\n",
       "song_length         274674.080472\n",
       "counter_user        207224.594504\n",
       "source_screen_name  118807.610065\n",
       "registration_day     89490.949817\n",
       "expiration_day       85655.890956\n",
       "bd                   82546.804894\n",
       "source_system_tab    56707.701243\n",
       "registration_year    25892.108803\n",
       "city                 22840.434104\n",
       "genre_ids            20971.813381\n",
       "registration_month   18701.198527\n",
       "song_year            17140.184943\n",
       "expiration_month     16121.770447\n",
       "expiration_year      12116.899383\n",
       "language              6429.690987\n",
       "registered_via        2989.451714\n",
       "song_name_length      1898.495577\n",
       "gender                 853.901129\n",
       "song_words_count       294.080535"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(sum(imps) / len(imps)).sort_values('importance', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7543445448173667"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "two_auc = np.mean(auc_score)\n",
    "two_auc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ВЫВОД"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Эмбединги дали больший скор и залетели в топ по импортансу. Я считал испортанс по гейну суммарному, а не по числу сплитов, тк он более релевантный"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}