notebooks/model_development.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b82d9cbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import GridSearchCV, StratifiedGroupKFold, GroupKFold, train_test_split, cross_val_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b10bd2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"../data/fake_users.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee9dd94e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Fake\"].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fad6005c",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(df[df[\"Fake\"] == 1][\"UserId\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "320996f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Undersampling\n",
    "df_fake = df[df[\"Fake\"] == 1]\n",
    "\n",
    "if False: # Undersample by User (Worse results)\n",
    "    fake_cnt = len(df_fake[\"UserId\"].unique())\n",
    "\n",
    "    df_real = df[df[\"Fake\"] == 0]\n",
    "    sampled_real_user_ids = df_real[\"UserId\"].sample(fake_cnt)\n",
    "\n",
    "    df_real_sampled = df_real[df_real[\"UserId\"].isin(sampled_real_user_ids)]\n",
    "\n",
    "else: # Undersample by transaction\n",
    "    df_real_sampled = df[df[\"Fake\"] == 0].sample(df_fake.shape[0], random_state=0)\n",
    "        \n",
    "\n",
    "df = pd.concat([df_fake, df_real_sampled], axis=0).sample(frac=1.)\n",
    "\n",
    "df[\"Fake\"].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75afd63b",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_ids = df.UserId\n",
    "df.drop(columns=\"UserId\", inplace=True)\n",
    "df.drop(columns=\"Unnamed: 0\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe13bb38",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop(columns=\"Fake\", axis=1)\n",
    "y = df.Fake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d2071c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_le = LabelEncoder()\n",
    "user_ids = user_le.fit_transform(user_ids)\n",
    "user_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fc22799",
   "metadata": {},
   "outputs": [],
   "source": [
    "event_le = LabelEncoder()\n",
    "cat_le = LabelEncoder()\n",
    "\n",
    "\n",
    "X[\"Event\"] = event_le.fit_transform(X.Event)\n",
    "X[\"Category\"] = cat_le.fit_transform(X.Category)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d350595",
   "metadata": {},
   "outputs": [],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d6960d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "ohe = OneHotEncoder(handle_unknown='ignore')\n",
    "\n",
    "X = ohe.fit_transform(X)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54d892e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "model = LogisticRegression()\n",
    "p_grid={\n",
    "    'C': [1, 2, 5, 7, 10, 20, 50],\n",
    "    'max_iter': [20, 50, 100, 200, 500],\n",
    "    'class_weight': [\"balanced\", None],\n",
    "}\n",
    "\n",
    "gkf = GroupKFold(n_splits=5)\n",
    "\n",
    "gscv = GridSearchCV(estimator=model, param_grid=p_grid, cv=gkf, scoring='roc_auc')\n",
    "\n",
    "lr = gscv.fit(X, y, groups=user_ids)\n",
    "\n",
    "lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78383de6",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr.best_score_, lr.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f91e21f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "model = RandomForestClassifier(random_state=0)\n",
    "p_grid={\n",
    "    'n_estimators': [1, 2, 5,],\n",
    "    'max_depth': [2, 3, 5, 10, 20],\n",
    "    'criterion': [\"gini\", \"entropy\"],\n",
    "    'class_weight': [\"balanced\", None],\n",
    "}\n",
    "\n",
    "gkf = GroupKFold(n_splits=5)\n",
    "\n",
    "gscv = GridSearchCV(estimator=model, param_grid=p_grid, cv=gkf, scoring='roc_auc')\n",
    "\n",
    "rf = gscv.fit(X, y, groups=user_ids)\n",
    "\n",
    "rf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3367feb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf.best_score_, rf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4085c9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.read_csv(\"../data/fake_users_test.csv\")\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a860358d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test[\"Event\"] = event_le.transform(df_test.Event)\n",
    "df_test[\"Category\"] = cat_le.transform(df_test.Category)\n",
    "\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93fd7f25",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test.sample(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8b5eedc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# test user\n",
    "test_user = df_test[df_test[\"UserId\"] == \"03E7EE785DT\"]\n",
    "test_user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e5dab6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_user = ohe.transform(test_user[[\"Event\", \"Category\"]])\n",
    "rf.predict_proba(X_test_user)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81a48b89",
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(rf.predict_proba(X_test_user)[:,1]) / len(rf.predict(X_test_user))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ac8820e",
   "metadata": {},
   "outputs": [],
   "source": [
    "output = {}\n",
    "df_test[\"pred\"] = None\n",
    "for user, group in df_test.groupby(\"UserId\"):\n",
    "    pred = rf.predict_proba(ohe.transform(group[[\"Event\", \"Category\"]]))\n",
    "    output[user] = sum(pred[:,1]) / pred.shape[0]\n",
    "    for idx, pred in zip(group.index, pred[:,1]):\n",
    "        df_test.at[idx, \"pred\"] = pred\n",
    "\n",
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cbf2704",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a472887a",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_preds = df_test.groupby(\"UserId\").first()\n",
    "user_preds[\"pred\"] = None\n",
    "user_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59e6af17",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_preds[\"pred\"] = [output[idx] for idx, val in user_preds[\"pred\"].iteritems()]\n",
    "\n",
    "user_preds.sample(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "884588a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(\n",
    "    classification_report(\n",
    "        user_preds[\"Fake\"], user_preds[\"pred\"].apply(lambda x: 1 if x>.5 else 0)\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ce53544",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1a29394",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\n",
    "    classification_report(\n",
    "        df_test[\"Fake\"], df_test[\"pred\"].apply(lambda x: 1 if x>.50 else 0)\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdc7b9ac",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}