P4-Xuefei-data process_clean.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4a52ea21",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "\n",
    "import math\n",
    "import scipy.stats as st\n",
    "import statsmodels.api as sm\n",
    "\n",
    "from datetime import datetime\n",
    "from IPython.display import display\n",
    "import warnings\n",
    "pd.set_option(\"display.max_rows\", None)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bd3fcf97",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       client_id   sex        birth\n",
      "count       8623  8623  8623.000000\n",
      "unique      8623     2          NaN\n",
      "top       c_1115     f          NaN\n",
      "freq           1  4491          NaN\n",
      "mean         NaN   NaN  1978.280877\n",
      "std          NaN   NaN    16.919535\n",
      "min          NaN   NaN  1929.000000\n",
      "25%          NaN   NaN  1966.000000\n",
      "50%          NaN   NaN  1979.000000\n",
      "75%          NaN   NaN  1992.000000\n",
      "max          NaN   NaN  2004.000000\n",
      "False\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>client_id</th>\n",
       "      <th>sex</th>\n",
       "      <th>birth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c_4410</td>\n",
       "      <td>f</td>\n",
       "      <td>1967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c_7839</td>\n",
       "      <td>f</td>\n",
       "      <td>1975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c_1699</td>\n",
       "      <td>f</td>\n",
       "      <td>1984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c_5961</td>\n",
       "      <td>f</td>\n",
       "      <td>1962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c_5320</td>\n",
       "      <td>m</td>\n",
       "      <td>1943</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  client_id sex  birth\n",
       "0    c_4410   f   1967\n",
       "1    c_7839   f   1975\n",
       "2    c_1699   f   1984\n",
       "3    c_5961   f   1962\n",
       "4    c_5320   m   1943"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "customers = pd.read_csv(\"customers.csv\")\n",
    "\n",
    "print(customers.describe(include=\"all\"))\n",
    "print(customers.isnull().values.any()) \n",
    "customers.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "080eb0fd",
   "metadata": {},
   "source": [
    "#### 3266 unique id_prod    vs  3287 in products\n",
    "#### 8602 unique client_id    vs  8623 in customers\n",
    "#### 336855 unique vs 337016  date   --- not a prob, real prob is related to date but not at this point\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "72c53c77",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       id_prod        price        categ\n",
      "count     3287  3287.000000  3287.000000\n",
      "unique    3287          NaN          NaN\n",
      "top      0_910          NaN          NaN\n",
      "freq         1          NaN          NaN\n",
      "mean       NaN    21.856641     0.370246\n",
      "std        NaN    29.847908     0.615387\n",
      "min        NaN    -1.000000     0.000000\n",
      "25%        NaN     6.990000     0.000000\n",
      "50%        NaN    13.060000     0.000000\n",
      "75%        NaN    22.990000     1.000000\n",
      "max        NaN   300.000000     2.000000\n",
      "False\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3287 entries, 0 to 3286\n",
      "Data columns (total 3 columns):\n",
      " #   Column   Non-Null Count  Dtype  \n",
      "---  ------   --------------  -----  \n",
      " 0   id_prod  3287 non-null   object \n",
      " 1   price    3287 non-null   float64\n",
      " 2   categ    3287 non-null   int64  \n",
      "dtypes: float64(1), int64(1), object(1)\n",
      "memory usage: 77.2+ KB\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>price</th>\n",
       "      <th>categ</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>731</th>\n",
       "      <td>T_0</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2355</th>\n",
       "      <td>0_202</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2272</th>\n",
       "      <td>0_528</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>370</th>\n",
       "      <td>0_120</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1211</th>\n",
       "      <td>0_1844</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id_prod  price  categ\n",
       "731      T_0  -1.00      0\n",
       "2355   0_202   0.62      0\n",
       "2272   0_528   0.62      0\n",
       "370    0_120   0.66      0\n",
       "1211  0_1844   0.77      0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "products= pd.read_csv(\"products.csv\")\n",
    "print(products.describe(include=\"all\"))\n",
    "print(products.isnull().values.any()) \n",
    "print(products.info())\n",
    "products.sort_values(by= 'price', ascending= True).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efa6c0dd",
   "metadata": {},
   "source": [
    "#### Remark: the product with id_prod = T_0 has its price negative,  to clean "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b6ef508f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 3286 entries, 0 to 3286\n",
      "Data columns (total 3 columns):\n",
      " #   Column   Non-Null Count  Dtype  \n",
      "---  ------   --------------  -----  \n",
      " 0   id_prod  3286 non-null   object \n",
      " 1   price    3286 non-null   float64\n",
      " 2   categ    3286 non-null   int64  \n",
      "dtypes: float64(1), int64(1), object(1)\n",
      "memory usage: 102.7+ KB\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>price</th>\n",
       "      <th>categ</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2272</th>\n",
       "      <td>0_528</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2355</th>\n",
       "      <td>0_202</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>370</th>\n",
       "      <td>0_120</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1211</th>\n",
       "      <td>0_1844</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1530</th>\n",
       "      <td>0_1620</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id_prod  price  categ\n",
       "2272   0_528   0.62      0\n",
       "2355   0_202   0.62      0\n",
       "370    0_120   0.66      0\n",
       "1211  0_1844   0.77      0\n",
       "1530  0_1620   0.80      0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "products = products[products.id_prod != \"T_0\"]\n",
    "print(products.info())\n",
    "products.sort_values(by= 'price', ascending= True).head()\n",
    "\n",
    "# the product with negative price is droped from the dataframe "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4f7f412b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([19.99,  5.13, 17.99, ..., 17.14, 11.22, 25.16])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# price in array of products categ 0 \n",
    "price0 = products.loc[products['categ']== 0 , 'price' ].values\n",
    "price0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "45ac4c45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average price for products in categ 0: 11.73\n",
      "Median price for products in categ 0: 10.32\n",
      "Mode of price for products in categ 0: 4.99\n"
     ]
    }
   ],
   "source": [
    "import statistics \n",
    "\n",
    "print(\"Average price for products in categ 0:\", round(statistics.mean(price0),2))\n",
    "print(\"Median price for products in categ 0:\", round(statistics.median(price0),2))\n",
    "print(\"Mode of price for products in categ 0:\", statistics.mode(price0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "95a400de",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ZoeyZhang/opt/anaconda3/lib/python3.8/site-packages/seaborn/distributions.py:2557: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVAAAAEvCAYAAAATnJnNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAtqUlEQVR4nO3dd3hcZ5n38e89o967rGpZiVwUXFFsZ0N6wQ4JBhZI2ZCFLcZLwsLu8u4b2AYsW2FD2+BsCFkIIQmhrgN+carjhI3jkrjbiuUqyeqS1bvu948ZOUKR7dFojs6MdH+uay5rzjkj3Sexfn7OOU8RVcUYY8zkedwuwBhjIpUFqDHGBMkC1BhjgmQBaowxQbIANcaYIFmAGmNMkKLcLiCUsrKytKSkxO0yjDEzzO7du5tVNXv89hkVoCUlJezatcvtMowxM4yInJpou13CG2NMkCxAjTEmSBagxhgTJAtQY4wJkgWoMcYEyQLUGGOCZAFqjDFBsgA1xpggWYAaY0yQLECNMSZIFqDGGBOkGTUWfiZ64vXTF9x/16riaarEGDOetUCNMSZIFqDGGBMkC1BjjAmSBagxxgTJAtQYY4LkaICKyBoRqRSRKhG5f4L9IiLf8u/fJyIr/NsXiMieMa8OEfmsk7UaY8xkOdaNSUS8wIPATUANsFNENqnqoTGHrQXK/K9VwEZglapWAsvGfJ9a4BdO1WqMMcFwsgW6EqhS1eOqOgA8Bawbd8w64DH12Q6kiUjeuGNuAI6p6oRrkhhjjFucDNACoHrM+xr/tskecwfwZMirM8aYKXIyQGWCbTqZY0QkBng/8JPz/hCR9SKyS0R2NTU1BVWoMcYEw8kArQGKxrwvBM5M8pi1wBuq2nC+H6KqD6tqhapWZGe/Y9lmY4xxjJMBuhMoE5F5/pbkHcCmccdsAu7xP41fDbSrat2Y/Xdil+/GmDDl2FN4VR0SkfuALYAXeFRVD4rIBv/+h4DNwC1AFdADfGL08yKSgO8J/iedqtEYY6bC0dmYVHUzvpAcu+2hMV8rcO95PtsDZDpZnzHGTIWNRDLGmCBZgBpjTJAsQI0xJkgWoMYYEyQLUGOMCZIFqDHGBMkC1BhjgmQBaowxQbIANcaYIFmAGmNMkCxAjTEmSBagxhgTJAtQY4wJkgWoMcYEyQLUGGOCZAFqjDFBsgA1xpggWYAaY0yQLECNMSZIFqDGGBMkC1BjjAmSBagxxgTJ0WWNjfOeeP30Bffftap4mioxZvaxFqgxxgTJAtQYY4JkAWqMMUFyNEBFZI2IVIpIlYjcP8F+EZFv+ffvE5EVY/alichPReSIiBwWkSucrNUYYybLsQAVES/wILAWKAfuFJHycYetBcr8r/XAxjH7vgn8RlUXAkuBw07VaowxwXCyBboSqFLV46o6ADwFrBt3zDrgMfXZDqSJSJ6IpABXA98DUNUBVT3rYK3GGDNpTgZoAVA95n2Nf1sgx5QCTcB/i8ibIvKIiCQ6WKsxxkyakwEqE2zTAI+JAlYAG1V1OdANvOMeKoCIrBeRXSKyq6mpaSr1GmPMpDgZoDVA0Zj3hcCZAI+pAWpU9XX/9p/iC9R3UNWHVbVCVSuys7NDUrgxxgTCyQDdCZSJyDwRiQHuADaNO2YTcI//afxqoF1V61S1HqgWkQX+424ADjlYqzHGTJpjQzlVdUhE7gO2AF7gUVU9KCIb/PsfAjYDtwBVQA/wiTHf4tPAj/zhe3zcPmOMcZ2jY+FVdTO+kBy77aExXytw73k+uweocLI+Y4yZChuJZIwxQbLZmCJMfXsfx5q6iIv2sLggjZgo+zfQGLdYgEaQl99q4rlD9Yz4O4M9d6iBj1YUUZqd5G5hxsxS1nyJEEfqOthysJ7yvBT+75qF/OlVpcRGe/nBayc51tTldnnGzEoWoBGgd2CYn71ZS15qHB+tKCI1Ppp5WYn86VWlpCfE8MTrp2nrHnC7TGNmHQvQCLDzZCvd/UN8cHkBUd63/5clxUbxsdVzUZQnd55meGT8QC9jjJMsQMPc8Iiy/XgL87ISKUxPeMf+zKRYPrCsgJq2Xv73WLMLFRoze1mAhrkj9R2c7R3kyksyz3vM4oJUFs5J5vnDDXYpb8w0sgANc/tr20mMjWJhXsp5jxER3r80H0HYtPcMvvEJxhinWYCGsYGhESrrO1k0JxmPTDRx1dvSEmK4qTyXyoZO9te2T1OFxsxuFqBh7PUTLfQPjbDoAq3PsVaXZpKfGsfm/XX0Dw07XJ0xxgI0jD13qIFor3BpTmAd5b0e4bal+XT0DbG10uZGNcZpFqBh7NWqZkqzkoj2Bv6/aW5mIiuK03j1aDPNnf0OVmeMsQANU81d/Rxv6qYka/Irmbz3sjlEeYVn9tkDJWOcZAEapnadbAWgJPOdfT8vJjkumhsX5XK0sYtnDzWEujRjjJ8FaJjacaKN2CgPBenxQX1+dWkmuSmxfPmZQ/QN2gMlY5xgszGFqZ0nW1lWlEaUJ7h/47we4bYl+Tzy6gk2bj3GX9w0/7zHPvH66Qt+r7tWFQdVgzEznbVAw1Df4DCH6jq4vCRjSt+nNDuJ25bms/HlY5xu6QlRdcaYURagYehwXQfDI8riwtQpf68v3LKQKI/w95sO2AMlY0LMAjQMHTzTAcBl+YF1oL+QvNR4PnfzArZWNvHzN2qn/P2MMW+zAA1DB8+0k5YQTUFacA+Qxvv475VQMTedLz1zkMaOvpB8T2OMBWhYOlDbwWX5KchFxr8HyuMR/v3DS+gfGuFvfmmX8saEigVomBkc9k0g8q78qd//HKs0O4m/unk+zx1q4Ce7a0L6vY2ZrSxAw8zRhi4GhkcoD8H9z/H++D2lXFGayT/8z0GqGm0dJWOmygI0zLzV0AkQ8AxMk+H1CN+4YxnxMV4+/eSb1sHemCmyAA0zRxs7ifIIJZmTHwMfiNyUOL72kSUcruvgn3592JGfYcxsYSORwszRhi7mZiYQE+Xcv23XL8xl/dWlPLzteEj6mhozWznaAhWRNSJSKSJVInL/BPtFRL7l379PRFaM2XdSRPaLyB4R2eVkneGkqrGLspxkx3/OX793AVdemsnf/uIA1a02SsmYYDgWoCLiBR4E1gLlwJ0iUj7usLVAmf+1Htg4bv91qrpMVSucqjOc9A8Nc7Klm7LcwCZQnooor4f/vHMFOSmx/Oj1U3T0DTr+M42ZaZxsga4EqlT1uKoOAE8B68Ydsw54TH22A2kikudgTWHtZHMPI0rAM9BPVXpiDA9/rILewWEe336KweGRafm5xswUTgZoAVA95n2Nf1ugxyjwrIjsFpH1jlUZRo42+p7AT8cl/Kjy/BRuryiitq2Xn+6uYcQ62RsTMCcDdKJhNON/Oy90zJWqugLfZf69InL1hD9EZL2I7BKRXU1Nkb0O0NGGLkSgNNuZJ/DnU56fynsvm8P+2nZePNI4rT/bmEjmZIDWAEVj3hcCZwI9RlVH/2wEfoHvlsA7qOrDqlqhqhXZ2dkhKt0dJ5q7KUiLJy7aO+0/+6qyLN5dnM6LRxrZW3122n++MZHIyQDdCZSJyDwRiQHuADaNO2YTcI//afxqoF1V60QkUUSSAUQkEbgZOOBgrWHhZEs384JYAykURIR1y/MpyUzkZ2/UcNqezBtzUY4FqKoOAfcBW4DDwNOqelBENojIBv9hm4HjQBXwXeBT/u25wKsishfYAfxaVX/jVK3hQFU50dzN3CDWQAqVKI+HP1hVTEp8ND/cfoq2ngHXajEmEjjakV5VN+MLybHbHhrztQL3TvC548BSJ2sLN209g3T2DTk2AilQibFR3LN6Lg9tO8YPXzvFJ68udbUeY8KZDeUMEydbugFcD1CAnJQ47lxZTGNnH0/trGZ4xJ7MGzMRC9AwcbLZH6Au3QMdrywnmVuX5FPZ0Mm/bLYx88ZMxAI0TJxs7sYjUJQRmlnoQ2F1aSZXlGbyyKsneHLHhVfuNGY2sslEwsTJlh7y0+KJjZr+LkwXcsviPGKiPPzdLw8wPzeZd89Nd7skY8KGtUDDxCkXuzBdiNcjfPuu5cxJjeOzP36TThszb8w5FqBhorqtl8J097owXUhKXDTfuH0ZtW29fHHTIbfLMSZs2CV8GOgZGKK1e4DC9PC5/zleRUkG9113Kd96sYrrFmZz65L8c/ueeP3C90fvWlXsdHnGuMJaoGGgtq0XIKwDFODTN5SxtCiNv/vlAVq7rZO9MRagYaDmbGQEaLTXw7///hI6+4b4Z+vaZIxdwrvtiddPs/14CwA7TrRRWR/eq2UumJPM+qtL+c7WY3xoRQG/d0mW2yUZ4xprgYaBsz0DeEVIjouMf8/+/IYy5mYm8De/OGAre5pZzQI0DLT1DJKaEI1HJpoeNfzERXv5ygfexYnmbr736gm3yzHGNRagYeBszwDpCdFulzEpV5Vlc1N5Lt95qcr6hppZywI0DJztGSQtIcbtMibt82sX0j80wvOHbRZ7MztZgLpscHiEzv6hiGuBApRmJ/GxK+ay62Qr9e19bpdjzLSzAHVZe4/v8jc9AlugAJ+5oYy4aC//70Cd26UYM+0sQF02Out7JF7Cg6/uaxdkc7Sxi+PN4d0Fy5hQswB1Wdu5FmjkXcKPWjUvk+S4KJ4/1IDasshmFrEAddnZngE8AslxkRugMVEerl2Qw8mWHqoarRVqZg8LUJe19QyQGh+N1xMZfUDP5/K56aTFR/PcYWuFmtkjoAAVkZ+JyPtExAI3xCK1C9N4UV4P1y/MoaatlyP1nW6XY8y0CDQQNwJ3AUdF5F9FZKGDNc0qbRHYif58lhenk5kYw3OHGhixVqiZBQIafK2qzwPPi0gqcCfwnIhU41vL/XFVtaEoQRgYGqGzbyjsW6AXm+9zlNcj3LAoh6d31XCgtp0lhWnOFmaMywK+JBeRTODjwJ8AbwLfBFYAzzlS2SxQ196LEtlP4MdbUphGTnIsLxxutFaomfECvQf6c+AVIAG4TVXfr6o/VtVPA0lOFjiTjU6kHO4t0MnwiHDDolyauvrZV9PudjnGOCrQ+dMeUdXNYzeISKyq9qtqhQN1zQqjEymnxc+cFijAZfkpzEmJ48UjDSwuSLUlP8yMFegl/Fcm2PZaKAuZjUbHj6fMsAD1tUJzaO4aYG/NWbfLMcYxFwxQEZkjIu8G4kVkuYis8L+uxXc5f0EiskZEKkWkSkTun2C/iMi3/Pv3iciKcfu9IvKmiPxqcqcVGeo7+kiI8RLtnXm9w8rzUshLjePFI40Mj9i9UDMzXewS/r34HhwVAg+M2d4JfOFCHxQRL/AgcBNQA+wUkU2qOnZd3LVAmf+1Cl93qVVj9n8GOAykXOxEIlFDex+pM6z1OUpEuHFRLj/cfoo91W28e26G2yUZE3IXbPqo6g9U9Trg46p63ZjX+1X15xf53iuBKlU9rqoDwFPAunHHrAMeU5/tQJqI5AGISCHwPuCRYE4sEtS195ESwUM4L2bhnGQK0uKtFWpmrAu2QEXkblV9HCgRkb8cv19VH5jgY6MKgOox72v43dbl+Y4pAOqAbwB/DSRfpMb1wHqA4uLIehjR0NFHabaznRgC7cPpBPHfC33stVO8cbqNy0usFWpmlovdfEv0/5mEL8jGvy5kosHd45shEx4jIrcCjaq6+yI/A1V9WFUrVLUiOzv7YoeHjf6hYVq6B0iJj4yF5IK1IDeZwvR4XjrSyNDIiNvlGBNSF/ztVdX/8v/5pSC+dw1QNOZ9IXAmwGM+DLxfRG4B4oAUEXlcVe8Ooo6w1NjRD0DqDL6Eh7fvhX7/f0+y+1Qbq+Zlul2SMSETaEf6fxeRFBGJFpEXRKRZRC4WZjuBMhGZJyIxwB3ApnHHbALu8T+NXw20q2qdqn5eVQtVtcT/uRdnUniC7wk8zLwuTBMpy0miOCOBrZVNDA1bK9TMHIH2n7lZVTuAW/G1GucD/+dCH1DVIeA+YAu+J+lPq+pBEdkgIhv8h20GjgNV+MbVf2rypxCZ6mZoH9CJjLZC23sH2Xmqze1yjAmZQG/Ajf6W3wI8qaqtEsAa5v7RS5vHbXtozNcK3HuR77EV2BpgnRGjwR+gM/0SftQl2YmUZCbwcmUjFXPTZ2TfVzP7BPq3+BkROQJUAC+ISDZgyzBOQX1HH/HRXuKiZ0eQiAg3lc+ho2+I31Y1u12OMSER0G+vqt4PXAFU+Keu6+adfTrNJNS395GXGkcgLfmZYl5WIovyUnj5rSa6+ofcLseYKZtM82cRcLuI3IPvKfnNzpQ0O9R39JGbEud2GdNuzWVzGBwe4cUjDW6XYsyUBfoU/ofA14D3AJf7XzYL0xTUt/cxJ3X2BWh2ciyXl2Sw40QrTZ39bpdjzJQE+hCpAihXWy0sJEZGlIaO2RmgADcsymVP9Vm2HKzn7tVz3S7HmKAFegl/AJjjZCGzSUv3AEMjypxZeAkPkBQbxdXzszlU18GJ5m63yzEmaIEGaBZwSES2iMim0ZeThc1ko/OAzsZ7oKOuvCSLlLgoNu+vY8QmGjERKtBL+C86WcRsMzoKKS81jtbuAZercUdMlIf3XjaHn+yu4eld1dyxMrImgjEGAu/G9DJwEoj2f70TeMPBuma00QCdrfdARy0rSqMkM4F/+80R2mbpPyQmsgXUAhWRP8U3ZVwGcAm+KeceAm5wrrSZq769F69HyEqKdbsUV4kIty3N5ztbj/HVZyv55w8uPu+xtq6SCUeB3gO9F7gS6ABQ1aNAjlNFzXT17f3kJMfi9cyeTvTnk5cazx9eUcKTO06zp/qs2+UYMymBBmi/f1Z5AEQkinfO7WkCVN/RO6sfII332ZvKyEmO5f6f7WNgyGZrMpEj0AB9WUS+gG9xuZuAnwDPOFfWzDY6jNP4pMRF85UPLOZIfScbtx5zuxxjAhZogN4PNAH7gU/im2Hpb50qaqZr6Oi3Fug4N5XnctvSfP7zpaNU1ne6XY4xAQn0KfwI8EvgU6r6YVX9ro1KCk5n3yBd/UPWAp3AF28rJzkumr/+6V6beNlEhIstKifAP+CbGFn8m4aBb6vql6ehvog3/ulxo78L07GmblcXfAtHmUmxfPH9l/HnT77Jo789wfqrL3G7JGMu6GIt0M/ie/p+uapmqmoGvpU1rxSRv3C6uJmovW8QYMYvJhes25bkceOiXP7j2bc41tTldjnGXNDFAvQe4E5VPTG6QVWPA3f795lJ6uj1zYM5W2ainywR4Z8++C7iY7x89qk99lTehLWLBWi0qr5j+nBVbeLtZT7MJHSca4Haf77zyU2J418/tIT9te184/m33C7HmPO6WIBeaHydjb0LQnvvIAkxXlsT6CLWvGsOt1cUsfHlY2w/3uJ2OcZM6GK/xUtFpGOCVydw/nF35rw6egdJscv3gPz9beXMzUjgL3+8h96BYbfLMeYdLhigqupV1ZQJXsmqaikQhI6+QXuAFKDE2Ci+ecdyGjv7+Z+9tVjPORNu7DpymrX3DpFq9z8DtrQojb+4aT77atp508bKmzBjATqNhkZG6O4fskv4SdpwzSWUZCayae8ZWrpsHSUTPixAp1HnaBcma4FOitcjfLSiEI/A07uqGbYZ7E2YsACdRtaFKXhpCTF8YFkB1W29tiSyCRuOBqiIrBGRShGpEpH7J9gvIvIt//59IrLCvz1ORHaIyF4ROSgiX3KyzunS3usPULuED8qSwjRWFKeztbLJFqMzYcGxABURL/AgsBYoB+4UkfJxh60Fyvyv9cBG//Z+4HpVXQosA9aIyGqnap0uHf4AtUv44N22JI/0xBie3lVtXZuM65zsT7MSqPIP/UREngLWAYfGHLMOeMw/s9N2EUkTkTxVrQNGB0JH+18Rf+Oro2+IaK8QF213ToIVG+3l9ooi/mvbMX65p5Y7Li9CRGzJD+MKJ3+TC4DqMe9r/NsCOkZEvCKyB2gEnlPV150rdXq0+zvR+ya5MsEqykjgxkW57K9tZ19tu9vlmFnMyQCdKCXGtyLPe4yqDqvqMqAQWCki75rwh4isF5FdIrKrqalpKvU6rqN30B4ghchVZdkUpsfzzN4zdPUPuV2OmaWcDNAaoGjM+0LgzGSPUdWzwFZgzUQ/RFUfVtUKVa3Izs6eYsnO6ugbtPufIeL1CB9aUUj/4Ai/3jf+r5Ux08PJAN0JlInIPBGJAe4ANo07ZhNwj/9p/GqgXVXrRCRbRNIARCQeuBE44mCtjhtRpaPXOtGH0pyUOK5ZkM3emnYO13W4XY6ZhRx7iKSqQyJyH7AF8AKPqupBEdng3/8QvrWVbgGqgB7gE/6P5wE/8D/J9wBPq+qvnKp1OvQMDDOsauPgQ+zaBdkcPNPOM3vPcEl2EjFR9oDOTB9Hf5tVdTO+kBy77aExXyu+NefHf24fsNzJ2qZbh/UBdUSUx8NtS/N55JUTbDvaxI2Lct0uycwi9s/1NGm3PqCOKc1KYklhKtveaqKt26apNdPHAnSa2DBOZ625bA4isPlAndulmFnEAnSatPcOIkBSrN0DdUJaQgzXLsjh4JkOW4zOTBsL0GnS0TtESnw0Xo91onfKey7NIjU+mmcP1tvky2ZaWIBOE99SHtb6dFK018P1C3Oobuu1bk1mWliATpN2G4U0LVYUp5OVFMOzhxoYsVaocZgF6DRpt1FI08LrEW4qn0NjZz97bAkQ4zAL0GnQNzjMwNCIBeg0uSw/hfy0OF443GCz1xtHWYBOA5tIeXp5RLhhYS5tPYPsrTnrdjlmBrMAnQbnRiFZC3TaLJyTTF5qHFsrG+1eqHGMBeg0GO1Eb5fw00dEuHZBDs1dAxywOUONQyxAp8Hbl/DWjWk6XZafQnZyLC9VNjJi90KNAyxAp0F77xCJMV6ivPafezp5RLh2fjYNHf08d9hW8jShZ02iadDRa12YLuRi6xlNxZLCNJ4/3MDGrce4uTzXllMxIWVNomnQ0Wed6N3i9QhXlWWzp/osr59odbscM8NYgE6DdmuBuurdc9PJTIzhoZePuV2KmWEsQB02ODxCz8CwtUBdFO318IkrS9ha2cShMzZG3oSOBajDRvuAploneld9bHUJiTFe/mubtUJN6FiAOqzdJlIOC6kJ0dy1qphn9p6hurXH7XLMDGEB6rAOW8ojbPzxe0rxeoTvvnLc7VLMDGEB6rD23iEAW40zDMxJjeODywv48c5qmrv63S7HzAAWoA7r6B0kLtpDbJTX7VIMsP7qSxgYHuEH/3vS7VLMDGAB6rD23kGbhSmMXJqTxM3luTz22im6+ofcLsdEOAtQh3XYRMphZ8M1l9DeO8hTO5wbAWVmBwtQh9lSHuFneXE6q0szeOSVEwwMjbhdjolgFqAOGhweoatvyFqgYWjDNZdQ39HHL/fUul2KiWAWoA6qb+9DgTQL0LBzzfxsFuWl8F8vH7Op7kzQHA1QEVkjIpUiUiUi90+wX0TkW/79+0RkhX97kYi8JCKHReSgiHzGyTqdUtPWC0BaQozLlZjxRIQN15RyrKmb522qOxMkxwJURLzAg8BaoBy4U0TKxx22Fijzv9YDG/3bh4C/UtVFwGrg3gk+G/bOnB0NUGuBhqP3Lc6jKCOejS8fQ23ZDxMEJ1ugK4EqVT2uqgPAU8C6ccesAx5Tn+1AmojkqWqdqr4BoKqdwGGgwMFaHVHrD1C7Bxqeorwe1l9Vypunz7LDprozQXAyQAuA6jHva3hnCF70GBEpAZYDr4e+RGfVtvWSHBtFtM1EH7Y+UlFEZmIMG22qOxMEJ8cXTjT19/jrpAseIyJJwM+Az6rqhPOQich6fJf/FBcXB1epQ2rP9trle5iLi/byiStL+Nqzb/Efz1aSlxo/4XF3rQqvv1smPDjZNKoBisa8LwTOBHqMiETjC88fqerPz/dDVPVhVa1Q1Yrs7OyQFB4qvgC1B0jh7mOrS4iJ8rDtrSa3SzERxskA3QmUicg8EYkB7gA2jTtmE3CP/2n8aqBdVevEt3DN94DDqvqAgzU6ZmREfQFq9z/DXmpCNCtLMthf205b94Db5ZgI4liAquoQcB+wBd9DoKdV9aCIbBCRDf7DNgPHgSrgu8Cn/NuvBD4GXC8ie/yvW5yq1QnN3f0MDI3YJXyEuPLSLAThlapmt0sxEcTROdZUdTO+kBy77aExXytw7wSfe5WJ749GjFrrAxpRUuOjWVacxu5TrVy/MIekWJt+0FycPR52SK31AY04V5VlMTSsvHasxe1STISwAHXIaCf6dGuBRoyc5DgW5aXw2vFmegZsqjtzcRagDqlt6yU5Loq4aJtIOZLcsCiH/sERXjlq90LNxVmAOqT2bC8FaRP3KTThKy81niWFqfzvsWY6/AsCGnM+FqAOqWnrpTDdAjQS3bgol+ERZWtlo9ulmDBnAeqQ2rO95FsLNCJlJsVSMTeDnSfaaLV+oeYCrK+GAzr6BunsG7JL+DDyxOuTW77juoU5vFndxpaD9dy50oZxmolZC9QBo31AC+wSPmKlxkdzVVk2+2vbOdXS7XY5JkxZgDrgXIBaCzSiXV2WTXJcFL/eX2ez1psJWYA6YLQTvbVAI1tMlIeby+dQ09bLpr3j58ExxgLUEbVne4mJ8pCVGOt2KWaKlhenkZ8Wx7/95gjdto68GccC1AGnW3ooSo/H44no4fwG8Ihw25J86tr7+OYLR90ux4QZC1AHnGrtYW5mottlmBCZm5nI7RVFfO/VE1TWd7pdjgkjFqAhpqqcbummOCPB7VJMCN2/diEpcVH87S/32wMlc44FaIi1dA/QPTBsATrDpCfG8Pm1i9h5so2nd1Vf/ANmVrAADbFTLT0AzM20AJ1pPvzuQlaXZvBPvz58brYtM7tZgIbY6VZfp2sL0JnH4xH+/feXMqzK/T/fb2vJGwvQUDvV0oMIFKZbgM5ExZkJ3L92IdvearJLeWMBGmqnW3qYkxJn84DOYHevmsvq0gy+8qvD5wZNmNnJAjTETrX22AOkGc7jEb76Yf+l/M/22aX8LGYBGmInm7uZl2V9QGe6oowEPr92Ia8cbebx7afcLse4xAI0hM72DNDSPUBptgXobHD36rlcuyCbf/z1YY7Ud7hdjnGBBWgIHWvyPYEvzUpyuRIzHUSEr31kKSlx0fz5k2/SOzDsdklmmtmEyiF0vKkLgEtyLEBni6ykWB746FLueXQHf/jfO/jAsoLzHnvXKpuYeaaxFmgIHW/uJtorFNk0drPK1fOzWX91KTtOtHKgtt3tcsw0sgANoeNNXRRnJBDltf+ss83nbl5AQVo8v3izljZbR2nWcPQ3XUTWiEiliFSJyP0T7BcR+ZZ//z4RWTFm36Mi0igiB5ysMZSON3VTmm2X77NRTJSHOy4vQlF+tOMUg8MjbpdkpoFjASoiXuBBYC1QDtwpIuXjDlsLlPlf64GNY/Z9H1jjVH2hNjQ8wsmWbnsCP4tlJsXykXcXceZsH5v2nLH+obOAky3QlUCVqh5X1QHgKWDduGPWAY+pz3YgTUTyAFR1G9DqYH0hVd3Wy+Cwcom1QGe1RXkpXLcgm92n29h5ss3tcozDnAzQAmDsYOEa/7bJHhMRKv39ABfOSXa5EuO2GxblUpaTxDP7zlDd2uN2OcZBTnZjmmg9i/HXNIEcc+EfIrIe3+U/xcXudRM5Ut+JCJTlWIDORJNZV94jwu2XF/HgS1U8seM0f3bNJaTERztYnXGLky3QGqBozPtCYPzShoEcc0Gq+rCqVqhqRXZ2dlCFhsJbDZ3MzUggPsYmETGQEBPF3avn0js4zGOvnaR/yDrZz0ROBuhOoExE5olIDHAHsGncMZuAe/xP41cD7apa52BNjjlS38n8XGt9mrflpcZz5+VF1LX38eOd1QzbUiAzjmMBqqpDwH3AFuAw8LSqHhSRDSKywX/YZuA4UAV8F/jU6OdF5EngNWCBiNSIyB87VetU9Q0Oc7K52+5/mndYMCeF25bmc6S+ky/83NZTmmkcHcqpqpvxheTYbQ+N+VqBe8/z2TudrC2Uqhq7GFGYbwFqJrC6NJPOvkF+vKua2GgPX3r/ZYjYktczgY2FD4Ej/qVurQVqzufGRbmU5Sbz8LbjRHs9/O37FlmIzgAWoCFwoLad+Ggv82wWJnMeIsLn1y5kYGiE7716gvbeQf7lQ4uJtmG/Ec0CNAT217bzroIUvB5rUZjze3JHNWU5SVy/MIef7q5hb/VZ7lpZTKx/+RebrSny2D9/UzQ0PMLBM+0sLkhzuxQTAUSEGxfl8sHlBRxr6mLjy8do7OhzuywTJAvQKapq6qJvcIQlhalul2IiyOUlGXz89+bR3T/Ed7Ye483TNuwzElmATtG+Gt/8j4stQM0kXZqTxH3Xl5GXFsdPdtfwZ4/vprHTWqORxAJ0ivbXtJMUG8W8TJuFyUxeanw0f/KeUm4uz+WFI43c9MA2frq7xmZyihAWoFO0+1QbS4tS8dgDJBMkr0e4dkEOm//8KspykvjcT/Zyz6M7ONHc7XZp5iIsQKegvXeQw/UdXF6S4XYpZga4NCeJpz95BV9edxl7Tp/lvV/fxle3HKFnYMjt0sx5WIBOwRun2lCFlfMsQE1oeDzCPVeU8MLnruHWJXk8+NIxbnpgG785UG+X9WHIAnQKdpxsJdorLC9Kd7sUM8PkJMfxwO3LePqTV5AcF8WGx3fz8f/eaZf1YcY60k/BjhOtLC5ItSnsTEicb87RP1g1l+3HW9j2VhPv/fo21l9dyr3XXWp/78KAtUCD1Nk3yN7qs6wqzXS7FDPDeT3ClZdmnbus/8+XqrjxgZftsj4MWAs0SL+tamZoRLl2vnuTOJvZ5flDjVSUZJCZFMsze8+w4fHdzM9N4tYl+WQlxdpQUBdYCzRILx5pJDkuinfPtfufZnrNy0rk3usu5X2L8zjV0sM3XzjKs4fq6R2wWe+nmwVoEEZGlJcqm7h6fjZRNpuOccHoZf1f3jSfJQWpbK1s4sYHXmbLQbusn0722x+E/bXtNHX2c92CHLdLMbNcclw0H6ko4k+vKiU5LopP/nA3n/j+Tk7a0/ppYQEahP/Zc4YYr4ebFuW6XYoxgO+y/leffg9/f2s5u062cfPXt/Efz1baZb3DLEAnaWh4hE17z3D9whxSE2ypWhM+orwe/ug983jxr67hfUvy+PaL9rTeaRagk/TbYy00d/XzgeUFbpdizIRyUuL4+u3L+PH61STF+jrhr3vwt7x4pMGCNMSsG9Mk/fC1U6QnRHPdQuu+ZMLLRB3x7149lzdPt/FSZSN/9P1dLClM5b7rLuWGRbm2gkIIWIBOQlVjF88fbuAzN5QRG2WjQEz483qEipIMlhenExvl4dsvHWX9D3dTkBbPXauK+dCKAvJS490uM2JZgE7Cd7cdJzbKw8eumOt2KcZMitcjfPTyIj64ooDnDjXw+PZTfHVLJV/dUknF3HRuWZzH1fOzeP146wVXC7XO+r/LAjRAh8508JPd1dxzRQlZSbFul2NMUKK9Hm5ZnMcti/M40dzNr/ae4df76/jyrw4BkBIXxaU5SczLSqQgPYGc5Fg8tvzyeVmABmBkRPniMwdJjY/mL26c73Y5xoTEvKxEPn1DGZ++oYzq1h5erWrmiddPc6S+kzdOnwUg2ivkp8UzJyWO7ORYCtLjKc1KJD8t3u6hYgEakO9srWLHiVb+7fcXW9clE7HON9vTWHeuLGZEleaufmrbeqk520ttWy97a87SNzjCr/bVARAT5aEkM4HijETmZib4vs5MpCQzgfy0+Fmz3r0F6EVs3l/Hfzz3FuuW5fPRiiK3yzHGcR4RcpLjyEmOY3mxb64HVaWrf4hlRWkcb+7meFMXJ5p7ON3azStHm+gfGjn3ea9HKEyPZ35uMpflp1Cel0J5fgoFafEXvL8aiRwNUBFZA3wT8AKPqOq/jtsv/v23AD3Ax1X1jUA+6zRV5YfbT/HFTQdZUZzOP39w8Yz7n29MoESE5LhojjX5hojOy0piXlYSACOqdPYN0do9QGt3Py3dAzR3DbDn9FmeP9TAaM/T+Ggvy4rSKM9PYeGcZBblpVCWmxTRPVocC1AR8QIPAjcBNcBOEdmkqofGHLYWKPO/VgEbgVUBftYRqsruU218/fm3+G1VC9cuyObBu1aQGGuNdWMm4hEhNT6a1Pho5mX97uq0A0Mj1Hf0UdfeS93ZPqrbeth1qpXBYfV/FrKSYslLjWNOShzrlhdQnJHA3MwEkuPC/3aZk6mwEqhS1eMAIvIUsA4YG4LrgMfUNzxiu4ikiUgeUBLAZ6dkaHiE1p4BWrsHaOka4GhDJ4fqOthxopWTLT1kJMbwj+su4w9WzbUVN40JUkyUh+KMBIozEs5tG1GlpWvgXLDWt/dxqqWHvTXtbDnUcO64jMQYijJ8PQGykmLJToohOzmWzKRYkuOiSIiJIjHWS2JMFPExXqK9HqI8gtf/ivKI41eNTgZoAVA95n0NvlbmxY4pCPCzU/LNF47y7RerfmdbZmIMSwpT+bNrL+HWJfnW6jTGAR4RspNjyU6OZXFB6rntfYPDrCrN4HRLD6daezjV0kNNWw/VrT28ebqNlu4BJjsS1SO+e7IeEURAEG5dksdXP7I0JOfiZEJMFP3jT/98xwTyWd83EFkPrPe/7RKRyoArHOcU8Abw/WC/wduygOapfxvXzZTzADuXcOTKeRwBvjb5j004esbJAK0Bxj62LgTOBHhMTACfBUBVHwYenmqxoSQiu1S1wu06pmqmnAfYuYSjmXAeTnbW2gmUicg8EYkB7gA2jTtmE3CP+KwG2lW1LsDPGmOMqxxrgarqkIjcB2zB1xXpUVU9KCIb/PsfAjbj68JUha8b0ycu9FmnajXGmGA4+pREVTfjC8mx2x4a87UC9wb62QgSVrcUpmCmnAfYuYSjiD8PsQlWjTEmOLNjwKoxxjjAAjSERGSNiFSKSJWI3O92PZMhIo+KSKOIHBizLUNEnhORo/4/092sMRAiUiQiL4nIYRE5KCKf8W+PxHOJE5EdIrLXfy5f8m+PuHMB3+hEEXlTRH7lfx+R5zGWBWiIjBl+uhYoB+4UkXJ3q5qU7wNrxm27H3hBVcuAF/zvw90Q8FequghYDdzr//8QiefSD1yvqkuBZcAaf2+VSDwXgM8Ah8e8j9TzOMcCNHTODV1V1QFgdPhpRFDVbUDruM3rgB/4v/4B8IHprCkYqlo3OiGNqnbi+4UtIDLPRVW1y/822v9SIvBcRKQQeB/wyJjNEXce41mAhs75hqVGslx/v1z8f+a4XM+kiEgJsBx4nQg9F/9l7x6gEXhOVSP1XL4B/DUwMmZbJJ7H77AADZ2Ah58a54lIEvAz4LOq2uF2PcFS1WFVXYZvNN5KEXmXyyVNmojcCjSq6m63awk1C9DQCWToaqRp8M+Ohf/PRpfrCYiIROMLzx+p6s/9myPyXEap6llgK7771JF2LlcC7xeRk/hubV0vIo8TeefxDhagoTMTh59uAv7Q//UfAv/jYi0B8U/S/T3gsKo+MGZXJJ5Ltoik+b+OB27ENxdGRJ2Lqn5eVQtVtQTf78WLqno3EXYeE7GO9CEkIrfgu9czOvz0n9ytKHAi8iRwLb4ZchqAfwB+CTwNFAOngY+o6vgHTWFFRN4DvALs5+37bV/Adx800s5lCb6HK158jZ2nVfXLIpJJhJ3LKBG5Fvicqt4ayecxygLUGGOCZJfwxhgTJAtQY4wJkgWoMcYEyQLUGGOCZAFqjDFBsgA1xpggWYAaY0yQLECNMSZI/x8Way/ZzoYmSgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig,ax = plt.subplots(figsize=(5,5))\n",
    "sns.distplot(price0)\n",
    "plt.savefig('pricecateg0.jpg')\n",
    "# median is better in this case \n",
    "# the distribution is unimodale and is quite right-skewed, and with the high-peak, thus it's reasonable to choose mean and/or median as the value to impute the nulls "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "23b0db2e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>price</th>\n",
       "      <th>categ</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [id_prod, price, categ]\n",
       "Index: []"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nullvalue = products.loc[products['id_prod'] == '0_2245']\n",
    "nullvalue\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfd9f71f",
   "metadata": {},
   "source": [
    "##### => there is NOT a product with id_prod = 0_2245, but it's sure that it is in categ 0. thus need to decide among mean/mode/median, which to use in categ 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d2716fab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       id_prod                             date session_id client_id\n",
      "count   337016                           337016     337016    337016\n",
      "unique    3266                           336855     169195      8602\n",
      "top      1_369  test_2021-03-01 02:30:02.237413        s_0    c_1609\n",
      "freq      1081                               13        200     12855\n",
      "False\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 337016 entries, 0 to 337015\n",
      "Data columns (total 4 columns):\n",
      " #   Column      Non-Null Count   Dtype \n",
      "---  ------      --------------   ----- \n",
      " 0   id_prod     337016 non-null  object\n",
      " 1   date        337016 non-null  object\n",
      " 2   session_id  337016 non-null  object\n",
      " 3   client_id   337016 non-null  object\n",
      "dtypes: object(4)\n",
      "memory usage: 10.3+ MB\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>date</th>\n",
       "      <th>session_id</th>\n",
       "      <th>client_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_1483</td>\n",
       "      <td>2021-04-10 18:37:28.723910</td>\n",
       "      <td>s_18746</td>\n",
       "      <td>c_4450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2_226</td>\n",
       "      <td>2022-02-03 01:55:53.276402</td>\n",
       "      <td>s_159142</td>\n",
       "      <td>c_277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1_374</td>\n",
       "      <td>2021-09-23 15:13:46.938559</td>\n",
       "      <td>s_94290</td>\n",
       "      <td>c_4270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_2186</td>\n",
       "      <td>2021-10-17 03:27:18.783634</td>\n",
       "      <td>s_105936</td>\n",
       "      <td>c_4597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_1351</td>\n",
       "      <td>2021-07-17 20:34:25.800563</td>\n",
       "      <td>s_63642</td>\n",
       "      <td>c_1242</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_prod                        date session_id client_id\n",
       "0  0_1483  2021-04-10 18:37:28.723910    s_18746    c_4450\n",
       "1   2_226  2022-02-03 01:55:53.276402   s_159142     c_277\n",
       "2   1_374  2021-09-23 15:13:46.938559    s_94290    c_4270\n",
       "3  0_2186  2021-10-17 03:27:18.783634   s_105936    c_4597\n",
       "4  0_1351  2021-07-17 20:34:25.800563    s_63642    c_1242"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transactions= pd.read_csv(\"transactions.csv\")\n",
    "print(transactions.describe(include=\"all\"), sep='\\n')\n",
    "print(products.isnull().values.any()) \n",
    "print(transactions.info())\n",
    "transactions.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "109274c0",
   "metadata": {},
   "source": [
    "####  Select all duplicate rows based on values of [date] column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2f7a42c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       id_prod                             date session_id client_id\n",
      "count      161                              161        161       161\n",
      "unique       1                               37          1         2\n",
      "top        T_0  test_2021-03-01 02:30:02.237413        s_0      ct_0\n",
      "freq       161                               12        161        84\n",
      "Duplicate rows based on the column 'date' are:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>date</th>\n",
       "      <th>session_id</th>\n",
       "      <th>client_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27161</th>\n",
       "      <td>T_0</td>\n",
       "      <td>test_2021-03-01 02:30:02.237434</td>\n",
       "      <td>s_0</td>\n",
       "      <td>ct_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34387</th>\n",
       "      <td>T_0</td>\n",
       "      <td>test_2021-03-01 02:30:02.237443</td>\n",
       "      <td>s_0</td>\n",
       "      <td>ct_0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48425</th>\n",
       "      <td>T_0</td>\n",
       "      <td>test_2021-03-01 02:30:02.237443</td>\n",
       "      <td>s_0</td>\n",
       "      <td>ct_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54813</th>\n",
       "      <td>T_0</td>\n",
       "      <td>test_2021-03-01 02:30:02.237412</td>\n",
       "      <td>s_0</td>\n",
       "      <td>ct_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56373</th>\n",
       "      <td>T_0</td>\n",
       "      <td>test_2021-03-01 02:30:02.237446</td>\n",
       "      <td>s_0</td>\n",
       "      <td>ct_0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id_prod                             date session_id client_id\n",
       "27161     T_0  test_2021-03-01 02:30:02.237434        s_0      ct_0\n",
       "34387     T_0  test_2021-03-01 02:30:02.237443        s_0      ct_0\n",
       "48425     T_0  test_2021-03-01 02:30:02.237443        s_0      ct_1\n",
       "54813     T_0  test_2021-03-01 02:30:02.237412        s_0      ct_1\n",
       "56373     T_0  test_2021-03-01 02:30:02.237446        s_0      ct_0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "date_duplicate = transactions[transactions.duplicated(['date'])]\n",
    "\n",
    "print(date_duplicate.describe(include=\"all\"))\n",
    "\n",
    "print(\"Duplicate rows based on the column 'date' are:\")\n",
    "date_duplicate.head()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5cfecff",
   "metadata": {},
   "source": [
    " it's clear that  all these duplicated rows have some points in common: id_prod = T_0, date started by 'test_',  session_id = 's=0'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3663f906",
   "metadata": {},
   "source": [
    "#### To drop duplicate rows in [date] column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fd2510db",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       id_prod                        date session_id client_id\n",
      "count   336816                      336816     336816    336816\n",
      "unique    3265                      336816     169194      8600\n",
      "top      1_369  2021-04-19 15:10:35.016717   s_118668    c_1609\n",
      "freq      1081                           1         14     12855\n",
      "       id_prod                        date session_id client_id\n",
      "count   336816                      336816     336816    336816\n",
      "unique    3265                      336816     169194      8600\n",
      "top      1_369  2021-04-19 15:10:35.016717   s_118668    c_1609\n",
      "freq      1081                           1         14     12855\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>date</th>\n",
       "      <th>session_id</th>\n",
       "      <th>client_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_1483</td>\n",
       "      <td>2021-04-10 18:37:28.723910</td>\n",
       "      <td>s_18746</td>\n",
       "      <td>c_4450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2_226</td>\n",
       "      <td>2022-02-03 01:55:53.276402</td>\n",
       "      <td>s_159142</td>\n",
       "      <td>c_277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1_374</td>\n",
       "      <td>2021-09-23 15:13:46.938559</td>\n",
       "      <td>s_94290</td>\n",
       "      <td>c_4270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_2186</td>\n",
       "      <td>2021-10-17 03:27:18.783634</td>\n",
       "      <td>s_105936</td>\n",
       "      <td>c_4597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_1351</td>\n",
       "      <td>2021-07-17 20:34:25.800563</td>\n",
       "      <td>s_63642</td>\n",
       "      <td>c_1242</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_prod                        date session_id client_id\n",
       "0  0_1483  2021-04-10 18:37:28.723910    s_18746    c_4450\n",
       "1   2_226  2022-02-03 01:55:53.276402   s_159142     c_277\n",
       "2   1_374  2021-09-23 15:13:46.938559    s_94290    c_4270\n",
       "3  0_2186  2021-10-17 03:27:18.783634   s_105936    c_4597\n",
       "4  0_1351  2021-07-17 20:34:25.800563    s_63642    c_1242"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transac=transactions[transactions.id_prod != \"T_0\"]\n",
    "print(transac.describe())\n",
    "transac.head()\n",
    "\n",
    "transac2 = transac[transac.session_id != \"s_0\"]\n",
    "print(transac2.describe())\n",
    "\n",
    "# => no difference in case we deduplicate by id_prod or by session_id\n",
    "\n",
    "transac.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "73ee51c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       id_prod                        date session_id client_id     sex  \\\n",
      "count   336816                      336816     336816    336816  336816   \n",
      "unique    3265                      336816     169194      8600       2   \n",
      "top      1_369  2021-04-19 15:10:35.016717   s_118668    c_1609       m   \n",
      "freq      1081                           1         14     12855  169241   \n",
      "mean       NaN                         NaN        NaN       NaN     NaN   \n",
      "std        NaN                         NaN        NaN       NaN     NaN   \n",
      "min        NaN                         NaN        NaN       NaN     NaN   \n",
      "25%        NaN                         NaN        NaN       NaN     NaN   \n",
      "50%        NaN                         NaN        NaN       NaN     NaN   \n",
      "75%        NaN                         NaN        NaN       NaN     NaN   \n",
      "max        NaN                         NaN        NaN       NaN     NaN   \n",
      "\n",
      "                birth  \n",
      "count   336816.000000  \n",
      "unique            NaN  \n",
      "top               NaN  \n",
      "freq              NaN  \n",
      "mean      1977.823396  \n",
      "std         13.523923  \n",
      "min       1929.000000  \n",
      "25%       1971.000000  \n",
      "50%       1980.000000  \n",
      "75%       1987.000000  \n",
      "max       2004.000000  \n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 336816 entries, 0 to 336815\n",
      "Data columns (total 6 columns):\n",
      " #   Column      Non-Null Count   Dtype \n",
      "---  ------      --------------   ----- \n",
      " 0   id_prod     336816 non-null  object\n",
      " 1   date        336816 non-null  object\n",
      " 2   session_id  336816 non-null  object\n",
      " 3   client_id   336816 non-null  object\n",
      " 4   sex         336816 non-null  object\n",
      " 5   birth       336816 non-null  int64 \n",
      "dtypes: int64(1), object(5)\n",
      "memory usage: 18.0+ MB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>date</th>\n",
       "      <th>session_id</th>\n",
       "      <th>client_id</th>\n",
       "      <th>sex</th>\n",
       "      <th>birth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_1483</td>\n",
       "      <td>2021-04-10 18:37:28.723910</td>\n",
       "      <td>s_18746</td>\n",
       "      <td>c_4450</td>\n",
       "      <td>f</td>\n",
       "      <td>1977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2_226</td>\n",
       "      <td>2022-02-03 01:55:53.276402</td>\n",
       "      <td>s_159142</td>\n",
       "      <td>c_277</td>\n",
       "      <td>f</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1_374</td>\n",
       "      <td>2021-09-23 15:13:46.938559</td>\n",
       "      <td>s_94290</td>\n",
       "      <td>c_4270</td>\n",
       "      <td>f</td>\n",
       "      <td>1979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_2186</td>\n",
       "      <td>2021-10-17 03:27:18.783634</td>\n",
       "      <td>s_105936</td>\n",
       "      <td>c_4597</td>\n",
       "      <td>m</td>\n",
       "      <td>1963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_1351</td>\n",
       "      <td>2021-07-17 20:34:25.800563</td>\n",
       "      <td>s_63642</td>\n",
       "      <td>c_1242</td>\n",
       "      <td>f</td>\n",
       "      <td>1980</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_prod                        date session_id client_id sex  birth\n",
       "0  0_1483  2021-04-10 18:37:28.723910    s_18746    c_4450   f   1977\n",
       "1   2_226  2022-02-03 01:55:53.276402   s_159142     c_277   f   2000\n",
       "2   1_374  2021-09-23 15:13:46.938559    s_94290    c_4270   f   1979\n",
       "3  0_2186  2021-10-17 03:27:18.783634   s_105936    c_4597   m   1963\n",
       "4  0_1351  2021-07-17 20:34:25.800563    s_63642    c_1242   f   1980"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trans_cus = pd.merge(transac, customers, on='client_id', how='left')\n",
    "\n",
    "print(trans_cus.describe(include=\"all\"))\n",
    "trans_cus.info()\n",
    "trans_cus.head() \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6673a82a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 336816 entries, 0 to 336815\n",
      "Data columns (total 6 columns):\n",
      " #   Column      Non-Null Count   Dtype  \n",
      "---  ------      --------------   -----  \n",
      " 0   id_prod     336816 non-null  object \n",
      " 1   date        336816 non-null  object \n",
      " 2   session_id  336816 non-null  object \n",
      " 3   client_id   336816 non-null  object \n",
      " 4   price       336713 non-null  float64\n",
      " 5   categ       336713 non-null  float64\n",
      "dtypes: float64(2), object(4)\n",
      "memory usage: 18.0+ MB\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>date</th>\n",
       "      <th>session_id</th>\n",
       "      <th>client_id</th>\n",
       "      <th>price</th>\n",
       "      <th>categ</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_1483</td>\n",
       "      <td>2021-04-10 18:37:28.723910</td>\n",
       "      <td>s_18746</td>\n",
       "      <td>c_4450</td>\n",
       "      <td>4.99</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2_226</td>\n",
       "      <td>2022-02-03 01:55:53.276402</td>\n",
       "      <td>s_159142</td>\n",
       "      <td>c_277</td>\n",
       "      <td>65.75</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1_374</td>\n",
       "      <td>2021-09-23 15:13:46.938559</td>\n",
       "      <td>s_94290</td>\n",
       "      <td>c_4270</td>\n",
       "      <td>10.71</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_2186</td>\n",
       "      <td>2021-10-17 03:27:18.783634</td>\n",
       "      <td>s_105936</td>\n",
       "      <td>c_4597</td>\n",
       "      <td>4.20</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_1351</td>\n",
       "      <td>2021-07-17 20:34:25.800563</td>\n",
       "      <td>s_63642</td>\n",
       "      <td>c_1242</td>\n",
       "      <td>8.99</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_prod                        date session_id client_id  price  categ\n",
       "0  0_1483  2021-04-10 18:37:28.723910    s_18746    c_4450   4.99    0.0\n",
       "1   2_226  2022-02-03 01:55:53.276402   s_159142     c_277  65.75    2.0\n",
       "2   1_374  2021-09-23 15:13:46.938559    s_94290    c_4270  10.71    1.0\n",
       "3  0_2186  2021-10-17 03:27:18.783634   s_105936    c_4597   4.20    0.0\n",
       "4  0_1351  2021-07-17 20:34:25.800563    s_63642    c_1242   8.99    0.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trans_prod = pd.merge(transac, products, on='id_prod', how='left')\n",
    "print(trans_prod.info())\n",
    "trans_prod.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96c68c53",
   "metadata": {},
   "source": [
    "#### price, categ: 336816-336713 = 103 "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c2e011c",
   "metadata": {},
   "source": [
    "#### to merge dataframes into one main dataframe \"trans_prod_cus\" via primary keys\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c23a4982",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 336816 entries, 0 to 336815\n",
      "Data columns (total 8 columns):\n",
      " #   Column      Non-Null Count   Dtype  \n",
      "---  ------      --------------   -----  \n",
      " 0   id_prod     336816 non-null  object \n",
      " 1   date        336816 non-null  object \n",
      " 2   session_id  336816 non-null  object \n",
      " 3   client_id   336816 non-null  object \n",
      " 4   price       336713 non-null  float64\n",
      " 5   categ       336713 non-null  float64\n",
      " 6   sex         336816 non-null  object \n",
      " 7   birth       336816 non-null  int64  \n",
      "dtypes: float64(2), int64(1), object(5)\n",
      "memory usage: 23.1+ MB\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>date</th>\n",
       "      <th>session_id</th>\n",
       "      <th>client_id</th>\n",
       "      <th>price</th>\n",
       "      <th>categ</th>\n",
       "      <th>sex</th>\n",
       "      <th>birth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_1483</td>\n",
       "      <td>2021-04-10 18:37:28.723910</td>\n",
       "      <td>s_18746</td>\n",
       "      <td>c_4450</td>\n",
       "      <td>4.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2_226</td>\n",
       "      <td>2022-02-03 01:55:53.276402</td>\n",
       "      <td>s_159142</td>\n",
       "      <td>c_277</td>\n",
       "      <td>65.75</td>\n",
       "      <td>2.0</td>\n",
       "      <td>f</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1_374</td>\n",
       "      <td>2021-09-23 15:13:46.938559</td>\n",
       "      <td>s_94290</td>\n",
       "      <td>c_4270</td>\n",
       "      <td>10.71</td>\n",
       "      <td>1.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_2186</td>\n",
       "      <td>2021-10-17 03:27:18.783634</td>\n",
       "      <td>s_105936</td>\n",
       "      <td>c_4597</td>\n",
       "      <td>4.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>m</td>\n",
       "      <td>1963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_1351</td>\n",
       "      <td>2021-07-17 20:34:25.800563</td>\n",
       "      <td>s_63642</td>\n",
       "      <td>c_1242</td>\n",
       "      <td>8.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1980</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_prod                        date session_id client_id  price  categ sex  \\\n",
       "0  0_1483  2021-04-10 18:37:28.723910    s_18746    c_4450   4.99    0.0   f   \n",
       "1   2_226  2022-02-03 01:55:53.276402   s_159142     c_277  65.75    2.0   f   \n",
       "2   1_374  2021-09-23 15:13:46.938559    s_94290    c_4270  10.71    1.0   f   \n",
       "3  0_2186  2021-10-17 03:27:18.783634   s_105936    c_4597   4.20    0.0   m   \n",
       "4  0_1351  2021-07-17 20:34:25.800563    s_63642    c_1242   8.99    0.0   f   \n",
       "\n",
       "   birth  \n",
       "0   1977  \n",
       "1   2000  \n",
       "2   1979  \n",
       "3   1963  \n",
       "4   1980  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trans_prod_cus = pd.merge(trans_prod, trans_cus, on = ['date', 'id_prod', 'session_id', 'client_id'])\n",
    "print(trans_prod_cus.info())\n",
    "trans_prod_cus.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a8ef437",
   "metadata": {},
   "source": [
    "#### in price and categ: 336816 - 336713 non-null  = 103 NaN  : Done before 3 df were merged into trans_prod_cus \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9ea04927",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 336816 entries, 0 to 336815\n",
      "Data columns (total 8 columns):\n",
      " #   Column      Non-Null Count   Dtype         \n",
      "---  ------      --------------   -----         \n",
      " 0   id_prod     336816 non-null  object        \n",
      " 1   date        336816 non-null  datetime64[ns]\n",
      " 2   session_id  336816 non-null  object        \n",
      " 3   client_id   336816 non-null  object        \n",
      " 4   price       336713 non-null  float64       \n",
      " 5   categ       336713 non-null  float64       \n",
      " 6   sex         336816 non-null  object        \n",
      " 7   birth       336816 non-null  datetime64[ns]\n",
      "dtypes: datetime64[ns](2), float64(2), object(4)\n",
      "memory usage: 23.1+ MB\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>date</th>\n",
       "      <th>session_id</th>\n",
       "      <th>client_id</th>\n",
       "      <th>price</th>\n",
       "      <th>categ</th>\n",
       "      <th>sex</th>\n",
       "      <th>birth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_1483</td>\n",
       "      <td>2021-04-10 18:37:28.723910</td>\n",
       "      <td>s_18746</td>\n",
       "      <td>c_4450</td>\n",
       "      <td>4.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1977-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2_226</td>\n",
       "      <td>2022-02-03 01:55:53.276402</td>\n",
       "      <td>s_159142</td>\n",
       "      <td>c_277</td>\n",
       "      <td>65.75</td>\n",
       "      <td>2.0</td>\n",
       "      <td>f</td>\n",
       "      <td>2000-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1_374</td>\n",
       "      <td>2021-09-23 15:13:46.938559</td>\n",
       "      <td>s_94290</td>\n",
       "      <td>c_4270</td>\n",
       "      <td>10.71</td>\n",
       "      <td>1.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1979-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_2186</td>\n",
       "      <td>2021-10-17 03:27:18.783634</td>\n",
       "      <td>s_105936</td>\n",
       "      <td>c_4597</td>\n",
       "      <td>4.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>m</td>\n",
       "      <td>1963-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_1351</td>\n",
       "      <td>2021-07-17 20:34:25.800563</td>\n",
       "      <td>s_63642</td>\n",
       "      <td>c_1242</td>\n",
       "      <td>8.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1980-01-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_prod                       date session_id client_id  price  categ sex  \\\n",
       "0  0_1483 2021-04-10 18:37:28.723910    s_18746    c_4450   4.99    0.0   f   \n",
       "1   2_226 2022-02-03 01:55:53.276402   s_159142     c_277  65.75    2.0   f   \n",
       "2   1_374 2021-09-23 15:13:46.938559    s_94290    c_4270  10.71    1.0   f   \n",
       "3  0_2186 2021-10-17 03:27:18.783634   s_105936    c_4597   4.20    0.0   m   \n",
       "4  0_1351 2021-07-17 20:34:25.800563    s_63642    c_1242   8.99    0.0   f   \n",
       "\n",
       "       birth  \n",
       "0 1977-01-01  \n",
       "1 2000-01-01  \n",
       "2 1979-01-01  \n",
       "3 1963-01-01  \n",
       "4 1980-01-01  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#convert the date and birth columns from object into datetime\n",
    "\n",
    "trans_prod_cus['date'] = pd.to_datetime(trans_prod_cus.date)\n",
    "trans_prod_cus['birth'] = pd.to_datetime(trans_prod_cus['birth'], format = '%Y', errors ='coerce')\n",
    "\n",
    "print(trans_prod_cus.info())\n",
    "trans_prod_cus.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c85e5e7b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "year: 2021\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>date</th>\n",
       "      <th>session_id</th>\n",
       "      <th>client_id</th>\n",
       "      <th>price</th>\n",
       "      <th>categ</th>\n",
       "      <th>sex</th>\n",
       "      <th>birth</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_1483</td>\n",
       "      <td>2021-04-10 18:37:28.723910</td>\n",
       "      <td>s_18746</td>\n",
       "      <td>c_4450</td>\n",
       "      <td>4.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1977-01-01</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2_226</td>\n",
       "      <td>2022-02-03 01:55:53.276402</td>\n",
       "      <td>s_159142</td>\n",
       "      <td>c_277</td>\n",
       "      <td>65.75</td>\n",
       "      <td>2.0</td>\n",
       "      <td>f</td>\n",
       "      <td>2000-01-01</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1_374</td>\n",
       "      <td>2021-09-23 15:13:46.938559</td>\n",
       "      <td>s_94290</td>\n",
       "      <td>c_4270</td>\n",
       "      <td>10.71</td>\n",
       "      <td>1.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1979-01-01</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_2186</td>\n",
       "      <td>2021-10-17 03:27:18.783634</td>\n",
       "      <td>s_105936</td>\n",
       "      <td>c_4597</td>\n",
       "      <td>4.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>m</td>\n",
       "      <td>1963-01-01</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_1351</td>\n",
       "      <td>2021-07-17 20:34:25.800563</td>\n",
       "      <td>s_63642</td>\n",
       "      <td>c_1242</td>\n",
       "      <td>8.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1980-01-01</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_prod                       date session_id client_id  price  categ sex  \\\n",
       "0  0_1483 2021-04-10 18:37:28.723910    s_18746    c_4450   4.99    0.0   f   \n",
       "1   2_226 2022-02-03 01:55:53.276402   s_159142     c_277  65.75    2.0   f   \n",
       "2   1_374 2021-09-23 15:13:46.938559    s_94290    c_4270  10.71    1.0   f   \n",
       "3  0_2186 2021-10-17 03:27:18.783634   s_105936    c_4597   4.20    0.0   m   \n",
       "4  0_1351 2021-07-17 20:34:25.800563    s_63642    c_1242   8.99    0.0   f   \n",
       "\n",
       "       birth  age  \n",
       "0 1977-01-01   44  \n",
       "1 2000-01-01   21  \n",
       "2 1979-01-01   42  \n",
       "3 1963-01-01   58  \n",
       "4 1980-01-01   41  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "from datetime import date\n",
    "\n",
    "now = datetime.now()    # current date and time\n",
    "year = now.strftime(\"%Y\")\n",
    "print(\"year:\", year)\n",
    "\n",
    "def calculate_age(birth):      # create a function to calculate age based on birth \n",
    "    today = date.today()\n",
    "    return today.year - birth.year - ((today.month, today.day) < (birth.month, birth.day))\n",
    "\n",
    "trans_prod_cus['age'] = trans_prod_cus['birth'].apply(calculate_age)    # column 'age' created\n",
    "trans_prod_cus.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a953e2d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 336816 entries, 0 to 336815\n",
      "Data columns (total 13 columns):\n",
      " #   Column          Non-Null Count   Dtype         \n",
      "---  ------          --------------   -----         \n",
      " 0   id_prod         336816 non-null  object        \n",
      " 1   date            336816 non-null  datetime64[ns]\n",
      " 2   session_id      336816 non-null  object        \n",
      " 3   client_id       336816 non-null  object        \n",
      " 4   price           336713 non-null  float64       \n",
      " 5   categ           336713 non-null  float64       \n",
      " 6   sex             336816 non-null  object        \n",
      " 7   birth           336816 non-null  datetime64[ns]\n",
      " 8   age             336816 non-null  int64         \n",
      " 9   year-month-day  336816 non-null  object        \n",
      " 10  year            336816 non-null  int64         \n",
      " 11  month           336816 non-null  int64         \n",
      " 12  period          336816 non-null  int64         \n",
      "dtypes: datetime64[ns](2), float64(2), int64(4), object(5)\n",
      "memory usage: 36.0+ MB\n",
      "None\n",
      "True\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>date</th>\n",
       "      <th>session_id</th>\n",
       "      <th>client_id</th>\n",
       "      <th>price</th>\n",
       "      <th>categ</th>\n",
       "      <th>sex</th>\n",
       "      <th>birth</th>\n",
       "      <th>age</th>\n",
       "      <th>year-month-day</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>period</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_1483</td>\n",
       "      <td>2021-04-10 18:37:28.723910</td>\n",
       "      <td>s_18746</td>\n",
       "      <td>c_4450</td>\n",
       "      <td>4.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1977-01-01</td>\n",
       "      <td>44</td>\n",
       "      <td>2021-04-10</td>\n",
       "      <td>2021</td>\n",
       "      <td>4</td>\n",
       "      <td>20214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2_226</td>\n",
       "      <td>2022-02-03 01:55:53.276402</td>\n",
       "      <td>s_159142</td>\n",
       "      <td>c_277</td>\n",
       "      <td>65.75</td>\n",
       "      <td>2.0</td>\n",
       "      <td>f</td>\n",
       "      <td>2000-01-01</td>\n",
       "      <td>21</td>\n",
       "      <td>2022-02-03</td>\n",
       "      <td>2022</td>\n",
       "      <td>2</td>\n",
       "      <td>20222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1_374</td>\n",
       "      <td>2021-09-23 15:13:46.938559</td>\n",
       "      <td>s_94290</td>\n",
       "      <td>c_4270</td>\n",
       "      <td>10.71</td>\n",
       "      <td>1.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1979-01-01</td>\n",
       "      <td>42</td>\n",
       "      <td>2021-09-23</td>\n",
       "      <td>2021</td>\n",
       "      <td>9</td>\n",
       "      <td>20219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_2186</td>\n",
       "      <td>2021-10-17 03:27:18.783634</td>\n",
       "      <td>s_105936</td>\n",
       "      <td>c_4597</td>\n",
       "      <td>4.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>m</td>\n",
       "      <td>1963-01-01</td>\n",
       "      <td>58</td>\n",
       "      <td>2021-10-17</td>\n",
       "      <td>2021</td>\n",
       "      <td>10</td>\n",
       "      <td>202110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_1351</td>\n",
       "      <td>2021-07-17 20:34:25.800563</td>\n",
       "      <td>s_63642</td>\n",
       "      <td>c_1242</td>\n",
       "      <td>8.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1980-01-01</td>\n",
       "      <td>41</td>\n",
       "      <td>2021-07-17</td>\n",
       "      <td>2021</td>\n",
       "      <td>7</td>\n",
       "      <td>20217</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_prod                       date session_id client_id  price  categ sex  \\\n",
       "0  0_1483 2021-04-10 18:37:28.723910    s_18746    c_4450   4.99    0.0   f   \n",
       "1   2_226 2022-02-03 01:55:53.276402   s_159142     c_277  65.75    2.0   f   \n",
       "2   1_374 2021-09-23 15:13:46.938559    s_94290    c_4270  10.71    1.0   f   \n",
       "3  0_2186 2021-10-17 03:27:18.783634   s_105936    c_4597   4.20    0.0   m   \n",
       "4  0_1351 2021-07-17 20:34:25.800563    s_63642    c_1242   8.99    0.0   f   \n",
       "\n",
       "       birth  age year-month-day  year  month  period  \n",
       "0 1977-01-01   44     2021-04-10  2021      4   20214  \n",
       "1 2000-01-01   21     2022-02-03  2022      2   20222  \n",
       "2 1979-01-01   42     2021-09-23  2021      9   20219  \n",
       "3 1963-01-01   58     2021-10-17  2021     10  202110  \n",
       "4 1980-01-01   41     2021-07-17  2021      7   20217  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trans_prod_cus['year-month-day'] = pd.to_datetime(trans_prod_cus ['date']).dt.date\n",
    "trans_prod_cus['year'] = pd.to_datetime(trans_prod_cus ['date']).dt.year\n",
    "trans_prod_cus['month'] = pd.to_datetime(trans_prod_cus ['date']).dt.month\n",
    "trans_prod_cus[\"period\"] = trans_prod_cus[\"year\"].astype(str)  + trans_prod_cus[\"month\"].astype(str)\n",
    "trans_prod_cus[\"period\"] = pd.to_numeric(trans_prod_cus[\"period\"])\n",
    "\n",
    "print(trans_prod_cus.info())\n",
    "print(trans_prod_cus.isnull().values.any()) \n",
    "trans_prod_cus.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60abf711",
   "metadata": {},
   "source": [
    "- create columns: [year-month-day], [year], [month], [period] \n",
    "- there're NaN in this dataframe. to be precise, in [price] and [categ] columns (seperately 103 NaN) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c363d438",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the number of NaN in price column is: 103\n",
      "the number of NaN in categ column is: 103\n"
     ]
    }
   ],
   "source": [
    "print(\"the number of NaN in price column is:\",  trans_prod_cus['price'].isnull().sum().sum())\n",
    "print(\"the number of NaN in categ column is:\", trans_prod_cus['categ'].isnull().sum().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d5efe710",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(103, 13)\n",
      "(103, 13)\n"
     ]
    }
   ],
   "source": [
    "pricenull = pd.isnull(trans_prod_cus['price'])\n",
    "categnull = pd.isnull(trans_prod_cus['categ'])\n",
    "print(trans_prod_cus[pricenull].shape)\n",
    "print(trans_prod_cus[categnull].shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c697d1f",
   "metadata": {},
   "source": [
    "##### => then we find that the product with price and categ = NaN is product 0_2245.\n",
    "##### given the id_prod starts always by the product categ, we can identify that product 0_2245 has categ 0. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "de83b8fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>date</th>\n",
       "      <th>session_id</th>\n",
       "      <th>client_id</th>\n",
       "      <th>price</th>\n",
       "      <th>categ</th>\n",
       "      <th>sex</th>\n",
       "      <th>birth</th>\n",
       "      <th>age</th>\n",
       "      <th>year-month-day</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>period</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6231</th>\n",
       "      <td>0_2245</td>\n",
       "      <td>2021-06-17 03:03:12.668129</td>\n",
       "      <td>s_49705</td>\n",
       "      <td>c_1533</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>m</td>\n",
       "      <td>1972-01-01</td>\n",
       "      <td>49</td>\n",
       "      <td>2021-06-17</td>\n",
       "      <td>2021</td>\n",
       "      <td>6</td>\n",
       "      <td>20216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10797</th>\n",
       "      <td>0_2245</td>\n",
       "      <td>2021-06-16 05:53:01.627491</td>\n",
       "      <td>s_49323</td>\n",
       "      <td>c_7954</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>m</td>\n",
       "      <td>1973-01-01</td>\n",
       "      <td>48</td>\n",
       "      <td>2021-06-16</td>\n",
       "      <td>2021</td>\n",
       "      <td>6</td>\n",
       "      <td>20216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14045</th>\n",
       "      <td>0_2245</td>\n",
       "      <td>2021-11-24 17:35:59.911427</td>\n",
       "      <td>s_124474</td>\n",
       "      <td>c_5120</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>f</td>\n",
       "      <td>1975-01-01</td>\n",
       "      <td>46</td>\n",
       "      <td>2021-11-24</td>\n",
       "      <td>2021</td>\n",
       "      <td>11</td>\n",
       "      <td>202111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17480</th>\n",
       "      <td>0_2245</td>\n",
       "      <td>2022-02-28 18:08:49.875709</td>\n",
       "      <td>s_172304</td>\n",
       "      <td>c_4964</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>f</td>\n",
       "      <td>1982-01-01</td>\n",
       "      <td>39</td>\n",
       "      <td>2022-02-28</td>\n",
       "      <td>2022</td>\n",
       "      <td>2</td>\n",
       "      <td>20222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21071</th>\n",
       "      <td>0_2245</td>\n",
       "      <td>2021-03-01 00:09:29.301897</td>\n",
       "      <td>s_3</td>\n",
       "      <td>c_580</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>m</td>\n",
       "      <td>1988-01-01</td>\n",
       "      <td>33</td>\n",
       "      <td>2021-03-01</td>\n",
       "      <td>2021</td>\n",
       "      <td>3</td>\n",
       "      <td>20213</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id_prod                       date session_id client_id  price  categ  \\\n",
       "6231   0_2245 2021-06-17 03:03:12.668129    s_49705    c_1533    NaN    NaN   \n",
       "10797  0_2245 2021-06-16 05:53:01.627491    s_49323    c_7954    NaN    NaN   \n",
       "14045  0_2245 2021-11-24 17:35:59.911427   s_124474    c_5120    NaN    NaN   \n",
       "17480  0_2245 2022-02-28 18:08:49.875709   s_172304    c_4964    NaN    NaN   \n",
       "21071  0_2245 2021-03-01 00:09:29.301897        s_3     c_580    NaN    NaN   \n",
       "\n",
       "      sex      birth  age year-month-day  year  month  period  \n",
       "6231    m 1972-01-01   49     2021-06-17  2021      6   20216  \n",
       "10797   m 1973-01-01   48     2021-06-16  2021      6   20216  \n",
       "14045   f 1975-01-01   46     2021-11-24  2021     11  202111  \n",
       "17480   f 1982-01-01   39     2022-02-28  2022      2   20222  \n",
       "21071   m 1988-01-01   33     2021-03-01  2021      3   20213  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trans_prod_cus[pricenull].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "770af63b",
   "metadata": {},
   "source": [
    "### Traitement des données manquantes: imputation \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1a1923f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after fillna, the number of null values in dataframe is: 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_prod</th>\n",
       "      <th>date</th>\n",
       "      <th>session_id</th>\n",
       "      <th>client_id</th>\n",
       "      <th>price</th>\n",
       "      <th>categ</th>\n",
       "      <th>sex</th>\n",
       "      <th>birth</th>\n",
       "      <th>age</th>\n",
       "      <th>year-month-day</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>period</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_1483</td>\n",
       "      <td>2021-04-10 18:37:28.723910</td>\n",
       "      <td>s_18746</td>\n",
       "      <td>c_4450</td>\n",
       "      <td>4.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1977-01-01</td>\n",
       "      <td>44</td>\n",
       "      <td>2021-04-10</td>\n",
       "      <td>2021</td>\n",
       "      <td>4</td>\n",
       "      <td>20214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2_226</td>\n",
       "      <td>2022-02-03 01:55:53.276402</td>\n",
       "      <td>s_159142</td>\n",
       "      <td>c_277</td>\n",
       "      <td>65.75</td>\n",
       "      <td>2.0</td>\n",
       "      <td>f</td>\n",
       "      <td>2000-01-01</td>\n",
       "      <td>21</td>\n",
       "      <td>2022-02-03</td>\n",
       "      <td>2022</td>\n",
       "      <td>2</td>\n",
       "      <td>20222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1_374</td>\n",
       "      <td>2021-09-23 15:13:46.938559</td>\n",
       "      <td>s_94290</td>\n",
       "      <td>c_4270</td>\n",
       "      <td>10.71</td>\n",
       "      <td>1.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1979-01-01</td>\n",
       "      <td>42</td>\n",
       "      <td>2021-09-23</td>\n",
       "      <td>2021</td>\n",
       "      <td>9</td>\n",
       "      <td>20219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0_2186</td>\n",
       "      <td>2021-10-17 03:27:18.783634</td>\n",
       "      <td>s_105936</td>\n",
       "      <td>c_4597</td>\n",
       "      <td>4.20</td>\n",
       "      <td>0.0</td>\n",
       "      <td>m</td>\n",
       "      <td>1963-01-01</td>\n",
       "      <td>58</td>\n",
       "      <td>2021-10-17</td>\n",
       "      <td>2021</td>\n",
       "      <td>10</td>\n",
       "      <td>202110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0_1351</td>\n",
       "      <td>2021-07-17 20:34:25.800563</td>\n",
       "      <td>s_63642</td>\n",
       "      <td>c_1242</td>\n",
       "      <td>8.99</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "      <td>1980-01-01</td>\n",
       "      <td>41</td>\n",
       "      <td>2021-07-17</td>\n",
       "      <td>2021</td>\n",
       "      <td>7</td>\n",
       "      <td>20217</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  id_prod                       date session_id client_id  price  categ sex  \\\n",
       "0  0_1483 2021-04-10 18:37:28.723910    s_18746    c_4450   4.99    0.0   f   \n",
       "1   2_226 2022-02-03 01:55:53.276402   s_159142     c_277  65.75    2.0   f   \n",
       "2   1_374 2021-09-23 15:13:46.938559    s_94290    c_4270  10.71    1.0   f   \n",
       "3  0_2186 2021-10-17 03:27:18.783634   s_105936    c_4597   4.20    0.0   m   \n",
       "4  0_1351 2021-07-17 20:34:25.800563    s_63642    c_1242   8.99    0.0   f   \n",
       "\n",
       "       birth  age year-month-day  year  month  period  \n",
       "0 1977-01-01   44     2021-04-10  2021      4   20214  \n",
       "1 2000-01-01   21     2022-02-03  2022      2   20222  \n",
       "2 1979-01-01   42     2021-09-23  2021      9   20219  \n",
       "3 1963-01-01   58     2021-10-17  2021     10  202110  \n",
       "4 1980-01-01   41     2021-07-17  2021      7   20217  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# to fill NaN in trans_prod_cus\n",
    "\n",
    "trans_prod_cus.loc[trans_prod_cus['id_prod'] == \"0_2245\", ['price'] ] =  trans_prod_cus['price'].fillna(10.32)\n",
    "trans_prod_cus.loc[trans_prod_cus['id_prod'] == \"0_2245\", ['categ'] ] =  trans_prod_cus['categ'].fillna(0)\n",
    "\n",
    "print('after fillna, the number of null values in dataframe is:', trans_prod_cus.isnull().sum().sum())\n",
    "trans_prod_cus.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e81f3418",
   "metadata": {},
   "source": [
    "#### Maintenant que le dataframe constitué, les données aberrantes nettoyées et les données manquantes imputées, on peut aller analyser les données de différentes variables de ce nouveau dataframe trans_prod_cus"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}