indicator_lags_num.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import talib as ta\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mysql.connector\n",
    "import ast\n",
    "with open('mysql_configure') as f:\n",
    "    mysqlkeys = f.read()\n",
    "    mysqlkeys = ast.literal_eval(mysqlkeys)\n",
    "mydb = mysql.connector.connect(**mysqlkeys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ASSET_DETAILS_CSV = './data/asset_details.csv'\n",
    "# df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\n",
    "df_asset_details = pd.read_sql(\"select * from asset_details\", mydb)\n",
    "add_weight_map = dict(zip(df_asset_details.Asset_ID, \n",
    "                        df_asset_details.Weight/df_asset_details.Weight.sum()))\n",
    "\n",
    "# sup_train = pd.read_csv('./data/supplemental_train.csv')\n",
    "sup_train = pd.read_sql(f\"select * from kaggle_train where timestamp >=1632182460\",mydb)\n",
    "sup_train['Weight'] = sup_train['Asset_ID'].map(add_weight_map)\n",
    "sup_train = sup_train.sort_values('timestamp').set_index([\"timestamp\",'Asset_ID'])\n",
    "sup_train.drop('Target',axis=1, inplace=True)\n",
    "\n",
    "\n",
    "def log_return(series, periods=5):\n",
    "    return np.log(series).diff(periods)\n",
    "lr_15 = sup_train.groupby('Asset_ID').apply( \n",
    "        lambda x: log_return(x[['Close']],15)\n",
    "        )\n",
    "sup_train['lr_15'] = lr_15['Close']\n",
    "\n",
    "mkt_lr_15 = sup_train.groupby('timestamp').apply( \n",
    "    lambda x: x[[\"lr_15\", \"Close\"]].multiply(x[\"Weight\"], axis=\"index\").sum(skipna=True)\n",
    "    )\n",
    "mkt_lr_15.columns = ['Mkt_lrt_15','Crypto_Index']\n",
    "firsts = sup_train.index.get_level_values('timestamp')\n",
    "sup_train[['Mkt_lrt_15','Crypto_Index']] = mkt_lr_15.loc[firsts].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1632182460, 1642982400)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sup_train.index.get_level_values('timestamp').min(),sup_train.index.get_level_values('timestamp').max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2518278, 11)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sup_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lag_features(df, fastk1,fastk2,adx,macd_s,macd_l,macd_sig,rsi,vol_sum,std_Crypto_Index,std_lr_15,std_Mkt_lrt_15, **kwargs):    \n",
    "    ####TECH indicators\n",
    "    df['slowK'], df['slowD'] = ta.STOCH(df.High, df.Low, df.Close, \n",
    "                                        fastk_period=fastk1, slowk_period=int(3*fastk1/5), slowd_period=int(3*fastk1/5),\n",
    "                                        slowk_matype=0, slowd_matype=0)\n",
    "    df['fastK'], df['fastD'] = ta.STOCHF(df.High, df.Low, df.Close,\n",
    "                                         fastk_period=fastk2, fastd_period=int(3*fastk2/5), \n",
    "                                         fastd_matype=0)\n",
    "    df[f'rsi_{rsi}'] = ta.RSI(df['Close'], timeperiod=rsi)\n",
    "    df[f'macd_{macd_s}_{macd_l}'],df[f'macd_signal_{macd_sig}'], df['macd_hist'] = \\\n",
    "                ta.MACD(df['Close'],fastperiod=macd_s, slowperiod=macd_l, signalperiod=macd_sig)\n",
    "    df[f'adx_{adx}'] = ta.ADX(df['High'], df['Low'],df['Close'], timeperiod=adx)#Average Directional Movement Index\n",
    "    #df['AD'] = ta.AD(df['High'], df['Low'],df['Close'], df['Volume'])#Accumulation Distribution Line\n",
    "    df[f'vol_sum_{vol_sum}'] = ta.SMA(df['Volume'],vol_sum)*vol_sum\n",
    "    ####std volatility\n",
    "    df[f'std_lr_15_{std_lr_15}'] = ta.STDDEV(df.lr_15,timeperiod=std_lr_15, nbdev=1)\n",
    "    df[f'std_Mkt_lrt_15_{std_Mkt_lrt_15}'] = ta.STDDEV(df.Mkt_lrt_15,timeperiod=std_Mkt_lrt_15, nbdev=1)\n",
    "    df[f'std_Crypto_Index_{std_Crypto_Index}'] = ta.STDDEV(df.Crypto_Index,timeperiod=std_Crypto_Index, nbdev=1)\n",
    "    ###NEW\n",
    "    df[f\"ATR_{kwargs['ATR']}\"] = ta.ATR(df['High'], df['Low'],df['Close'], timeperiod=kwargs['ATR'])\n",
    "    df['TRENDLINE'] =ta.HT_TRENDLINE(df['Open'])\n",
    "    df[f\"willr_{kwargs['willr']}\"] = ta.WILLR(df['High'], df['Low'],df['Close'], timeperiod=kwargs['willr'])\n",
    "    return df\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Count: 0 rows, 0.0 mins\n",
      "Open: 0 rows, 0.0 mins\n",
      "High: 0 rows, 0.0 mins\n",
      "Low: 0 rows, 0.0 mins\n",
      "Close: 0 rows, 0.0 mins\n",
      "Volume: 0 rows, 0.0 mins\n",
      "VWAP: 0 rows, 0.0 mins\n",
      "Weight: 0 rows, 0.0 mins\n",
      "lr_15: 0 rows, 0.0 mins\n",
      "Mkt_lrt_15: 0 rows, 0.0 mins\n",
      "Crypto_Index: 0 rows, 0.0 mins\n",
      "slowK: 420 rows, 30.0 mins\n",
      "slowD: 420 rows, 30.0 mins\n",
      "fastK: 196 rows, 14.0 mins\n",
      "fastD: 196 rows, 14.0 mins\n",
      "rsi_30: 420 rows, 30.0 mins\n",
      "macd_10_60: 1022 rows, 73.0 mins\n",
      "macd_signal_15: 1022 rows, 73.0 mins\n",
      "macd_hist: 1022 rows, 73.0 mins\n",
      "adx_50: 1386 rows, 99.0 mins\n",
      "vol_sum_15: 196 rows, 14.0 mins\n",
      "std_lr_15_30: 406 rows, 29.0 mins\n",
      "std_Mkt_lrt_15_10: 126 rows, 9.0 mins\n",
      "std_Crypto_Index_30: 406 rows, 29.0 mins\n",
      "ATR_60: 840 rows, 60.0 mins\n",
      "TRENDLINE: 882 rows, 63.0 mins\n",
      "willr_60: 826 rows, 59.0 mins\n",
      "max nan mins=99.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>lr_15</th>\n",
       "      <th>Mkt_lrt_15</th>\n",
       "      <th>Crypto_Index</th>\n",
       "      <th>slowK</th>\n",
       "      <th>slowD</th>\n",
       "      <th>fastK</th>\n",
       "      <th>fastD</th>\n",
       "      <th>rsi_30</th>\n",
       "      <th>macd_10_60</th>\n",
       "      <th>macd_signal_15</th>\n",
       "      <th>macd_hist</th>\n",
       "      <th>adx_50</th>\n",
       "      <th>vol_sum_15</th>\n",
       "      <th>std_lr_15_30</th>\n",
       "      <th>std_Mkt_lrt_15_10</th>\n",
       "      <th>std_Crypto_Index_30</th>\n",
       "      <th>ATR_60</th>\n",
       "      <th>TRENDLINE</th>\n",
       "      <th>willr_60</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>timestamp</th>\n",
       "      <th>Asset_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"14\" valign=\"top\">1642982400</th>\n",
       "      <th>13</th>\n",
       "      <td>-0.000622</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>59.905404</td>\n",
       "      <td>69.549659</td>\n",
       "      <td>33.418367</td>\n",
       "      <td>40.759107</td>\n",
       "      <td>52.838253</td>\n",
       "      <td>0.000142</td>\n",
       "      <td>0.000206</td>\n",
       "      <td>-0.000063</td>\n",
       "      <td>17.109192</td>\n",
       "      <td>2.142484e+07</td>\n",
       "      <td>0.005697</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.000182</td>\n",
       "      <td>0.057491</td>\n",
       "      <td>-39.491889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-0.002008</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>49.483092</td>\n",
       "      <td>61.626629</td>\n",
       "      <td>61.600136</td>\n",
       "      <td>43.506434</td>\n",
       "      <td>54.529856</td>\n",
       "      <td>5.285849</td>\n",
       "      <td>7.954096</td>\n",
       "      <td>-2.668247</td>\n",
       "      <td>14.244778</td>\n",
       "      <td>8.734481e+01</td>\n",
       "      <td>0.004661</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>5.954260</td>\n",
       "      <td>1832.947071</td>\n",
       "      <td>-31.552210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-0.001254</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>65.325459</td>\n",
       "      <td>72.574171</td>\n",
       "      <td>22.751323</td>\n",
       "      <td>48.151957</td>\n",
       "      <td>57.476814</td>\n",
       "      <td>0.794795</td>\n",
       "      <td>0.933060</td>\n",
       "      <td>-0.138265</td>\n",
       "      <td>34.110821</td>\n",
       "      <td>1.857978e+04</td>\n",
       "      <td>0.006084</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.444724</td>\n",
       "      <td>111.926883</td>\n",
       "      <td>-21.180556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-0.001189</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>68.490289</td>\n",
       "      <td>76.730922</td>\n",
       "      <td>21.428571</td>\n",
       "      <td>53.335891</td>\n",
       "      <td>59.018494</td>\n",
       "      <td>0.008934</td>\n",
       "      <td>0.010039</td>\n",
       "      <td>-0.001105</td>\n",
       "      <td>28.819432</td>\n",
       "      <td>1.076816e+05</td>\n",
       "      <td>0.007250</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.003728</td>\n",
       "      <td>0.811013</td>\n",
       "      <td>-12.207792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.000148</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>55.321706</td>\n",
       "      <td>61.557516</td>\n",
       "      <td>44.483031</td>\n",
       "      <td>52.695343</td>\n",
       "      <td>58.505906</td>\n",
       "      <td>19.188244</td>\n",
       "      <td>23.032518</td>\n",
       "      <td>-3.844274</td>\n",
       "      <td>38.360700</td>\n",
       "      <td>1.910385e+04</td>\n",
       "      <td>0.003680</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>9.827786</td>\n",
       "      <td>2535.533645</td>\n",
       "      <td>-13.015960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-0.004492</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>45.866008</td>\n",
       "      <td>55.486098</td>\n",
       "      <td>20.496124</td>\n",
       "      <td>44.379531</td>\n",
       "      <td>54.217994</td>\n",
       "      <td>0.159849</td>\n",
       "      <td>0.202309</td>\n",
       "      <td>-0.042460</td>\n",
       "      <td>25.371531</td>\n",
       "      <td>4.182528e+04</td>\n",
       "      <td>0.005412</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.099856</td>\n",
       "      <td>25.074783</td>\n",
       "      <td>-24.857834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.001337</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>56.802987</td>\n",
       "      <td>65.693501</td>\n",
       "      <td>25.221347</td>\n",
       "      <td>45.744561</td>\n",
       "      <td>54.342695</td>\n",
       "      <td>131.897521</td>\n",
       "      <td>173.636165</td>\n",
       "      <td>-41.738643</td>\n",
       "      <td>31.022512</td>\n",
       "      <td>1.216447e+03</td>\n",
       "      <td>0.004165</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>122.043891</td>\n",
       "      <td>36252.195780</td>\n",
       "      <td>-30.736478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.000066</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>59.490364</td>\n",
       "      <td>67.344531</td>\n",
       "      <td>10.682353</td>\n",
       "      <td>39.765283</td>\n",
       "      <td>52.390914</td>\n",
       "      <td>0.000592</td>\n",
       "      <td>0.000676</td>\n",
       "      <td>-0.000084</td>\n",
       "      <td>24.574737</td>\n",
       "      <td>2.143891e+07</td>\n",
       "      <td>0.008182</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.000633</td>\n",
       "      <td>0.141385</td>\n",
       "      <td>-45.175258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.001288</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>67.653708</td>\n",
       "      <td>77.355154</td>\n",
       "      <td>23.777116</td>\n",
       "      <td>44.703780</td>\n",
       "      <td>56.280375</td>\n",
       "      <td>2.667685</td>\n",
       "      <td>3.317364</td>\n",
       "      <td>-0.649680</td>\n",
       "      <td>33.059342</td>\n",
       "      <td>1.203241e+04</td>\n",
       "      <td>0.005818</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>1.081932</td>\n",
       "      <td>383.292427</td>\n",
       "      <td>-23.562500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.001066</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>65.342120</td>\n",
       "      <td>69.915164</td>\n",
       "      <td>44.500000</td>\n",
       "      <td>49.284144</td>\n",
       "      <td>62.271397</td>\n",
       "      <td>2.409332</td>\n",
       "      <td>2.786770</td>\n",
       "      <td>-0.377437</td>\n",
       "      <td>28.322375</td>\n",
       "      <td>3.515688e+03</td>\n",
       "      <td>0.004626</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.887057</td>\n",
       "      <td>301.624064</td>\n",
       "      <td>-11.411501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.001616</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>63.897603</td>\n",
       "      <td>66.865815</td>\n",
       "      <td>30.763152</td>\n",
       "      <td>58.653781</td>\n",
       "      <td>55.828744</td>\n",
       "      <td>0.008168</td>\n",
       "      <td>0.009504</td>\n",
       "      <td>-0.001335</td>\n",
       "      <td>29.428618</td>\n",
       "      <td>5.311577e+06</td>\n",
       "      <td>0.010606</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.005002</td>\n",
       "      <td>1.119505</td>\n",
       "      <td>-30.848291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>-0.002306</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>72.619823</td>\n",
       "      <td>79.507949</td>\n",
       "      <td>40.376227</td>\n",
       "      <td>55.228135</td>\n",
       "      <td>56.636070</td>\n",
       "      <td>0.001636</td>\n",
       "      <td>0.001943</td>\n",
       "      <td>-0.000307</td>\n",
       "      <td>22.105329</td>\n",
       "      <td>7.621051e+06</td>\n",
       "      <td>0.004580</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.000799</td>\n",
       "      <td>0.200496</td>\n",
       "      <td>-14.777575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.002785</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>54.562963</td>\n",
       "      <td>59.836502</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>42.285897</td>\n",
       "      <td>53.119654</td>\n",
       "      <td>0.011772</td>\n",
       "      <td>0.015650</td>\n",
       "      <td>-0.003878</td>\n",
       "      <td>10.145732</td>\n",
       "      <td>6.608970e+05</td>\n",
       "      <td>0.004211</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.010838</td>\n",
       "      <td>2.270651</td>\n",
       "      <td>-26.526316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>-0.000843</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>66.714137</td>\n",
       "      <td>75.638878</td>\n",
       "      <td>52.884615</td>\n",
       "      <td>62.373355</td>\n",
       "      <td>59.480381</td>\n",
       "      <td>1.370963</td>\n",
       "      <td>1.564095</td>\n",
       "      <td>-0.193132</td>\n",
       "      <td>27.463692</td>\n",
       "      <td>1.018333e+03</td>\n",
       "      <td>0.007119</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.606897</td>\n",
       "      <td>156.141154</td>\n",
       "      <td>-9.606299</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        lr_15  Mkt_lrt_15  Crypto_Index      slowK      slowD  \\\n",
       "timestamp  Asset_ID                                                             \n",
       "1642982400 13       -0.000622   -0.001114   6494.648269  59.905404  69.549659   \n",
       "           10       -0.002008   -0.001114   6494.648269  49.483092  61.626629   \n",
       "           9        -0.001254   -0.001114   6494.648269  65.325459  72.574171   \n",
       "           8        -0.001189   -0.001114   6494.648269  68.490289  76.730922   \n",
       "           6         0.000148   -0.001114   6494.648269  55.321706  61.557516   \n",
       "           7        -0.004492   -0.001114   6494.648269  45.866008  55.486098   \n",
       "           1        -0.001337   -0.001114   6494.648269  56.802987  65.693501   \n",
       "           4        -0.000066   -0.001114   6494.648269  59.490364  67.344531   \n",
       "           0        -0.001288   -0.001114   6494.648269  67.653708  77.355154   \n",
       "           2         0.001066   -0.001114   6494.648269  65.342120  69.915164   \n",
       "           3        -0.001616   -0.001114   6494.648269  63.897603  66.865815   \n",
       "           12       -0.002306   -0.001114   6494.648269  72.619823  79.507949   \n",
       "           5        -0.002785   -0.001114   6494.648269  54.562963  59.836502   \n",
       "           11       -0.000843   -0.001114   6494.648269  66.714137  75.638878   \n",
       "\n",
       "                         fastK      fastD     rsi_30  macd_10_60  \\\n",
       "timestamp  Asset_ID                                                \n",
       "1642982400 13        33.418367  40.759107  52.838253    0.000142   \n",
       "           10        61.600136  43.506434  54.529856    5.285849   \n",
       "           9         22.751323  48.151957  57.476814    0.794795   \n",
       "           8         21.428571  53.335891  59.018494    0.008934   \n",
       "           6         44.483031  52.695343  58.505906   19.188244   \n",
       "           7         20.496124  44.379531  54.217994    0.159849   \n",
       "           1         25.221347  45.744561  54.342695  131.897521   \n",
       "           4         10.682353  39.765283  52.390914    0.000592   \n",
       "           0         23.777116  44.703780  56.280375    2.667685   \n",
       "           2         44.500000  49.284144  62.271397    2.409332   \n",
       "           3         30.763152  58.653781  55.828744    0.008168   \n",
       "           12        40.376227  55.228135  56.636070    0.001636   \n",
       "           5         24.000000  42.285897  53.119654    0.011772   \n",
       "           11        52.884615  62.373355  59.480381    1.370963   \n",
       "\n",
       "                     macd_signal_15  macd_hist     adx_50    vol_sum_15  \\\n",
       "timestamp  Asset_ID                                                       \n",
       "1642982400 13              0.000206  -0.000063  17.109192  2.142484e+07   \n",
       "           10              7.954096  -2.668247  14.244778  8.734481e+01   \n",
       "           9               0.933060  -0.138265  34.110821  1.857978e+04   \n",
       "           8               0.010039  -0.001105  28.819432  1.076816e+05   \n",
       "           6              23.032518  -3.844274  38.360700  1.910385e+04   \n",
       "           7               0.202309  -0.042460  25.371531  4.182528e+04   \n",
       "           1             173.636165 -41.738643  31.022512  1.216447e+03   \n",
       "           4               0.000676  -0.000084  24.574737  2.143891e+07   \n",
       "           0               3.317364  -0.649680  33.059342  1.203241e+04   \n",
       "           2               2.786770  -0.377437  28.322375  3.515688e+03   \n",
       "           3               0.009504  -0.001335  29.428618  5.311577e+06   \n",
       "           12              0.001943  -0.000307  22.105329  7.621051e+06   \n",
       "           5               0.015650  -0.003878  10.145732  6.608970e+05   \n",
       "           11              1.564095  -0.193132  27.463692  1.018333e+03   \n",
       "\n",
       "                     std_lr_15_30  std_Mkt_lrt_15_10  std_Crypto_Index_30  \\\n",
       "timestamp  Asset_ID                                                         \n",
       "1642982400 13            0.005697           0.001848             8.745965   \n",
       "           10            0.004661           0.001848             8.745965   \n",
       "           9             0.006084           0.001848             8.745965   \n",
       "           8             0.007250           0.001848             8.745965   \n",
       "           6             0.003680           0.001848             8.745965   \n",
       "           7             0.005412           0.001848             8.745965   \n",
       "           1             0.004165           0.001848             8.745965   \n",
       "           4             0.008182           0.001848             8.745965   \n",
       "           0             0.005818           0.001848             8.745965   \n",
       "           2             0.004626           0.001848             8.745965   \n",
       "           3             0.010606           0.001848             8.745965   \n",
       "           12            0.004580           0.001848             8.745965   \n",
       "           5             0.004211           0.001848             8.745965   \n",
       "           11            0.007119           0.001848             8.745965   \n",
       "\n",
       "                         ATR_60     TRENDLINE   willr_60  \n",
       "timestamp  Asset_ID                                       \n",
       "1642982400 13          0.000182      0.057491 -39.491889  \n",
       "           10          5.954260   1832.947071 -31.552210  \n",
       "           9           0.444724    111.926883 -21.180556  \n",
       "           8           0.003728      0.811013 -12.207792  \n",
       "           6           9.827786   2535.533645 -13.015960  \n",
       "           7           0.099856     25.074783 -24.857834  \n",
       "           1         122.043891  36252.195780 -30.736478  \n",
       "           4           0.000633      0.141385 -45.175258  \n",
       "           0           1.081932    383.292427 -23.562500  \n",
       "           2           0.887057    301.624064 -11.411501  \n",
       "           3           0.005002      1.119505 -30.848291  \n",
       "           12          0.000799      0.200496 -14.777575  \n",
       "           5           0.010838      2.270651 -26.526316  \n",
       "           11          0.606897    156.141154  -9.606299  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fdict={'willr': 60,\n",
    " 'vol_sum': 15,\n",
    " 'std_lr_15': 30,\n",
    " 'std_Mkt_lrt_15': 10,\n",
    " 'std_Crypto_Index': 30,\n",
    " 'rsi': 30,\n",
    " 'macd_sig': 15,\n",
    " 'macd_s': 10,\n",
    " 'macd_l': 60,\n",
    " 'lrtn': 50,\n",
    " 'fastk2': 10,\n",
    " 'fastk1': 15,\n",
    " 'beta_s': '6h',\n",
    " 'beta_l': '2d',\n",
    " 'adx': 50,\n",
    " 'ATR': 60}\n",
    "\n",
    "pre_minute = 2000\n",
    "sup_train2 = sup_train.iloc[-14*pre_minute:,:]\n",
    "sup_train2=sup_train2.groupby('Asset_ID').apply(lambda x: lag_features(x,**fdict))\n",
    "nan_num = []\n",
    "nan_mins = []\n",
    "for col in sup_train2.columns:\n",
    "    nan_num.append(sup_train2[col].isin([np.nan]).sum())\n",
    "    nan_mins.append(nan_num[-1]/14)\n",
    "    print(f\"{col}: {nan_num[-1]} rows, {nan_mins[-1]} mins\")\n",
    "print(f\"max nan mins={max(nan_mins)}\")\n",
    "\n",
    "sup_train2[sup_train2.isin([np.nan]).any(axis=1)]\n",
    "sup_train2.iloc[-14:,8:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>lr_15</th>\n",
       "      <th>Mkt_lrt_15</th>\n",
       "      <th>Crypto_Index</th>\n",
       "      <th>slowK</th>\n",
       "      <th>slowD</th>\n",
       "      <th>fastK</th>\n",
       "      <th>fastD</th>\n",
       "      <th>rsi_30</th>\n",
       "      <th>macd_10_60</th>\n",
       "      <th>macd_signal_15</th>\n",
       "      <th>macd_hist</th>\n",
       "      <th>adx_50</th>\n",
       "      <th>vol_sum_15</th>\n",
       "      <th>std_lr_15_30</th>\n",
       "      <th>std_Mkt_lrt_15_10</th>\n",
       "      <th>std_Crypto_Index_30</th>\n",
       "      <th>ATR_60</th>\n",
       "      <th>TRENDLINE</th>\n",
       "      <th>willr_60</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>timestamp</th>\n",
       "      <th>Asset_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"14\" valign=\"top\">1642982400</th>\n",
       "      <th>13</th>\n",
       "      <td>-0.000622</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>59.905404</td>\n",
       "      <td>69.549659</td>\n",
       "      <td>33.418367</td>\n",
       "      <td>40.759107</td>\n",
       "      <td>52.833090</td>\n",
       "      <td>0.000142</td>\n",
       "      <td>0.000206</td>\n",
       "      <td>-0.000063</td>\n",
       "      <td>17.723073</td>\n",
       "      <td>2.142484e+07</td>\n",
       "      <td>0.005697</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.000181</td>\n",
       "      <td>0.057491</td>\n",
       "      <td>-39.491889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-0.002008</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>49.483092</td>\n",
       "      <td>61.626629</td>\n",
       "      <td>61.600136</td>\n",
       "      <td>43.506434</td>\n",
       "      <td>54.522230</td>\n",
       "      <td>5.275542</td>\n",
       "      <td>7.940582</td>\n",
       "      <td>-2.665040</td>\n",
       "      <td>14.361313</td>\n",
       "      <td>8.734481e+01</td>\n",
       "      <td>0.004661</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>5.912055</td>\n",
       "      <td>1832.947071</td>\n",
       "      <td>-31.552210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-0.001254</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>65.325459</td>\n",
       "      <td>72.574171</td>\n",
       "      <td>22.751323</td>\n",
       "      <td>48.151957</td>\n",
       "      <td>57.466329</td>\n",
       "      <td>0.794298</td>\n",
       "      <td>0.932409</td>\n",
       "      <td>-0.138111</td>\n",
       "      <td>32.435046</td>\n",
       "      <td>1.857978e+04</td>\n",
       "      <td>0.006084</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.442222</td>\n",
       "      <td>111.926883</td>\n",
       "      <td>-21.180556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-0.001189</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>68.490289</td>\n",
       "      <td>76.730922</td>\n",
       "      <td>21.428571</td>\n",
       "      <td>53.335891</td>\n",
       "      <td>59.029531</td>\n",
       "      <td>0.008932</td>\n",
       "      <td>0.010037</td>\n",
       "      <td>-0.001105</td>\n",
       "      <td>30.288226</td>\n",
       "      <td>1.076816e+05</td>\n",
       "      <td>0.007250</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.003706</td>\n",
       "      <td>0.811013</td>\n",
       "      <td>-12.207792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.000148</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>55.321706</td>\n",
       "      <td>61.557516</td>\n",
       "      <td>44.483031</td>\n",
       "      <td>52.695343</td>\n",
       "      <td>58.504715</td>\n",
       "      <td>19.171123</td>\n",
       "      <td>23.010071</td>\n",
       "      <td>-3.838948</td>\n",
       "      <td>38.541788</td>\n",
       "      <td>1.910385e+04</td>\n",
       "      <td>0.003680</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>9.847167</td>\n",
       "      <td>2535.533645</td>\n",
       "      <td>-13.015960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-0.004492</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>45.866008</td>\n",
       "      <td>55.486098</td>\n",
       "      <td>20.496124</td>\n",
       "      <td>44.379531</td>\n",
       "      <td>54.211100</td>\n",
       "      <td>0.159903</td>\n",
       "      <td>0.202380</td>\n",
       "      <td>-0.042477</td>\n",
       "      <td>25.632911</td>\n",
       "      <td>4.182528e+04</td>\n",
       "      <td>0.005412</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.099073</td>\n",
       "      <td>25.074783</td>\n",
       "      <td>-24.857834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.001337</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>56.802987</td>\n",
       "      <td>65.693501</td>\n",
       "      <td>25.221347</td>\n",
       "      <td>45.744561</td>\n",
       "      <td>54.341882</td>\n",
       "      <td>131.800449</td>\n",
       "      <td>173.508891</td>\n",
       "      <td>-41.708443</td>\n",
       "      <td>31.827496</td>\n",
       "      <td>1.216447e+03</td>\n",
       "      <td>0.004165</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>121.287226</td>\n",
       "      <td>36252.195780</td>\n",
       "      <td>-30.736478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.000066</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>59.490364</td>\n",
       "      <td>67.344531</td>\n",
       "      <td>10.682353</td>\n",
       "      <td>39.765283</td>\n",
       "      <td>52.379913</td>\n",
       "      <td>0.000591</td>\n",
       "      <td>0.000674</td>\n",
       "      <td>-0.000084</td>\n",
       "      <td>22.677791</td>\n",
       "      <td>2.143891e+07</td>\n",
       "      <td>0.008182</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.000628</td>\n",
       "      <td>0.141385</td>\n",
       "      <td>-45.175258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.001288</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>67.653708</td>\n",
       "      <td>77.355154</td>\n",
       "      <td>23.777116</td>\n",
       "      <td>44.703780</td>\n",
       "      <td>56.285974</td>\n",
       "      <td>2.667455</td>\n",
       "      <td>3.317063</td>\n",
       "      <td>-0.649608</td>\n",
       "      <td>33.871578</td>\n",
       "      <td>1.203241e+04</td>\n",
       "      <td>0.005818</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>1.073387</td>\n",
       "      <td>383.292427</td>\n",
       "      <td>-23.562500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.001066</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>65.342120</td>\n",
       "      <td>69.915164</td>\n",
       "      <td>44.500000</td>\n",
       "      <td>49.284144</td>\n",
       "      <td>62.266294</td>\n",
       "      <td>2.409192</td>\n",
       "      <td>2.786585</td>\n",
       "      <td>-0.377394</td>\n",
       "      <td>28.383728</td>\n",
       "      <td>3.515688e+03</td>\n",
       "      <td>0.004626</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.883349</td>\n",
       "      <td>301.624064</td>\n",
       "      <td>-11.411501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.001616</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>63.897603</td>\n",
       "      <td>66.865815</td>\n",
       "      <td>30.763152</td>\n",
       "      <td>58.653781</td>\n",
       "      <td>55.830567</td>\n",
       "      <td>0.008160</td>\n",
       "      <td>0.009493</td>\n",
       "      <td>-0.001333</td>\n",
       "      <td>28.617802</td>\n",
       "      <td>5.311577e+06</td>\n",
       "      <td>0.010606</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.004961</td>\n",
       "      <td>1.119505</td>\n",
       "      <td>-30.848291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>-0.002306</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>72.619823</td>\n",
       "      <td>79.507949</td>\n",
       "      <td>40.376227</td>\n",
       "      <td>55.228135</td>\n",
       "      <td>56.629005</td>\n",
       "      <td>0.001634</td>\n",
       "      <td>0.001941</td>\n",
       "      <td>-0.000307</td>\n",
       "      <td>22.102572</td>\n",
       "      <td>7.621051e+06</td>\n",
       "      <td>0.004580</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.000795</td>\n",
       "      <td>0.200496</td>\n",
       "      <td>-14.777575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.002785</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>54.562963</td>\n",
       "      <td>59.836502</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>42.285897</td>\n",
       "      <td>53.119781</td>\n",
       "      <td>0.011758</td>\n",
       "      <td>0.015632</td>\n",
       "      <td>-0.003874</td>\n",
       "      <td>11.243197</td>\n",
       "      <td>6.608970e+05</td>\n",
       "      <td>0.004211</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.010797</td>\n",
       "      <td>2.270651</td>\n",
       "      <td>-26.526316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>-0.000843</td>\n",
       "      <td>-0.001114</td>\n",
       "      <td>6494.648269</td>\n",
       "      <td>66.714137</td>\n",
       "      <td>75.638878</td>\n",
       "      <td>52.884615</td>\n",
       "      <td>62.373355</td>\n",
       "      <td>59.485752</td>\n",
       "      <td>1.369266</td>\n",
       "      <td>1.561871</td>\n",
       "      <td>-0.192605</td>\n",
       "      <td>28.155992</td>\n",
       "      <td>1.018333e+03</td>\n",
       "      <td>0.007119</td>\n",
       "      <td>0.001848</td>\n",
       "      <td>8.745965</td>\n",
       "      <td>0.602774</td>\n",
       "      <td>156.141154</td>\n",
       "      <td>-9.606299</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        lr_15  Mkt_lrt_15  Crypto_Index      slowK      slowD  \\\n",
       "timestamp  Asset_ID                                                             \n",
       "1642982400 13       -0.000622   -0.001114   6494.648269  59.905404  69.549659   \n",
       "           10       -0.002008   -0.001114   6494.648269  49.483092  61.626629   \n",
       "           9        -0.001254   -0.001114   6494.648269  65.325459  72.574171   \n",
       "           8        -0.001189   -0.001114   6494.648269  68.490289  76.730922   \n",
       "           6         0.000148   -0.001114   6494.648269  55.321706  61.557516   \n",
       "           7        -0.004492   -0.001114   6494.648269  45.866008  55.486098   \n",
       "           1        -0.001337   -0.001114   6494.648269  56.802987  65.693501   \n",
       "           4        -0.000066   -0.001114   6494.648269  59.490364  67.344531   \n",
       "           0        -0.001288   -0.001114   6494.648269  67.653708  77.355154   \n",
       "           2         0.001066   -0.001114   6494.648269  65.342120  69.915164   \n",
       "           3        -0.001616   -0.001114   6494.648269  63.897603  66.865815   \n",
       "           12       -0.002306   -0.001114   6494.648269  72.619823  79.507949   \n",
       "           5        -0.002785   -0.001114   6494.648269  54.562963  59.836502   \n",
       "           11       -0.000843   -0.001114   6494.648269  66.714137  75.638878   \n",
       "\n",
       "                         fastK      fastD     rsi_30  macd_10_60  \\\n",
       "timestamp  Asset_ID                                                \n",
       "1642982400 13        33.418367  40.759107  52.833090    0.000142   \n",
       "           10        61.600136  43.506434  54.522230    5.275542   \n",
       "           9         22.751323  48.151957  57.466329    0.794298   \n",
       "           8         21.428571  53.335891  59.029531    0.008932   \n",
       "           6         44.483031  52.695343  58.504715   19.171123   \n",
       "           7         20.496124  44.379531  54.211100    0.159903   \n",
       "           1         25.221347  45.744561  54.341882  131.800449   \n",
       "           4         10.682353  39.765283  52.379913    0.000591   \n",
       "           0         23.777116  44.703780  56.285974    2.667455   \n",
       "           2         44.500000  49.284144  62.266294    2.409192   \n",
       "           3         30.763152  58.653781  55.830567    0.008160   \n",
       "           12        40.376227  55.228135  56.629005    0.001634   \n",
       "           5         24.000000  42.285897  53.119781    0.011758   \n",
       "           11        52.884615  62.373355  59.485752    1.369266   \n",
       "\n",
       "                     macd_signal_15  macd_hist     adx_50    vol_sum_15  \\\n",
       "timestamp  Asset_ID                                                       \n",
       "1642982400 13              0.000206  -0.000063  17.723073  2.142484e+07   \n",
       "           10              7.940582  -2.665040  14.361313  8.734481e+01   \n",
       "           9               0.932409  -0.138111  32.435046  1.857978e+04   \n",
       "           8               0.010037  -0.001105  30.288226  1.076816e+05   \n",
       "           6              23.010071  -3.838948  38.541788  1.910385e+04   \n",
       "           7               0.202380  -0.042477  25.632911  4.182528e+04   \n",
       "           1             173.508891 -41.708443  31.827496  1.216447e+03   \n",
       "           4               0.000674  -0.000084  22.677791  2.143891e+07   \n",
       "           0               3.317063  -0.649608  33.871578  1.203241e+04   \n",
       "           2               2.786585  -0.377394  28.383728  3.515688e+03   \n",
       "           3               0.009493  -0.001333  28.617802  5.311577e+06   \n",
       "           12              0.001941  -0.000307  22.102572  7.621051e+06   \n",
       "           5               0.015632  -0.003874  11.243197  6.608970e+05   \n",
       "           11              1.561871  -0.192605  28.155992  1.018333e+03   \n",
       "\n",
       "                     std_lr_15_30  std_Mkt_lrt_15_10  std_Crypto_Index_30  \\\n",
       "timestamp  Asset_ID                                                         \n",
       "1642982400 13            0.005697           0.001848             8.745965   \n",
       "           10            0.004661           0.001848             8.745965   \n",
       "           9             0.006084           0.001848             8.745965   \n",
       "           8             0.007250           0.001848             8.745965   \n",
       "           6             0.003680           0.001848             8.745965   \n",
       "           7             0.005412           0.001848             8.745965   \n",
       "           1             0.004165           0.001848             8.745965   \n",
       "           4             0.008182           0.001848             8.745965   \n",
       "           0             0.005818           0.001848             8.745965   \n",
       "           2             0.004626           0.001848             8.745965   \n",
       "           3             0.010606           0.001848             8.745965   \n",
       "           12            0.004580           0.001848             8.745965   \n",
       "           5             0.004211           0.001848             8.745965   \n",
       "           11            0.007119           0.001848             8.745965   \n",
       "\n",
       "                         ATR_60     TRENDLINE   willr_60  \n",
       "timestamp  Asset_ID                                       \n",
       "1642982400 13          0.000181      0.057491 -39.491889  \n",
       "           10          5.912055   1832.947071 -31.552210  \n",
       "           9           0.442222    111.926883 -21.180556  \n",
       "           8           0.003706      0.811013 -12.207792  \n",
       "           6           9.847167   2535.533645 -13.015960  \n",
       "           7           0.099073     25.074783 -24.857834  \n",
       "           1         121.287226  36252.195780 -30.736478  \n",
       "           4           0.000628      0.141385 -45.175258  \n",
       "           0           1.073387    383.292427 -23.562500  \n",
       "           2           0.883349    301.624064 -11.411501  \n",
       "           3           0.004961      1.119505 -30.848291  \n",
       "           12          0.000795      0.200496 -14.777575  \n",
       "           5           0.010797      2.270651 -26.526316  \n",
       "           11          0.602774    156.141154  -9.606299  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre_minute = 200\n",
    "sup_train2 = sup_train.iloc[-14*pre_minute:,:]\n",
    "sup_train2= sup_train2.groupby('Asset_ID').apply(lambda x: lag_features(x,**fdict))\n",
    "sup_train2[sup_train2.isin([np.nan]).any(axis=1)]\n",
    "sup_train2.iloc[-14:,8:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['willr', 'vol_sum', 'std_lr_15', 'std_Mkt_lrt_15', 'std_Crypto_Index', 'rsi', 'macd_sig', 'macd_s', 'macd_l', 'lrtn', 'fastk2', 'fastk1', 'beta_s', 'beta_l', 'adx', 'ATR'])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fdict.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "101"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##pre_minute make sure the last minute can get non-nan values\n",
    "pre_minute = max([fdict['std_lr_15']+15,fdict['std_Mkt_lrt_15'],fdict['std_Crypto_Index'],\n",
    "                  fdict['rsi']+1,fdict['macd_l']+5,fdict['lrtn']+1,\n",
    "                  fdict['fastk2']+10,fdict['fastk1']+10,fdict['adx']*2+1])\n",
    "pre_minute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'b': 2}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd= {\"a\":1,\"b\":2}\n",
    "del dd['a']\n",
    "dd"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "4c1b049caad8a8b325ed882a549e2be8617c954355cf49088b2db08ccb0041d6"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit ('myML': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}