c3_analysis_python.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9082fc49",
   "metadata": {},
   "outputs": [],
   "source": [
    "import RENT \n",
    "import BIC_hyperparameter_search\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.metrics import f1_score, matthews_corrcoef, accuracy_score, matthews_corrcoef\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7210bfab",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv('data\\\\c3\\\\wisconsin_train.csv').iloc[:,1:]\n",
    "train_labels = 1 - pd.read_csv('data\\\\c3\\\\wisconsin_train_labels.csv').iloc[:,1]\n",
    "test_data = pd.read_csv('data\\\\c3\\\\wisconsin_test.csv').iloc[:,1:]\n",
    "test_labels = 1 - pd.read_csv('data\\\\c3\\\\wisconsin_test_labels.csv').iloc[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b1f62a16",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data dimension: (399, 30)  data type: <class 'pandas.core.frame.DataFrame'>\n",
      "target dimension: (399,)\n",
      "regularization parameters C: [0.1, 1, 10]\n",
      "elastic net l1_ratios: [0, 0.1, 0.25, 0.5, 0.75, 0.9, 1]\n",
      "poly: OFF\n",
      "number of models in ensemble: 100\n",
      "random state: 0\n",
      "verbose: 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 4 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "classifier: logreg\n",
      "scoring: mcc\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   7 out of   7 | elapsed:    0.2s finished\n"
     ]
    }
   ],
   "source": [
    "# Define a range of regualarization parameters for the penalty term. A minimum of at least one value is required.\n",
    "my_C_params = [0.1, 1, 10]\n",
    "\n",
    "# Define a range of l1-ratios for elastic net.  A minimum of at least one value is required.\n",
    "my_l1_ratios = [0, 0.1, 0.25, 0.5, 0.75, 0.9, 1]\n",
    "\n",
    "# Define setting for RENT\n",
    "model = RENT.RENT_Classification(data=train_data, \n",
    "                                 target=train_labels, \n",
    "                                 feat_names=train_data.columns, \n",
    "                                 C=my_C_params,\n",
    "                                 l1_ratios=my_l1_ratios,\n",
    "                                 autoEnetParSel=True,\n",
    "                                 BIC=True,\n",
    "                                 poly='OFF',\n",
    "                                 testsize_range=(0.25,0.25),\n",
    "                                 scoring='mcc',\n",
    "                                 classifier='logreg',\n",
    "                                 K=100,\n",
    "                                 random_state=0,\n",
    "                                 verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "02e50801",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0.1</th>\n",
       "      <th>1.0</th>\n",
       "      <th>10.0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.00</th>\n",
       "      <td>260.603784</td>\n",
       "      <td>237.937293</td>\n",
       "      <td>235.038652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.10</th>\n",
       "      <td>251.850404</td>\n",
       "      <td>238.18584</td>\n",
       "      <td>235.054014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.25</th>\n",
       "      <td>221.010979</td>\n",
       "      <td>226.638275</td>\n",
       "      <td>235.077773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.50</th>\n",
       "      <td>192.422296</td>\n",
       "      <td>215.515464</td>\n",
       "      <td>235.119253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.75</th>\n",
       "      <td>193.187215</td>\n",
       "      <td>216.504365</td>\n",
       "      <td>235.162998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.90</th>\n",
       "      <td>173.269467</td>\n",
       "      <td>211.131411</td>\n",
       "      <td>235.190304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.00</th>\n",
       "      <td>169.750094</td>\n",
       "      <td>205.529354</td>\n",
       "      <td>235.208948</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0.1         1.0         10.0\n",
       "0.00  260.603784  237.937293  235.038652\n",
       "0.10  251.850404   238.18584  235.054014\n",
       "0.25  221.010979  226.638275  235.077773\n",
       "0.50  192.422296  215.515464  235.119253\n",
       "0.75  193.187215  216.504365  235.162998\n",
       "0.90  173.269467  211.131411  235.190304\n",
       "1.00  169.750094  205.529354  235.208948"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model._BIC_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bc1bde19",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "938680d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.1, 1.0)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_enet_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d2a4a24f",
   "metadata": {},
   "outputs": [],
   "source": [
    "cutoff_parameters = {'t1': np.arange(0.2,1.05,0.05),\n",
    "                     't2': np.arange(0.2,1.05,0.05),\n",
    "                     't3': [0.9, 0.95, 0.975, 0.99]}\n",
    "\n",
    "result = BIC_hyperparameter_search.BIC_hyperparameter_search(model, cutoff_parameters, test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "15c512fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "BIC = result['BIC']\n",
    "#indices = np.where(BIC == np.min(BIC))\n",
    "indices_matrix = np.stack(np.where(BIC == np.min(BIC)), axis=0)\n",
    "tau1 = np.max(indices_matrix[0,:])\n",
    "indices_matrix = indices_matrix[:,np.where(indices_matrix[0,:] == tau1)[0]]\n",
    "tau2 = np.max(indices_matrix[1,:])\n",
    "indices_matrix = indices_matrix[:,np.where(indices_matrix[1,:] == tau2)[0]]\n",
    "tau3 = np.max(indices_matrix[2,:])\n",
    "\n",
    "tau1 = np.round(cutoff_parameters['t1'][tau1], 2)\n",
    "tau2 = np.round(cutoff_parameters['t2'][tau2], 2)\n",
    "tau3 = np.round(cutoff_parameters['t3'][tau3], 2)\n",
    "\n",
    "\n",
    "selected_features = model.select_features(tau_1_cutoff=tau1, tau_2_cutoff=tau2, tau_3_cutoff=tau3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "054f7ab5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 7, 20, 21, 27], dtype=int64)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e230ca95",
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = StandardScaler()\n",
    "train_data_sc = sc.fit_transform(train_data.iloc[:, selected_features])\n",
    "test_data_sc = sc.transform(test_data.iloc[:, selected_features])\n",
    "\n",
    "# Train model with \n",
    "model = LogisticRegression(penalty='none', max_iter=8000, solver=\"saga\", random_state=0).\\\n",
    "        fit(train_data_sc, train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "652d385a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predictions():\n",
    "    print(\"f1 1: \", f1_score(test_labels, model.predict(test_data_sc)))\n",
    "    print(\"f1 0: \", f1_score(1 - test_labels, 1 - model.predict(test_data_sc)))\n",
    "    print(\"Accuracy: \", accuracy_score(test_labels, model.predict(test_data_sc)))\n",
    "    print(\"Matthews correlation coefficient: \", matthews_corrcoef(test_labels, model.predict(test_data_sc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "70d67432",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1 1:  0.9865470852017937\n",
      "f1 0:  0.9743589743589743\n",
      "Accuracy:  0.9823529411764705\n",
      "Matthews correlation coefficient:  0.9609891503308582\n"
     ]
    }
   ],
   "source": [
    "predictions()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f973f8a7",
   "metadata": {},
   "source": [
    "# Competing methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "833a30e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from ReliefF import ReliefF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3054bbde",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1 1:  0.9819819819819819\n",
      "f1 0:  0.9661016949152542\n",
      "Accuracy:  0.9764705882352941\n",
      "Matthews correlation coefficient:  0.9480836768972363\n"
     ]
    }
   ],
   "source": [
    "# M0\n",
    "\n",
    "sc=StandardScaler()\n",
    "scaled_data = sc.fit_transform(train_data)\n",
    "\n",
    "sgd = LogisticRegression(penalty=\"elasticnet\", C=0.1, l1_ratio=1, solver=\"saga\", random_state=0)\n",
    "sgd.fit(scaled_data, train_labels)\n",
    "params = np.where(sgd.coef_ != 0)[1]\n",
    "\n",
    "sc = StandardScaler()\n",
    "train_data_sc = sc.fit_transform(train_data.iloc[:,params])\n",
    "test_data_sc = sc.transform(test_data.iloc[:, params])\n",
    "model = LogisticRegression(penalty='none', max_iter=8000, solver=\"saga\", random_state=0).\\\n",
    "        fit(train_data_sc,train_labels)\n",
    "\n",
    "predictions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "06756adb",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_features = len(selected_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "556a7315",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1 1:  0.9515418502202643\n",
      "f1 0:  0.9026548672566371\n",
      "Accuracy:  0.9352941176470588\n",
      "Matthews correlation coefficient:  0.8562174533946231\n"
     ]
    }
   ],
   "source": [
    "# RFE\n",
    "\n",
    "svc = SVC(kernel=\"linear\", C=1)\n",
    "rfe = RFE(estimator=svc, n_features_to_select=num_features, step=1)\n",
    "rfe.fit(scaled_data, train_labels)\n",
    "params = np.where(rfe.ranking_ == 1)[0]\n",
    "\n",
    "sc = StandardScaler()\n",
    "train_data_sc = sc.fit_transform(train_data.iloc[:,params])\n",
    "test_data_sc = sc.transform(test_data.iloc[:, params])\n",
    "model = LogisticRegression(penalty='none', max_iter=8000, solver=\"saga\").\\\n",
    "        fit(train_data_sc,train_labels)\n",
    "\n",
    "predictions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9d1d6e39",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1 1:  0.926605504587156\n",
      "f1 0:  0.8688524590163935\n",
      "Accuracy:  0.9058823529411765\n",
      "Matthews correlation coefficient:  0.7966218885241891\n"
     ]
    }
   ],
   "source": [
    "# reliefF\n",
    "rfF = ReliefF(n_neighbors=np.round(np.shape(train_data)[0]/3).astype(int), n_features_to_keep=num_features)\n",
    "rfF.fit(scaled_data, train_labels)\n",
    "params = rfF.top_features[0:num_features]\n",
    "\n",
    "\n",
    "sc = StandardScaler()\n",
    "train_data_sc = sc.fit_transform(train_data.iloc[:,params])\n",
    "test_data_sc = sc.transform(test_data.iloc[:, params])\n",
    "model = LogisticRegression(penalty='none', max_iter=8000, solver=\"saga\", random_state=0).\\\n",
    "        fit(train_data_sc,train_labels)\n",
    "\n",
    "predictions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d54aff5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1 1:  0.9589041095890412\n",
      "f1 0:  0.9256198347107438\n",
      "Accuracy:  0.9470588235294117\n",
      "Matthews correlation coefficient:  0.8852187340243354\n"
     ]
    }
   ],
   "source": [
    "# random forest\n",
    "clf = RandomForestClassifier(random_state=0)\n",
    "model = clf.fit(scaled_data, train_labels)\n",
    "ind = np.where(np.argsort(np.argsort(-1*(model.feature_importances_)))<num_features)[0]\n",
    "\n",
    "sc = StandardScaler()\n",
    "train_data_sc = sc.fit_transform(train_data.iloc[:,ind])\n",
    "test_data_sc = sc.transform(test_data.iloc[:, ind])\n",
    "model = LogisticRegression(penalty='none', max_iter=8000, solver=\"saga\").\\\n",
    "        fit(train_data_sc,train_labels)\n",
    "\n",
    "predictions()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}