model_training/notebooks/Dense Layer - Model Tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add directory above current directory to path\n",
    "import sys; sys.path.insert(0, '..')\n",
    "\n",
    "from functools import partial\n",
    "from pathlib import Path\n",
    "from pprint import pprint\n",
    "\n",
    "import optuna\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "\n",
    "from config.dense_features import *\n",
    "from data_preparation.io_utils import read_yaml_file\n",
    "from dataset.dataset_builder import DatasetBuilder\n",
    "from model.model_builder import ModelBuilder\n",
    "from model.model_tuning import OptunaOptimizer\n",
    "from pipeline.trainer import Trainer\n",
    "\n",
    "CONFIG_PATH = Path('../config/config.yaml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data Configuration\n",
    "config = read_yaml_file(CONFIG_PATH)\n",
    "data_config = config['data']\n",
    "print(data_config)\n",
    "conv_pattern = \"**/dense.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_builder = DatasetBuilder(features=DENSE_FEATURES)\n",
    "dense_dataset = dataset_builder.create_dataset(\n",
    "            data_dir=Path('../training_data'),\n",
    "            test_models=data_config[\"test_models\"],\n",
    "            pattern=conv_pattern,\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"Number of training samples: {len(dense_dataset.train.input_features)}\")\n",
    "print(f\"Number of testing samples: {len(dense_dataset.test.input_features)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4",
   "metadata": {},
   "source": [
    "## Baseline Models\n",
    "\n",
    "\n",
    "Baseline models for power and runtime models, we will use the mean of train dataset as prediction.\n",
    "\n",
    "\n",
    "### Baseline Result\n",
    "\n",
    "Test dataset RMSPE Power : 243.72%\n",
    "\n",
    "Test dataset RMSPE Runtime: 416.88%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Trainer(data_config=data_config, model_config=config['model'], features=DENSE_FEATURES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6",
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_power = dense_dataset.train.power.mean()\n",
    "target = dense_dataset.test.power"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7",
   "metadata": {},
   "outputs": [],
   "source": [
    "pprint(trainer.eval_metrics(actual=target, pred=[mean_power]*len(target)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8",
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_runtime = dense_dataset.train.runtime.mean()\n",
    "target = dense_dataset.test.runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9",
   "metadata": {},
   "outputs": [],
   "source": [
    "pprint(trainer.eval_metrics(actual=target, pred=[mean_runtime]*len(target)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10",
   "metadata": {},
   "source": [
    "## Power\n",
    "\n",
    "[Optuna](https://optuna.org/) library is used to perform hyperparameter tuning for power model.\n",
    "\n",
    "Best trial configuration\n",
    "\n",
    "```json\n",
    "FrozenTrial(number=83, state=1, values=[0.3461064156797421], datetime_start=datetime.datetime(2024, 11, 26, 16, 57, 32, 119055), datetime_complete=datetime.datetime(2024, 11, 26, 16, 57, 32, 304749), params={'degree': 4, 'log_scale': True, 'special_features': True, 'scalers': 'robust', 'max_iter': 1616, 'n_alphas': 102, 'fit_intercept': False, 'positive': False}, user_attrs={'testing_mean_absolute_error': 2.1073726925781435, 'testing_mean_absolute_percentage_error': 0.5881811267685447, 'testing_mean_squared_error': 6.104086079316093, 'testing_r2_score': 0.3461064156797421, 'testing_root_mean_squared_error': 2.4706448711452023, 'testing_root_mean_squared_percentage_error': 67.98575856021878}, system_attrs={}, intermediate_values={}, distributions={'degree': IntDistribution(high=4, log=False, low=1, step=1), 'log_scale': CategoricalDistribution(choices=(True, False)), 'special_features': CategoricalDistribution(choices=(True, False)), 'scalers': CategoricalDistribution(choices=('minmax', 'standard', 'robust')), 'max_iter': IntDistribution(high=50000, log=True, low=1000, step=1), 'n_alphas': IntDistribution(high=1000, log=True, low=100, step=1), 'fit_intercept': CategoricalDistribution(choices=(True, False)), 'positive': CategoricalDistribution(choices=(True, False))}, trial_id=84, value=None)\n",
    "```\n",
    "\n",
    "Test dataset metrics for the best trial\n",
    "\n",
    "```json\n",
    "{   \n",
    "    'testing_mean_absolute_error': 2.1073726925781435, \n",
    "    'testing_mean_absolute_percentage_error': 0.5881811267685447, \n",
    "    'testing_mean_squared_error': 6.104086079316093, \n",
    "    'testing_r2_score': 0.3461064156797421, \n",
    "    'testing_root_mean_squared_error': 2.4706448711452023, \n",
    "    'testing_root_mean_squared_percentage_error': 67.98575856021878\n",
    "}\n",
    "```\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = dense_dataset.train.input_features.values\n",
    "y_power_train = dense_dataset.train.power.values\n",
    "print(f\"Training shape: {X_train.shape}, {y_power_train.shape}\")\n",
    "\n",
    "X_test = dense_dataset.test.input_features.values\n",
    "y_power_test = dense_dataset.test.power.values\n",
    "print(f\"Testing shape: {X_test.shape}, {y_power_test.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_builder = ModelBuilder()\n",
    "optimizer = OptunaOptimizer(X_train=X_train, y_train=y_power_train, X_test=X_test, y_test=y_power_test, model_builder=model_builder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Takes about 40 secs to complete\n",
    "\n",
    "# Ignore ConvergenceWarning from sklearn to avoid tab crash\n",
    "from warnings import filterwarnings\n",
    "filterwarnings('ignore')\n",
    "\n",
    "# Maximize test R^2 score during tuning\n",
    "power_study = optuna.create_study(study_name='dense_power_model_tuning', direction=\"maximize\", storage=\"sqlite:///dense_power_model_tuning.db\")\n",
    "# Run study for 100 trials\n",
    "power_study.optimize(partial(optimizer.objective, \n",
    "                             features_mapping=dataset_builder.features_mapping,\n",
    "                             special_terms_list=None),\n",
    "               n_trials=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print the best performing pipeline\n",
    "pprint(power_study.best_trial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15",
   "metadata": {},
   "outputs": [],
   "source": [
    "from optuna.visualization import plot_optimization_history\n",
    "\n",
    "plot_optimization_history(power_study)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "17",
   "metadata": {},
   "source": [
    "## Runtime\n",
    "\n",
    "[Optuna](https://optuna.org/) library is used to perform hyperparameter tuning for runtime model.\n",
    "\n",
    "Best trial\n",
    "\n",
    "```json\n",
    "FrozenTrial(number=67, state=1, values=[-0.0995709243010976], datetime_start=datetime.datetime(2024, 11, 26, 17, 2, 18, 862242), datetime_complete=datetime.datetime(2024, 11, 26, 17, 2, 30, 551254), params={'degree': 4, 'log_scale': False, 'special_features': False, 'scalers': 'standard', 'max_iter': 46296, 'n_alphas': 112, 'fit_intercept': True, 'positive': False}, user_attrs={'testing_mean_absolute_error': 0.608014977251175, 'testing_mean_absolute_percentage_error': 7.304681270592265, 'testing_mean_squared_error': 1.4327889539950647, 'testing_r2_score': -0.0995709243010976, 'testing_root_mean_squared_error': 1.1969916265350666, 'testing_root_mean_squared_percentage_error': 998.2098464331851}, system_attrs={}, intermediate_values={}, distributions={'degree': IntDistribution(high=4, log=False, low=1, step=1), 'log_scale': CategoricalDistribution(choices=(True, False)), 'special_features': CategoricalDistribution(choices=(True, False)), 'scalers': CategoricalDistribution(choices=('minmax', 'standard', 'robust')), 'max_iter': IntDistribution(high=50000, log=True, low=1000, step=1), 'n_alphas': IntDistribution(high=1000, log=True, low=100, step=1), 'fit_intercept': CategoricalDistribution(choices=(True, False)), 'positive': CategoricalDistribution(choices=(True, False))}, trial_id=68, value=None)\n",
    "```\n",
    "\n",
    "Test dataset metrics for best trial\n",
    "\n",
    "```json\n",
    "{   \n",
    "    'testing_mean_absolute_error': 0.608014977251175, \n",
    "    'testing_mean_absolute_percentage_error': 7.304681270592265, \n",
    "    'testing_mean_squared_error': 1.4327889539950647, \n",
    "    'testing_r2_score': -0.0995709243010976, \n",
    "    'testing_root_mean_squared_error': 1.1969916265350666, \n",
    "    'testing_root_mean_squared_percentage_error': 998.2098464331851\n",
    "}\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = dense_dataset.train.input_features.values\n",
    "y_runtime_train = dense_dataset.train.runtime.values\n",
    "print(f\"Training shape: {X_train.shape}, {y_runtime_train.shape}\")\n",
    "\n",
    "X_test = dense_dataset.test.input_features.values\n",
    "y_runtime_test = dense_dataset.test.runtime.values\n",
    "print(f\"Testing shape: {X_test.shape}, {y_runtime_test.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_builder = ModelBuilder()\n",
    "optimizer = OptunaOptimizer(X_train=X_train, y_train=y_runtime_train, X_test=X_test, y_test=y_runtime_test, model_builder=model_builder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Takes about 10 min secs to complete\n",
    "\n",
    "# Ignore ConvergenceWarning from sklearn to avoid tab crash\n",
    "from warnings import filterwarnings\n",
    "filterwarnings('ignore')\n",
    "\n",
    "# Maximize the test R^2 score during tuning\n",
    "runtime_study = optuna.create_study(study_name='dense_runtime_model_tuning', direction=\"maximize\", storage=\"sqlite:///dense_runtime_model_tuning.db\")\n",
    "# Run study for 100 trials\n",
    "runtime_study.optimize(partial(optimizer.objective, \n",
    "                             features_mapping=dataset_builder.features_mapping,\n",
    "                             special_terms_list=None),\n",
    "               n_trials=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print the best performing pipeline\n",
    "pprint(runtime_study.best_trial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22",
   "metadata": {},
   "outputs": [],
   "source": [
    "from optuna.visualization import plot_optimization_history\n",
    "\n",
    "plot_optimization_history(runtime_study)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}