dataset_transformation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv('dataset/3894044.csv')\n",
    "# Take this features\n",
    "features_need = ['STATION', 'NAME', 'DLY-TAVG-NORMAL', 'DLY-TMAX-NORMAL', 'DLY-TMIN-NORMAL', 'DLY-DUTR-NORMAL', 'DLY-TAVG-STDDEV', 'DLY-TMAX-STDDEV', 'DLY-TMIN-STDDEV', 'DLY-DUTR-STDDEV', 'DLY-PRCP-PCTALL-GE010HI', 'DLY-PRCP-PCTALL-GE025HI', 'DLY-PRCP-PCTALL-GE050HI']\n",
    "dataset = dataset[features_need]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Take only the stations without missing values\n",
    "dataset = dataset.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          STATION                                  NAME  DLY-TAVG-NORMAL  \\\n",
      "0     USC00084050                      HOLLYWOOD, FL US             20.4   \n",
      "1     USC00084050                      HOLLYWOOD, FL US             20.3   \n",
      "2     USC00084050                      HOLLYWOOD, FL US             20.3   \n",
      "3     USC00084050                      HOLLYWOOD, FL US             20.2   \n",
      "4     USC00084050                      HOLLYWOOD, FL US             20.1   \n",
      "...           ...                                   ...              ...   \n",
      "5485  USW00012888  MIAMI KENDALL TAMIAMI EXEC AP, FL US             20.5   \n",
      "5486  USW00012888  MIAMI KENDALL TAMIAMI EXEC AP, FL US             20.4   \n",
      "5487  USW00012888  MIAMI KENDALL TAMIAMI EXEC AP, FL US             20.3   \n",
      "5488  USW00012888  MIAMI KENDALL TAMIAMI EXEC AP, FL US             20.2   \n",
      "5489  USW00012888  MIAMI KENDALL TAMIAMI EXEC AP, FL US             20.1   \n",
      "\n",
      "      DLY-TMAX-NORMAL  DLY-TMIN-NORMAL  DLY-DUTR-NORMAL  DLY-TAVG-STDDEV  \\\n",
      "0                24.8             16.1             -9.0            -13.7   \n",
      "1                24.8             15.9             -8.9            -13.6   \n",
      "2                24.7             15.8             -8.9            -13.6   \n",
      "3                24.7             15.7             -8.8            -13.6   \n",
      "4                24.6             15.6             -8.8            -13.6   \n",
      "...               ...              ...              ...              ...   \n",
      "5485             25.6             15.4             -7.6            -14.0   \n",
      "5486             25.5             15.3             -7.6            -14.0   \n",
      "5487             25.4             15.2             -7.5            -14.0   \n",
      "5488             25.4             15.1             -7.4            -14.0   \n",
      "5489             25.3             14.9             -7.4            -14.1   \n",
      "\n",
      "      DLY-TMAX-STDDEV  DLY-TMIN-STDDEV  DLY-DUTR-STDDEV  \\\n",
      "0               -14.3            -12.4            -14.2   \n",
      "1               -14.3            -12.4            -14.2   \n",
      "2               -14.2            -12.4            -14.2   \n",
      "3               -14.2            -12.4            -14.2   \n",
      "4               -14.2            -12.4            -14.2   \n",
      "...               ...              ...              ...   \n",
      "5485            -14.6            -13.0            -14.7   \n",
      "5486            -14.6            -13.0            -14.7   \n",
      "5487            -14.6            -13.0            -14.7   \n",
      "5488            -14.6            -13.1            -14.7   \n",
      "5489            -14.6            -13.1            -14.7   \n",
      "\n",
      "      DLY-PRCP-PCTALL-GE010HI  DLY-PRCP-PCTALL-GE025HI  \\\n",
      "0                        16.1                      8.5   \n",
      "1                        16.1                      8.6   \n",
      "2                        16.1                      8.6   \n",
      "3                        16.2                      8.7   \n",
      "4                        16.2                      8.7   \n",
      "...                       ...                      ...   \n",
      "5485                     11.1                      5.5   \n",
      "5486                     11.0                      5.4   \n",
      "5487                     10.8                      5.2   \n",
      "5488                     10.7                      5.1   \n",
      "5489                     10.6                      5.0   \n",
      "\n",
      "      DLY-PRCP-PCTALL-GE050HI  \n",
      "0                         4.2  \n",
      "1                         4.2  \n",
      "2                         4.3  \n",
      "3                         4.3  \n",
      "4                         4.3  \n",
      "...                       ...  \n",
      "5485                      2.7  \n",
      "5486                      2.6  \n",
      "5487                      2.5  \n",
      "5488                      2.5  \n",
      "5489                      2.4  \n",
      "\n",
      "[4392 rows x 13 columns]\n"
     ]
    }
   ],
   "source": [
    "print(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Target_Temperature  (avg temperature of the next day)\n",
    "dataset[\"Target_Temperature\"] = dataset[\"DLY-TAVG-NORMAL\"].shift(-1)\n",
    "# Create Target_Rain (if the next day has more than 30% of rain)\n",
    "dataset[\"Rain_Tomorrow\"] = (dataset[\"DLY-PRCP-PCTALL-GE010HI\"].shift(-1) > 30).astype(int)\n",
    "# Remove the last row\n",
    "dataset = dataset.iloc[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the dataset with different stations on different files and folders\n",
    "stations = dataset['STATION'].unique()\n",
    "if not os.path.exists('dataset/stations'):\n",
    "    os.makedirs('dataset/stations')\n",
    "for station in stations:\n",
    "    dataset_station = dataset[dataset['STATION'] == station]\n",
    "    dataset_station.to_csv(f'dataset/stations/{station}.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the dataset\n",
    "dataset.to_csv('dataset/dataset.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "SDC",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}