M3-competition. Arima model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import os\n",
    "import time\n",
    "from math import sqrt\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from functools import reduce\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "from ArimaModel import ArimaModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_file = 'data/M3C.xls'\n",
    "categories = ['MICRO', 'INDUSTRY', 'MACRO', 'FINANCE', 'DEMOGRAPHIC', 'OTHER']\n",
    "frequencies = ['Year', 'Quart', 'Month', 'Other']\n",
    "forecast_type = 'full' #or 'one_step'\n",
    "validation_criteria = 'cross' #or 'aic'\n",
    "seasonal_arima_model = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(input_file, categories, frequency):\n",
    "\n",
    "    m3c_file = pd.ExcelFile(input_file)\n",
    "\n",
    "    sheet_name = 'M3' + frequency\n",
    "\n",
    "    m3c_month_df = m3c_file.parse(sheet_name)\n",
    "\n",
    "    # strip unnecessary spaces from 'Category' column\n",
    "    m3c_month_df['Category'] = m3c_month_df['Category'].apply(lambda x: x.strip())\n",
    "\n",
    "    return m3c_month_df[m3c_month_df['Category'].isin(categories)]\n",
    "\n",
    "def next_series_generator(data_df, test_data_size):\n",
    "\n",
    "    scaler = preprocessing.MinMaxScaler()\n",
    "\n",
    "    for _, row in data_df.iterrows():\n",
    "        # series_size incremented for the purpose of range\n",
    "        series_size = row['N'] + 1\n",
    "        # conversion to type float64 to silence the warning when fitting scaler\n",
    "        train_data = row.loc[range(1, series_size - test_data_size)].values.astype(np.float64)\n",
    "        test_data = row.loc[range(series_size - test_data_size, series_size)].values.astype(np.float64)\n",
    "\n",
    "        # normalise data to range [0, 1]\n",
    "        scaler.fit(train_data.reshape(-1, 1))\n",
    "        train_data = scaler.transform(train_data.reshape(1, -1)).tolist()[0]\n",
    "        test_data = scaler.transform(test_data.reshape(1, -1)).tolist()[0]\n",
    "\n",
    "        yield train_data, test_data, scaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for frequency in frequencies:\n",
    "    for category in categories:\n",
    "        data_df = load_data(input_file, [category], frequency)\n",
    "\n",
    "        if frequency == 'Year':\n",
    "            test_data_size = 6\n",
    "            data_frequency = 1\n",
    "        elif frequency == 'Quart':\n",
    "            test_data_size = 8\n",
    "            data_frequency = 4\n",
    "        elif frequency == 'Month':\n",
    "            test_data_size = 18\n",
    "            data_frequency = 12\n",
    "        else:\n",
    "            test_data_size = 8\n",
    "            data_frequency = 0\n",
    "\n",
    "        if not seasonal_arima_model:\n",
    "            data_frequency = 0\n",
    "\n",
    "        predicted_data_matrix = []\n",
    "        expected_data_matrix = []\n",
    "\n",
    "        normalized_forecast_data_matrix = []\n",
    "        normalized_test_data_matrix = []\n",
    "\n",
    "        # list of dicts of parameters of each trained model\n",
    "        parameters_dicts = []\n",
    "\n",
    "        rmse_prediction = []\n",
    "        rmse_forecast = []\n",
    "        rmse_total = []\n",
    "\n",
    "        training_time = 0\n",
    "        testing_time = 0\n",
    "\n",
    "        for train_data, test_data, scaler in next_series_generator(data_df, test_data_size):\n",
    "            training_start = time.time()\n",
    "            model = ArimaModel(train_data, validation_criteria, data_frequency)\n",
    "            training_end = time.time()\n",
    "\n",
    "            prediction, forecast = model.make_prediction(test_data, forecast_type)\n",
    "            testing_end = time.time()\n",
    "\n",
    "            training_time += training_end - training_start\n",
    "            testing_time += testing_end - training_end\n",
    "            parameters_dicts.append(model.get_parameters())\n",
    "\n",
    "            # rmse per series\n",
    "            rmse_prediction.append(sqrt(mean_squared_error(train_data, prediction)))\n",
    "            rmse_forecast.append(sqrt(mean_squared_error(test_data, forecast)))\n",
    "            rmse_total.append(sqrt(mean_squared_error(train_data + test_data, prediction + forecast)))\n",
    "\n",
    "            normalized_forecast_data_matrix.append(forecast)\n",
    "            normalized_test_data_matrix.append(test_data)\n",
    "\n",
    "            # denormalization of data\n",
    "            predicted_data = scaler.inverse_transform([prediction + forecast]).tolist()[0]\n",
    "            expected_data = scaler.inverse_transform([train_data + test_data]).tolist()[0]\n",
    "\n",
    "            predicted_data_matrix.append(predicted_data)\n",
    "            expected_data_matrix.append(expected_data)\n",
    "\n",
    "        # names of data columns in form of integers\n",
    "        data_columns = list(range(1, max(len(x) for x in predicted_data_matrix) + 1))\n",
    "        series_index = list(range(len(predicted_data_matrix)))\n",
    "        series_length = [len(x) for x in predicted_data_matrix]\n",
    "\n",
    "        predicted_data_df = pd.DataFrame(predicted_data_matrix, columns=data_columns)\n",
    "        predicted_data_df['series'] = series_index\n",
    "        predicted_data_df['N'] = series_length\n",
    "        predicted_data_df['NF'] = test_data_size\n",
    "\n",
    "        # take parameters names from first dict\n",
    "        parameters_names = list(parameters_dicts[0].keys())\n",
    "        for parameter in parameters_names:\n",
    "            predicted_data_df[parameter] = [x[parameter] for x in parameters_dicts]\n",
    "\n",
    "        predicted_data_df['rmse_prediction'] = rmse_prediction\n",
    "        predicted_data_df['rmse_forecast'] = rmse_forecast\n",
    "        predicted_data_df['rmse_total'] = rmse_total\n",
    "\n",
    "        # change the order of columns\n",
    "        output_columns = ['series', 'N', 'NF'] + parameters_names + ['rmse_prediction', 'rmse_forecast', 'rmse_total'] \\\n",
    "                         + data_columns\n",
    "        predicted_data_df = predicted_data_df[output_columns]\n",
    "\n",
    "        expected_data_df = pd.DataFrame(expected_data_matrix, columns=data_columns)\n",
    "        expected_data_df['series'] = series_index\n",
    "        expected_data_df['N'] = series_length\n",
    "        expected_data_df['NF'] = test_data_size\n",
    "\n",
    "        output_columns = ['series', 'N', 'NF'] + data_columns\n",
    "        expected_data_df = expected_data_df[output_columns]\n",
    "\n",
    "        # calculate error separately for each month of forecast\n",
    "        rmse_per_month = []\n",
    "        for i in range(test_data_size):\n",
    "            month_forecast = [x[i] for x in normalized_forecast_data_matrix]\n",
    "            month_expected = [x[i] for x in normalized_test_data_matrix]\n",
    "            rmse_per_month.append(sqrt(mean_squared_error(month_expected, month_forecast)))\n",
    "\n",
    "        # calculate total error of all forecasts\n",
    "        all_forecast = reduce(lambda x, y: x + y, normalized_forecast_data_matrix)\n",
    "        all_test = reduce(lambda x, y: x + y, normalized_test_data_matrix)\n",
    "\n",
    "        total_rmse = sqrt(mean_squared_error(all_test, all_forecast))\n",
    "\n",
    "        output_dir = '{frequency}_{category}_{forecast_type}_{validation_criteria}'.format(\n",
    "                    frequency=frequency, category=category,\n",
    "                    forecast_type=forecast_type, validation_criteria=validation_criteria)\n",
    "        \n",
    "        if not os.path.isdir(output_dir):\n",
    "            os.makedirs(output_dir)\n",
    "\n",
    "        normalized_forecast_data_df = pd.DataFrame(normalized_forecast_data_matrix)\n",
    "        normalized_test_dat_df = pd.DataFrame(normalized_test_data_matrix)\n",
    "\n",
    "        normalized_forecast_data_df.to_csv(os.path.join(output_dir, 'normalized_forecast.tsv'), sep='\\t', index=False)\n",
    "        normalized_test_dat_df.to_csv(os.path.join(output_dir, 'normalized_test_data.tsv'), sep='\\t', index=False)\n",
    "\n",
    "        predicted_data_df.to_csv(os.path.join(output_dir, 'predictions.tsv'), sep='\\t', index=False)\n",
    "        expected_data_df.to_csv(os.path.join(output_dir, 'expected.tsv'), sep='\\t', index=False)\n",
    "\n",
    "        with open(os.path.join(output_dir, 'rmse_file.tsv'), 'w') as f:\n",
    "            f.write('\\t'.join([str(x) for x in rmse_per_month]))\n",
    "            f.write('\\n' + str(total_rmse))\n",
    "\n",
    "        mean_training_time = training_time / len(predicted_data_df)\n",
    "        mean_testing_time = testing_time / len(predicted_data_df)\n",
    "\n",
    "        with open(os.path.join(output_dir, 'time.tsv'), 'w') as f:\n",
    "            f.write('mean_training_time\\tmean_testing_time\\n{training_time}\\t{testing_time}\\n'.format(training_time=mean_training_time, testing_time=mean_testing_time))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}