HD_Model_Summary_Gen.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hard Drive Model Summary Generator\n",
    "* makes a summary of hard drive models for each data period\n",
    "* writes a .csv into OUTPUT_DIR for each folder in DATA_FOLDERS \n",
    "* the output data summary columns might include:\n",
    " ['model', 'size_tb', 'drive_count', 'days_observed', 'days_elapsed','runtime_total', 'runtime_elapsed', 'failure_rate', 'percent_total', 'failures']\n",
    " \n",
    "* count is number of drives of a given model, in particualr the total count of unique serial_numbers that pandas aggregated over\n",
    "* obs_days in the sumver over all drives of max(date) - min(date)\n",
    "* obs_runtime is the sum over all drives of max(runtime) - min(runtime)\n",
    "* runtime is the sum over all dirves of max(runtime)\n",
    "* obs_failure_rate `100.0*nfailures/(obs_runtime/8760.0)` \n",
    "* failure rate is `100.0*nfailures/(runtime/8760.0)` really makes no sense because it uses assumption of complete observation coverage\n",
    "* a unique drive is a unique serial_number\n",
    "* if we had complete observation coverage of a drive since the begining then obs_runtime would be equal to runtime\n",
    "* the annual failure rate of a drive can be greater than 100%, for example in Q1 of 2016 model ST320LT007 had 31 failures out of 73 drives in the 64 days of the quarter, thus the failure rate was approximately 242% `(100*31*365/(73*64)=242)` if we assume that each drive had 24 hour uptime and we had complete observation coverage.\n",
    "\n",
    " observations coverage would be number of days in obs period time number of drives / days, which is often greater than 1, this should not be so, do duplicates need to be removed? Likely!\n",
    " \n",
    "An outstanding confusion in the data is why the number of drives, count, times the number of days in the obs period is less than both obs_days and days\n",
    "\n",
    "why is `count*24*(number of days in obs period)` less than both obs_runtime and runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "from time import localtime, strftime\n",
    "import logging\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "TEST = False\n",
    "DATA_DIR = \"data/\"\n",
    "DATA_FOLDERS = [\"2014\",\"2015\",\"data_Q1_2016\",\"data_Q2_2016\",\"data_Q3_2016\",\"data_Q4_2016\"]\n",
    "#DATA_FOLDERS = [\"data_Q2_2016\"]\n",
    "FEATURE_COLS = ['date','capacity_bytes','smart_9_raw','model','failure','serial_number']\n",
    "THE_TIME = strftime(\"%Y-%m-%d-%H-%M\", localtime())\n",
    "OUTPUT_DIR = \"summary_data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def start_logger():\n",
    "    if not os.path.isdir(OUTPUT_DIR):\n",
    "        os.system(\"mkdir \" + OUTPUT_DIR)\n",
    "    root_logger = logging.getLogger()\n",
    "    root_logger.setLevel(logging.DEBUG)\n",
    "    logger = logging.getLogger(__name__)\n",
    "    logfile = OUTPUT_DIR + '/' + THE_TIME + \".log\"\n",
    "    handler = logging.FileHandler(logfile, 'w')\n",
    "    handler.setLevel(logging.INFO)\n",
    "    logger.addHandler(handler)\n",
    "    logger.info(\"TEST\\t\\t=\\t\" + str(TEST))\n",
    "    logger.info(\"DATA_DIR\\t=\\t\" + DATA_DIR)\n",
    "    logger.info(\"DATA_FOLDERS\\t=\\t\" + str(DATA_FOLDERS))\n",
    "    logger.info(\"FEATURE_COLS\\t=\\t\" + str(FEATURE_COLS))\n",
    "    logger.info(\"THE_TIME\\t=\\t\" + THE_TIME)\n",
    "    logger.info(\"OUTPUT_DIR\\t=\\t\" + OUTPUT_DIR)\n",
    "    return logger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sift_data(data):\n",
    "    for ikey in data.keys():\n",
    "        if ikey not in FEATURE_COLS:\n",
    "            del data[ikey]\n",
    "    return data\n",
    "\n",
    "\n",
    "def tb_capacity(x):\n",
    "    \"\"\"\n",
    "    1 gig is np.power(2, 30) bytes, but maybe it is 10^9 bytes. Who you asking? Whatever.\n",
    "    \"\"\"\n",
    "    tb = np.power(10, 12)\n",
    "    #if not math.isnan(x):\n",
    "    return x/tb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def summarize(data):\n",
    "    aggregations = {\n",
    "    'failure': {\n",
    "     'failure': 'sum'\n",
    "    },\n",
    "    'capacity_tb':{\n",
    "        'mean_cap': 'mean'\n",
    "    },\n",
    "    'date': { \n",
    "        #'max_days': 'max',\n",
    "        #'min_days': 'min',\n",
    "        #'unique_days': lambda x: x.nunique(),\n",
    "        'days_obs': 'count', #count returns series with number of non-NA/null observations over requested axis.\n",
    "        'days_elap': lambda x: max(x) - min(x)  \n",
    "    },\n",
    "    'smart_9_raw': {\n",
    "        #'min_runtime': 'min'\n",
    "        'runtime_max': 'max',  \n",
    "        'runtime_elap': lambda x: max(x) - min(x)}    \n",
    "    }\n",
    "\n",
    "    by_model_serial = data.groupby(['model', 'serial_number']).agg(aggregations)#.reset_index()\n",
    "    by_model_serial.columns = by_model_serial.columns.droplevel()\n",
    "    models = by_model_serial.index.levels[0].tolist()\n",
    "    ### The annualized failure rate is: 100 * Failures/(Drive Days/365)\n",
    "    summary_cols = ['model', 'size_tb', 'drive_count', 'days_observed', 'days_elapsed','runtime_total', 'runtime_elapsed',\n",
    "                    'failure_rate', 'percent_total', 'failures']\n",
    "    summary = pd.DataFrame([], columns=summary_cols)\n",
    "    ntot_drives = float(data['serial_number'].value_counts().count())\n",
    "    \n",
    "    for i in models:\n",
    "        tmp = by_model_serial.xs(i)\n",
    "        drive_count = len(tmp)\n",
    "        nfailures = np.sum(tmp['failure'])\n",
    "        runtime_elap =  np.sum(tmp['runtime_elap'])\n",
    "        if runtime_elap != 0:\n",
    "            failrate = 100.0 * nfailures/(runtime_elap/8760.0) \n",
    "        else: \n",
    "            failrate  = 'NaN'\n",
    "        df_tmp = pd.DataFrame([[i, stats.mode(tmp['mean_cap'])[0][0], drive_count, \n",
    "                                np.nansum(tmp['days_obs']), np.nansum(tmp['days_elap']),\n",
    "                                np.sum(tmp['runtime_max']), runtime_elap, failrate, \n",
    "                                drive_count/ntot_drives, nfailures]], columns=summary_cols)\n",
    "\n",
    "        summary = summary.append(df_tmp, ignore_index=True)\n",
    "\n",
    "    return summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "logger = start_logger()\n",
    "\n",
    "total_summary = []\n",
    "for data_dir in DATA_FOLDERS:\n",
    "    data_path = DATA_DIR + data_dir + \"/\"\n",
    "    logger.info(\" * * *\")\n",
    "    logger.info(\"Loading and working with: %s\" % data_path)\n",
    "    logger.info(\"Current time: %s\" % strftime(\"%Y-%m-%d-%H-%M\", localtime()))\n",
    "    hd = pd.DataFrame()\n",
    "    for data_file in os.listdir(data_path):\n",
    "        if data_file.split('.')[1] == 'csv':\n",
    "            if TEST:\n",
    "                temp = pd.read_csv(data_path + data_file, header=0, nrows=1000)\n",
    "                temp = sift_data(temp)\n",
    "            else:\n",
    "                temp = pd.read_csv(data_path + data_file, header=0)\n",
    "                temp = sift_data(temp)\n",
    "            hd = hd.append(temp)\n",
    "    hd.capacity_bytes = hd.capacity_bytes.map(tb_capacity)\n",
    "    hd.rename(columns={'capacity_bytes': 'capacity_tb'}, inplace=True)\n",
    "    hd['date'] = pd.to_datetime(hd['date'], format='%Y-%m-%d')\n",
    "    _sn = hd['serial_number'].value_counts().count()\n",
    "    _models = hd['model'].value_counts().count()\n",
    "    _dates = hd['date'].value_counts().count()\n",
    "    _fail = hd['failure'].sum()\n",
    "    logger.info(\"There are %d unique drives. \" % _sn)\n",
    "    logger.info(\"There are %d unique models. \" % _models)\n",
    "    logger.info(\"There are %d unique dates. \" % _dates)\n",
    "    logger.info(\"There are %d failures.\" % _fail)\n",
    "    total_summary.append([data_dir, _sn, _dates, _models, _fail])\n",
    "    \n",
    "    hd[\"quarter\"] = hd['date'].dt.quarter\n",
    "    quarters = hd['quarter'].unique()\n",
    "    logger.info(\"There are %d quarters.\" % len(quarters))\n",
    "    if len(quarters) > 1:\n",
    "        for q in quarters:\n",
    "            qhd = hd[hd['quarter'] == q]\n",
    "            \n",
    "            summary = summarize(qhd)\n",
    "            summary.to_csv(OUTPUT_DIR + \"/\" + \"Q\" + str(q) + \"_\" + data_dir + \".csv\", index = False)\n",
    "    else:\n",
    "        summary = summarize(hd)\n",
    "        summary.to_csv(OUTPUT_DIR + \"/\" + data_dir  + \".csv\", index = False)\n",
    "\n",
    "f =open(OUTPUT_DIR + \"/total_summary.txt','w')\n",
    "for item in total_summary:\n",
    "    f.write(str(item))\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}