Data Arranging.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'speedchallenge'...\n",
      "remote: Enumerating objects: 11, done.\u001b[K\n",
      "remote: Total 11 (delta 0), reused 0 (delta 0), pack-reused 11\u001b[K\n",
      "Unpacking objects: 100% (11/11), done.\n",
      "Checking connectivity... done.\n",
      "Downloading data/test.mp4 (67 MB)\n",
      "Downloading data/train.mp4 (130 MB)\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/commaai/speedchallenge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/media/sandeepgogadi/Data/Study/DL/Imitation/speedchallenge\n"
     ]
    }
   ],
   "source": [
    "cd speedchallenge/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir images/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "\n",
    "vidcap = cv2.VideoCapture('data/train.mp4')\n",
    "success, image = vidcap.read()\n",
    "count = 0\n",
    "while success:\n",
    "    cv2.imwrite('images/{:06d}.jpg'.format(count), image)\n",
    "    success, image = vidcap.read()\n",
    "    count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20400\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "\n",
    "X = sorted(glob.glob('images/*'))\n",
    "print(len(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20400\n"
     ]
    }
   ],
   "source": [
    "f = open('data/train.txt', 'r')\n",
    "y = f.read().split('\\n')\n",
    "f.close()\n",
    "\n",
    "print(len(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train, X_valid, y_valid, X_test, y_test = [], [], [], [], [], []\n",
    "\n",
    "for i in range(len(X)):\n",
    "    if i%4 in [0, 3]:\n",
    "        X_train.append(X[i])\n",
    "        y_train.append(float(y[i]))\n",
    "    elif i%4 == 1:\n",
    "        X_valid.append(X[i])\n",
    "        y_valid.append(float(y[i]))        \n",
    "    elif i%4 == 2: \n",
    "        X_test.append(X[i])\n",
    "        y_test.append(float(y[i]))        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10200 10200 5100 5100 5100 5100\n"
     ]
    }
   ],
   "source": [
    "print(len(X_train), len(y_train), \n",
    "      len(X_valid), len(y_valid),\n",
    "      len(X_test), len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "images/000000.jpg 28.105569 \n",
      " images/000001.jpg 28.105569 \n",
      " images/000002.jpg 28.106527\n"
     ]
    }
   ],
   "source": [
    "print(X_train[0], y_train[0], '\\n',\n",
    "      X_valid[0], y_valid[0], '\\n',\n",
    "      X_test[0], y_test[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {'train': [X_train, y_train],\n",
    "        'valid': [X_valid, y_valid],\n",
    "        'test': [X_test, y_test]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(data, open('data.p', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "data_train, data_valid, data_test = [], [], []\n",
    "\n",
    "for i in range(len(X)):\n",
    "    if i%4 in [0, 3]:\n",
    "        data_train.append([X[i], float(y[i])])\n",
    "    elif i%4 == 1:\n",
    "        data_valid.append([X[i], float(y[i])])       \n",
    "    elif i%4 == 2: \n",
    "        data_test.append([X[i], float(y[i])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10200 5100 5100\n",
      "['images/000000.jpg', 28.105569] ['images/000001.jpg', 28.105569] ['images/000002.jpg', 28.106527]\n"
     ]
    }
   ],
   "source": [
    "print(len(data_train), len(data_valid), len(data_test))\n",
    "print(data_train[0], data_valid[0], data_test[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_frame = pd.DataFrame(data_train, columns=['file_name', 'value'])\n",
    "valid_frame = pd.DataFrame(data_valid, columns=['file_name', 'value'])\n",
    "test_frame = pd.DataFrame(data_test, columns=['file_name', 'value'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_frame.to_csv('train.csv')\n",
    "valid_frame.to_csv('valid.csv')\n",
    "test_frame.to_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              file_name      value\n",
      "0     images/000002.jpg  28.106527\n",
      "1     images/000006.jpg  28.034211\n",
      "2     images/000010.jpg  27.981986\n",
      "3     images/000014.jpg  27.937178\n",
      "4     images/000018.jpg  27.855982\n",
      "5     images/000022.jpg  27.824348\n",
      "6     images/000026.jpg  27.694763\n",
      "7     images/000030.jpg  27.723912\n",
      "8     images/000034.jpg  27.606098\n",
      "9     images/000038.jpg  27.559232\n",
      "10    images/000042.jpg  27.493015\n",
      "11    images/000046.jpg  27.412907\n",
      "12    images/000050.jpg  27.365647\n",
      "13    images/000054.jpg  27.266277\n",
      "14    images/000058.jpg  27.227439\n",
      "15    images/000062.jpg  27.170096\n",
      "16    images/000066.jpg  27.071874\n",
      "17    images/000070.jpg  27.018145\n",
      "18    images/000074.jpg  26.968576\n",
      "19    images/000078.jpg  26.860309\n",
      "20    images/000082.jpg  26.836415\n",
      "21    images/000086.jpg  26.730095\n",
      "22    images/000090.jpg  26.690703\n",
      "23    images/000094.jpg  26.569781\n",
      "24    images/000098.jpg  26.541647\n",
      "25    images/000102.jpg  26.456369\n",
      "26    images/000106.jpg  26.432169\n",
      "27    images/000110.jpg  26.287243\n",
      "28    images/000114.jpg  26.264020\n",
      "29    images/000118.jpg  26.184952\n",
      "...                 ...        ...\n",
      "5070  images/020282.jpg   5.603890\n",
      "5071  images/020286.jpg   5.532767\n",
      "5072  images/020290.jpg   5.454502\n",
      "5073  images/020294.jpg   5.362389\n",
      "5074  images/020298.jpg   5.303368\n",
      "5075  images/020302.jpg   5.188777\n",
      "5076  images/020306.jpg   5.145942\n",
      "5077  images/020310.jpg   5.024955\n",
      "5078  images/020314.jpg   4.933479\n",
      "5079  images/020318.jpg   4.810176\n",
      "5080  images/020322.jpg   4.729469\n",
      "5081  images/020326.jpg   4.578079\n",
      "5082  images/020330.jpg   4.510230\n",
      "5083  images/020334.jpg   4.373685\n",
      "5084  images/020338.jpg   4.241361\n",
      "5085  images/020342.jpg   4.112127\n",
      "5086  images/020346.jpg   3.980881\n",
      "5087  images/020350.jpg   3.829010\n",
      "5088  images/020354.jpg   3.645290\n",
      "5089  images/020358.jpg   3.562821\n",
      "5090  images/020362.jpg   3.394105\n",
      "5091  images/020366.jpg   3.272431\n",
      "5092  images/020370.jpg   3.104029\n",
      "5093  images/020374.jpg   2.979671\n",
      "5094  images/020378.jpg   2.825360\n",
      "5095  images/020382.jpg   2.746720\n",
      "5096  images/020386.jpg   2.594662\n",
      "5097  images/020390.jpg   2.479404\n",
      "5098  images/020394.jpg   2.364811\n",
      "5099  images/020398.jpg   2.260600\n",
      "\n",
      "[5100 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "print(test_frame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}