createa_categories.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import unicodedata\n",
    "import glob\n",
    "from pathlib import Path\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create functions for load raw csv as dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LoadRawDF:\n",
    "    def __init__(self):\n",
    "        \n",
    "        # Function to create raw dataframes\n",
    "\n",
    "        self.a = 0\n",
    "        self.pattern_name = re.compile(\"[A-Z]+[a-z]+\")\n",
    "\n",
    "    def read_csv(self, filename):\n",
    "        df = pd.read_csv(filename, sep='delimiter', header=None)\n",
    "        return df\n",
    "\n",
    "    def set_column_name(self, df, colname):\n",
    "        df.columns = [colname]\n",
    "        return df\n",
    "\n",
    "    def extract_categories(self, df):\n",
    "        unique = df.data.unique()\n",
    "        categories = []\n",
    "        for i in range(len(unique)):\n",
    "            if (\"►\" in unique[i]):\n",
    "                categories.append(unique[i])\n",
    "            else:\n",
    "                pass\n",
    "        return categories\n",
    "\n",
    "    def _label_race(self, row, categories):\n",
    "        if row['data'] in categories:\n",
    "            self.a +=1\n",
    "            if self.a == 0:\n",
    "                pass\n",
    "            elif self.a >1:\n",
    "                self.a = 0\n",
    "            else:\n",
    "                pass\n",
    "            return self.a\n",
    "        else:\n",
    "            return -1\n",
    "\n",
    "    def create_tag_column(self, df, categories):\n",
    "        df['tag'] = df.apply (lambda row: self._label_race(row, categories), axis=1)\n",
    "        return df\n",
    "\n",
    "    def create_dataframe_set(self, df):\n",
    "        a = pd.factorize(df['tag'].isin([0,1]).iloc[::-1].cumsum().sort_index())[0]\n",
    "        dfs = dict(tuple(df.groupby(a)))\n",
    "        return dfs\n",
    "\n",
    "    def create_dfs_by_cat(self, dfs, categories):\n",
    "        new_cats = []\n",
    "        for i in range(len(dfs)):\n",
    "            key = dfs[i][\"data\"].tolist()[-1]\n",
    "            if key in categories:\n",
    "                n_c = dict()\n",
    "                n_c[key] = dfs[i+1]\n",
    "            else:\n",
    "                pass\n",
    "            new_cats.append(n_c)\n",
    "\n",
    "        return new_cats\n",
    "\n",
    "    def create_csv(self, new_cats):\n",
    "        for i, d in enumerate(new_cats):\n",
    "            for k, v in d.items():\n",
    "                n_df = pd.DataFrame(v)\n",
    "                \n",
    "                if (\"/\" in k):\n",
    "                    new_name = k.replace(\"/\",\"_\")\n",
    "                else:\n",
    "                    new_name = k\n",
    "\n",
    "                export_csv = n_df.to_csv (f\"data/{new_name}.csv\", index = None, header=True) #Don't forget to add '.csv' at the end of the path \n",
    "    \n",
    "    def create_raw_dfs(self, new_cats):\n",
    "        raw_df=[]\n",
    "        for i, d in enumerate(new_cats):\n",
    "            for k, v in d.items():\n",
    "                n_df = pd.DataFrame(v)\n",
    "                raw_df.append(n_df)\n",
    "    \n",
    "\n",
    "    def remove_accent(self, text):\n",
    "        \n",
    "        output = unicodedata.normalize('NFD', text).encode('ascii', 'ignore')\n",
    "        decoded = output.decode('utf-8')\n",
    "        return decoded\n",
    "    \n",
    "    def get_name(self, filename):\n",
    "        result = self.pattern_name.findall(filename)\n",
    "        name = result[-1]\n",
    "        return result\n",
    "    \n",
    "    def __call__(self, fileName):\n",
    "        \"\"\"\n",
    "        Return a list of dfs\n",
    "        \"\"\"\n",
    "        output_dfs_list = None\n",
    "        \n",
    "        # Load dataframe\n",
    "        df = self.read_csv(fileName)\n",
    "        \n",
    "        # Create column name by using the filename\n",
    "        \n",
    "        # Change column name\n",
    "        column_name = \"data\"\n",
    "        df = self.set_column_name(df, column_name)\n",
    "        \n",
    "        # Exctrac all the catogories as list\n",
    "        categories = self.extract_categories(df)\n",
    "        \n",
    "        if len(categories) > 0:\n",
    "            \n",
    "            # Create Tag column with if -1 is general, 0 if is category,\n",
    "            # 1 next category and 0 again for next cat.\n",
    "            df = self.create_tag_column(df, categories)\n",
    "\n",
    "            # Create dict of dictionaries by categorie where Key is the index 0 or 1 for categories\n",
    "            dfs = self.create_dataframe_set(df)\n",
    "\n",
    "            # Replace the keys (0 or 1) by his real value in categoire list.\n",
    "            new_cats = self.create_dfs_by_cat(dfs, categories)\n",
    "        \n",
    "            return new_cats\n",
    "        else :\n",
    "            return pd.DataFrame(columns= df.columns.tolist())\n",
    "        \n",
    "df_raw_loader = LoadRawDF()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions for clean the data and save as csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Functions for clean the data\n",
    "\n",
    "class CleanerDF:\n",
    "    \n",
    "    def __init__(self):\n",
    "\n",
    "        self.pattern_name = re.compile(\"[A-Z]+[a-z]+\")\n",
    "        self.pattern_unused_rows = \"\\[+[0-9]+-[0-9]+\\]\"\n",
    "\n",
    "    def _remove_accent(self, text):\n",
    "        output = unicodedata.normalize('NFD', text).encode('ascii', 'ignore')\n",
    "        decoded = output.decode('utf-8')\n",
    "        return decoded\n",
    "\n",
    "    def read_csv(self, file):\n",
    "        df = pd.read_csv(file)\n",
    "        return df\n",
    "\n",
    "    def unused_df_rows(self, dff):\n",
    "        aux_df = dff[dff['data'].astype(str).str.match(self.pattern_unused_rows)]\n",
    "        return aux_df\n",
    "\n",
    "    def get_index_to_filter(self, u_df, dff):\n",
    "        \"\"\"\n",
    "        Return list of indices to remove\n",
    "        \"\"\"\n",
    "        indexNames = u_df.index\n",
    "        indexUnused = dff[-1:].index\n",
    "\n",
    "        indexes = indexNames.tolist() + indexUnused.tolist()\n",
    "        return indexes\n",
    "\n",
    "    def _drop_rows(self, dff, indexes):\n",
    "        # Delete these row indexes from dataFrame\n",
    "        dff.drop(indexes, inplace=True)\n",
    "        return dff\n",
    "\n",
    "    def _drop_column(self, dff):\n",
    "        dff = dff.drop(columns=['tag'])\n",
    "        return dff\n",
    "\n",
    "    def _rename_column(self, dff, name):\n",
    "        dff = dff.rename(columns={\"data\": name})\n",
    "        return dff\n",
    "\n",
    "    def save_clean_df(self, dff, filename):\n",
    "        dff.to_csv(f'clean_data/{filename}.csv', index = None, header=True)\n",
    "\n",
    "    def get_name(self, filename):\n",
    "        \"\"\"\n",
    "        Return Column name as string\n",
    "        \"\"\"\n",
    "        result = self.pattern_name.findall(filename)[-1]\n",
    "        return result\n",
    "\n",
    "\n",
    "    def clean_df(self, df, fileName):\n",
    "\n",
    "        f = self._remove_accent(fileName)\n",
    "        \n",
    "        column_name = self.get_name(f)\n",
    "\n",
    "        aux_df = self.unused_df_rows(df)\n",
    "        \n",
    "        indexes = self.get_index_to_filter(aux_df, df)\n",
    "        n_df = self._drop_rows(df, indexes)\n",
    "        df = self._drop_column(n_df)\n",
    "        \n",
    "        \n",
    "        df = self._rename_column(df, column_name)\n",
    "        \n",
    "        return df\n",
    "\n",
    "\n",
    "cleaner = CleanerDF()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Obtain all the clean dataframes as a list\n",
    "\n",
    "def create_clean_df_as_list(raw_dfs_list):\n",
    "    dfs = []\n",
    "    for i, df in enumerate(raw_dfs_list):\n",
    "        for k, v in df.items():\n",
    "            df = cleaner.clean_df(v, k)\n",
    "            dfs.append(df)\n",
    "    return dfs\n",
    "\n",
    "#dfs = create_clean_df_as_list(raw_dfs_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create unique dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_joint_dataframe(dfs, date):\n",
    "    \"\"\"\n",
    "    Return the concatenate dataframe for all \n",
    "    \"\"\"\n",
    "    join_df = []\n",
    "\n",
    "    for df in dfs:\n",
    "        column_name = df.columns.tolist()[0]\n",
    "        data = df[column_name].tolist()\n",
    "        if (len(data) > 0 ):\n",
    "            df_n = pd.DataFrame(data[1:], columns=[\"document\"])\n",
    "            df_n[\"date\"] = date\n",
    "            df_n[\"label_1\"]= column_name\n",
    "            df_n[\"label_2\"] = data[0]\n",
    "            join_df.append(df_n)\n",
    "        else:\n",
    "            pass\n",
    "\n",
    "    \n",
    "    new_df = pd.concat(join_df)\n",
    "    new_df = new_df.reset_index()\n",
    "    \n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data/2019-01-01/2019-01-01.csv',\n",
       " 'data/2017-07-13/2017-07-13.csv',\n",
       " 'data/2017-07-21/2017-07-21.csv']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_files_in_dir(src):\n",
    "\n",
    "    filenames = []\n",
    "    for filename in Path(src).glob(\"**/*.csv\"):\n",
    "        filenames.append(str(filename))\n",
    "    return filenames\n",
    "\n",
    "filenames = get_files_in_dir(\"data\")\n",
    "\n",
    "filenames[:4]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../data_raw/2018-10-02/2018-10-02.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/stanlee321/anaconda3/envs/nlp/lib/python3.7/site-packages/ipykernel_launcher.py:10: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  # Remove the CWD from sys.path while we load stuff.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../data_raw/2018-04-18/2018-04-18.csv\n",
      "../data_raw/2018-05-11/2018-05-11.csv\n",
      "../data_raw/2018-06-02/2018-06-02.csv\n",
      "../data_raw/2017-12-13/2017-12-13.csv\n",
      "../data_raw/2017-07-04/2017-07-04.csv\n",
      "THIS  ../data_raw/2017-07-04/2017-07-04.csv\n",
      "../data_raw/2018-05-04/2018-05-04.csv\n",
      "../data_raw/2018-02-04/2018-02-04.csv\n",
      "../data_raw/2019-02-28/2019-02-28.csv\n",
      "../data_raw/2019-03-11/2019-03-11.csv\n",
      "../data_raw/2017-09-16/2017-09-16.csv\n",
      "../data_raw/2017-08-30/2017-08-30.csv\n",
      "../data_raw/2017-11-30/2017-11-30.csv\n",
      "../data_raw/2018-10-06/2018-10-06.csv\n",
      "../data_raw/2019-01-18/2019-01-18.csv\n",
      "../data_raw/2018-02-23/2018-02-23.csv\n",
      "../data_raw/2018-12-18/2018-12-18.csv\n",
      "../data_raw/2018-10-15/2018-10-15.csv\n",
      "../data_raw/2018-11-11/2018-11-11.csv\n",
      "../data_raw/2017-07-27/2017-07-27.csv\n",
      "../data_raw/2018-04-16/2018-04-16.csv\n",
      "../data_raw/2019-07-07/2019-07-07.csv\n",
      "../data_raw/2019-02-14/2019-02-14.csv\n",
      "../data_raw/2018-12-04/2018-12-04.csv\n",
      "../data_raw/2018-03-05/2018-03-05.csv\n",
      "../data_raw/2017-11-11/2017-11-11.csv\n",
      "../data_raw/2017-09-19/2017-09-19.csv\n",
      "../data_raw/2018-05-12/2018-05-12.csv\n",
      "../data_raw/2018-12-19/2018-12-19.csv\n",
      "../data_raw/2019-06-03/2019-06-03.csv\n",
      "../data_raw/2019-04-24/2019-04-24.csv\n",
      "../data_raw/2019-07-15/2019-07-15.csv\n",
      "../data_raw/2017-07-08/2017-07-08.csv\n",
      "THIS  ../data_raw/2017-07-08/2017-07-08.csv\n",
      "../data_raw/2018-03-21/2018-03-21.csv\n",
      "../data_raw/2018-04-15/2018-04-15.csv\n",
      "../data_raw/2018-12-12/2018-12-12.csv\n",
      "../data_raw/2018-10-18/2018-10-18.csv\n",
      "../data_raw/2019-02-17/2019-02-17.csv\n",
      "../data_raw/2017-09-10/2017-09-10.csv\n",
      "../data_raw/2018-05-18/2018-05-18.csv\n",
      "../data_raw/2018-12-26/2018-12-26.csv\n",
      "../data_raw/2018-02-11/2018-02-11.csv\n",
      "../data_raw/2018-02-03/2018-02-03.csv\n",
      "../data_raw/2018-09-17/2018-09-17.csv\n",
      "../data_raw/2017-12-09/2017-12-09.csv\n",
      "../data_raw/2018-02-13/2018-02-13.csv\n",
      "../data_raw/2017-10-22/2017-10-22.csv\n",
      "../data_raw/2018-05-22/2018-05-22.csv\n",
      "../data_raw/2017-07-06/2017-07-06.csv\n",
      "THIS  ../data_raw/2017-07-06/2017-07-06.csv\n",
      "../data_raw/2017-10-14/2017-10-14.csv\n",
      "../data_raw/2018-12-09/2018-12-09.csv\n",
      "../data_raw/2018-02-25/2018-02-25.csv\n",
      "../data_raw/2017-10-06/2017-10-06.csv\n",
      "../data_raw/2018-12-29/2018-12-29.csv\n",
      "../data_raw/2018-05-17/2018-05-17.csv\n",
      "../data_raw/2019-04-27/2019-04-27.csv\n",
      "../data_raw/2018-11-08/2018-11-08.csv\n",
      "../data_raw/2018-05-30/2018-05-30.csv\n",
      "../data_raw/2019-02-20/2019-02-20.csv\n",
      "../data_raw/2017-11-02/2017-11-02.csv\n",
      "../data_raw/2018-04-03/2018-04-03.csv\n",
      "../data_raw/2019-01-28/2019-01-28.csv\n",
      "../data_raw/2018-10-11/2018-10-11.csv\n",
      "../data_raw/2018-10-25/2018-10-25.csv\n",
      "../data_raw/2018-02-29/2018-02-29.csv\n",
      "../data_raw/2019-07-30/2019-07-30.csv\n",
      "THIS  ../data_raw/2019-07-30/2019-07-30.csv\n",
      "../data_raw/2018-02-26/2018-02-26.csv\n",
      "../data_raw/2017-07-09/2017-07-09.csv\n",
      "THIS  ../data_raw/2017-07-09/2017-07-09.csv\n",
      "../data_raw/2018-03-14/2018-03-14.csv\n",
      "../data_raw/2019-05-19/2019-05-19.csv\n",
      "THIS  ../data_raw/2019-05-19/2019-05-19.csv\n",
      "../data_raw/2018-06-03/2018-06-03.csv\n",
      "../data_raw/2017-12-08/2017-12-08.csv\n",
      "../data_raw/2018-10-05/2018-10-05.csv\n",
      "../data_raw/2019-01-01/2019-01-01.csv\n",
      "../data_raw/2018-12-01/2018-12-01.csv\n",
      "../data_raw/2019-07-11/2019-07-11.csv\n",
      "../data_raw/2018-09-15/2018-09-15.csv\n",
      "../data_raw/2019-01-22/2019-01-22.csv\n",
      "../data_raw/2017-09-12/2017-09-12.csv\n",
      "../data_raw/2018-01-19/2018-01-19.csv\n",
      "THIS  ../data_raw/2018-01-19/2018-01-19.csv\n",
      "../data_raw/2017-11-12/2017-11-12.csv\n",
      "../data_raw/2017-12-21/2017-12-21.csv\n",
      "../data_raw/2017-10-19/2017-10-19.csv\n",
      "../data_raw/2018-04-12/2018-04-12.csv\n",
      "THIS  ../data_raw/2018-04-12/2018-04-12.csv\n",
      "../data_raw/2018-03-24/2018-03-24.csv\n",
      "../data_raw/2019-01-09/2019-01-09.csv\n",
      "../data_raw/2019-07-03/2019-07-03.csv\n",
      "THIS  ../data_raw/2019-07-03/2019-07-03.csv\n",
      "../data_raw/2019-03-10/2019-03-10.csv\n",
      "../data_raw/2018-03-09/2018-03-09.csv\n",
      "../data_raw/2019-03-15/2019-03-15.csv\n",
      "../data_raw/2019-01-26/2019-01-26.csv\n",
      "../data_raw/2017-08-28/2017-08-28.csv\n",
      "../data_raw/2017-11-08/2017-11-08.csv\n",
      "../data_raw/2017-08-24/2017-08-24.csv\n",
      "../data_raw/2018-12-25/2018-12-25.csv\n",
      "../data_raw/2017-11-28/2017-11-28.csv\n",
      "../data_raw/2018-05-03/2018-05-03.csv\n",
      "../data_raw/2018-10-28/2018-10-28.csv\n",
      "../data_raw/2018-03-28/2018-03-28.csv\n",
      "../data_raw/2017-08-18/2017-08-18.csv\n",
      "../data_raw/2019-02-12/2019-02-12.csv\n",
      "../data_raw/2018-02-07/2018-02-07.csv\n",
      "../data_raw/2018-03-23/2018-03-23.csv\n",
      "../data_raw/2017-12-24/2017-12-24.csv\n",
      "../data_raw/2018-05-02/2018-05-02.csv\n",
      "../data_raw/2019-03-31/2019-03-31.csv\n",
      "../data_raw/2019-05-26/2019-05-26.csv\n",
      "THIS  ../data_raw/2019-05-26/2019-05-26.csv\n",
      "../data_raw/2019-05-03/2019-05-03.csv\n",
      "../data_raw/2018-04-14/2018-04-14.csv\n",
      "../data_raw/2019-04-26/2019-04-26.csv\n",
      "../data_raw/2019-05-15/2019-05-15.csv\n",
      "../data_raw/2018-01-10/2018-01-10.csv\n",
      "../data_raw/2019-05-22/2019-05-22.csv\n",
      "THIS  ../data_raw/2019-05-22/2019-05-22.csv\n",
      "../data_raw/2017-12-20/2017-12-20.csv\n",
      "../data_raw/2019-06-02/2019-06-02.csv\n",
      "../data_raw/2018-09-26/2018-09-26.csv\n",
      "../data_raw/2018-10-22/2018-10-22.csv\n",
      "../data_raw/2018-12-11/2018-12-11.csv\n",
      "../data_raw/2019-01-14/2019-01-14.csv\n",
      "../data_raw/2019-06-01/2019-06-01.csv\n",
      "../data_raw/2018-06-06/2018-06-06.csv\n",
      "../data_raw/2017-12-05/2017-12-05.csv\n",
      "../data_raw/2018-02-08/2018-02-08.csv\n",
      "../data_raw/2018-05-08/2018-05-08.csv\n",
      "../data_raw/2019-02-27/2019-02-27.csv\n",
      "../data_raw/2019-04-17/2019-04-17.csv\n",
      "THIS  ../data_raw/2019-04-17/2019-04-17.csv\n",
      "../data_raw/2019-06-10/2019-06-10.csv\n",
      "../data_raw/2018-12-30/2018-12-30.csv\n",
      "../data_raw/2018-11-09/2018-11-09.csv\n",
      "../data_raw/2018-05-25/2018-05-25.csv\n",
      "../data_raw/2019-07-01/2019-07-01.csv\n",
      "../data_raw/2019-03-04/2019-03-04.csv\n",
      "../data_raw/2019-04-20/2019-04-20.csv\n",
      "../data_raw/2017-11-16/2017-11-16.csv\n",
      "../data_raw/2017-12-02/2017-12-02.csv\n",
      "../data_raw/2017-11-04/2017-11-04.csv\n",
      "../data_raw/2019-07-17/2019-07-17.csv\n",
      "THIS  ../data_raw/2019-07-17/2019-07-17.csv\n",
      "../data_raw/2019-05-16/2019-05-16.csv\n",
      "../data_raw/2019-07-02/2019-07-02.csv\n",
      "../data_raw/2018-06-24/2018-06-24.csv\n",
      "../data_raw/2018-10-03/2018-10-03.csv\n",
      "../data_raw/2018-01-01/2018-01-01.csv\n",
      "../data_raw/2019-02-18/2019-02-18.csv\n",
      "../data_raw/2017-12-30/2017-12-30.csv\n",
      "../data_raw/2018-05-13/2018-05-13.csv\n",
      "../data_raw/2018-05-01/2018-05-01.csv\n",
      "../data_raw/2018-11-05/2018-11-05.csv\n",
      "../data_raw/2017-10-16/2017-10-16.csv\n",
      "../data_raw/2019-04-01/2019-04-01.csv\n",
      "../data_raw/2017-09-24/2017-09-24.csv\n",
      "../data_raw/2019-01-24/2019-01-24.csv\n",
      "../data_raw/2018-05-21/2018-05-21.csv\n",
      "../data_raw/2018-12-23/2018-12-23.csv\n",
      "../data_raw/2018-01-13/2018-01-13.csv\n",
      "../data_raw/2019-02-01/2019-02-01.csv\n",
      "../data_raw/2019-06-28/2019-06-28.csv\n",
      "../data_raw/2017-10-12/2017-10-12.csv\n",
      "../data_raw/2019-07-06/2019-07-06.csv\n",
      "../data_raw/2019-05-08/2019-05-08.csv\n",
      "../data_raw/2018-12-14/2018-12-14.csv\n",
      "../data_raw/2017-10-24/2017-10-24.csv\n",
      "../data_raw/2018-09-08/2018-09-08.csv\n",
      "../data_raw/2017-09-02/2017-09-02.csv\n",
      "../data_raw/2018-11-26/2018-11-26.csv\n",
      "../data_raw/2019-04-05/2019-04-05.csv\n",
      "../data_raw/2018-06-16/2018-06-16.csv\n",
      "../data_raw/2019-05-13/2019-05-13.csv\n",
      "../data_raw/2017-07-14/2017-07-14.csv\n",
      "THIS  ../data_raw/2017-07-14/2017-07-14.csv\n",
      "../data_raw/2017-09-07/2017-09-07.csv\n",
      "../data_raw/2017-08-04/2017-08-04.csv\n",
      "../data_raw/2017-10-15/2017-10-15.csv\n",
      "../data_raw/2019-02-04/2019-02-04.csv\n",
      "../data_raw/2019-05-09/2019-05-09.csv\n",
      "../data_raw/2018-03-02/2018-03-02.csv\n",
      "../data_raw/2017-11-29/2017-11-29.csv\n",
      "../data_raw/2017-09-11/2017-09-11.csv\n",
      "../data_raw/2018-11-28/2018-11-28.csv\n",
      "../data_raw/2017-09-18/2017-09-18.csv\n",
      "../data_raw/2019-03-18/2019-03-18.csv\n",
      "../data_raw/2018-01-28/2018-01-28.csv\n",
      "../data_raw/2017-07-12/2017-07-12.csv\n",
      "THIS  ../data_raw/2017-07-12/2017-07-12.csv\n",
      "../data_raw/2017-11-18/2017-11-18.csv\n",
      "../data_raw/2019-07-23/2019-07-23.csv\n",
      "../data_raw/2018-11-24/2018-11-24.csv\n",
      "../data_raw/2019-02-13/2019-02-13.csv\n",
      "../data_raw/2019-06-27/2019-06-27.csv\n",
      "../data_raw/2019-04-22/2019-04-22.csv\n",
      "../data_raw/2018-09-20/2018-09-20.csv\n",
      "../data_raw/2017-10-11/2017-10-11.csv\n",
      "../data_raw/2019-01-04/2019-01-04.csv\n",
      "../data_raw/2017-10-25/2017-10-25.csv\n",
      "../data_raw/2019-02-24/2019-02-24.csv\n",
      "../data_raw/2017-12-07/2017-12-07.csv\n",
      "../data_raw/2017-07-29/2017-07-29.csv\n",
      "../data_raw/2017-09-20/2017-09-20.csv\n",
      "../data_raw/2019-01-20/2019-01-20.csv\n",
      "../data_raw/2017-08-25/2017-08-25.csv\n",
      "../data_raw/2017-12-15/2017-12-15.csv\n",
      "../data_raw/2017-10-03/2017-10-03.csv\n",
      "../data_raw/2018-03-06/2018-03-06.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../data_raw/2017-07-10/2017-07-10.csv\n",
      "THIS  ../data_raw/2017-07-10/2017-07-10.csv\n",
      "../data_raw/2019-07-29/2019-07-29.csv\n",
      "THIS  ../data_raw/2019-07-29/2019-07-29.csv\n",
      "../data_raw/2018-05-24/2018-05-24.csv\n",
      "../data_raw/2018-10-01/2018-10-01.csv\n",
      "../data_raw/2018-04-31/2018-04-31.csv\n",
      "../data_raw/2019-01-19/2019-01-19.csv\n",
      "../data_raw/2019-07-05/2019-07-05.csv\n",
      "../data_raw/2019-03-07/2019-03-07.csv\n",
      "../data_raw/2019-02-22/2019-02-22.csv\n",
      "../data_raw/2018-01-06/2018-01-06.csv\n",
      "../data_raw/2019-04-14/2019-04-14.csv\n",
      "THIS  ../data_raw/2019-04-14/2019-04-14.csv\n",
      "../data_raw/2017-07-02/2017-07-02.csv\n",
      "THIS  ../data_raw/2017-07-02/2017-07-02.csv\n",
      "../data_raw/2019-05-12/2019-05-12.csv\n",
      "THIS  ../data_raw/2019-05-12/2019-05-12.csv\n",
      "../data_raw/2018-03-20/2018-03-20.csv\n",
      "../data_raw/2017-07-11/2017-07-11.csv\n",
      "THIS  ../data_raw/2017-07-11/2017-07-11.csv\n",
      "../data_raw/2018-10-26/2018-10-26.csv\n",
      "../data_raw/2018-10-23/2018-10-23.csv\n",
      "../data_raw/2019-02-25/2019-02-25.csv\n",
      "../data_raw/2018-10-27/2018-10-27.csv\n",
      "../data_raw/2018-05-09/2018-05-09.csv\n",
      "../data_raw/2019-04-18/2019-04-18.csv\n",
      "../data_raw/2017-10-27/2017-10-27.csv\n",
      "../data_raw/2018-09-16/2018-09-16.csv\n",
      "../data_raw/2019-01-16/2019-01-16.csv\n",
      "../data_raw/2019-06-21/2019-06-21.csv\n",
      "../data_raw/2018-02-28/2018-02-28.csv\n",
      "../data_raw/2017-09-17/2017-09-17.csv\n",
      "../data_raw/2019-03-19/2019-03-19.csv\n",
      "../data_raw/2018-01-16/2018-01-16.csv\n",
      "../data_raw/2019-04-15/2019-04-15.csv\n",
      "../data_raw/2017-08-03/2017-08-03.csv\n",
      "../data_raw/2018-04-07/2018-04-07.csv\n",
      "../data_raw/2017-09-06/2017-09-06.csv\n",
      "../data_raw/2019-02-19/2019-02-19.csv\n",
      "../data_raw/2018-01-15/2018-01-15.csv\n",
      "../data_raw/2018-03-26/2018-03-26.csv\n",
      "../data_raw/2018-12-03/2018-12-03.csv\n",
      "../data_raw/2018-12-27/2018-12-27.csv\n",
      "../data_raw/2018-03-03/2018-03-03.csv\n",
      "../data_raw/2018-09-24/2018-09-24.csv\n",
      "../data_raw/2019-06-13/2019-06-13.csv\n",
      "../data_raw/2019-07-13/2019-07-13.csv\n",
      "THIS  ../data_raw/2019-07-13/2019-07-13.csv\n",
      "../data_raw/2017-07-16/2017-07-16.csv\n",
      "THIS  ../data_raw/2017-07-16/2017-07-16.csv\n",
      "../data_raw/2019-06-29/2019-06-29.csv\n",
      "../data_raw/2018-05-29/2018-05-29.csv\n",
      "../data_raw/2019-04-12/2019-04-12.csv\n",
      "../data_raw/2018-05-07/2018-05-07.csv\n",
      "../data_raw/2019-05-20/2019-05-20.csv\n",
      "THIS  ../data_raw/2019-05-20/2019-05-20.csv\n",
      "../data_raw/2019-01-08/2019-01-08.csv\n",
      "../data_raw/2017-09-29/2017-09-29.csv\n",
      "../data_raw/2018-02-17/2018-02-17.csv\n",
      "../data_raw/2019-03-23/2019-03-23.csv\n",
      "../data_raw/2019-03-17/2019-03-17.csv\n",
      "../data_raw/2017-10-17/2017-10-17.csv\n",
      "../data_raw/2018-06-20/2018-06-20.csv\n",
      "../data_raw/2018-02-18/2018-02-18.csv\n",
      "../data_raw/2019-02-31/2019-02-31.csv\n",
      "../data_raw/2018-06-07/2018-06-07.csv\n",
      "../data_raw/2018-01-27/2018-01-27.csv\n",
      "../data_raw/2017-12-26/2017-12-26.csv\n",
      "../data_raw/2017-09-28/2017-09-28.csv\n",
      "../data_raw/2018-01-09/2018-01-09.csv\n",
      "../data_raw/2018-02-06/2018-02-06.csv\n",
      "../data_raw/2019-04-11/2019-04-11.csv\n",
      "../data_raw/2019-04-10/2019-04-10.csv\n",
      "../data_raw/2018-12-17/2018-12-17.csv\n",
      "../data_raw/2019-01-11/2019-01-11.csv\n",
      "../data_raw/2017-10-30/2017-10-30.csv\n",
      "../data_raw/2019-07-27/2019-07-27.csv\n",
      "THIS  ../data_raw/2019-07-27/2019-07-27.csv\n",
      "../data_raw/2018-02-12/2018-02-12.csv\n",
      "../data_raw/2018-06-21/2018-06-21.csv\n",
      "../data_raw/2018-01-02/2018-01-02.csv\n",
      "../data_raw/2017-11-09/2017-11-09.csv\n",
      "../data_raw/2017-11-25/2017-11-25.csv\n",
      "../data_raw/2017-12-27/2017-12-27.csv\n",
      "../data_raw/2019-06-12/2019-06-12.csv\n",
      "../data_raw/2018-10-07/2018-10-07.csv\n",
      "../data_raw/2018-09-31/2018-09-31.csv\n",
      "../data_raw/2018-05-10/2018-05-10.csv\n",
      "../data_raw/2018-01-08/2018-01-08.csv\n",
      "../data_raw/2018-06-22/2018-06-22.csv\n",
      "../data_raw/2018-05-05/2018-05-05.csv\n",
      "../data_raw/2017-07-13/2017-07-13.csv\n",
      "THIS  ../data_raw/2017-07-13/2017-07-13.csv\n",
      "../data_raw/2018-06-17/2018-06-17.csv\n",
      "../data_raw/2018-01-20/2018-01-20.csv\n",
      "THIS  ../data_raw/2018-01-20/2018-01-20.csv\n",
      "../data_raw/2018-05-16/2018-05-16.csv\n",
      "../data_raw/2019-07-19/2019-07-19.csv\n",
      "../data_raw/2018-10-08/2018-10-08.csv\n",
      "../data_raw/2019-01-27/2019-01-27.csv\n",
      "../data_raw/2019-04-03/2019-04-03.csv\n",
      "../data_raw/2019-06-08/2019-06-08.csv\n",
      "../data_raw/2019-04-28/2019-04-28.csv\n",
      "../data_raw/2017-08-12/2017-08-12.csv\n",
      "../data_raw/2018-02-09/2018-02-09.csv\n",
      "../data_raw/2018-04-20/2018-04-20.csv\n",
      "../data_raw/2017-11-15/2017-11-15.csv\n",
      "../data_raw/2017-08-22/2017-08-22.csv\n",
      "../data_raw/2017-09-14/2017-09-14.csv\n",
      "../data_raw/2018-12-21/2018-12-21.csv\n",
      "../data_raw/2018-04-26/2018-04-26.csv\n",
      "../data_raw/2018-03-19/2018-03-19.csv\n",
      "../data_raw/2019-06-18/2019-06-18.csv\n",
      "../data_raw/2017-10-08/2017-10-08.csv\n",
      "../data_raw/2018-10-09/2018-10-09.csv\n",
      "../data_raw/2018-04-24/2018-04-24.csv\n",
      "../data_raw/2018-12-16/2018-12-16.csv\n",
      "../data_raw/2017-09-22/2017-09-22.csv\n",
      "../data_raw/2017-11-17/2017-11-17.csv\n",
      "../data_raw/2018-01-22/2018-01-22.csv\n",
      "../data_raw/2017-09-27/2017-09-27.csv\n",
      "../data_raw/2019-05-10/2019-05-10.csv\n",
      "../data_raw/2019-05-28/2019-05-28.csv\n",
      "../data_raw/2018-03-29/2018-03-29.csv\n",
      "../data_raw/2017-12-06/2017-12-06.csv\n",
      "../data_raw/2017-07-22/2017-07-22.csv\n",
      "../data_raw/2018-12-13/2018-12-13.csv\n",
      "../data_raw/2018-09-27/2018-09-27.csv\n",
      "../data_raw/2018-02-22/2018-02-22.csv\n",
      "../data_raw/2018-11-16/2018-11-16.csv\n",
      "../data_raw/2019-01-23/2019-01-23.csv\n",
      "../data_raw/2018-03-15/2018-03-15.csv\n",
      "../data_raw/2017-11-23/2017-11-23.csv\n",
      "../data_raw/2018-12-28/2018-12-28.csv\n",
      "../data_raw/2019-02-06/2019-02-06.csv\n",
      "../data_raw/2017-12-16/2017-12-16.csv\n",
      "../data_raw/2019-05-02/2019-05-02.csv\n",
      "../data_raw/2017-08-14/2017-08-14.csv\n",
      "../data_raw/2019-03-20/2019-03-20.csv\n",
      "../data_raw/2018-09-30/2018-09-30.csv\n",
      "../data_raw/2019-05-27/2019-05-27.csv\n",
      "../data_raw/2018-11-10/2018-11-10.csv\n",
      "../data_raw/2018-04-27/2018-04-27.csv\n",
      "../data_raw/2018-10-24/2018-10-24.csv\n",
      "../data_raw/2018-06-19/2018-06-19.csv\n",
      "../data_raw/2017-08-09/2017-08-09.csv\n",
      "../data_raw/2018-11-19/2018-11-19.csv\n",
      "../data_raw/2018-10-21/2018-10-21.csv\n",
      "../data_raw/2017-10-05/2017-10-05.csv\n",
      "../data_raw/2017-12-23/2017-12-23.csv\n",
      "../data_raw/2018-04-02/2018-04-02.csv\n",
      "../data_raw/2018-06-15/2018-06-15.csv\n",
      "../data_raw/2017-11-13/2017-11-13.csv\n",
      "../data_raw/2019-04-23/2019-04-23.csv\n",
      "../data_raw/2018-03-22/2018-03-22.csv\n",
      "../data_raw/2019-05-06/2019-05-06.csv\n",
      "../data_raw/2017-07-30/2017-07-30.csv\n",
      "../data_raw/2017-10-28/2017-10-28.csv\n",
      "../data_raw/2017-09-21/2017-09-21.csv\n",
      "../data_raw/2019-02-29/2019-02-29.csv\n",
      "../data_raw/2018-05-23/2018-05-23.csv\n",
      "../data_raw/2017-12-10/2017-12-10.csv\n",
      "../data_raw/2017-07-23/2017-07-23.csv\n",
      "../data_raw/2018-01-21/2018-01-21.csv\n",
      "THIS  ../data_raw/2018-01-21/2018-01-21.csv\n",
      "../data_raw/2019-07-16/2019-07-16.csv\n",
      "../data_raw/2019-01-12/2019-01-12.csv\n",
      "../data_raw/2018-10-14/2018-10-14.csv\n",
      "../data_raw/2017-08-10/2017-08-10.csv\n",
      "../data_raw/2018-04-06/2018-04-06.csv\n",
      "../data_raw/2018-04-09/2018-04-09.csv\n",
      "../data_raw/2018-11-13/2018-11-13.csv\n",
      "../data_raw/2018-11-20/2018-11-20.csv\n",
      "../data_raw/2017-12-22/2017-12-22.csv\n",
      "../data_raw/2018-06-25/2018-06-25.csv\n",
      "../data_raw/2019-02-16/2019-02-16.csv\n",
      "../data_raw/2018-10-12/2018-10-12.csv\n",
      "../data_raw/2018-04-19/2018-04-19.csv\n",
      "../data_raw/2018-02-19/2018-02-19.csv\n",
      "../data_raw/2018-03-08/2018-03-08.csv\n",
      "../data_raw/2018-04-21/2018-04-21.csv\n",
      "../data_raw/2017-08-11/2017-08-11.csv\n",
      "../data_raw/2019-05-24/2019-05-24.csv\n",
      "../data_raw/2018-04-17/2018-04-17.csv\n",
      "../data_raw/2019-06-04/2019-06-04.csv\n",
      "../data_raw/2018-01-18/2018-01-18.csv\n",
      "THIS  ../data_raw/2018-01-18/2018-01-18.csv\n",
      "../data_raw/2018-03-16/2018-03-16.csv\n",
      "../data_raw/2018-11-14/2018-11-14.csv\n",
      "../data_raw/2018-11-15/2018-11-15.csv\n",
      "../data_raw/2018-12-24/2018-12-24.csv\n",
      "../data_raw/2019-02-08/2019-02-08.csv\n",
      "../data_raw/2018-11-02/2018-11-02.csv\n",
      "../data_raw/2017-10-10/2017-10-10.csv\n",
      "../data_raw/2017-09-26/2017-09-26.csv\n",
      "../data_raw/2017-10-09/2017-10-09.csv\n",
      "../data_raw/2018-06-14/2018-06-14.csv\n",
      "../data_raw/2019-03-01/2019-03-01.csv\n",
      "../data_raw/2017-08-06/2017-08-06.csv\n",
      "../data_raw/2018-02-27/2018-02-27.csv\n",
      "../data_raw/2019-06-23/2019-06-23.csv\n",
      "../data_raw/2018-06-09/2018-06-09.csv\n",
      "../data_raw/2018-12-20/2018-12-20.csv\n",
      "../data_raw/2018-01-29/2018-01-29.csv\n",
      "../data_raw/2017-09-01/2017-09-01.csv\n",
      "../data_raw/2017-07-25/2017-07-25.csv\n",
      "../data_raw/2018-06-01/2018-06-01.csv\n",
      "../data_raw/2017-11-24/2017-11-24.csv\n",
      "../data_raw/2017-09-13/2017-09-13.csv\n",
      "../data_raw/2018-12-22/2018-12-22.csv\n",
      "../data_raw/2019-05-21/2019-05-21.csv\n",
      "THIS  ../data_raw/2019-05-21/2019-05-21.csv\n",
      "../data_raw/2019-04-04/2019-04-04.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../data_raw/2019-05-14/2019-05-14.csv\n",
      "../data_raw/2018-12-10/2018-12-10.csv\n",
      "../data_raw/2019-04-19/2019-04-19.csv\n",
      "../data_raw/2019-07-21/2019-07-21.csv\n",
      "THIS  ../data_raw/2019-07-21/2019-07-21.csv\n",
      "../data_raw/2017-08-16/2017-08-16.csv\n",
      "../data_raw/2017-07-24/2017-07-24.csv\n",
      "../data_raw/2018-03-10/2018-03-10.csv\n",
      "../data_raw/2018-05-31/2018-05-31.csv\n",
      "../data_raw/2018-11-23/2018-11-23.csv\n",
      "../data_raw/2019-01-10/2019-01-10.csv\n",
      "../data_raw/2019-01-17/2019-01-17.csv\n",
      "../data_raw/2019-01-21/2019-01-21.csv\n",
      "../data_raw/2017-07-20/2017-07-20.csv\n",
      "../data_raw/2018-03-27/2018-03-27.csv\n",
      "THIS  ../data_raw/2018-03-27/2018-03-27.csv\n",
      "../data_raw/2018-01-11/2018-01-11.csv\n",
      "../data_raw/2019-07-18/2019-07-18.csv\n",
      "../data_raw/2018-06-08/2018-06-08.csv\n",
      "../data_raw/2017-09-03/2017-09-03.csv\n",
      "../data_raw/2019-04-08/2019-04-08.csv\n",
      "../data_raw/2018-10-04/2018-10-04.csv\n",
      "../data_raw/2019-02-10/2019-02-10.csv\n",
      "../data_raw/2019-04-07/2019-04-07.csv\n",
      "../data_raw/2017-11-10/2017-11-10.csv\n",
      "../data_raw/2017-07-17/2017-07-17.csv\n",
      "THIS  ../data_raw/2017-07-17/2017-07-17.csv\n",
      "../data_raw/2018-02-05/2018-02-05.csv\n",
      "../data_raw/2019-03-12/2019-03-12.csv\n",
      "../data_raw/2018-02-30/2018-02-30.csv\n",
      "../data_raw/2017-10-01/2017-10-01.csv\n",
      "../data_raw/2017-10-04/2017-10-04.csv\n",
      "../data_raw/2018-09-12/2018-09-12.csv\n",
      "../data_raw/2018-03-07/2018-03-07.csv\n",
      "../data_raw/2017-07-01/2017-07-01.csv\n",
      "THIS  ../data_raw/2017-07-01/2017-07-01.csv\n",
      "../data_raw/2018-02-15/2018-02-15.csv\n",
      "../data_raw/2017-07-18/2017-07-18.csv\n",
      "../data_raw/2019-03-27/2019-03-27.csv\n",
      "../data_raw/2017-09-30/2017-09-30.csv\n",
      "../data_raw/2019-06-30/2019-06-30.csv\n",
      "THIS  ../data_raw/2019-06-30/2019-06-30.csv\n",
      "../data_raw/2019-05-07/2019-05-07.csv\n",
      "../data_raw/2019-05-25/2019-05-25.csv\n",
      "THIS  ../data_raw/2019-05-25/2019-05-25.csv\n",
      "../data_raw/2019-06-05/2019-06-05.csv\n",
      "THIS  ../data_raw/2019-06-05/2019-06-05.csv\n",
      "../data_raw/2018-12-02/2018-12-02.csv\n",
      "../data_raw/2019-06-19/2019-06-19.csv\n",
      "../data_raw/2019-06-07/2019-06-07.csv\n",
      "../data_raw/2018-11-25/2018-11-25.csv\n",
      "THIS  ../data_raw/2018-11-25/2018-11-25.csv\n",
      "../data_raw/2017-07-19/2017-07-19.csv\n",
      "../data_raw/2018-06-18/2018-06-18.csv\n",
      "../data_raw/2019-02-05/2019-02-05.csv\n",
      "../data_raw/2017-12-17/2017-12-17.csv\n",
      "../data_raw/2018-06-13/2018-06-13.csv\n",
      "../data_raw/2017-09-25/2017-09-25.csv\n",
      "../data_raw/2018-03-17/2018-03-17.csv\n",
      "../data_raw/2017-08-07/2017-08-07.csv\n",
      "../data_raw/2018-05-20/2018-05-20.csv\n",
      "../data_raw/2019-04-30/2019-04-30.csv\n",
      "../data_raw/2017-10-21/2017-10-21.csv\n",
      "../data_raw/2018-05-28/2018-05-28.csv\n",
      "../data_raw/2018-11-30/2018-11-30.csv\n",
      "../data_raw/2019-03-05/2019-03-05.csv\n",
      "../data_raw/2017-08-23/2017-08-23.csv\n",
      "../data_raw/2018-01-14/2018-01-14.csv\n",
      "../data_raw/2017-11-20/2017-11-20.csv\n",
      "../data_raw/2019-04-29/2019-04-29.csv\n",
      "../data_raw/2018-06-11/2018-06-11.csv\n",
      "../data_raw/2018-05-14/2018-05-14.csv\n",
      "../data_raw/2019-07-31/2019-07-31.csv\n",
      "THIS  ../data_raw/2019-07-31/2019-07-31.csv\n",
      "../data_raw/2019-03-13/2019-03-13.csv\n",
      "../data_raw/2017-07-07/2017-07-07.csv\n",
      "THIS  ../data_raw/2017-07-07/2017-07-07.csv\n",
      "../data_raw/2019-03-25/2019-03-25.csv\n",
      "../data_raw/2019-06-31/2019-06-31.csv\n",
      "THIS  ../data_raw/2019-06-31/2019-06-31.csv\n",
      "../data_raw/2018-09-29/2018-09-29.csv\n",
      "../data_raw/2019-01-05/2019-01-05.csv\n",
      "../data_raw/2018-10-10/2018-10-10.csv\n",
      "../data_raw/2017-11-21/2017-11-21.csv\n",
      "../data_raw/2017-08-08/2017-08-08.csv\n",
      "../data_raw/2019-05-04/2019-05-04.csv\n",
      "../data_raw/2019-03-24/2019-03-24.csv\n",
      "../data_raw/2019-03-16/2019-03-16.csv\n",
      "../data_raw/2019-05-18/2019-05-18.csv\n",
      "THIS  ../data_raw/2019-05-18/2019-05-18.csv\n",
      "../data_raw/2019-07-24/2019-07-24.csv\n",
      "THIS  ../data_raw/2019-07-24/2019-07-24.csv\n",
      "../data_raw/2019-06-17/2019-06-17.csv\n",
      "../data_raw/2018-10-20/2018-10-20.csv\n",
      "../data_raw/2019-06-20/2019-06-20.csv\n",
      "../data_raw/2019-02-23/2019-02-23.csv\n",
      "../data_raw/2017-08-19/2017-08-19.csv\n",
      "../data_raw/2017-10-18/2017-10-18.csv\n",
      "../data_raw/2018-11-31/2018-11-31.csv\n",
      "../data_raw/2017-12-18/2017-12-18.csv\n",
      "../data_raw/2018-10-29/2018-10-29.csv\n",
      "../data_raw/2017-11-14/2017-11-14.csv\n",
      "../data_raw/2017-10-07/2017-10-07.csv\n",
      "../data_raw/2018-12-31/2018-12-31.csv\n",
      "../data_raw/2019-06-11/2019-06-11.csv\n",
      "../data_raw/2018-04-28/2018-04-28.csv\n",
      "../data_raw/2018-06-05/2018-06-05.csv\n",
      "../data_raw/2019-06-22/2019-06-22.csv\n",
      "../data_raw/2018-03-12/2018-03-12.csv\n",
      "../data_raw/2017-12-12/2017-12-12.csv\n",
      "../data_raw/2019-07-28/2019-07-28.csv\n",
      "THIS  ../data_raw/2019-07-28/2019-07-28.csv\n",
      "../data_raw/2019-04-06/2019-04-06.csv\n",
      "../data_raw/2019-07-20/2019-07-20.csv\n",
      "THIS  ../data_raw/2019-07-20/2019-07-20.csv\n",
      "../data_raw/2018-05-27/2018-05-27.csv\n",
      "../data_raw/2017-08-21/2017-08-21.csv\n",
      "../data_raw/2018-12-05/2018-12-05.csv\n",
      "../data_raw/2017-08-01/2017-08-01.csv\n",
      "../data_raw/2019-02-15/2019-02-15.csv\n",
      "../data_raw/2017-07-15/2017-07-15.csv\n",
      "THIS  ../data_raw/2017-07-15/2017-07-15.csv\n",
      "../data_raw/2018-02-24/2018-02-24.csv\n",
      "../data_raw/2019-06-15/2019-06-15.csv\n",
      "../data_raw/2018-05-19/2018-05-19.csv\n",
      "../data_raw/2018-09-19/2018-09-19.csv\n",
      "../data_raw/2018-10-31/2018-10-31.csv\n",
      "../data_raw/2019-03-28/2019-03-28.csv\n",
      "../data_raw/2019-02-21/2019-02-21.csv\n",
      "../data_raw/2017-11-06/2017-11-06.csv\n",
      "../data_raw/2018-03-01/2018-03-01.csv\n",
      "../data_raw/2019-05-17/2019-05-17.csv\n",
      "../data_raw/2017-08-27/2017-08-27.csv\n",
      "../data_raw/2019-07-22/2019-07-22.csv\n",
      "../data_raw/2018-04-29/2018-04-29.csv\n",
      "../data_raw/2018-09-18/2018-09-18.csv\n",
      "../data_raw/2018-10-30/2018-10-30.csv\n",
      "../data_raw/2017-11-01/2017-11-01.csv\n",
      "../data_raw/2019-02-09/2019-02-09.csv\n",
      "../data_raw/2019-03-08/2019-03-08.csv\n",
      "../data_raw/2018-09-28/2018-09-28.csv\n",
      "../data_raw/2018-10-16/2018-10-16.csv\n",
      "../data_raw/2018-03-30/2018-03-30.csv\n",
      "../data_raw/2017-12-28/2017-12-28.csv\n",
      "THIS  ../data_raw/2017-12-28/2017-12-28.csv\n",
      "../data_raw/2018-02-10/2018-02-10.csv\n",
      "../data_raw/2018-04-30/2018-04-30.csv\n",
      "../data_raw/2019-03-03/2019-03-03.csv\n",
      "../data_raw/2017-08-17/2017-08-17.csv\n",
      "../data_raw/2017-11-26/2017-11-26.csv\n",
      "../data_raw/2018-06-04/2018-06-04.csv\n",
      "../data_raw/2019-07-12/2019-07-12.csv\n",
      "../data_raw/2019-04-25/2019-04-25.csv\n",
      "../data_raw/2019-07-26/2019-07-26.csv\n",
      "THIS  ../data_raw/2019-07-26/2019-07-26.csv\n",
      "../data_raw/2018-11-04/2018-11-04.csv\n",
      "../data_raw/2019-05-31/2019-05-31.csv\n",
      "../data_raw/2019-06-26/2019-06-26.csv\n",
      "../data_raw/2018-06-23/2018-06-23.csv\n",
      "../data_raw/2017-08-29/2017-08-29.csv\n",
      "../data_raw/2017-12-29/2017-12-29.csv\n",
      "../data_raw/2018-02-02/2018-02-02.csv\n",
      "../data_raw/2017-09-04/2017-09-04.csv\n",
      "../data_raw/2018-04-11/2018-04-11.csv\n",
      "../data_raw/2019-01-31/2019-01-31.csv\n",
      "../data_raw/2018-01-23/2018-01-23.csv\n",
      "../data_raw/2019-03-14/2019-03-14.csv\n",
      "../data_raw/2018-12-07/2018-12-07.csv\n",
      "../data_raw/2017-07-28/2017-07-28.csv\n",
      "../data_raw/2019-01-13/2019-01-13.csv\n",
      "../data_raw/2017-07-03/2017-07-03.csv\n",
      "THIS  ../data_raw/2017-07-03/2017-07-03.csv\n",
      "../data_raw/2018-09-07/2018-09-07.csv\n",
      "../data_raw/2018-01-07/2018-01-07.csv\n",
      "../data_raw/2018-11-06/2018-11-06.csv\n",
      "../data_raw/2017-10-13/2017-10-13.csv\n",
      "../data_raw/2018-03-11/2018-03-11.csv\n",
      "../data_raw/2019-02-11/2019-02-11.csv\n",
      "../data_raw/2018-10-13/2018-10-13.csv\n",
      "../data_raw/2018-05-15/2018-05-15.csv\n",
      "../data_raw/2018-01-24/2018-01-24.csv\n",
      "../data_raw/2019-05-30/2019-05-30.csv\n",
      "../data_raw/2018-09-05/2018-09-05.csv\n",
      "../data_raw/2018-04-01/2018-04-01.csv\n",
      "../data_raw/2017-12-01/2017-12-01.csv\n",
      "../data_raw/2018-04-10/2018-04-10.csv\n",
      "../data_raw/2018-10-17/2018-10-17.csv\n",
      "../data_raw/2018-04-25/2018-04-25.csv\n",
      "THIS  ../data_raw/2018-04-25/2018-04-25.csv\n",
      "../data_raw/2017-07-21/2017-07-21.csv\n",
      "../data_raw/2019-04-21/2019-04-21.csv\n",
      "THIS  ../data_raw/2019-04-21/2019-04-21.csv\n",
      "../data_raw/2018-11-21/2018-11-21.csv\n",
      "../data_raw/2017-10-02/2017-10-02.csv\n",
      "../data_raw/2018-02-20/2018-02-20.csv\n",
      "../data_raw/2017-10-20/2017-10-20.csv\n",
      "../data_raw/2019-06-09/2019-06-09.csv\n",
      "../data_raw/2018-11-29/2018-11-29.csv\n",
      "../data_raw/2017-12-11/2017-12-11.csv\n",
      "../data_raw/2017-11-05/2017-11-05.csv\n",
      "../data_raw/2018-10-19/2018-10-19.csv\n",
      "../data_raw/2018-09-10/2018-09-10.csv\n",
      "../data_raw/2019-05-23/2019-05-23.csv\n",
      "../data_raw/2019-02-03/2019-02-03.csv\n",
      "../data_raw/2018-04-05/2018-04-05.csv\n",
      "../data_raw/2019-06-16/2019-06-16.csv\n",
      "../data_raw/2018-06-10/2018-06-10.csv\n",
      "../data_raw/2018-01-03/2018-01-03.csv\n",
      "../data_raw/2019-01-15/2019-01-15.csv\n",
      "../data_raw/2018-01-26/2018-01-26.csv\n",
      "../data_raw/2017-11-19/2017-11-19.csv\n",
      "../data_raw/2017-11-22/2017-11-22.csv\n",
      "../data_raw/2018-04-08/2018-04-08.csv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../data_raw/2017-08-26/2017-08-26.csv\n",
      "../data_raw/2017-09-15/2017-09-15.csv\n",
      "../data_raw/2018-01-17/2018-01-17.csv\n",
      "../data_raw/2019-04-13/2019-04-13.csv\n",
      "../data_raw/2018-01-25/2018-01-25.csv\n",
      "../data_raw/2018-09-06/2018-09-06.csv\n",
      "../data_raw/2017-10-26/2017-10-26.csv\n",
      "../data_raw/2019-05-11/2019-05-11.csv\n",
      "THIS  ../data_raw/2019-05-11/2019-05-11.csv\n",
      "../data_raw/2018-11-18/2018-11-18.csv\n",
      "../data_raw/2017-11-27/2017-11-27.csv\n",
      "../data_raw/2018-05-06/2018-05-06.csv\n",
      "../data_raw/2019-03-30/2019-03-30.csv\n",
      "../data_raw/2017-09-23/2017-09-23.csv\n",
      "../data_raw/2018-09-11/2018-09-11.csv\n",
      "../data_raw/2017-11-07/2017-11-07.csv\n",
      "../data_raw/2018-12-06/2018-12-06.csv\n",
      "../data_raw/2018-03-13/2018-03-13.csv\n",
      "../data_raw/2019-01-02/2019-01-02.csv\n",
      "../data_raw/2019-05-01/2019-05-01.csv\n",
      "../data_raw/2018-01-04/2018-01-04.csv\n",
      "../data_raw/2019-01-07/2019-01-07.csv\n",
      "../data_raw/2019-07-25/2019-07-25.csv\n",
      "../data_raw/2019-02-07/2019-02-07.csv\n",
      "../data_raw/2017-11-03/2017-11-03.csv\n",
      "../data_raw/2019-05-05/2019-05-05.csv\n",
      "../data_raw/2019-06-24/2019-06-24.csv\n",
      "../data_raw/2019-03-22/2019-03-22.csv\n",
      "../data_raw/2018-11-27/2018-11-27.csv\n",
      "../data_raw/2019-01-30/2019-01-30.csv\n",
      "../data_raw/2017-12-19/2017-12-19.csv\n",
      "../data_raw/2019-07-14/2019-07-14.csv\n",
      "THIS  ../data_raw/2019-07-14/2019-07-14.csv\n",
      "../data_raw/2019-02-02/2019-02-02.csv\n",
      "../data_raw/2018-03-25/2018-03-25.csv\n",
      "../data_raw/2017-12-14/2017-12-14.csv\n",
      "../data_raw/2018-09-25/2018-09-25.csv\n",
      "../data_raw/2018-04-13/2018-04-13.csv\n",
      "../data_raw/2018-01-05/2018-01-05.csv\n",
      "../data_raw/2019-01-29/2019-01-29.csv\n",
      "../data_raw/2019-01-06/2019-01-06.csv\n",
      "../data_raw/2018-12-08/2018-12-08.csv\n",
      "../data_raw/2017-12-04/2017-12-04.csv\n",
      "../data_raw/2018-01-30/2018-01-30.csv\n",
      "../data_raw/2018-01-12/2018-01-12.csv\n",
      "../data_raw/2018-09-23/2018-09-23.csv\n",
      "../data_raw/2018-12-15/2018-12-15.csv\n",
      "../data_raw/2019-06-14/2019-06-14.csv\n",
      "../data_raw/2018-11-03/2018-11-03.csv\n",
      "../data_raw/2019-07-04/2019-07-04.csv\n",
      "../data_raw/2018-02-16/2018-02-16.csv\n",
      "../data_raw/2019-04-16/2019-04-16.csv\n",
      "../data_raw/2017-08-02/2017-08-02.csv\n",
      "../data_raw/2019-07-08/2019-07-08.csv\n",
      "../data_raw/2018-06-12/2018-06-12.csv\n",
      "../data_raw/2017-10-29/2017-10-29.csv\n",
      "../data_raw/2019-07-10/2019-07-10.csv\n",
      "THIS  ../data_raw/2019-07-10/2019-07-10.csv\n",
      "../data_raw/2017-10-23/2017-10-23.csv\n",
      "../data_raw/2019-04-31/2019-04-31.csv\n",
      "../data_raw/2018-03-18/2018-03-18.csv\n",
      "../data_raw/2019-01-25/2019-01-25.csv\n",
      "../data_raw/2018-11-12/2018-11-12.csv\n",
      "../data_raw/2019-06-25/2019-06-25.csv\n",
      "../data_raw/2019-03-09/2019-03-09.csv\n",
      "../data_raw/2019-03-26/2019-03-26.csv\n",
      "../data_raw/2017-08-05/2017-08-05.csv\n",
      "../data_raw/2019-03-21/2019-03-21.csv\n",
      "../data_raw/2017-08-15/2017-08-15.csv\n",
      "../data_raw/2018-11-17/2018-11-17.csv\n",
      "../data_raw/2018-04-22/2018-04-22.csv\n",
      "../data_raw/2017-09-05/2017-09-05.csv\n",
      "../data_raw/2019-04-09/2019-04-09.csv\n",
      "../data_raw/2019-02-30/2019-02-30.csv\n",
      "../data_raw/2017-09-09/2017-09-09.csv\n",
      "../data_raw/2018-11-07/2018-11-07.csv\n",
      "../data_raw/2017-12-03/2017-12-03.csv\n",
      "../data_raw/2018-02-01/2018-02-01.csv\n",
      "../data_raw/2018-11-22/2018-11-22.csv\n",
      "../data_raw/2017-07-26/2017-07-26.csv\n",
      "../data_raw/2019-07-09/2019-07-09.csv\n",
      "../data_raw/2017-08-13/2017-08-13.csv\n",
      "../data_raw/2019-03-02/2019-03-02.csv\n",
      "../data_raw/2018-09-22/2018-09-22.csv\n",
      "../data_raw/2018-05-26/2018-05-26.csv\n",
      "../data_raw/2017-12-25/2017-12-25.csv\n",
      "../data_raw/2019-06-06/2019-06-06.csv\n",
      "../data_raw/2018-11-01/2018-11-01.csv\n",
      "../data_raw/2018-09-09/2018-09-09.csv\n",
      "../data_raw/2018-09-13/2018-09-13.csv\n",
      "../data_raw/2019-03-29/2019-03-29.csv\n",
      "../data_raw/2018-03-04/2018-03-04.csv\n",
      "../data_raw/2019-04-02/2019-04-02.csv\n",
      "../data_raw/2019-05-29/2019-05-29.csv\n"
     ]
    }
   ],
   "source": [
    "df_raw_loader = LoadRawDF()\n",
    "cleaner = CleanerDF()\n",
    "\n",
    "def create_huge_df(folder):\n",
    "    filenames = get_files_in_dir(folder)\n",
    "    all_df=[]\n",
    "    for file in filenames:\n",
    "        print(file)\n",
    "        date = str(file).split(\"/\")[-1].split(\".\")[0]       \n",
    "        raw_dfs_list = df_raw_loader(file)\n",
    "        try:\n",
    "            dfs = create_clean_df_as_list(raw_dfs_list)\n",
    "            new_df = create_joint_dataframe(dfs,  date)\n",
    "            all_df.append(new_df)\n",
    "        except:\n",
    "            print(\"THIS \", file)\n",
    "\n",
    "        \n",
    "    return all_df\n",
    "\n",
    "\n",
    "all_df_as_list = create_huge_df(\"../data_raw\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df = pd.concat(all_df_as_list)\n",
    "new_df = new_df.reset_index()\n",
    "\n",
    "new_df.to_csv(f'clean_data/all_data.csv', index = None, header=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp (p37)",
   "language": "python",
   "name": "nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}