DownloadYahooFinanceData.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f424567a",
   "metadata": {},
   "source": [
    "# Download Yahoo Finance Data\n",
    "Downloading stock price data from Yahoo Finance using yfinance library.\n",
    "\n",
    "Sources:\n",
    "* [yfinance library](https://pypi.org/project/yfinance/)\n",
    "* [Tutorial](https://analyzingalpha.com/yfinance-python)\n",
    "\n",
    "The notebook contains code used to get company codes, download data from Yahoo, and merge datasets in one file. If you want to load the downloaded data and explore it, you can use ExploreYahooFinanceData notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b4e87a06",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import yfinance as yf\n",
    "import matplotlib.pyplot as plt\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "66e0d1c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4466, 7)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bim = yf.Ticker('BIMAS.IS')\n",
    "bim_history = bim.history(period='max')\n",
    "bim_history.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a3886966",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_yahoo_data(codes, path='data/yahoo_finance_unmerged'):\n",
    "    for code in codes:\n",
    "        # Turkish company symbols have \".IS\" suffix on Yahoo Finance (meaning Istanbul probably)\n",
    "        ticker = yf.Ticker(code + '.IS')\n",
    "        df = ticker.history(period='max')\n",
    "        \n",
    "        if not df.empty:\n",
    "            df = df.reset_index()\n",
    "            df.columns = df.columns.map(lambda x: x.lower())\n",
    "            df.to_csv(os.path.join(path, code+'.csv'), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2252cde6",
   "metadata": {},
   "source": [
    "Company codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "be30e0e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki_codes = pd.read_csv('data/wiki_company_symbols.csv').squeeze()\n",
    "bist_codes = pd.read_csv('data/bist100_company_symbols.csv').squeeze()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5503e3e2",
   "metadata": {},
   "source": [
    "### Loading company codes from Wikipedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "769bc8ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Code to get company symbols from Wikipedia\n",
    "\n",
    "# url = 'https://en.wikipedia.org/wiki/List_of_companies_listed_on_the_Istanbul_Stock_Exchange'\n",
    "# wiki_codes = pd.read_html(url)\n",
    "\n",
    "# wiki_codes = pd.concat(wiki_codes, ignore_index=False)\n",
    "\n",
    "# wiki_codes = wiki_codes.dropna(subset='Symbol')\n",
    "\n",
    "# # Removing edit button\n",
    "# wiki_codes = wiki_codes['Symbol'].str.replace('[edit]', '', regex=False)\n",
    "\n",
    "# # Removing letter cells\n",
    "# cond = wiki_codes.map(len) > 1\n",
    "# wiki_codes = wiki_codes[cond]\n",
    "\n",
    "# # These are cells with several company wiki_codes separated by a comma and a space\n",
    "# cond = wiki_codes.map(len) > 5\n",
    "# multi_wiki_codes = wiki_codes[cond].copy()\n",
    "# wiki_codes = wiki_codes[~cond]\n",
    "\n",
    "# # Splitting the multi_wiki_codes and reappending them\n",
    "# code_list = []\n",
    "# for code in multi_wiki_codes:\n",
    "#     code_list += code.split(', ')\n",
    "# wiki_codes = pd.concat([wiki_codes, pd.Series(code_list)])\n",
    "\n",
    "# # Removing special letters\n",
    "# cond = wiki_codes.map(len) < 4\n",
    "# wiki_codes = wiki_codes[~cond]\n",
    "\n",
    "# wiki_codes = wiki_codes.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "id": "f9ee7eea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "76"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 76 out of 100 are in the wiki_codes\n",
    "# Maybe some were lost in cleaning or were not read by pandas.read_html\n",
    "wiki_codes.isin(bist_100).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d1f8918",
   "metadata": {},
   "source": [
    "Download log (Do not run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "dd3ce925",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- USDTR.IS: 1d data not available for startTime=-2208994789 and endTime=1669575868. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker AKGUV.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- AKGUV.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker AKFEN.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- AKFEN.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker AKSEL.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- AKSEL.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker AKPAZ.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- AKPAZ.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker ALYAG.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- ALYAG.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker ANACM.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- ANACM.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker ANSA.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- ANSA.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker ARTOG.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- ARTOG.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker ARBUL.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- ARBUL.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker ASCEL.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- ASCEL.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker ASLAN.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- ASLAN.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker ASYAB.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- ASYAB.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker ATAC.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- ATAC.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker ATPET.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- ATPET.IS: No data found for this date range, symbol may be delisted\n",
      "- AVIVA.IS: 1d data not available for startTime=-2208994789 and endTime=1669575899. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker BMEKS.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- BMEKS.IS: No data found for this date range, symbol may be delisted\n",
      "- BISAS.IS: 1d data not available for startTime=-2208994789 and endTime=1669575915. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker BOLUC.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- BOLUC.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker CLKHO.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- CLKHO.IS: No data found for this date range, symbol may be delisted\n",
      "- COMDO.IS: 1d data not available for startTime=-2208994789 and endTime=1669575925. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker DENCM.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- DENCM.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker DGKLB.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- DGKLB.IS: No data found for this date range, symbol may be delisted\n",
      "- DJIST.IS: 1d data not available for startTime=-2208994789 and endTime=1669575942. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- ESEMS.IS: 1d data not available for startTime=-2208994789 and endTime=1669575956. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker EVKUR.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- EVKUR.IS: No data found for this date range, symbol may be delisted\n",
      "- FVORI.IS: 1d data not available for startTime=-2208994789 and endTime=1669575957. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker FONFK.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- FONFK.IS: No data found for this date range, symbol may be delisted\n",
      "- FFKRL.IS: 1d data not available for startTime=-2208994789 and endTime=1669575961. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- FBIST.IS: 1d data not available for startTime=-2208994789 and endTime=1669575963. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- GDKGS.IS: 1d data not available for startTime=-2208994789 and endTime=1669575966. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker GUSGR.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- GUSGR.IS: No data found for this date range, symbol may be delisted\n",
      "- HZNDR.IS: 1d data not available for startTime=-2208994789 and endTime=1669575974. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- IDAS.IS: 1d data not available for startTime=-2208994789 and endTime=1669575975. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker IHMAD.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- IHMAD.IS: No data found for this date range, symbol may be delisted\n",
      "- GLDTR.IS: 1d data not available for startTime=-2208994789 and endTime=1669575982. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- ISGSY.IS: 1d data not available for startTime=-2208994789 and endTime=1669575984. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker KPHOL.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- KPHOL.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker KRATL.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- KRATL.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker KRSAN.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- KRSAN.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker KILER.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- KILER.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker KOMHL.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- KOMHL.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker LATEK.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- LATEK.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker MANGO.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- MANGO.IS: No data found for this date range, symbol may be delisted\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Got error from yahoo api for ticker MRDIN.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- MRDIN.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker MRTGG.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- MRTGG.IS: No data found for this date range, symbol may be delisted\n",
      "- NTTUR.IS: 1d data not available for startTime=-2208994789 and endTime=1669576021. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- PTOFS.IS: 1d data not available for startTime=-2208994789 and endTime=1669576027. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker TKSTL.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- TKSTL.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker TRCAM.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- TRCAM.IS: No data found for this date range, symbol may be delisted\n",
      "- TRNSK.IS: 1d data not available for startTime=-2208994789 and endTime=1669576037. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker UNYEC.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- UNYEC.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker VKBYO.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- VKBYO.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker VKNG.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- VKNG.IS: No data found for this date range, symbol may be delisted\n",
      "- YKSGR.IS: 1d data not available for startTime=-2208994789 and endTime=1669576042. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker YKBYO.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- YKBYO.IS: No data found for this date range, symbol may be delisted\n",
      "- YAZIC.IS: 1d data not available for startTime=-2208994789 and endTime=1669576043. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker ADANA.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- ADANA.IS: No data found for this date range, symbol may be delisted\n",
      "Failed to get ticker 'ADBGR.IS' reason: Expecting value: line 1 column 1 (char 0)\n",
      "- ADBGR.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker ADNAC.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- ADNAC.IS: No data found for this date range, symbol may be delisted\n",
      "- ALNTF.IS: 1d data not available for startTime=-2208994789 and endTime=1669576045. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- ARFYO.IS: 1d data not available for startTime=-2208994789 and endTime=1669576046. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- DJIMT.IS: 1d data not available for startTime=-2208994789 and endTime=1669576047. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- KTLME.IS: 1d data not available for startTime=-2208994789 and endTime=1669576047. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- BOYNR.IS: 1d data not available for startTime=-2208994789 and endTime=1669576048. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker BOYP.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- BOYP.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker CARFA.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- CARFA.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker CARFB.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- CARFB.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker DENİZ.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- DENİZ.IS: No data found for this date range, symbol may be delisted\n",
      "- DYHOL.IS: 1d data not available for startTime=-2208994789 and endTime=1669576054. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker DGZTE.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- DGZTE.IS: No data found for this date range, symbol may be delisted\n",
      "- ECBYO.IS: 1d data not available for startTime=-2208994789 and endTime=1669576056. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- EGLYO.IS: 1d data not available for startTime=-2208994789 and endTime=1669576057. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker EUROM.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- EUROM.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker EMBYO.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- EMBYO.IS: No data found for this date range, symbol may be delisted\n",
      "- FINBN.IS: 1d data not available for startTime=-2208994789 and endTime=1669576061. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- IST30.IS: 1d data not available for startTime=-2208994789 and endTime=1669576062. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker GT30.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- GT30.IS: No data found for this date range, symbol may be delisted\n",
      "- GDKYO.IS: 1d data not available for startTime=-2208994789 and endTime=1669576064. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker GENPWR.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- GENPWR.IS: No data found for this date range, symbol may be delisted\n",
      "Got error from yahoo api for ticker ISYHO.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- ISYHO.IS: No data found for this date range, symbol may be delisted\n",
      "- ISY30.IS: 1d data not available for startTime=-2208994789 and endTime=1669576069. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- ISUVT.IS: 1d data not available for startTime=-2208994789 and endTime=1669576071. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- GOLDP.IS: 1d data not available for startTime=-2208994789 and endTime=1669576074. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- SLVRP.IS: 1d data not available for startTime=-2208994789 and endTime=1669576074. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "- MUTLU.IS: 1d data not available for startTime=-2208994789 and endTime=1669576077. Only 100 years worth of day granularity data are allowed to be fetched per request.\n",
      "Got error from yahoo api for ticker MUTGG.IS, Error: {'code': 'Not Found', 'description': 'No data found, symbol may be delisted'}\n",
      "- MUTGG.IS: No data found for this date range, symbol may be delisted\n"
     ]
    }
   ],
   "source": [
    "# Some companies were not downloaded for different reasons (see log below)\n",
    "# download_yahoo_data(codes)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ff11da2",
   "metadata": {},
   "source": [
    "Creating a summary of the datasets downloaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "89df1478",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_file_summary(path='data/yahoo_finance_unmerged'):\n",
    "    \n",
    "    files = os.listdir(path)\n",
    "    \n",
    "    data_details = {\n",
    "        'name': [],\n",
    "        'start_date': [],\n",
    "        'end_date': [],\n",
    "        'size': [],\n",
    "        'na_in_close': []\n",
    "    }\n",
    "    \n",
    "    for file in files:\n",
    "        file_path = os.path.join(path, file)\n",
    "        df = pd.read_csv(file_path, parse_dates=['date'])\n",
    "    \n",
    "        data_details['name'].append(file[:-4])\n",
    "        data_details['start_date'].append(df['date'].min().strftime('%Y-%m-%d'))\n",
    "        data_details['end_date'].append(df['date'].max().strftime('%Y-%m-%d'))\n",
    "        data_details['size'].append(df.shape[0])\n",
    "        data_details['na_in_close'].append(df['close'].isna().sum())\n",
    "        \n",
    "    return data_details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fce41d78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>start_date</th>\n",
       "      <th>end_date</th>\n",
       "      <th>size</th>\n",
       "      <th>na_in_close</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ACSEL</td>\n",
       "      <td>2012-07-06</td>\n",
       "      <td>2022-11-25</td>\n",
       "      <td>2680</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ADEL</td>\n",
       "      <td>2000-05-10</td>\n",
       "      <td>2022-11-25</td>\n",
       "      <td>5792</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ADESE</td>\n",
       "      <td>2011-11-03</td>\n",
       "      <td>2022-11-25</td>\n",
       "      <td>2856</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AEFES</td>\n",
       "      <td>2000-05-10</td>\n",
       "      <td>2022-11-25</td>\n",
       "      <td>5709</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AFYON</td>\n",
       "      <td>2000-05-10</td>\n",
       "      <td>2022-11-25</td>\n",
       "      <td>5774</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    name start_date   end_date  size  na_in_close\n",
       "0  ACSEL 2012-07-06 2022-11-25  2680            0\n",
       "1   ADEL 2000-05-10 2022-11-25  5792            0\n",
       "2  ADESE 2011-11-03 2022-11-25  2856            0\n",
       "3  AEFES 2000-05-10 2022-11-25  5709            0\n",
       "4  AFYON 2000-05-10 2022-11-25  5774            0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# data_details = create_file_summary()\n",
    "# df = pd.DataFrame.from_dict(data_details)\n",
    "# df.to_csv('data/yahoo_finance/data_summary.csv', index=False)\n",
    "\n",
    "df = pd.read_csv('data/yahoo_finance/data_summary.csv', parse_dates=['start_date', 'end_date'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "307c1237",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     284.000000\n",
       "mean     4821.961268\n",
       "std      1299.515768\n",
       "min        83.000000\n",
       "25%      3941.250000\n",
       "50%      5648.000000\n",
       "75%      5713.000000\n",
       "max      5824.000000\n",
       "Name: size, dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['size'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb300d71",
   "metadata": {},
   "source": [
    "### Merging files\n",
    "Do not run this code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "43678e1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# path = 'data/yahoo_finance'\n",
    "# files = os.listdir(path)\n",
    "\n",
    "# df_list = []\n",
    "\n",
    "# for file in files:\n",
    "#     file_path = os.path.join(path, file)\n",
    "#     df = pd.read_csv(file_path)\n",
    "    \n",
    "#     # Adding company name column\n",
    "#     df.insert(0, 'name', file.replace('.csv', ''))\n",
    "#     df_list.append(df)\n",
    "    \n",
    "# merged_df = pd.concat(df_list, ignore_index=True)\n",
    "# merged_df.to_csv('data/yahoo_finance/merged_yahoo_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d7ce268",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}